1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 #include <sys/types.h>
  27 #include <sys/systm.h>
  28 #include <sys/ddi.h>
  29 #include <sys/sysmacros.h>
  30 #include <sys/archsystm.h>
  31 #include <sys/vmsystm.h>
  32 #include <sys/machparam.h>
  33 #include <sys/machsystm.h>
  34 #include <sys/machthread.h>
  35 #include <sys/cpu.h>
  36 #include <sys/cmp.h>
  37 #include <sys/elf_SPARC.h>
  38 #include <vm/vm_dep.h>
  39 #include <vm/hat_sfmmu.h>
  40 #include <vm/seg_kpm.h>
  41 #include <sys/cpuvar.h>
  42 #include <sys/cheetahregs.h>
  43 #include <sys/us3_module.h>
  44 #include <sys/async.h>
  45 #include <sys/cmn_err.h>
  46 #include <sys/debug.h>
  47 #include <sys/dditypes.h>
  48 #include <sys/prom_debug.h>
  49 #include <sys/prom_plat.h>
  50 #include <sys/cpu_module.h>
  51 #include <sys/sysmacros.h>
  52 #include <sys/intreg.h>
  53 #include <sys/clock.h>
  54 #include <sys/platform_module.h>
  55 #include <sys/machtrap.h>
  56 #include <sys/ontrap.h>
  57 #include <sys/panic.h>
  58 #include <sys/memlist.h>
  59 #include <sys/bootconf.h>
  60 #include <sys/ivintr.h>
  61 #include <sys/atomic.h>
  62 #include <sys/taskq.h>
  63 #include <sys/note.h>
  64 #include <sys/ndifm.h>
  65 #include <sys/ddifm.h>
  66 #include <sys/fm/protocol.h>
  67 #include <sys/fm/util.h>
  68 #include <sys/fm/cpu/UltraSPARC-III.h>
  69 #include <sys/fpras_impl.h>
  70 #include <sys/dtrace.h>
  71 #include <sys/watchpoint.h>
  72 #include <sys/plat_ecc_unum.h>
  73 #include <sys/cyclic.h>
  74 #include <sys/errorq.h>
  75 #include <sys/errclassify.h>
  76 #include <sys/pghw.h>
  77 #include <sys/clock_impl.h>
  78 
  79 #ifdef  CHEETAHPLUS_ERRATUM_25
  80 #include <sys/xc_impl.h>
  81 #endif  /* CHEETAHPLUS_ERRATUM_25 */
  82 
  83 ch_cpu_logout_t clop_before_flush;
  84 ch_cpu_logout_t clop_after_flush;
  85 uint_t  flush_retries_done = 0;
  86 /*
  87  * Note that 'Cheetah PRM' refers to:
  88  *   SPARC V9 JPS1 Implementation Supplement: Sun UltraSPARC-III
  89  */
  90 
  91 /*
  92  * Per CPU pointers to physical address of TL>0 logout data areas.
  93  * These pointers have to be in the kernel nucleus to avoid MMU
  94  * misses.
  95  */
  96 uint64_t ch_err_tl1_paddrs[NCPU];
  97 
  98 /*
  99  * One statically allocated structure to use during startup/DR
 100  * to prevent unnecessary panics.
 101  */
 102 ch_err_tl1_data_t ch_err_tl1_data;
 103 
 104 /*
 105  * Per CPU pending error at TL>0, used by level15 softint handler
 106  */
 107 uchar_t ch_err_tl1_pending[NCPU];
 108 
 109 /*
 110  * For deferred CE re-enable after trap.
 111  */
 112 taskq_t         *ch_check_ce_tq;
 113 
 114 /*
 115  * Internal functions.
 116  */
 117 static int cpu_async_log_err(void *flt, errorq_elem_t *eqep);
 118 static void cpu_log_diag_info(ch_async_flt_t *ch_flt);
 119 static void cpu_queue_one_event(ch_async_flt_t *ch_flt, char *reason,
 120     ecc_type_to_info_t *eccp, ch_diag_data_t *cdp);
 121 static int cpu_flt_in_memory_one_event(ch_async_flt_t *ch_flt,
 122     uint64_t t_afsr_bit);
 123 static int clear_ecc(struct async_flt *ecc);
 124 #if defined(CPU_IMP_ECACHE_ASSOC)
 125 static int cpu_ecache_line_valid(ch_async_flt_t *ch_flt);
 126 #endif
 127 int cpu_ecache_set_size(struct cpu *cp);
 128 static int cpu_ectag_line_invalid(int cachesize, uint64_t tag);
 129 int cpu_ectag_pa_to_subblk(int cachesize, uint64_t subaddr);
 130 uint64_t cpu_ectag_to_pa(int setsize, uint64_t tag);
 131 int cpu_ectag_pa_to_subblk_state(int cachesize,
 132                                 uint64_t subaddr, uint64_t tag);
 133 static void cpu_flush_ecache_line(ch_async_flt_t *ch_flt);
 134 static int afsr_to_afar_status(uint64_t afsr, uint64_t afsr_bit);
 135 static int afsr_to_esynd_status(uint64_t afsr, uint64_t afsr_bit);
 136 static int afsr_to_msynd_status(uint64_t afsr, uint64_t afsr_bit);
 137 static int afsr_to_synd_status(uint_t cpuid, uint64_t afsr, uint64_t afsr_bit);
 138 static int synd_to_synd_code(int synd_status, ushort_t synd, uint64_t afsr_bit);
 139 static int cpu_get_mem_unum_synd(int synd_code, struct async_flt *, char *buf);
 140 static void cpu_uninit_ecache_scrub_dr(struct cpu *cp);
 141 static void cpu_scrubphys(struct async_flt *aflt);
 142 static void cpu_payload_add_aflt(struct async_flt *, nvlist_t *, nvlist_t *,
 143     int *, int *);
 144 static void cpu_payload_add_ecache(struct async_flt *, nvlist_t *);
 145 static void cpu_ereport_init(struct async_flt *aflt);
 146 static int cpu_check_secondary_errors(ch_async_flt_t *, uint64_t, uint64_t);
 147 static uint8_t cpu_flt_bit_to_plat_error(struct async_flt *aflt);
 148 static void cpu_log_fast_ecc_error(caddr_t tpc, int priv, int tl, uint64_t ceen,
 149     uint64_t nceen, ch_cpu_logout_t *clop);
 150 static int cpu_ce_delayed_ec_logout(uint64_t);
 151 static int cpu_matching_ecache_line(uint64_t, void *, int, int *);
 152 static int cpu_error_is_ecache_data(int, uint64_t);
 153 static void cpu_fmri_cpu_set(nvlist_t *, int);
 154 static int cpu_error_to_resource_type(struct async_flt *aflt);
 155 
 156 #ifdef  CHEETAHPLUS_ERRATUM_25
 157 static int mondo_recover_proc(uint16_t, int);
 158 static void cheetah_nudge_init(void);
 159 static void cheetah_nudge_onln(void *arg, cpu_t *cpu, cyc_handler_t *hdlr,
 160     cyc_time_t *when);
 161 static void cheetah_nudge_buddy(void);
 162 #endif  /* CHEETAHPLUS_ERRATUM_25 */
 163 
 164 #if defined(CPU_IMP_L1_CACHE_PARITY)
 165 static void cpu_dcache_parity_info(ch_async_flt_t *ch_flt);
 166 static void cpu_dcache_parity_check(ch_async_flt_t *ch_flt, int index);
 167 static void cpu_record_dc_data_parity(ch_async_flt_t *ch_flt,
 168     ch_dc_data_t *dest_dcp, ch_dc_data_t *src_dcp, int way, int word);
 169 static void cpu_icache_parity_info(ch_async_flt_t *ch_flt);
 170 static void cpu_icache_parity_check(ch_async_flt_t *ch_flt, int index);
 171 static void cpu_pcache_parity_info(ch_async_flt_t *ch_flt);
 172 static void cpu_pcache_parity_check(ch_async_flt_t *ch_flt, int index);
 173 static void cpu_payload_add_dcache(struct async_flt *, nvlist_t *);
 174 static void cpu_payload_add_icache(struct async_flt *, nvlist_t *);
 175 #endif  /* CPU_IMP_L1_CACHE_PARITY */
 176 
 177 int (*p2get_mem_info)(int synd_code, uint64_t paddr,
 178     uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep,
 179     int *segsp, int *banksp, int *mcidp);
 180 
 181 /*
 182  * This table is used to determine which bit(s) is(are) bad when an ECC
 183  * error occurs.  The array is indexed by an 9-bit syndrome.  The entries
 184  * of this array have the following semantics:
 185  *
 186  *      00-127  The number of the bad bit, when only one bit is bad.
 187  *      128     ECC bit C0 is bad.
 188  *      129     ECC bit C1 is bad.
 189  *      130     ECC bit C2 is bad.
 190  *      131     ECC bit C3 is bad.
 191  *      132     ECC bit C4 is bad.
 192  *      133     ECC bit C5 is bad.
 193  *      134     ECC bit C6 is bad.
 194  *      135     ECC bit C7 is bad.
 195  *      136     ECC bit C8 is bad.
 196  *      137-143 reserved for Mtag Data and ECC.
 197  *      144(M2) Two bits are bad within a nibble.
 198  *      145(M3) Three bits are bad within a nibble.
 199  *      146(M3) Four bits are bad within a nibble.
 200  *      147(M)  Multiple bits (5 or more) are bad.
 201  *      148     NO bits are bad.
 202  * Based on "Cheetah Programmer's Reference Manual" rev 1.1, Tables 11-4,11-5.
 203  */
 204 
 205 #define C0      128
 206 #define C1      129
 207 #define C2      130
 208 #define C3      131
 209 #define C4      132
 210 #define C5      133
 211 #define C6      134
 212 #define C7      135
 213 #define C8      136
 214 #define MT0     137     /* Mtag Data bit 0 */
 215 #define MT1     138
 216 #define MT2     139
 217 #define MTC0    140     /* Mtag Check bit 0 */
 218 #define MTC1    141
 219 #define MTC2    142
 220 #define MTC3    143
 221 #define M2      144
 222 #define M3      145
 223 #define M4      146
 224 #define M       147
 225 #define NA      148
 226 #if defined(JALAPENO) || defined(SERRANO)
 227 #define S003    149     /* Syndrome 0x003 => likely from CPU/EDU:ST/FRU/BP */
 228 #define S003MEM 150     /* Syndrome 0x003 => likely from WDU/WBP */
 229 #define SLAST   S003MEM /* last special syndrome */
 230 #else /* JALAPENO || SERRANO */
 231 #define S003    149     /* Syndrome 0x003 => likely from EDU:ST */
 232 #define S071    150     /* Syndrome 0x071 => likely from WDU/CPU */
 233 #define S11C    151     /* Syndrome 0x11c => likely from BERR/DBERR */
 234 #define SLAST   S11C    /* last special syndrome */
 235 #endif /* JALAPENO || SERRANO */
 236 #if defined(JALAPENO) || defined(SERRANO)
 237 #define BPAR0   152     /* syndrom 152 through 167 for bus parity */
 238 #define BPAR15  167
 239 #endif  /* JALAPENO || SERRANO */
 240 
 241 static uint8_t ecc_syndrome_tab[] =
 242 {
 243 NA,  C0,  C1, S003, C2,  M2,  M3,  47,  C3,  M2,  M2,  53,  M2,  41,  29,   M,
 244 C4,   M,   M,  50,  M2,  38,  25,  M2,  M2,  33,  24,  M2,  11,   M,  M2,  16,
 245 C5,   M,   M,  46,  M2,  37,  19,  M2,   M,  31,  32,   M,   7,  M2,  M2,  10,
 246 M2,  40,  13,  M2,  59,   M,  M2,  66,   M,  M2,  M2,   0,  M2,  67,  71,   M,
 247 C6,   M,   M,  43,   M,  36,  18,   M,  M2,  49,  15,   M,  63,  M2,  M2,   6,
 248 M2,  44,  28,  M2,   M,  M2,  M2,  52,  68,  M2,  M2,  62,  M2,  M3,  M3,  M4,
 249 M2,  26, 106,  M2,  64,   M,  M2,   2, 120,   M,  M2,  M3,   M,  M3,  M3,  M4,
 250 #if defined(JALAPENO) || defined(SERRANO)
 251 116, M2,  M2,  M3,  M2,  M3,   M,  M4,  M2,  58,  54,  M2,   M,  M4,  M4,  M3,
 252 #else   /* JALAPENO || SERRANO */
 253 116, S071, M2,  M3,  M2,  M3,   M,  M4,  M2,  58,  54,  M2,   M,  M4,  M4,  M3,
 254 #endif  /* JALAPENO || SERRANO */
 255 C7,  M2,   M,  42,   M,  35,  17,  M2,   M,  45,  14,  M2,  21,  M2,  M2,   5,
 256 M,   27,   M,   M,  99,   M,   M,   3, 114,  M2,  M2,  20,  M2,  M3,  M3,   M,
 257 M2,  23, 113,  M2, 112,  M2,   M,  51,  95,   M,  M2,  M3,  M2,  M3,  M3,  M2,
 258 103,  M,  M2,  M3,  M2,  M3,  M3,  M4,  M2,  48,   M,   M,  73,  M2,   M,  M3,
 259 M2,  22, 110,  M2, 109,  M2,   M,   9, 108,  M2,   M,  M3,  M2,  M3,  M3,   M,
 260 102, M2,   M,   M,  M2,  M3,  M3,   M,  M2,  M3,  M3,  M2,   M,  M4,   M,  M3,
 261 98,   M,  M2,  M3,  M2,   M,  M3,  M4,  M2,  M3,  M3,  M4,  M3,   M,   M,   M,
 262 M2,  M3,  M3,   M,  M3,   M,   M,   M,  56,  M4,   M,  M3,  M4,   M,   M,   M,
 263 C8,   M,  M2,  39,   M,  34, 105,  M2,   M,  30, 104,   M, 101,   M,   M,   4,
 264 #if defined(JALAPENO) || defined(SERRANO)
 265 M,    M, 100,   M,  83,   M,  M2,  12,  87,   M,   M,  57,  M2,   M,  M3,   M,
 266 #else   /* JALAPENO || SERRANO */
 267 M,    M, 100,   M,  83,   M,  M2,  12,  87,   M,   M,  57, S11C,  M,  M3,   M,
 268 #endif  /* JALAPENO || SERRANO */
 269 M2,  97,  82,  M2,  78,  M2,  M2,   1,  96,   M,   M,   M,   M,   M,  M3,  M2,
 270 94,   M,  M2,  M3,  M2,   M,  M3,   M,  M2,   M,  79,   M,  69,   M,  M4,   M,
 271 M2,  93,  92,   M,  91,   M,  M2,   8,  90,  M2,  M2,   M,   M,   M,   M,  M4,
 272 89,   M,   M,  M3,  M2,  M3,  M3,   M,   M,   M,  M3,  M2,  M3,  M2,   M,  M3,
 273 86,   M,  M2,  M3,  M2,   M,  M3,   M,  M2,   M,  M3,   M,  M3,   M,   M,  M3,
 274 M,    M,  M3,  M2,  M3,  M2,  M4,   M,  60,   M,  M2,  M3,  M4,   M,   M,  M2,
 275 M2,  88,  85,  M2,  84,   M,  M2,  55,  81,  M2,  M2,  M3,  M2,  M3,  M3,  M4,
 276 77,   M,   M,   M,  M2,  M3,   M,   M,  M2,  M3,  M3,  M4,  M3,  M2,   M,   M,
 277 74,   M,  M2,  M3,   M,   M,  M3,   M,   M,   M,  M3,   M,  M3,   M,  M4,  M3,
 278 M2,  70, 107,  M4,  65,  M2,  M2,   M, 127,   M,   M,   M,  M2,  M3,  M3,   M,
 279 80,  M2,  M2,  72,   M, 119, 118,   M,  M2, 126,  76,   M, 125,   M,  M4,  M3,
 280 M2, 115, 124,   M,  75,   M,   M,  M3,  61,   M,  M4,   M,  M4,   M,   M,   M,
 281 M,  123, 122,  M4, 121,  M4,   M,  M3, 117,  M2,  M2,  M3,  M4,  M3,   M,   M,
 282 111,  M,   M,   M,  M4,  M3,  M3,   M,   M,   M,  M3,   M,  M3,  M2,   M,   M
 283 };
 284 
 285 #define ESYND_TBL_SIZE  (sizeof (ecc_syndrome_tab) / sizeof (uint8_t))
 286 
 287 #if !(defined(JALAPENO) || defined(SERRANO))
 288 /*
 289  * This table is used to determine which bit(s) is(are) bad when a Mtag
 290  * error occurs.  The array is indexed by an 4-bit ECC syndrome. The entries
 291  * of this array have the following semantics:
 292  *
 293  *      -1      Invalid mtag syndrome.
 294  *      137     Mtag Data 0 is bad.
 295  *      138     Mtag Data 1 is bad.
 296  *      139     Mtag Data 2 is bad.
 297  *      140     Mtag ECC 0 is bad.
 298  *      141     Mtag ECC 1 is bad.
 299  *      142     Mtag ECC 2 is bad.
 300  *      143     Mtag ECC 3 is bad.
 301  * Based on "Cheetah Programmer's Reference Manual" rev 1.1, Tables 11-6.
 302  */
 303 short mtag_syndrome_tab[] =
 304 {
 305 NA, MTC0, MTC1, M2, MTC2, M2, M2, MT0, MTC3, M2, M2,  MT1, M2, MT2, M2, M2
 306 };
 307 
 308 #define MSYND_TBL_SIZE  (sizeof (mtag_syndrome_tab) / sizeof (short))
 309 
 310 #else /* !(JALAPENO || SERRANO) */
 311 
 312 #define BSYND_TBL_SIZE  16
 313 
 314 #endif /* !(JALAPENO || SERRANO) */
 315 
 316 /*
 317  * Virtual Address bit flag in the data cache. This is actually bit 2 in the
 318  * dcache data tag.
 319  */
 320 #define VA13    INT64_C(0x0000000000000002)
 321 
 322 /*
 323  * Types returned from cpu_error_to_resource_type()
 324  */
 325 #define ERRTYPE_UNKNOWN         0
 326 #define ERRTYPE_CPU             1
 327 #define ERRTYPE_MEMORY          2
 328 #define ERRTYPE_ECACHE_DATA     3
 329 
 330 /*
 331  * CE initial classification and subsequent action lookup table
 332  */
 333 static ce_dispact_t ce_disp_table[CE_INITDISPTBL_SIZE];
 334 static int ce_disp_inited;
 335 
 336 /*
 337  * Set to disable leaky and partner check for memory correctables
 338  */
 339 int ce_xdiag_off;
 340 
 341 /*
 342  * The following are not incremented atomically so are indicative only
 343  */
 344 static int ce_xdiag_drops;
 345 static int ce_xdiag_lkydrops;
 346 static int ce_xdiag_ptnrdrops;
 347 static int ce_xdiag_bad;
 348 
 349 /*
 350  * CE leaky check callback structure
 351  */
 352 typedef struct {
 353         struct async_flt *lkycb_aflt;
 354         errorq_t *lkycb_eqp;
 355         errorq_elem_t *lkycb_eqep;
 356 } ce_lkychk_cb_t;
 357 
 358 /*
 359  * defines for various ecache_flush_flag's
 360  */
 361 #define ECACHE_FLUSH_LINE       1
 362 #define ECACHE_FLUSH_ALL        2
 363 
 364 /*
 365  * STICK sync
 366  */
 367 #define STICK_ITERATION 10
 368 #define MAX_TSKEW       1
 369 #define EV_A_START      0
 370 #define EV_A_END        1
 371 #define EV_B_START      2
 372 #define EV_B_END        3
 373 #define EVENTS          4
 374 
 375 static int64_t stick_iter = STICK_ITERATION;
 376 static int64_t stick_tsk = MAX_TSKEW;
 377 
 378 typedef enum {
 379         EVENT_NULL = 0,
 380         SLAVE_START,
 381         SLAVE_CONT,
 382         MASTER_START
 383 } event_cmd_t;
 384 
 385 static volatile event_cmd_t stick_sync_cmd = EVENT_NULL;
 386 static int64_t timestamp[EVENTS];
 387 static volatile int slave_done;
 388 
 389 #ifdef DEBUG
 390 #define DSYNC_ATTEMPTS 64
 391 typedef struct {
 392         int64_t skew_val[DSYNC_ATTEMPTS];
 393 } ss_t;
 394 
 395 ss_t stick_sync_stats[NCPU];
 396 #endif /* DEBUG */
 397 
 398 uint_t cpu_impl_dual_pgsz = 0;
 399 #if defined(CPU_IMP_DUAL_PAGESIZE)
 400 uint_t disable_dual_pgsz = 0;
 401 #endif  /* CPU_IMP_DUAL_PAGESIZE */
 402 
 403 /*
 404  * Save the cache bootup state for use when internal
 405  * caches are to be re-enabled after an error occurs.
 406  */
 407 uint64_t cache_boot_state;
 408 
 409 /*
 410  * PA[22:0] represent Displacement in Safari configuration space.
 411  */
 412 uint_t  root_phys_addr_lo_mask = 0x7fffffu;
 413 
 414 bus_config_eclk_t bus_config_eclk[] = {
 415 #if defined(JALAPENO) || defined(SERRANO)
 416         {JBUS_CONFIG_ECLK_1_DIV, JBUS_CONFIG_ECLK_1},
 417         {JBUS_CONFIG_ECLK_2_DIV, JBUS_CONFIG_ECLK_2},
 418         {JBUS_CONFIG_ECLK_32_DIV, JBUS_CONFIG_ECLK_32},
 419 #else /* JALAPENO || SERRANO */
 420         {SAFARI_CONFIG_ECLK_1_DIV, SAFARI_CONFIG_ECLK_1},
 421         {SAFARI_CONFIG_ECLK_2_DIV, SAFARI_CONFIG_ECLK_2},
 422         {SAFARI_CONFIG_ECLK_32_DIV, SAFARI_CONFIG_ECLK_32},
 423 #endif /* JALAPENO || SERRANO */
 424         {0, 0}
 425 };
 426 
 427 /*
 428  * Interval for deferred CEEN reenable
 429  */
 430 int cpu_ceen_delay_secs = CPU_CEEN_DELAY_SECS;
 431 
 432 /*
 433  * set in /etc/system to control logging of user BERR/TO's
 434  */
 435 int cpu_berr_to_verbose = 0;
 436 
 437 /*
 438  * set to 0 in /etc/system to defer CEEN reenable for all CEs
 439  */
 440 uint64_t cpu_ce_not_deferred = CPU_CE_NOT_DEFERRED;
 441 uint64_t cpu_ce_not_deferred_ext = CPU_CE_NOT_DEFERRED_EXT;
 442 
 443 /*
 444  * Set of all offline cpus
 445  */
 446 cpuset_t cpu_offline_set;
 447 
 448 static void cpu_delayed_check_ce_errors(void *);
 449 static void cpu_check_ce_errors(void *);
 450 void cpu_error_ecache_flush(ch_async_flt_t *);
 451 static int cpu_error_ecache_flush_required(ch_async_flt_t *);
 452 static void cpu_log_and_clear_ce(ch_async_flt_t *);
 453 void cpu_ce_detected(ch_cpu_errors_t *, int);
 454 
 455 /*
 456  * CE Leaky check timeout in microseconds.  This is chosen to be twice the
 457  * memory refresh interval of current DIMMs (64ms).  After initial fix that
 458  * gives at least one full refresh cycle in which the cell can leak
 459  * (whereafter further refreshes simply reinforce any incorrect bit value).
 460  */
 461 clock_t cpu_ce_lkychk_timeout_usec = 128000;
 462 
 463 /*
 464  * CE partner check partner caching period in seconds
 465  */
 466 int cpu_ce_ptnr_cachetime_sec = 60;
 467 
 468 /*
 469  * Sets trap table entry ttentry by overwriting eight instructions from ttlabel
 470  */
 471 #define CH_SET_TRAP(ttentry, ttlabel)                   \
 472                 bcopy((const void *)&ttlabel, &ttentry, 32);            \
 473                 flush_instr_mem((caddr_t)&ttentry, 32);
 474 
 475 static int min_ecache_size;
 476 static uint_t priv_hcl_1;
 477 static uint_t priv_hcl_2;
 478 static uint_t priv_hcl_4;
 479 static uint_t priv_hcl_8;
 480 
 481 void
 482 cpu_setup(void)
 483 {
 484         extern int at_flags;
 485         extern int cpc_has_overflow_intr;
 486 
 487         /*
 488          * Setup chip-specific trap handlers.
 489          */
 490         cpu_init_trap();
 491 
 492         cache |= (CACHE_VAC | CACHE_PTAG | CACHE_IOCOHERENT);
 493 
 494         at_flags = EF_SPARC_32PLUS | EF_SPARC_SUN_US1 | EF_SPARC_SUN_US3;
 495 
 496         /*
 497          * save the cache bootup state.
 498          */
 499         cache_boot_state = get_dcu() & DCU_CACHE;
 500 
 501         /*
 502          * Due to the number of entries in the fully-associative tlb
 503          * this may have to be tuned lower than in spitfire.
 504          */
 505         pp_slots = MIN(8, MAXPP_SLOTS);
 506 
 507         /*
 508          * Block stores do not invalidate all pages of the d$, pagecopy
 509          * et. al. need virtual translations with virtual coloring taken
 510          * into consideration.  prefetch/ldd will pollute the d$ on the
 511          * load side.
 512          */
 513         pp_consistent_coloring = PPAGE_STORE_VCOLORING | PPAGE_LOADS_POLLUTE;
 514 
 515         if (use_page_coloring) {
 516                 do_pg_coloring = 1;
 517         }
 518 
 519         isa_list =
 520             "sparcv9+vis2 sparcv9+vis sparcv9 "
 521             "sparcv8plus+vis2 sparcv8plus+vis sparcv8plus "
 522             "sparcv8 sparcv8-fsmuld sparcv7 sparc";
 523 
 524         /*
 525          * On Panther-based machines, this should
 526          * also include AV_SPARC_POPC too
 527          */
 528         cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2;
 529 
 530         /*
 531          * On cheetah, there's no hole in the virtual address space
 532          */
 533         hole_start = hole_end = 0;
 534 
 535         /*
 536          * The kpm mapping window.
 537          * kpm_size:
 538          *      The size of a single kpm range.
 539          *      The overall size will be: kpm_size * vac_colors.
 540          * kpm_vbase:
 541          *      The virtual start address of the kpm range within the kernel
 542          *      virtual address space. kpm_vbase has to be kpm_size aligned.
 543          */
 544         kpm_size = (size_t)(8ull * 1024 * 1024 * 1024 * 1024); /* 8TB */
 545         kpm_size_shift = 43;
 546         kpm_vbase = (caddr_t)0x8000000000000000ull; /* 8EB */
 547         kpm_smallpages = 1;
 548 
 549         /*
 550          * The traptrace code uses either %tick or %stick for
 551          * timestamping.  We have %stick so we can use it.
 552          */
 553         traptrace_use_stick = 1;
 554 
 555         /*
 556          * Cheetah has a performance counter overflow interrupt
 557          */
 558         cpc_has_overflow_intr = 1;
 559 
 560 #if defined(CPU_IMP_DUAL_PAGESIZE)
 561         /*
 562          * Use Cheetah+ and later dual page size support.
 563          */
 564         if (!disable_dual_pgsz) {
 565                 cpu_impl_dual_pgsz = 1;
 566         }
 567 #endif  /* CPU_IMP_DUAL_PAGESIZE */
 568 
 569         /*
 570          * Declare that this architecture/cpu combination does fpRAS.
 571          */
 572         fpras_implemented = 1;
 573 
 574         /*
 575          * Setup CE lookup table
 576          */
 577         CE_INITDISPTBL_POPULATE(ce_disp_table);
 578         ce_disp_inited = 1;
 579 }
 580 
 581 /*
 582  * Called by setcpudelay
 583  */
 584 void
 585 cpu_init_tick_freq(void)
 586 {
 587         /*
 588          * For UltraSPARC III and beyond we want to use the
 589          * system clock rate as the basis for low level timing,
 590          * due to support of mixed speed CPUs and power managment.
 591          */
 592         if (system_clock_freq == 0)
 593                 cmn_err(CE_PANIC, "setcpudelay: invalid system_clock_freq");
 594 
 595         sys_tick_freq = system_clock_freq;
 596 }
 597 
 598 #ifdef CHEETAHPLUS_ERRATUM_25
 599 /*
 600  * Tunables
 601  */
 602 int cheetah_bpe_off = 0;
 603 int cheetah_sendmondo_recover = 1;
 604 int cheetah_sendmondo_fullscan = 0;
 605 int cheetah_sendmondo_recover_delay = 5;
 606 
 607 #define CHEETAH_LIVELOCK_MIN_DELAY      1
 608 
 609 /*
 610  * Recovery Statistics
 611  */
 612 typedef struct cheetah_livelock_entry   {
 613         int cpuid;              /* fallen cpu */
 614         int buddy;              /* cpu that ran recovery */
 615         clock_t lbolt;          /* when recovery started */
 616         hrtime_t recovery_time; /* time spent in recovery */
 617 } cheetah_livelock_entry_t;
 618 
 619 #define CHEETAH_LIVELOCK_NENTRY 32
 620 
 621 cheetah_livelock_entry_t cheetah_livelock_hist[CHEETAH_LIVELOCK_NENTRY];
 622 int cheetah_livelock_entry_nxt;
 623 
 624 #define CHEETAH_LIVELOCK_ENTRY_NEXT(statp)      {                       \
 625         statp = cheetah_livelock_hist + cheetah_livelock_entry_nxt;     \
 626         if (++cheetah_livelock_entry_nxt >= CHEETAH_LIVELOCK_NENTRY) {       \
 627                 cheetah_livelock_entry_nxt = 0;                         \
 628         }                                                               \
 629 }
 630 
 631 #define CHEETAH_LIVELOCK_ENTRY_SET(statp, item, val)    statp->item = val
 632 
 633 struct {
 634         hrtime_t hrt;           /* maximum recovery time */
 635         int recovery;           /* recovered */
 636         int full_claimed;       /* maximum pages claimed in full recovery */
 637         int proc_entry;         /* attempted to claim TSB */
 638         int proc_tsb_scan;      /* tsb scanned */
 639         int proc_tsb_partscan;  /* tsb partially scanned */
 640         int proc_tsb_fullscan;  /* whole tsb scanned */
 641         int proc_claimed;       /* maximum pages claimed in tsb scan */
 642         int proc_user;          /* user thread */
 643         int proc_kernel;        /* kernel thread */
 644         int proc_onflt;         /* bad stack */
 645         int proc_cpu;           /* null cpu */
 646         int proc_thread;        /* null thread */
 647         int proc_proc;          /* null proc */
 648         int proc_as;            /* null as */
 649         int proc_hat;           /* null hat */
 650         int proc_hat_inval;     /* hat contents don't make sense */
 651         int proc_hat_busy;      /* hat is changing TSBs */
 652         int proc_tsb_reloc;     /* TSB skipped because being relocated */
 653         int proc_cnum_bad;      /* cnum out of range */
 654         int proc_cnum;          /* last cnum processed */
 655         tte_t proc_tte;         /* last tte processed */
 656 } cheetah_livelock_stat;
 657 
 658 #define CHEETAH_LIVELOCK_STAT(item)     cheetah_livelock_stat.item++
 659 
 660 #define CHEETAH_LIVELOCK_STATSET(item, value)           \
 661         cheetah_livelock_stat.item = value
 662 
 663 #define CHEETAH_LIVELOCK_MAXSTAT(item, value)   {       \
 664         if (value > cheetah_livelock_stat.item)              \
 665                 cheetah_livelock_stat.item = value;     \
 666 }
 667 
 668 /*
 669  * Attempt to recover a cpu by claiming every cache line as saved
 670  * in the TSB that the non-responsive cpu is using. Since we can't
 671  * grab any adaptive lock, this is at best an attempt to do so. Because
 672  * we don't grab any locks, we must operate under the protection of
 673  * on_fault().
 674  *
 675  * Return 1 if cpuid could be recovered, 0 if failed.
 676  */
 677 int
 678 mondo_recover_proc(uint16_t cpuid, int bn)
 679 {
 680         label_t ljb;
 681         cpu_t *cp;
 682         kthread_t *t;
 683         proc_t *p;
 684         struct as *as;
 685         struct hat *hat;
 686         uint_t  cnum;
 687         struct tsb_info *tsbinfop;
 688         struct tsbe *tsbep;
 689         caddr_t tsbp;
 690         caddr_t end_tsbp;
 691         uint64_t paddr;
 692         uint64_t idsr;
 693         u_longlong_t pahi, palo;
 694         int pages_claimed = 0;
 695         tte_t tsbe_tte;
 696         int tried_kernel_tsb = 0;
 697         mmu_ctx_t *mmu_ctxp;
 698 
 699         CHEETAH_LIVELOCK_STAT(proc_entry);
 700 
 701         if (on_fault(&ljb)) {
 702                 CHEETAH_LIVELOCK_STAT(proc_onflt);
 703                 goto badstruct;
 704         }
 705 
 706         if ((cp = cpu[cpuid]) == NULL) {
 707                 CHEETAH_LIVELOCK_STAT(proc_cpu);
 708                 goto badstruct;
 709         }
 710 
 711         if ((t = cp->cpu_thread) == NULL) {
 712                 CHEETAH_LIVELOCK_STAT(proc_thread);
 713                 goto badstruct;
 714         }
 715 
 716         if ((p = ttoproc(t)) == NULL) {
 717                 CHEETAH_LIVELOCK_STAT(proc_proc);
 718                 goto badstruct;
 719         }
 720 
 721         if ((as = p->p_as) == NULL) {
 722                 CHEETAH_LIVELOCK_STAT(proc_as);
 723                 goto badstruct;
 724         }
 725 
 726         if ((hat = as->a_hat) == NULL) {
 727                 CHEETAH_LIVELOCK_STAT(proc_hat);
 728                 goto badstruct;
 729         }
 730 
 731         if (hat != ksfmmup) {
 732                 CHEETAH_LIVELOCK_STAT(proc_user);
 733                 if (hat->sfmmu_flags & (HAT_BUSY | HAT_SWAPPED | HAT_SWAPIN)) {
 734                         CHEETAH_LIVELOCK_STAT(proc_hat_busy);
 735                         goto badstruct;
 736                 }
 737                 tsbinfop = hat->sfmmu_tsb;
 738                 if (tsbinfop == NULL) {
 739                         CHEETAH_LIVELOCK_STAT(proc_hat_inval);
 740                         goto badstruct;
 741                 }
 742                 tsbp = tsbinfop->tsb_va;
 743                 end_tsbp = tsbp + TSB_BYTES(tsbinfop->tsb_szc);
 744         } else {
 745                 CHEETAH_LIVELOCK_STAT(proc_kernel);
 746                 tsbinfop = NULL;
 747                 tsbp = ktsb_base;
 748                 end_tsbp = tsbp + TSB_BYTES(ktsb_sz);
 749         }
 750 
 751         /* Verify as */
 752         if (hat->sfmmu_as != as) {
 753                 CHEETAH_LIVELOCK_STAT(proc_hat_inval);
 754                 goto badstruct;
 755         }
 756 
 757         mmu_ctxp = CPU_MMU_CTXP(cp);
 758         ASSERT(mmu_ctxp);
 759         cnum = hat->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum;
 760         CHEETAH_LIVELOCK_STATSET(proc_cnum, cnum);
 761 
 762         if ((cnum < 0) || (cnum == INVALID_CONTEXT) ||
 763             (cnum >= mmu_ctxp->mmu_nctxs)) {
 764                 CHEETAH_LIVELOCK_STAT(proc_cnum_bad);
 765                 goto badstruct;
 766         }
 767 
 768         do {
 769                 CHEETAH_LIVELOCK_STAT(proc_tsb_scan);
 770 
 771                 /*
 772                  * Skip TSBs being relocated.  This is important because
 773                  * we want to avoid the following deadlock scenario:
 774                  *
 775                  * 1) when we came in we set ourselves to "in recover" state.
 776                  * 2) when we try to touch TSB being relocated the mapping
 777                  *    will be in the suspended state so we'll spin waiting
 778                  *    for it to be unlocked.
 779                  * 3) when the CPU that holds the TSB mapping locked tries to
 780                  *    unlock it it will send a xtrap which will fail to xcall
 781                  *    us or the CPU we're trying to recover, and will in turn
 782                  *    enter the mondo code.
 783                  * 4) since we are still spinning on the locked mapping
 784                  *    no further progress will be made and the system will
 785                  *    inevitably hard hang.
 786                  *
 787                  * A TSB not being relocated can't begin being relocated
 788                  * while we're accessing it because we check
 789                  * sendmondo_in_recover before relocating TSBs.
 790                  */
 791                 if (hat != ksfmmup &&
 792                     (tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) {
 793                         CHEETAH_LIVELOCK_STAT(proc_tsb_reloc);
 794                         goto next_tsbinfo;
 795                 }
 796 
 797                 for (tsbep = (struct tsbe *)tsbp;
 798                     tsbep < (struct tsbe *)end_tsbp; tsbep++) {
 799                         tsbe_tte = tsbep->tte_data;
 800 
 801                         if (tsbe_tte.tte_val == 0) {
 802                                 /*
 803                                  * Invalid tte
 804                                  */
 805                                 continue;
 806                         }
 807                         if (tsbe_tte.tte_se) {
 808                                 /*
 809                                  * Don't want device registers
 810                                  */
 811                                 continue;
 812                         }
 813                         if (tsbe_tte.tte_cp == 0) {
 814                                 /*
 815                                  * Must be cached in E$
 816                                  */
 817                                 continue;
 818                         }
 819                         if (tsbep->tte_tag.tag_invalid != 0) {
 820                                 /*
 821                                  * Invalid tag, ingnore this entry.
 822                                  */
 823                                 continue;
 824                         }
 825                         CHEETAH_LIVELOCK_STATSET(proc_tte, tsbe_tte);
 826                         idsr = getidsr();
 827                         if ((idsr & (IDSR_NACK_BIT(bn) |
 828                             IDSR_BUSY_BIT(bn))) == 0) {
 829                                 CHEETAH_LIVELOCK_STAT(proc_tsb_partscan);
 830                                 goto done;
 831                         }
 832                         pahi = tsbe_tte.tte_pahi;
 833                         palo = tsbe_tte.tte_palo;
 834                         paddr = (uint64_t)((pahi << 32) |
 835                             (palo << MMU_PAGESHIFT));
 836                         claimlines(paddr, TTEBYTES(TTE_CSZ(&tsbe_tte)),
 837                             CH_ECACHE_SUBBLK_SIZE);
 838                         if ((idsr & IDSR_BUSY_BIT(bn)) == 0) {
 839                                 shipit(cpuid, bn);
 840                         }
 841                         pages_claimed++;
 842                 }
 843 next_tsbinfo:
 844                 if (tsbinfop != NULL)
 845                         tsbinfop = tsbinfop->tsb_next;
 846                 if (tsbinfop != NULL) {
 847                         tsbp = tsbinfop->tsb_va;
 848                         end_tsbp = tsbp + TSB_BYTES(tsbinfop->tsb_szc);
 849                 } else if (tsbp == ktsb_base) {
 850                         tried_kernel_tsb = 1;
 851                 } else if (!tried_kernel_tsb) {
 852                         tsbp = ktsb_base;
 853                         end_tsbp = tsbp + TSB_BYTES(ktsb_sz);
 854                         hat = ksfmmup;
 855                         tsbinfop = NULL;
 856                 }
 857         } while (tsbinfop != NULL ||
 858             ((tsbp == ktsb_base) && !tried_kernel_tsb));
 859 
 860         CHEETAH_LIVELOCK_STAT(proc_tsb_fullscan);
 861         CHEETAH_LIVELOCK_MAXSTAT(proc_claimed, pages_claimed);
 862         no_fault();
 863         idsr = getidsr();
 864         if ((idsr & (IDSR_NACK_BIT(bn) |
 865             IDSR_BUSY_BIT(bn))) == 0) {
 866                 return (1);
 867         } else {
 868                 return (0);
 869         }
 870 
 871 done:
 872         no_fault();
 873         CHEETAH_LIVELOCK_MAXSTAT(proc_claimed, pages_claimed);
 874         return (1);
 875 
 876 badstruct:
 877         no_fault();
 878         return (0);
 879 }
 880 
 881 /*
 882  * Attempt to claim ownership, temporarily, of every cache line that a
 883  * non-responsive cpu might be using.  This might kick that cpu out of
 884  * this state.
 885  *
 886  * The return value indicates to the caller if we have exhausted all recovery
 887  * techniques. If 1 is returned, it is useless to call this function again
 888  * even for a different target CPU.
 889  */
 890 int
 891 mondo_recover(uint16_t cpuid, int bn)
 892 {
 893         struct memseg *seg;
 894         uint64_t begin_pa, end_pa, cur_pa;
 895         hrtime_t begin_hrt, end_hrt;
 896         int retval = 0;
 897         int pages_claimed = 0;
 898         cheetah_livelock_entry_t *histp;
 899         uint64_t idsr;
 900 
 901         if (atomic_cas_32(&sendmondo_in_recover, 0, 1) != 0) {
 902                 /*
 903                  * Wait while recovery takes place
 904                  */
 905                 while (sendmondo_in_recover) {
 906                         drv_usecwait(1);
 907                 }
 908                 /*
 909                  * Assume we didn't claim the whole memory. If
 910                  * the target of this caller is not recovered,
 911                  * it will come back.
 912                  */
 913                 return (retval);
 914         }
 915 
 916         CHEETAH_LIVELOCK_ENTRY_NEXT(histp);
 917         CHEETAH_LIVELOCK_ENTRY_SET(histp, lbolt, LBOLT_WAITFREE);
 918         CHEETAH_LIVELOCK_ENTRY_SET(histp, cpuid, cpuid);
 919         CHEETAH_LIVELOCK_ENTRY_SET(histp, buddy, CPU->cpu_id);
 920 
 921         begin_hrt = gethrtime_waitfree();
 922         /*
 923          * First try to claim the lines in the TSB the target
 924          * may have been using.
 925          */
 926         if (mondo_recover_proc(cpuid, bn) == 1) {
 927                 /*
 928                  * Didn't claim the whole memory
 929                  */
 930                 goto done;
 931         }
 932 
 933         /*
 934          * We tried using the TSB. The target is still
 935          * not recovered. Check if complete memory scan is
 936          * enabled.
 937          */
 938         if (cheetah_sendmondo_fullscan == 0) {
 939                 /*
 940                  * Full memory scan is disabled.
 941                  */
 942                 retval = 1;
 943                 goto done;
 944         }
 945 
 946         /*
 947          * Try claiming the whole memory.
 948          */
 949         for (seg = memsegs; seg; seg = seg->next) {
 950                 begin_pa = (uint64_t)(seg->pages_base) << MMU_PAGESHIFT;
 951                 end_pa = (uint64_t)(seg->pages_end) << MMU_PAGESHIFT;
 952                 for (cur_pa = begin_pa; cur_pa < end_pa;
 953                     cur_pa += MMU_PAGESIZE) {
 954                         idsr = getidsr();
 955                         if ((idsr & (IDSR_NACK_BIT(bn) |
 956                             IDSR_BUSY_BIT(bn))) == 0) {
 957                                 /*
 958                                  * Didn't claim all memory
 959                                  */
 960                                 goto done;
 961                         }
 962                         claimlines(cur_pa, MMU_PAGESIZE,
 963                             CH_ECACHE_SUBBLK_SIZE);
 964                         if ((idsr & IDSR_BUSY_BIT(bn)) == 0) {
 965                                 shipit(cpuid, bn);
 966                         }
 967                         pages_claimed++;
 968                 }
 969         }
 970 
 971         /*
 972          * We did all we could.
 973          */
 974         retval = 1;
 975 
 976 done:
 977         /*
 978          * Update statistics
 979          */
 980         end_hrt = gethrtime_waitfree();
 981         CHEETAH_LIVELOCK_STAT(recovery);
 982         CHEETAH_LIVELOCK_MAXSTAT(hrt, (end_hrt - begin_hrt));
 983         CHEETAH_LIVELOCK_MAXSTAT(full_claimed, pages_claimed);
 984         CHEETAH_LIVELOCK_ENTRY_SET(histp, recovery_time, \
 985             (end_hrt -  begin_hrt));
 986 
 987         while (atomic_cas_32(&sendmondo_in_recover, 1, 0) != 1)
 988                 ;
 989 
 990         return (retval);
 991 }
 992 
 993 /*
 994  * This is called by the cyclic framework when this CPU becomes online
 995  */
 996 /*ARGSUSED*/
 997 static void
 998 cheetah_nudge_onln(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when)
 999 {
1000 
1001         hdlr->cyh_func = (cyc_func_t)cheetah_nudge_buddy;
1002         hdlr->cyh_level = CY_LOW_LEVEL;
1003         hdlr->cyh_arg = NULL;
1004 
1005         /*
1006          * Stagger the start time
1007          */
1008         when->cyt_when = cpu->cpu_id * (NANOSEC / NCPU);
1009         if (cheetah_sendmondo_recover_delay < CHEETAH_LIVELOCK_MIN_DELAY) {
1010                 cheetah_sendmondo_recover_delay = CHEETAH_LIVELOCK_MIN_DELAY;
1011         }
1012         when->cyt_interval = cheetah_sendmondo_recover_delay * NANOSEC;
1013 }
1014 
1015 /*
1016  * Create a low level cyclic to send a xtrap to the next cpu online.
1017  * However, there's no need to have this running on a uniprocessor system.
1018  */
1019 static void
1020 cheetah_nudge_init(void)
1021 {
1022         cyc_omni_handler_t hdlr;
1023 
1024         if (max_ncpus == 1) {
1025                 return;
1026         }
1027 
1028         hdlr.cyo_online = cheetah_nudge_onln;
1029         hdlr.cyo_offline = NULL;
1030         hdlr.cyo_arg = NULL;
1031 
1032         mutex_enter(&cpu_lock);
1033         (void) cyclic_add_omni(&hdlr);
1034         mutex_exit(&cpu_lock);
1035 }
1036 
1037 /*
1038  * Cyclic handler to wake up buddy
1039  */
1040 void
1041 cheetah_nudge_buddy(void)
1042 {
1043         /*
1044          * Disable kernel preemption to protect the cpu list
1045          */
1046         kpreempt_disable();
1047         if ((CPU->cpu_next_onln != CPU) && (sendmondo_in_recover == 0)) {
1048                 xt_one(CPU->cpu_next_onln->cpu_id, (xcfunc_t *)xt_sync_tl1,
1049                     0, 0);
1050         }
1051         kpreempt_enable();
1052 }
1053 
1054 #endif  /* CHEETAHPLUS_ERRATUM_25 */
1055 
1056 #ifdef SEND_MONDO_STATS
1057 uint32_t x_one_stimes[64];
1058 uint32_t x_one_ltimes[16];
1059 uint32_t x_set_stimes[64];
1060 uint32_t x_set_ltimes[16];
1061 uint32_t x_set_cpus[NCPU];
1062 uint32_t x_nack_stimes[64];
1063 #endif
1064 
1065 /*
1066  * Note: A version of this function is used by the debugger via the KDI,
1067  * and must be kept in sync with this version.  Any changes made to this
1068  * function to support new chips or to accomodate errata must also be included
1069  * in the KDI-specific version.  See us3_kdi.c.
1070  */
1071 void
1072 send_one_mondo(int cpuid)
1073 {
1074         int busy, nack;
1075         uint64_t idsr, starttick, endtick, tick, lasttick;
1076         uint64_t busymask;
1077 #ifdef  CHEETAHPLUS_ERRATUM_25
1078         int recovered = 0;
1079 #endif
1080 
1081         CPU_STATS_ADDQ(CPU, sys, xcalls, 1);
1082         starttick = lasttick = gettick();
1083         shipit(cpuid, 0);
1084         endtick = starttick + xc_tick_limit;
1085         busy = nack = 0;
1086 #if defined(JALAPENO) || defined(SERRANO)
1087         /*
1088          * Lower 2 bits of the agent ID determine which BUSY/NACK pair
1089          * will be used for dispatching interrupt. For now, assume
1090          * there are no more than IDSR_BN_SETS CPUs, hence no aliasing
1091          * issues with respect to BUSY/NACK pair usage.
1092          */
1093         busymask  = IDSR_BUSY_BIT(cpuid);
1094 #else /* JALAPENO || SERRANO */
1095         busymask = IDSR_BUSY;
1096 #endif /* JALAPENO || SERRANO */
1097         for (;;) {
1098                 idsr = getidsr();
1099                 if (idsr == 0)
1100                         break;
1101 
1102                 tick = gettick();
1103                 /*
1104                  * If there is a big jump between the current tick
1105                  * count and lasttick, we have probably hit a break
1106                  * point.  Adjust endtick accordingly to avoid panic.
1107                  */
1108                 if (tick > (lasttick + xc_tick_jump_limit))
1109                         endtick += (tick - lasttick);
1110                 lasttick = tick;
1111                 if (tick > endtick) {
1112                         if (panic_quiesce)
1113                                 return;
1114 #ifdef  CHEETAHPLUS_ERRATUM_25
1115                         if (cheetah_sendmondo_recover && recovered == 0) {
1116                                 if (mondo_recover(cpuid, 0)) {
1117                                         /*
1118                                          * We claimed the whole memory or
1119                                          * full scan is disabled.
1120                                          */
1121                                         recovered++;
1122                                 }
1123                                 tick = gettick();
1124                                 endtick = tick + xc_tick_limit;
1125                                 lasttick = tick;
1126                                 /*
1127                                  * Recheck idsr
1128                                  */
1129                                 continue;
1130                         } else
1131 #endif  /* CHEETAHPLUS_ERRATUM_25 */
1132                         {
1133                                 cmn_err(CE_PANIC, "send mondo timeout "
1134                                     "(target 0x%x) [%d NACK %d BUSY]",
1135                                     cpuid, nack, busy);
1136                         }
1137                 }
1138 
1139                 if (idsr & busymask) {
1140                         busy++;
1141                         continue;
1142                 }
1143                 drv_usecwait(1);
1144                 shipit(cpuid, 0);
1145                 nack++;
1146                 busy = 0;
1147         }
1148 #ifdef SEND_MONDO_STATS
1149         {
1150                 int n = gettick() - starttick;
1151                 if (n < 8192)
1152                         x_one_stimes[n >> 7]++;
1153                 else
1154                         x_one_ltimes[(n >> 13) & 0xf]++;
1155         }
1156 #endif
1157 }
1158 
1159 void
1160 syncfpu(void)
1161 {
1162 }
1163 
1164 /*
1165  * Return processor specific async error structure
1166  * size used.
1167  */
1168 int
1169 cpu_aflt_size(void)
1170 {
1171         return (sizeof (ch_async_flt_t));
1172 }
1173 
1174 /*
1175  * Tunable to disable the checking of other cpu logout areas during panic for
1176  * potential syndrome 71 generating errors.
1177  */
1178 int enable_check_other_cpus_logout = 1;
1179 
1180 /*
1181  * Check other cpus logout area for potential synd 71 generating
1182  * errors.
1183  */
1184 static void
1185 cpu_check_cpu_logout(int cpuid, caddr_t tpc, int tl, int ecc_type,
1186     ch_cpu_logout_t *clop)
1187 {
1188         struct async_flt *aflt;
1189         ch_async_flt_t ch_flt;
1190         uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs;
1191 
1192         if (clop == NULL || clop->clo_data.chd_afar == LOGOUT_INVALID) {
1193                 return;
1194         }
1195 
1196         bzero(&ch_flt, sizeof (ch_async_flt_t));
1197 
1198         t_afar = clop->clo_data.chd_afar;
1199         t_afsr = clop->clo_data.chd_afsr;
1200         t_afsr_ext = clop->clo_data.chd_afsr_ext;
1201 #if defined(SERRANO)
1202         ch_flt.afar2 = clop->clo_data.chd_afar2;
1203 #endif  /* SERRANO */
1204 
1205         /*
1206          * In order to simplify code, we maintain this afsr_errs
1207          * variable which holds the aggregate of AFSR and AFSR_EXT
1208          * sticky bits.
1209          */
1210         t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) |
1211             (t_afsr & C_AFSR_ALL_ERRS);
1212 
1213         /* Setup the async fault structure */
1214         aflt = (struct async_flt *)&ch_flt;
1215         aflt->flt_id = gethrtime_waitfree();
1216         ch_flt.afsr_ext = t_afsr_ext;
1217         ch_flt.afsr_errs = t_afsr_errs;
1218         aflt->flt_stat = t_afsr;
1219         aflt->flt_addr = t_afar;
1220         aflt->flt_bus_id = cpuid;
1221         aflt->flt_inst = cpuid;
1222         aflt->flt_pc = tpc;
1223         aflt->flt_prot = AFLT_PROT_NONE;
1224         aflt->flt_class = CPU_FAULT;
1225         aflt->flt_priv = ((t_afsr & C_AFSR_PRIV) != 0);
1226         aflt->flt_tl = tl;
1227         aflt->flt_status = ecc_type;
1228         aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs);
1229 
1230         /*
1231          * Queue events on the async event queue, one event per error bit.
1232          * If no events are queued, queue an event to complain.
1233          */
1234         if (cpu_queue_events(&ch_flt, NULL, t_afsr_errs, clop) == 0) {
1235                 ch_flt.flt_type = CPU_INV_AFSR;
1236                 cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR,
1237                     (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue,
1238                     aflt->flt_panic);
1239         }
1240 
1241         /*
1242          * Zero out + invalidate CPU logout.
1243          */
1244         bzero(clop, sizeof (ch_cpu_logout_t));
1245         clop->clo_data.chd_afar = LOGOUT_INVALID;
1246 }
1247 
1248 /*
1249  * Check the logout areas of all other cpus for unlogged errors.
1250  */
1251 static void
1252 cpu_check_other_cpus_logout(void)
1253 {
1254         int i, j;
1255         processorid_t myid;
1256         struct cpu *cp;
1257         ch_err_tl1_data_t *cl1p;
1258 
1259         myid = CPU->cpu_id;
1260         for (i = 0; i < NCPU; i++) {
1261                 cp = cpu[i];
1262 
1263                 if ((cp == NULL) || !(cp->cpu_flags & CPU_EXISTS) ||
1264                     (cp->cpu_id == myid) || (CPU_PRIVATE(cp) == NULL)) {
1265                         continue;
1266                 }
1267 
1268                 /*
1269                  * Check each of the tl>0 logout areas
1270                  */
1271                 cl1p = CPU_PRIVATE_PTR(cp, chpr_tl1_err_data[0]);
1272                 for (j = 0; j < CH_ERR_TL1_TLMAX; j++, cl1p++) {
1273                         if (cl1p->ch_err_tl1_flags == 0)
1274                                 continue;
1275 
1276                         cpu_check_cpu_logout(i, (caddr_t)cl1p->ch_err_tl1_tpc,
1277                             1, ECC_F_TRAP, &cl1p->ch_err_tl1_logout);
1278                 }
1279 
1280                 /*
1281                  * Check each of the remaining logout areas
1282                  */
1283                 cpu_check_cpu_logout(i, NULL, 0, ECC_F_TRAP,
1284                     CPU_PRIVATE_PTR(cp, chpr_fecctl0_logout));
1285                 cpu_check_cpu_logout(i, NULL, 0, ECC_C_TRAP,
1286                     CPU_PRIVATE_PTR(cp, chpr_cecc_logout));
1287                 cpu_check_cpu_logout(i, NULL, 0, ECC_D_TRAP,
1288                     CPU_PRIVATE_PTR(cp, chpr_async_logout));
1289         }
1290 }
1291 
1292 /*
1293  * The fast_ecc_err handler transfers control here for UCU, UCC events.
1294  * Note that we flush Ecache twice, once in the fast_ecc_err handler to
1295  * flush the error that caused the UCU/UCC, then again here at the end to
1296  * flush the TL=1 trap handler code out of the Ecache, so we can minimize
1297  * the probability of getting a TL>1 Fast ECC trap when we're fielding
1298  * another Fast ECC trap.
1299  *
1300  * Cheetah+ also handles: TSCE: No additional processing required.
1301  * Panther adds L3_UCU and L3_UCC which are reported in AFSR_EXT.
1302  *
1303  * Note that the p_clo_flags input is only valid in cases where the
1304  * cpu_private struct is not yet initialized (since that is the only
1305  * time that information cannot be obtained from the logout struct.)
1306  */
1307 /*ARGSUSED*/
1308 void
1309 cpu_fast_ecc_error(struct regs *rp, ulong_t p_clo_flags)
1310 {
1311         ch_cpu_logout_t *clop;
1312         uint64_t ceen, nceen;
1313 
1314         /*
1315          * Get the CPU log out info. If we can't find our CPU private
1316          * pointer, then we will have to make due without any detailed
1317          * logout information.
1318          */
1319         if (CPU_PRIVATE(CPU) == NULL) {
1320                 clop = NULL;
1321                 ceen = p_clo_flags & EN_REG_CEEN;
1322                 nceen = p_clo_flags & EN_REG_NCEEN;
1323         } else {
1324                 clop = CPU_PRIVATE_PTR(CPU, chpr_fecctl0_logout);
1325                 ceen = clop->clo_flags & EN_REG_CEEN;
1326                 nceen = clop->clo_flags & EN_REG_NCEEN;
1327         }
1328 
1329         cpu_log_fast_ecc_error((caddr_t)rp->r_pc,
1330             (rp->r_tstate & TSTATE_PRIV) ? 1 : 0, 0, ceen, nceen, clop);
1331 }
1332 
1333 /*
1334  * Log fast ecc error, called from either Fast ECC at TL=0 or Fast
1335  * ECC at TL>0.  Need to supply either a error register pointer or a
1336  * cpu logout structure pointer.
1337  */
1338 static void
1339 cpu_log_fast_ecc_error(caddr_t tpc, int priv, int tl, uint64_t ceen,
1340     uint64_t nceen, ch_cpu_logout_t *clop)
1341 {
1342         struct async_flt *aflt;
1343         ch_async_flt_t ch_flt;
1344         uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs;
1345         char pr_reason[MAX_REASON_STRING];
1346         ch_cpu_errors_t cpu_error_regs;
1347 
1348         bzero(&ch_flt, sizeof (ch_async_flt_t));
1349         /*
1350          * If no cpu logout data, then we will have to make due without
1351          * any detailed logout information.
1352          */
1353         if (clop == NULL) {
1354                 ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID;
1355                 get_cpu_error_state(&cpu_error_regs);
1356                 set_cpu_error_state(&cpu_error_regs);
1357                 t_afar = cpu_error_regs.afar;
1358                 t_afsr = cpu_error_regs.afsr;
1359                 t_afsr_ext = cpu_error_regs.afsr_ext;
1360 #if defined(SERRANO)
1361                 ch_flt.afar2 = cpu_error_regs.afar2;
1362 #endif  /* SERRANO */
1363         } else {
1364                 t_afar = clop->clo_data.chd_afar;
1365                 t_afsr = clop->clo_data.chd_afsr;
1366                 t_afsr_ext = clop->clo_data.chd_afsr_ext;
1367 #if defined(SERRANO)
1368                 ch_flt.afar2 = clop->clo_data.chd_afar2;
1369 #endif  /* SERRANO */
1370         }
1371 
1372         /*
1373          * In order to simplify code, we maintain this afsr_errs
1374          * variable which holds the aggregate of AFSR and AFSR_EXT
1375          * sticky bits.
1376          */
1377         t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) |
1378             (t_afsr & C_AFSR_ALL_ERRS);
1379         pr_reason[0] = '\0';
1380 
1381         /* Setup the async fault structure */
1382         aflt = (struct async_flt *)&ch_flt;
1383         aflt->flt_id = gethrtime_waitfree();
1384         ch_flt.afsr_ext = t_afsr_ext;
1385         ch_flt.afsr_errs = t_afsr_errs;
1386         aflt->flt_stat = t_afsr;
1387         aflt->flt_addr = t_afar;
1388         aflt->flt_bus_id = getprocessorid();
1389         aflt->flt_inst = CPU->cpu_id;
1390         aflt->flt_pc = tpc;
1391         aflt->flt_prot = AFLT_PROT_NONE;
1392         aflt->flt_class = CPU_FAULT;
1393         aflt->flt_priv = priv;
1394         aflt->flt_tl = tl;
1395         aflt->flt_status = ECC_F_TRAP;
1396         aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs);
1397 
1398         /*
1399          * XXXX - Phenomenal hack to get around Solaris not getting all the
1400          * cmn_err messages out to the console.  The situation is a UCU (in
1401          * priv mode) which causes a WDU which causes a UE (on the retry).
1402          * The messages for the UCU and WDU are enqueued and then pulled off
1403          * the async queue via softint and syslogd starts to process them
1404          * but doesn't get them to the console.  The UE causes a panic, but
1405          * since the UCU/WDU messages are already in transit, those aren't
1406          * on the async queue.  The hack is to check if we have a matching
1407          * WDU event for the UCU, and if it matches, we're more than likely
1408          * going to panic with a UE, unless we're under protection.  So, we
1409          * check to see if we got a matching WDU event and if we're under
1410          * protection.
1411          *
1412          * For Cheetah/Cheetah+/Jaguar/Jalapeno, the sequence we care about
1413          * looks like this:
1414          *    UCU->WDU->UE
1415          * For Panther, it could look like either of these:
1416          *    UCU---->WDU->L3_WDU->UE
1417          *    L3_UCU->WDU->L3_WDU->UE
1418          */
1419         if ((t_afsr_errs & (C_AFSR_UCU | C_AFSR_L3_UCU)) &&
1420             aflt->flt_panic == 0 && aflt->flt_priv != 0 &&
1421             curthread->t_ontrap == NULL && curthread->t_lofault == NULL) {
1422                 get_cpu_error_state(&cpu_error_regs);
1423                 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) {
1424                         aflt->flt_panic |=
1425                             ((cpu_error_regs.afsr & C_AFSR_WDU) &&
1426                             (cpu_error_regs.afsr_ext & C_AFSR_L3_WDU) &&
1427                             (cpu_error_regs.afar == t_afar));
1428                         aflt->flt_panic |= ((clop == NULL) &&
1429                             (t_afsr_errs & C_AFSR_WDU) &&
1430                             (t_afsr_errs & C_AFSR_L3_WDU));
1431                 } else {
1432                         aflt->flt_panic |=
1433                             ((cpu_error_regs.afsr & C_AFSR_WDU) &&
1434                             (cpu_error_regs.afar == t_afar));
1435                         aflt->flt_panic |= ((clop == NULL) &&
1436                             (t_afsr_errs & C_AFSR_WDU));
1437                 }
1438         }
1439 
1440         /*
1441          * Queue events on the async event queue, one event per error bit.
1442          * If no events are queued or no Fast ECC events are on in the AFSR,
1443          * queue an event to complain.
1444          */
1445         if (cpu_queue_events(&ch_flt, pr_reason, t_afsr_errs, clop) == 0 ||
1446             ((t_afsr_errs & (C_AFSR_FECC_ERRS | C_AFSR_EXT_FECC_ERRS)) == 0)) {
1447                 ch_flt.flt_type = CPU_INV_AFSR;
1448                 cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR,
1449                     (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue,
1450                     aflt->flt_panic);
1451         }
1452 
1453         /*
1454          * Zero out + invalidate CPU logout.
1455          */
1456         if (clop) {
1457                 bzero(clop, sizeof (ch_cpu_logout_t));
1458                 clop->clo_data.chd_afar = LOGOUT_INVALID;
1459         }
1460 
1461         /*
1462          * We carefully re-enable NCEEN and CEEN and then check if any deferred
1463          * or disrupting errors have happened.  We do this because if a
1464          * deferred or disrupting error had occurred with NCEEN/CEEN off, the
1465          * trap will not be taken when NCEEN/CEEN is re-enabled.  Note that
1466          * CEEN works differently on Cheetah than on Spitfire.  Also, we enable
1467          * NCEEN/CEEN *before* checking the AFSR to avoid the small window of a
1468          * deferred or disrupting error happening between checking the AFSR and
1469          * enabling NCEEN/CEEN.
1470          *
1471          * Note: CEEN and NCEEN are only reenabled if they were on when trap
1472          * taken.
1473          */
1474         set_error_enable(get_error_enable() | (nceen | ceen));
1475         if (clear_errors(&ch_flt)) {
1476                 aflt->flt_panic |= ((ch_flt.afsr_errs &
1477                     (C_AFSR_EXT_ASYNC_ERRS | C_AFSR_ASYNC_ERRS)) != 0);
1478                 (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs,
1479                     NULL);
1480         }
1481 
1482         /*
1483          * Panic here if aflt->flt_panic has been set.  Enqueued errors will
1484          * be logged as part of the panic flow.
1485          */
1486         if (aflt->flt_panic)
1487                 fm_panic("%sError(s)", pr_reason);
1488 
1489         /*
1490          * Flushing the Ecache here gets the part of the trap handler that
1491          * is run at TL=1 out of the Ecache.
1492          */
1493         cpu_flush_ecache();
1494 }
1495 
1496 /*
1497  * This is called via sys_trap from pil15_interrupt code if the
1498  * corresponding entry in ch_err_tl1_pending is set.  Checks the
1499  * various ch_err_tl1_data structures for valid entries based on the bit
1500  * settings in the ch_err_tl1_flags entry of the structure.
1501  */
1502 /*ARGSUSED*/
1503 void
1504 cpu_tl1_error(struct regs *rp, int panic)
1505 {
1506         ch_err_tl1_data_t *cl1p, cl1;
1507         int i, ncl1ps;
1508         uint64_t me_flags;
1509         uint64_t ceen, nceen;
1510 
1511         if (ch_err_tl1_paddrs[CPU->cpu_id] == 0) {
1512                 cl1p = &ch_err_tl1_data;
1513                 ncl1ps = 1;
1514         } else if (CPU_PRIVATE(CPU) != NULL) {
1515                 cl1p = CPU_PRIVATE_PTR(CPU, chpr_tl1_err_data[0]);
1516                 ncl1ps = CH_ERR_TL1_TLMAX;
1517         } else {
1518                 ncl1ps = 0;
1519         }
1520 
1521         for (i = 0; i < ncl1ps; i++, cl1p++) {
1522                 if (cl1p->ch_err_tl1_flags == 0)
1523                         continue;
1524 
1525                 /*
1526                  * Grab a copy of the logout data and invalidate
1527                  * the logout area.
1528                  */
1529                 cl1 = *cl1p;
1530                 bzero(cl1p, sizeof (ch_err_tl1_data_t));
1531                 cl1p->ch_err_tl1_logout.clo_data.chd_afar = LOGOUT_INVALID;
1532                 me_flags = CH_ERR_ME_FLAGS(cl1.ch_err_tl1_flags);
1533 
1534                 /*
1535                  * Log "first error" in ch_err_tl1_data.
1536                  */
1537                 if (cl1.ch_err_tl1_flags & CH_ERR_FECC) {
1538                         ceen = get_error_enable() & EN_REG_CEEN;
1539                         nceen = get_error_enable() & EN_REG_NCEEN;
1540                         cpu_log_fast_ecc_error((caddr_t)cl1.ch_err_tl1_tpc, 1,
1541                             1, ceen, nceen, &cl1.ch_err_tl1_logout);
1542                 }
1543 #if defined(CPU_IMP_L1_CACHE_PARITY)
1544                 if (cl1.ch_err_tl1_flags & (CH_ERR_IPE | CH_ERR_DPE)) {
1545                         cpu_parity_error(rp, cl1.ch_err_tl1_flags,
1546                             (caddr_t)cl1.ch_err_tl1_tpc);
1547                 }
1548 #endif  /* CPU_IMP_L1_CACHE_PARITY */
1549 
1550                 /*
1551                  * Log "multiple events" in ch_err_tl1_data.  Note that
1552                  * we don't read and clear the AFSR/AFAR in the TL>0 code
1553                  * if the structure is busy, we just do the cache flushing
1554                  * we have to do and then do the retry.  So the AFSR/AFAR
1555                  * at this point *should* have some relevant info.  If there
1556                  * are no valid errors in the AFSR, we'll assume they've
1557                  * already been picked up and logged.  For I$/D$ parity,
1558                  * we just log an event with an "Unknown" (NULL) TPC.
1559                  */
1560                 if (me_flags & CH_ERR_FECC) {
1561                         ch_cpu_errors_t cpu_error_regs;
1562                         uint64_t t_afsr_errs;
1563 
1564                         /*
1565                          * Get the error registers and see if there's
1566                          * a pending error.  If not, don't bother
1567                          * generating an "Invalid AFSR" error event.
1568                          */
1569                         get_cpu_error_state(&cpu_error_regs);
1570                         t_afsr_errs = (cpu_error_regs.afsr_ext &
1571                             C_AFSR_EXT_ALL_ERRS) |
1572                             (cpu_error_regs.afsr & C_AFSR_ALL_ERRS);
1573                         if (t_afsr_errs != 0) {
1574                                 ceen = get_error_enable() & EN_REG_CEEN;
1575                                 nceen = get_error_enable() & EN_REG_NCEEN;
1576                                 cpu_log_fast_ecc_error((caddr_t)NULL, 1,
1577                                     1, ceen, nceen, NULL);
1578                         }
1579                 }
1580 #if defined(CPU_IMP_L1_CACHE_PARITY)
1581                 if (me_flags & (CH_ERR_IPE | CH_ERR_DPE)) {
1582                         cpu_parity_error(rp, me_flags, (caddr_t)NULL);
1583                 }
1584 #endif  /* CPU_IMP_L1_CACHE_PARITY */
1585         }
1586 }
1587 
1588 /*
1589  * Called from Fast ECC TL>0 handler in case of fatal error.
1590  * cpu_tl1_error should always find an associated ch_err_tl1_data structure,
1591  * but if we don't, we'll panic with something reasonable.
1592  */
1593 /*ARGSUSED*/
1594 void
1595 cpu_tl1_err_panic(struct regs *rp, ulong_t flags)
1596 {
1597         cpu_tl1_error(rp, 1);
1598         /*
1599          * Should never return, but just in case.
1600          */
1601         fm_panic("Unsurvivable ECC Error at TL>0");
1602 }
1603 
1604 /*
1605  * The ce_err/ce_err_tl1 handlers transfer control here for CE, EMC, EDU:ST,
1606  * EDC, WDU, WDC, CPU, CPC, IVU, IVC events.
1607  * Disrupting errors controlled by NCEEN: EDU:ST, WDU, CPU, IVU
1608  * Disrupting errors controlled by CEEN: CE, EMC, EDC, WDC, CPC, IVC
1609  *
1610  * Cheetah+ also handles (No additional processing required):
1611  *    DUE, DTO, DBERR   (NCEEN controlled)
1612  *    THCE              (CEEN and ET_ECC_en controlled)
1613  *    TUE               (ET_ECC_en controlled)
1614  *
1615  * Panther further adds:
1616  *    IMU, L3_EDU, L3_WDU, L3_CPU               (NCEEN controlled)
1617  *    IMC, L3_EDC, L3_WDC, L3_CPC, L3_THCE      (CEEN controlled)
1618  *    TUE_SH, TUE               (NCEEN and L2_tag_ECC_en controlled)
1619  *    L3_TUE, L3_TUE_SH         (NCEEN and ET_ECC_en controlled)
1620  *    THCE                      (CEEN and L2_tag_ECC_en controlled)
1621  *    L3_THCE                   (CEEN and ET_ECC_en controlled)
1622  *
1623  * Note that the p_clo_flags input is only valid in cases where the
1624  * cpu_private struct is not yet initialized (since that is the only
1625  * time that information cannot be obtained from the logout struct.)
1626  */
1627 /*ARGSUSED*/
1628 void
1629 cpu_disrupting_error(struct regs *rp, ulong_t p_clo_flags)
1630 {
1631         struct async_flt *aflt;
1632         ch_async_flt_t ch_flt;
1633         char pr_reason[MAX_REASON_STRING];
1634         ch_cpu_logout_t *clop;
1635         uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs;
1636         ch_cpu_errors_t cpu_error_regs;
1637 
1638         bzero(&ch_flt, sizeof (ch_async_flt_t));
1639         /*
1640          * Get the CPU log out info. If we can't find our CPU private
1641          * pointer, then we will have to make due without any detailed
1642          * logout information.
1643          */
1644         if (CPU_PRIVATE(CPU) == NULL) {
1645                 clop = NULL;
1646                 ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID;
1647                 get_cpu_error_state(&cpu_error_regs);
1648                 set_cpu_error_state(&cpu_error_regs);
1649                 t_afar = cpu_error_regs.afar;
1650                 t_afsr = cpu_error_regs.afsr;
1651                 t_afsr_ext = cpu_error_regs.afsr_ext;
1652 #if defined(SERRANO)
1653                 ch_flt.afar2 = cpu_error_regs.afar2;
1654 #endif  /* SERRANO */
1655         } else {
1656                 clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout);
1657                 t_afar = clop->clo_data.chd_afar;
1658                 t_afsr = clop->clo_data.chd_afsr;
1659                 t_afsr_ext = clop->clo_data.chd_afsr_ext;
1660 #if defined(SERRANO)
1661                 ch_flt.afar2 = clop->clo_data.chd_afar2;
1662 #endif  /* SERRANO */
1663         }
1664 
1665         /*
1666          * In order to simplify code, we maintain this afsr_errs
1667          * variable which holds the aggregate of AFSR and AFSR_EXT
1668          * sticky bits.
1669          */
1670         t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) |
1671             (t_afsr & C_AFSR_ALL_ERRS);
1672 
1673         pr_reason[0] = '\0';
1674         /* Setup the async fault structure */
1675         aflt = (struct async_flt *)&ch_flt;
1676         ch_flt.afsr_ext = t_afsr_ext;
1677         ch_flt.afsr_errs = t_afsr_errs;
1678         aflt->flt_stat = t_afsr;
1679         aflt->flt_addr = t_afar;
1680         aflt->flt_pc = (caddr_t)rp->r_pc;
1681         aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ?  1 : 0;
1682         aflt->flt_tl = 0;
1683         aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs);
1684 
1685         /*
1686          * If this trap is a result of one of the errors not masked
1687          * by cpu_ce_not_deferred, we don't reenable CEEN. Instead
1688          * indicate that a timeout is to be set later.
1689          */
1690         if (!(t_afsr_errs & (cpu_ce_not_deferred | cpu_ce_not_deferred_ext)) &&
1691             !aflt->flt_panic)
1692                 ch_flt.flt_trapped_ce = CE_CEEN_DEFER | CE_CEEN_TRAPPED;
1693         else
1694                 ch_flt.flt_trapped_ce = CE_CEEN_NODEFER | CE_CEEN_TRAPPED;
1695 
1696         /*
1697          * log the CE and clean up
1698          */
1699         cpu_log_and_clear_ce(&ch_flt);
1700 
1701         /*
1702          * We re-enable CEEN (if required) and check if any disrupting errors
1703          * have happened.  We do this because if a disrupting error had occurred
1704          * with CEEN off, the trap will not be taken when CEEN is re-enabled.
1705          * Note that CEEN works differently on Cheetah than on Spitfire.  Also,
1706          * we enable CEEN *before* checking the AFSR to avoid the small window
1707          * of a error happening between checking the AFSR and enabling CEEN.
1708          */
1709         if (ch_flt.flt_trapped_ce & CE_CEEN_NODEFER)
1710                 set_error_enable(get_error_enable() | EN_REG_CEEN);
1711         if (clear_errors(&ch_flt)) {
1712                 (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs,
1713                     NULL);
1714         }
1715 
1716         /*
1717          * Panic here if aflt->flt_panic has been set.  Enqueued errors will
1718          * be logged as part of the panic flow.
1719          */
1720         if (aflt->flt_panic)
1721                 fm_panic("%sError(s)", pr_reason);
1722 }
1723 
1724 /*
1725  * The async_err handler transfers control here for UE, EMU, EDU:BLD,
1726  * L3_EDU:BLD, TO, and BERR events.
1727  * Deferred errors controlled by NCEEN: UE, EMU, EDU:BLD, L3_EDU:BLD, TO, BERR
1728  *
1729  * Cheetah+: No additional errors handled.
1730  *
1731  * Note that the p_clo_flags input is only valid in cases where the
1732  * cpu_private struct is not yet initialized (since that is the only
1733  * time that information cannot be obtained from the logout struct.)
1734  */
1735 /*ARGSUSED*/
1736 void
1737 cpu_deferred_error(struct regs *rp, ulong_t p_clo_flags)
1738 {
1739         ushort_t ttype, tl;
1740         ch_async_flt_t ch_flt;
1741         struct async_flt *aflt;
1742         int trampolined = 0;
1743         char pr_reason[MAX_REASON_STRING];
1744         ch_cpu_logout_t *clop;
1745         uint64_t ceen, clo_flags;
1746         uint64_t log_afsr;
1747         uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs;
1748         ch_cpu_errors_t cpu_error_regs;
1749         int expected = DDI_FM_ERR_UNEXPECTED;
1750         ddi_acc_hdl_t *hp;
1751 
1752         /*
1753          * We need to look at p_flag to determine if the thread detected an
1754          * error while dumping core.  We can't grab p_lock here, but it's ok
1755          * because we just need a consistent snapshot and we know that everyone
1756          * else will store a consistent set of bits while holding p_lock.  We
1757          * don't have to worry about a race because SDOCORE is set once prior
1758          * to doing i/o from the process's address space and is never cleared.
1759          */
1760         uint_t pflag = ttoproc(curthread)->p_flag;
1761 
1762         bzero(&ch_flt, sizeof (ch_async_flt_t));
1763         /*
1764          * Get the CPU log out info. If we can't find our CPU private
1765          * pointer then we will have to make due without any detailed
1766          * logout information.
1767          */
1768         if (CPU_PRIVATE(CPU) == NULL) {
1769                 clop = NULL;
1770                 ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID;
1771                 get_cpu_error_state(&cpu_error_regs);
1772                 set_cpu_error_state(&cpu_error_regs);
1773                 t_afar = cpu_error_regs.afar;
1774                 t_afsr = cpu_error_regs.afsr;
1775                 t_afsr_ext = cpu_error_regs.afsr_ext;
1776 #if defined(SERRANO)
1777                 ch_flt.afar2 = cpu_error_regs.afar2;
1778 #endif  /* SERRANO */
1779                 clo_flags = p_clo_flags;
1780         } else {
1781                 clop = CPU_PRIVATE_PTR(CPU, chpr_async_logout);
1782                 t_afar = clop->clo_data.chd_afar;
1783                 t_afsr = clop->clo_data.chd_afsr;
1784                 t_afsr_ext = clop->clo_data.chd_afsr_ext;
1785 #if defined(SERRANO)
1786                 ch_flt.afar2 = clop->clo_data.chd_afar2;
1787 #endif  /* SERRANO */
1788                 clo_flags = clop->clo_flags;
1789         }
1790 
1791         /*
1792          * In order to simplify code, we maintain this afsr_errs
1793          * variable which holds the aggregate of AFSR and AFSR_EXT
1794          * sticky bits.
1795          */
1796         t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) |
1797             (t_afsr & C_AFSR_ALL_ERRS);
1798         pr_reason[0] = '\0';
1799 
1800         /*
1801          * Grab information encoded into our clo_flags field.
1802          */
1803         ceen = clo_flags & EN_REG_CEEN;
1804         tl = (clo_flags & CLO_FLAGS_TL_MASK) >> CLO_FLAGS_TL_SHIFT;
1805         ttype = (clo_flags & CLO_FLAGS_TT_MASK) >> CLO_FLAGS_TT_SHIFT;
1806 
1807         /*
1808          * handle the specific error
1809          */
1810         aflt = (struct async_flt *)&ch_flt;
1811         aflt->flt_id = gethrtime_waitfree();
1812         aflt->flt_bus_id = getprocessorid();
1813         aflt->flt_inst = CPU->cpu_id;
1814         ch_flt.afsr_ext = t_afsr_ext;
1815         ch_flt.afsr_errs = t_afsr_errs;
1816         aflt->flt_stat = t_afsr;
1817         aflt->flt_addr = t_afar;
1818         aflt->flt_pc = (caddr_t)rp->r_pc;
1819         aflt->flt_prot = AFLT_PROT_NONE;
1820         aflt->flt_class = CPU_FAULT;
1821         aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ?  1 : 0;
1822         aflt->flt_tl = (uchar_t)tl;
1823         aflt->flt_panic = ((tl != 0) || (aft_testfatal != 0) ||
1824             C_AFSR_PANIC(t_afsr_errs));
1825         aflt->flt_core = (pflag & SDOCORE) ? 1 : 0;
1826         aflt->flt_status = ((ttype == T_DATA_ERROR) ? ECC_D_TRAP : ECC_I_TRAP);
1827 
1828         /*
1829          * If the trap occurred in privileged mode at TL=0, we need to check to
1830          * see if we were executing in the kernel under on_trap() or t_lofault
1831          * protection.  If so, modify the saved registers so that we return
1832          * from the trap to the appropriate trampoline routine.
1833          */
1834         if (aflt->flt_priv && tl == 0) {
1835                 if (curthread->t_ontrap != NULL) {
1836                         on_trap_data_t *otp = curthread->t_ontrap;
1837 
1838                         if (otp->ot_prot & OT_DATA_EC) {
1839                                 aflt->flt_prot = AFLT_PROT_EC;
1840                                 otp->ot_trap |= OT_DATA_EC;
1841                                 rp->r_pc = otp->ot_trampoline;
1842                                 rp->r_npc = rp->r_pc + 4;
1843                                 trampolined = 1;
1844                         }
1845 
1846                         if ((t_afsr & (C_AFSR_TO | C_AFSR_BERR)) &&
1847                             (otp->ot_prot & OT_DATA_ACCESS)) {
1848                                 aflt->flt_prot = AFLT_PROT_ACCESS;
1849                                 otp->ot_trap |= OT_DATA_ACCESS;
1850                                 rp->r_pc = otp->ot_trampoline;
1851                                 rp->r_npc = rp->r_pc + 4;
1852                                 trampolined = 1;
1853                                 /*
1854                                  * for peeks and caut_gets errors are expected
1855                                  */
1856                                 hp = (ddi_acc_hdl_t *)otp->ot_handle;
1857                                 if (!hp)
1858                                         expected = DDI_FM_ERR_PEEK;
1859                                 else if (hp->ah_acc.devacc_attr_access ==
1860                                     DDI_CAUTIOUS_ACC)
1861                                         expected = DDI_FM_ERR_EXPECTED;
1862                         }
1863 
1864                 } else if (curthread->t_lofault) {
1865                         aflt->flt_prot = AFLT_PROT_COPY;
1866                         rp->r_g1 = EFAULT;
1867                         rp->r_pc = curthread->t_lofault;
1868                         rp->r_npc = rp->r_pc + 4;
1869                         trampolined = 1;
1870                 }
1871         }
1872 
1873         /*
1874          * If we're in user mode or we're doing a protected copy, we either
1875          * want the ASTON code below to send a signal to the user process
1876          * or we want to panic if aft_panic is set.
1877          *
1878          * If we're in privileged mode and we're not doing a copy, then we
1879          * need to check if we've trampolined.  If we haven't trampolined,
1880          * we should panic.
1881          */
1882         if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY) {
1883                 if (t_afsr_errs &
1884                     ((C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS) &
1885                     ~(C_AFSR_BERR | C_AFSR_TO)))
1886                         aflt->flt_panic |= aft_panic;
1887         } else if (!trampolined) {
1888                         aflt->flt_panic = 1;
1889         }
1890 
1891         /*
1892          * If we've trampolined due to a privileged TO or BERR, or if an
1893          * unprivileged TO or BERR occurred, we don't want to enqueue an
1894          * event for that TO or BERR.  Queue all other events (if any) besides
1895          * the TO/BERR.  Since we may not be enqueing any events, we need to
1896          * ignore the number of events queued.  If we haven't trampolined due
1897          * to a TO or BERR, just enqueue events normally.
1898          */
1899         log_afsr = t_afsr_errs;
1900         if (trampolined) {
1901                 log_afsr &= ~(C_AFSR_TO | C_AFSR_BERR);
1902         } else if (!aflt->flt_priv) {
1903                 /*
1904                  * User mode, suppress messages if
1905                  * cpu_berr_to_verbose is not set.
1906                  */
1907                 if (!cpu_berr_to_verbose)
1908                         log_afsr &= ~(C_AFSR_TO | C_AFSR_BERR);
1909         }
1910 
1911         /*
1912          * Log any errors that occurred
1913          */
1914         if (((log_afsr &
1915             ((C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS) & ~C_AFSR_ME)) &&
1916             cpu_queue_events(&ch_flt, pr_reason, log_afsr, clop) == 0) ||
1917             (t_afsr_errs & (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS)) == 0) {
1918                 ch_flt.flt_type = CPU_INV_AFSR;
1919                 cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR,
1920                     (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue,
1921                     aflt->flt_panic);
1922         }
1923 
1924         /*
1925          * Zero out + invalidate CPU logout.
1926          */
1927         if (clop) {
1928                 bzero(clop, sizeof (ch_cpu_logout_t));
1929                 clop->clo_data.chd_afar = LOGOUT_INVALID;
1930         }
1931 
1932 #if defined(JALAPENO) || defined(SERRANO)
1933         /*
1934          * UE/RUE/BERR/TO: Call our bus nexus friends to check for
1935          * IO errors that may have resulted in this trap.
1936          */
1937         if (t_afsr & (C_AFSR_UE|C_AFSR_RUE|C_AFSR_TO|C_AFSR_BERR)) {
1938                 cpu_run_bus_error_handlers(aflt, expected);
1939         }
1940 
1941         /*
1942          * UE/RUE: If UE or RUE is in memory, we need to flush the bad
1943          * line from the Ecache.  We also need to query the bus nexus for
1944          * fatal errors.  Attempts to do diagnostic read on caches may
1945          * introduce more errors (especially when the module is bad).
1946          */
1947         if (t_afsr & (C_AFSR_UE|C_AFSR_RUE)) {
1948                 /*
1949                  * Ask our bus nexus friends if they have any fatal errors.  If
1950                  * so, they will log appropriate error messages.
1951                  */
1952                 if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL)
1953                         aflt->flt_panic = 1;
1954 
1955                 /*
1956                  * We got a UE or RUE and are panicking, save the fault PA in
1957                  * a known location so that the platform specific panic code
1958                  * can check for copyback errors.
1959                  */
1960                 if (aflt->flt_panic && cpu_flt_in_memory(&ch_flt, C_AFSR_UE)) {
1961                         panic_aflt = *aflt;
1962                 }
1963         }
1964 
1965         /*
1966          * Flush Ecache line or entire Ecache
1967          */
1968         if (t_afsr & (C_AFSR_UE | C_AFSR_RUE | C_AFSR_EDU | C_AFSR_BERR))
1969                 cpu_error_ecache_flush(&ch_flt);
1970 #else /* JALAPENO || SERRANO */
1971         /*
1972          * UE/BERR/TO: Call our bus nexus friends to check for
1973          * IO errors that may have resulted in this trap.
1974          */
1975         if (t_afsr & (C_AFSR_UE|C_AFSR_TO|C_AFSR_BERR)) {
1976                 cpu_run_bus_error_handlers(aflt, expected);
1977         }
1978 
1979         /*
1980          * UE: If the UE is in memory, we need to flush the bad
1981          * line from the Ecache.  We also need to query the bus nexus for
1982          * fatal errors.  Attempts to do diagnostic read on caches may
1983          * introduce more errors (especially when the module is bad).
1984          */
1985         if (t_afsr & C_AFSR_UE) {
1986                 /*
1987                  * Ask our legacy bus nexus friends if they have any fatal
1988                  * errors.  If so, they will log appropriate error messages.
1989                  */
1990                 if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL)
1991                         aflt->flt_panic = 1;
1992 
1993                 /*
1994                  * We got a UE and are panicking, save the fault PA in a known
1995                  * location so that the platform specific panic code can check
1996                  * for copyback errors.
1997                  */
1998                 if (aflt->flt_panic && cpu_flt_in_memory(&ch_flt, C_AFSR_UE)) {
1999                         panic_aflt = *aflt;
2000                 }
2001         }
2002 
2003         /*
2004          * Flush Ecache line or entire Ecache
2005          */
2006         if (t_afsr_errs &
2007             (C_AFSR_UE | C_AFSR_EDU | C_AFSR_BERR | C_AFSR_L3_EDU))
2008                 cpu_error_ecache_flush(&ch_flt);
2009 #endif /* JALAPENO || SERRANO */
2010 
2011         /*
2012          * We carefully re-enable NCEEN and CEEN and then check if any deferred
2013          * or disrupting errors have happened.  We do this because if a
2014          * deferred or disrupting error had occurred with NCEEN/CEEN off, the
2015          * trap will not be taken when NCEEN/CEEN is re-enabled.  Note that
2016          * CEEN works differently on Cheetah than on Spitfire.  Also, we enable
2017          * NCEEN/CEEN *before* checking the AFSR to avoid the small window of a
2018          * deferred or disrupting error happening between checking the AFSR and
2019          * enabling NCEEN/CEEN.
2020          *
2021          * Note: CEEN reenabled only if it was on when trap taken.
2022          */
2023         set_error_enable(get_error_enable() | (EN_REG_NCEEN | ceen));
2024         if (clear_errors(&ch_flt)) {
2025                 /*
2026                  * Check for secondary errors, and avoid panicking if we
2027                  * have them
2028                  */
2029                 if (cpu_check_secondary_errors(&ch_flt, t_afsr_errs,
2030                     t_afar) == 0) {
2031                         aflt->flt_panic |= ((ch_flt.afsr_errs &
2032                             (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS)) != 0);
2033                 }
2034                 (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs,
2035                     NULL);
2036         }
2037 
2038         /*
2039          * Panic here if aflt->flt_panic has been set.  Enqueued errors will
2040          * be logged as part of the panic flow.
2041          */
2042         if (aflt->flt_panic)
2043                 fm_panic("%sError(s)", pr_reason);
2044 
2045         /*
2046          * If we queued an error and we are going to return from the trap and
2047          * the error was in user mode or inside of a copy routine, set AST flag
2048          * so the queue will be drained before returning to user mode.  The
2049          * AST processing will also act on our failure policy.
2050          */
2051         if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY) {
2052                 int pcb_flag = 0;
2053 
2054                 if (t_afsr_errs &
2055                     (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS &
2056                     ~(C_AFSR_BERR | C_AFSR_TO)))
2057                         pcb_flag |= ASYNC_HWERR;
2058 
2059                 if (t_afsr & C_AFSR_BERR)
2060                         pcb_flag |= ASYNC_BERR;
2061 
2062                 if (t_afsr & C_AFSR_TO)
2063                         pcb_flag |= ASYNC_BTO;
2064 
2065                 ttolwp(curthread)->lwp_pcb.pcb_flags |= pcb_flag;
2066                 aston(curthread);
2067         }
2068 }
2069 
2070 #if defined(CPU_IMP_L1_CACHE_PARITY)
2071 /*
2072  * Handling of data and instruction parity errors (traps 0x71, 0x72).
2073  *
2074  * For Panther, P$ data parity errors during floating point load hits
2075  * are also detected (reported as TT 0x71) and handled by this trap
2076  * handler.
2077  *
2078  * AFSR/AFAR are not set for parity errors, only TPC (a virtual address)
2079  * is available.
2080  */
2081 /*ARGSUSED*/
2082 void
2083 cpu_parity_error(struct regs *rp, uint_t flags, caddr_t tpc)
2084 {
2085         ch_async_flt_t ch_flt;
2086         struct async_flt *aflt;
2087         uchar_t tl = ((flags & CH_ERR_TL) != 0);
2088         uchar_t iparity = ((flags & CH_ERR_IPE) != 0);
2089         uchar_t panic = ((flags & CH_ERR_PANIC) != 0);
2090         char *error_class;
2091         int index, way, word;
2092         ch_dc_data_t tmp_dcp;
2093         int dc_set_size = dcache_size / CH_DCACHE_NWAY;
2094         uint64_t parity_bits, pbits;
2095         /* The parity bit array corresponds to the result of summing two bits */
2096         static int parity_bits_popc[] = { 0, 1, 1, 0 };
2097 
2098         /*
2099          * Log the error.
2100          * For icache parity errors the fault address is the trap PC.
2101          * For dcache/pcache parity errors the instruction would have to
2102          * be decoded to determine the address and that isn't possible
2103          * at high PIL.
2104          */
2105         bzero(&ch_flt, sizeof (ch_async_flt_t));
2106         aflt = (struct async_flt *)&ch_flt;
2107         aflt->flt_id = gethrtime_waitfree();
2108         aflt->flt_bus_id = getprocessorid();
2109         aflt->flt_inst = CPU->cpu_id;
2110         aflt->flt_pc = tpc;
2111         aflt->flt_addr = iparity ? (uint64_t)tpc : AFLT_INV_ADDR;
2112         aflt->flt_prot = AFLT_PROT_NONE;
2113         aflt->flt_class = CPU_FAULT;
2114         aflt->flt_priv = (tl || (rp->r_tstate & TSTATE_PRIV)) ?  1 : 0;
2115         aflt->flt_tl = tl;
2116         aflt->flt_panic = panic;
2117         aflt->flt_status = iparity ? ECC_IP_TRAP : ECC_DP_TRAP;
2118         ch_flt.flt_type = iparity ? CPU_IC_PARITY : CPU_DC_PARITY;
2119 
2120         if (iparity) {
2121                 cpu_icache_parity_info(&ch_flt);
2122                 if (ch_flt.parity_data.ipe.cpl_off != -1)
2123                         error_class = FM_EREPORT_CPU_USIII_IDSPE;
2124                 else if (ch_flt.parity_data.ipe.cpl_way != -1)
2125                         error_class = FM_EREPORT_CPU_USIII_ITSPE;
2126                 else
2127                         error_class = FM_EREPORT_CPU_USIII_IPE;
2128                 aflt->flt_payload = FM_EREPORT_PAYLOAD_ICACHE_PE;
2129         } else {
2130                 cpu_dcache_parity_info(&ch_flt);
2131                 if (ch_flt.parity_data.dpe.cpl_off != -1) {
2132                         /*
2133                          * If not at TL 0 and running on a Jalapeno processor,
2134                          * then process as a true ddspe.  A true
2135                          * ddspe error can only occur if the way == 0
2136                          */
2137                         way = ch_flt.parity_data.dpe.cpl_way;
2138                         if ((tl == 0) && (way != 0) &&
2139                             IS_JALAPENO(cpunodes[CPU->cpu_id].implementation)) {
2140                                 for (index = 0; index < dc_set_size;
2141                                     index += dcache_linesize) {
2142                                         get_dcache_dtag(index + way *
2143                                             dc_set_size,
2144                                             (uint64_t *)&tmp_dcp);
2145                                         /*
2146                                          * Check data array for even parity.
2147                                          * The 8 parity bits are grouped into
2148                                          * 4 pairs each of which covers a 64-bit
2149                                          * word.  The endianness is reversed
2150                                          * -- the low-order parity bits cover
2151                                          *  the high-order data words.
2152                                          */
2153                                         parity_bits = tmp_dcp.dc_utag >> 8;
2154                                         for (word = 0; word < 4; word++) {
2155                                                 pbits = (parity_bits >>
2156                                                     (6 - word * 2)) & 3;
2157                                                 if (((popc64(
2158                                                     tmp_dcp.dc_data[word]) +
2159                                                     parity_bits_popc[pbits]) &
2160                                                     1) && (tmp_dcp.dc_tag &
2161                                                     VA13)) {
2162                                                         /* cleanup */
2163                                                         correct_dcache_parity(
2164                                                             dcache_size,
2165                                                             dcache_linesize);
2166                                                         if (cache_boot_state &
2167                                                             DCU_DC) {
2168                                                                 flush_dcache();
2169                                                         }
2170 
2171                                                         set_dcu(get_dcu() |
2172                                                             cache_boot_state);
2173                                                         return;
2174                                                 }
2175                                         }
2176                                 }
2177                         } /* (tl == 0) && (way != 0) && IS JALAPENO */
2178                         error_class = FM_EREPORT_CPU_USIII_DDSPE;
2179                 } else if (ch_flt.parity_data.dpe.cpl_way != -1)
2180                         error_class = FM_EREPORT_CPU_USIII_DTSPE;
2181                 else
2182                         error_class = FM_EREPORT_CPU_USIII_DPE;
2183                 aflt->flt_payload = FM_EREPORT_PAYLOAD_DCACHE_PE;
2184                 /*
2185                  * For panther we also need to check the P$ for parity errors.
2186                  */
2187                 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) {
2188                         cpu_pcache_parity_info(&ch_flt);
2189                         if (ch_flt.parity_data.dpe.cpl_cache == CPU_PC_PARITY) {
2190                                 error_class = FM_EREPORT_CPU_USIII_PDSPE;
2191                                 aflt->flt_payload =
2192                                     FM_EREPORT_PAYLOAD_PCACHE_PE;
2193                         }
2194                 }
2195         }
2196 
2197         cpu_errorq_dispatch(error_class, (void *)&ch_flt,
2198             sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic);
2199 
2200         if (iparity) {
2201                 /*
2202                  * Invalidate entire I$.
2203                  * This is required due to the use of diagnostic ASI
2204                  * accesses that may result in a loss of I$ coherency.
2205                  */
2206                 if (cache_boot_state & DCU_IC) {
2207                         flush_icache();
2208                 }
2209                 /*
2210                  * According to section P.3.1 of the Panther PRM, we
2211                  * need to do a little more for recovery on those
2212                  * CPUs after encountering an I$ parity error.
2213                  */
2214                 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) {
2215                         flush_ipb();
2216                         correct_dcache_parity(dcache_size,
2217                             dcache_linesize);
2218                         flush_pcache();
2219                 }
2220         } else {
2221                 /*
2222                  * Since the valid bit is ignored when checking parity the
2223                  * D$ data and tag must also be corrected.  Set D$ data bits
2224                  * to zero and set utag to 0, 1, 2, 3.
2225                  */
2226                 correct_dcache_parity(dcache_size, dcache_linesize);
2227 
2228                 /*
2229                  * According to section P.3.3 of the Panther PRM, we
2230                  * need to do a little more for recovery on those
2231                  * CPUs after encountering a D$ or P$ parity error.
2232                  *
2233                  * As far as clearing P$ parity errors, it is enough to
2234                  * simply invalidate all entries in the P$ since P$ parity
2235                  * error traps are only generated for floating point load
2236                  * hits.
2237                  */
2238                 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) {
2239                         flush_icache();
2240                         flush_ipb();
2241                         flush_pcache();
2242                 }
2243         }
2244 
2245         /*
2246          * Invalidate entire D$ if it was enabled.
2247          * This is done to avoid stale data in the D$ which might
2248          * occur with the D$ disabled and the trap handler doing
2249          * stores affecting lines already in the D$.
2250          */
2251         if (cache_boot_state & DCU_DC) {
2252                 flush_dcache();
2253         }
2254 
2255         /*
2256          * Restore caches to their bootup state.
2257          */
2258         set_dcu(get_dcu() | cache_boot_state);
2259 
2260         /*
2261          * Panic here if aflt->flt_panic has been set.  Enqueued errors will
2262          * be logged as part of the panic flow.
2263          */
2264         if (aflt->flt_panic)
2265                 fm_panic("%sError(s)", iparity ? "IPE " : "DPE ");
2266 
2267         /*
2268          * If this error occurred at TL>0 then flush the E$ here to reduce
2269          * the chance of getting an unrecoverable Fast ECC error.  This
2270          * flush will evict the part of the parity trap handler that is run
2271          * at TL>1.
2272          */
2273         if (tl) {
2274                 cpu_flush_ecache();
2275         }
2276 }
2277 
2278 /*
2279  * On an I$ parity error, mark the appropriate entries in the ch_async_flt_t
2280  * to indicate which portions of the captured data should be in the ereport.
2281  */
2282 void
2283 cpu_async_log_ic_parity_err(ch_async_flt_t *ch_flt)
2284 {
2285         int way = ch_flt->parity_data.ipe.cpl_way;
2286         int offset = ch_flt->parity_data.ipe.cpl_off;
2287         int tag_index;
2288         struct async_flt *aflt = (struct async_flt *)ch_flt;
2289 
2290 
2291         if ((offset != -1) || (way != -1)) {
2292                 /*
2293                  * Parity error in I$ tag or data
2294                  */
2295                 tag_index = ch_flt->parity_data.ipe.cpl_ic[way].ic_idx;
2296                 if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation))
2297                         ch_flt->parity_data.ipe.cpl_ic[way].ic_way =
2298                             PN_ICIDX_TO_WAY(tag_index);
2299                 else
2300                         ch_flt->parity_data.ipe.cpl_ic[way].ic_way =
2301                             CH_ICIDX_TO_WAY(tag_index);
2302                 ch_flt->parity_data.ipe.cpl_ic[way].ic_logflag =
2303                     IC_LOGFLAG_MAGIC;
2304         } else {
2305                 /*
2306                  * Parity error was not identified.
2307                  * Log tags and data for all ways.
2308                  */
2309                 for (way = 0; way < CH_ICACHE_NWAY; way++) {
2310                         tag_index = ch_flt->parity_data.ipe.cpl_ic[way].ic_idx;
2311                         if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation))
2312                                 ch_flt->parity_data.ipe.cpl_ic[way].ic_way =
2313                                     PN_ICIDX_TO_WAY(tag_index);
2314                         else
2315                                 ch_flt->parity_data.ipe.cpl_ic[way].ic_way =
2316                                     CH_ICIDX_TO_WAY(tag_index);
2317                         ch_flt->parity_data.ipe.cpl_ic[way].ic_logflag =
2318                             IC_LOGFLAG_MAGIC;
2319                 }
2320         }
2321 }
2322 
2323 /*
2324  * On an D$ parity error, mark the appropriate entries in the ch_async_flt_t
2325  * to indicate which portions of the captured data should be in the ereport.
2326  */
2327 void
2328 cpu_async_log_dc_parity_err(ch_async_flt_t *ch_flt)
2329 {
2330         int way = ch_flt->parity_data.dpe.cpl_way;
2331         int offset = ch_flt->parity_data.dpe.cpl_off;
2332         int tag_index;
2333 
2334         if (offset != -1) {
2335                 /*
2336                  * Parity error in D$ or P$ data array.
2337                  *
2338                  * First check to see whether the parity error is in D$ or P$
2339                  * since P$ data parity errors are reported in Panther using
2340                  * the same trap.
2341                  */
2342                 if (ch_flt->parity_data.dpe.cpl_cache == CPU_PC_PARITY) {
2343                         tag_index = ch_flt->parity_data.dpe.cpl_pc[way].pc_idx;
2344                         ch_flt->parity_data.dpe.cpl_pc[way].pc_way =
2345                             CH_PCIDX_TO_WAY(tag_index);
2346                         ch_flt->parity_data.dpe.cpl_pc[way].pc_logflag =
2347                             PC_LOGFLAG_MAGIC;
2348                 } else {
2349                         tag_index = ch_flt->parity_data.dpe.cpl_dc[way].dc_idx;
2350                         ch_flt->parity_data.dpe.cpl_dc[way].dc_way =
2351                             CH_DCIDX_TO_WAY(tag_index);
2352                         ch_flt->parity_data.dpe.cpl_dc[way].dc_logflag =
2353                             DC_LOGFLAG_MAGIC;
2354                 }
2355         } else if (way != -1) {
2356                 /*
2357                  * Parity error in D$ tag.
2358                  */
2359                 tag_index = ch_flt->parity_data.dpe.cpl_dc[way].dc_idx;
2360                 ch_flt->parity_data.dpe.cpl_dc[way].dc_way =
2361                     CH_DCIDX_TO_WAY(tag_index);
2362                 ch_flt->parity_data.dpe.cpl_dc[way].dc_logflag =
2363                     DC_LOGFLAG_MAGIC;
2364         }
2365 }
2366 #endif  /* CPU_IMP_L1_CACHE_PARITY */
2367 
2368 /*
2369  * The cpu_async_log_err() function is called via the [uc]e_drain() function to
2370  * post-process CPU events that are dequeued.  As such, it can be invoked
2371  * from softint context, from AST processing in the trap() flow, or from the
2372  * panic flow.  We decode the CPU-specific data, and take appropriate actions.
2373  * Historically this entry point was used to log the actual cmn_err(9F) text;
2374  * now with FMA it is used to prepare 'flt' to be converted into an ereport.
2375  * With FMA this function now also returns a flag which indicates to the
2376  * caller whether the ereport should be posted (1) or suppressed (0).
2377  */
2378 static int
2379 cpu_async_log_err(void *flt, errorq_elem_t *eqep)
2380 {
2381         ch_async_flt_t *ch_flt = (ch_async_flt_t *)flt;
2382         struct async_flt *aflt = (struct async_flt *)flt;
2383         uint64_t errors;
2384         extern void memscrub_induced_error(void);
2385 
2386         switch (ch_flt->flt_type) {
2387         case CPU_INV_AFSR:
2388                 /*
2389                  * If it is a disrupting trap and the AFSR is zero, then
2390                  * the event has probably already been noted. Do not post
2391                  * an ereport.
2392                  */
2393                 if ((aflt->flt_status & ECC_C_TRAP) &&
2394                     (!(aflt->flt_stat & C_AFSR_MASK)))
2395                         return (0);
2396                 else
2397                         return (1);
2398         case CPU_TO:
2399         case CPU_BERR:
2400         case CPU_FATAL:
2401         case CPU_FPUERR:
2402                 return (1);
2403 
2404         case CPU_UE_ECACHE_RETIRE:
2405                 cpu_log_err(aflt);
2406                 cpu_page_retire(ch_flt);
2407                 return (1);
2408 
2409         /*
2410          * Cases where we may want to suppress logging or perform
2411          * extended diagnostics.
2412          */
2413         case CPU_CE:
2414         case CPU_EMC:
2415                 /*
2416                  * We want to skip logging and further classification
2417                  * only if ALL the following conditions are true:
2418                  *
2419                  *      1. There is only one error
2420                  *      2. That error is a correctable memory error
2421                  *      3. The error is caused by the memory scrubber (in
2422                  *         which case the error will have occurred under
2423                  *         on_trap protection)
2424                  *      4. The error is on a retired page
2425                  *
2426                  * Note: AFLT_PROT_EC is used places other than the memory
2427                  * scrubber.  However, none of those errors should occur
2428                  * on a retired page.
2429                  */
2430                 if ((ch_flt->afsr_errs &
2431                     (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_CE &&
2432                     aflt->flt_prot == AFLT_PROT_EC) {
2433 
2434                         if (page_retire_check(aflt->flt_addr, NULL) == 0) {
2435                                 if (ch_flt->flt_trapped_ce & CE_CEEN_DEFER) {
2436 
2437                                 /*
2438                                  * Since we're skipping logging, we'll need
2439                                  * to schedule the re-enabling of CEEN
2440                                  */
2441                                 (void) timeout(cpu_delayed_check_ce_errors,
2442                                     (void *)(uintptr_t)aflt->flt_inst,
2443                                     drv_sectohz((clock_t)cpu_ceen_delay_secs));
2444                                 }
2445 
2446                                 /*
2447                                  * Inform memscrubber - scrubbing induced
2448                                  * CE on a retired page.
2449                                  */
2450                                 memscrub_induced_error();
2451                                 return (0);
2452                         }
2453                 }
2454 
2455                 /*
2456                  * Perform/schedule further classification actions, but
2457                  * only if the page is healthy (we don't want bad
2458                  * pages inducing too much diagnostic activity).  If we could
2459                  * not find a page pointer then we also skip this.  If
2460                  * ce_scrub_xdiag_recirc returns nonzero then it has chosen
2461                  * to copy and recirculate the event (for further diagnostics)
2462                  * and we should not proceed to log it here.
2463                  *
2464                  * This must be the last step here before the cpu_log_err()
2465                  * below - if an event recirculates cpu_ce_log_err() will
2466                  * not call the current function but just proceed directly
2467                  * to cpu_ereport_post after the cpu_log_err() avoided below.
2468                  *
2469                  * Note: Check cpu_impl_async_log_err if changing this
2470                  */
2471                 if (page_retire_check(aflt->flt_addr, &errors) == EINVAL) {
2472                         CE_XDIAG_SETSKIPCODE(aflt->flt_disp,
2473                             CE_XDIAG_SKIP_NOPP);
2474                 } else {
2475                         if (errors != PR_OK) {
2476                                 CE_XDIAG_SETSKIPCODE(aflt->flt_disp,
2477                                     CE_XDIAG_SKIP_PAGEDET);
2478                         } else if (ce_scrub_xdiag_recirc(aflt, ce_queue, eqep,
2479                             offsetof(ch_async_flt_t, cmn_asyncflt))) {
2480                                 return (0);
2481                         }
2482                 }
2483                 /*FALLTHRU*/
2484 
2485         /*
2486          * Cases where we just want to report the error and continue.
2487          */
2488         case CPU_CE_ECACHE:
2489         case CPU_UE_ECACHE:
2490         case CPU_IV:
2491         case CPU_ORPH:
2492                 cpu_log_err(aflt);
2493                 return (1);
2494 
2495         /*
2496          * Cases where we want to fall through to handle panicking.
2497          */
2498         case CPU_UE:
2499                 /*
2500                  * We want to skip logging in the same conditions as the
2501                  * CE case.  In addition, we want to make sure we're not
2502                  * panicking.
2503                  */
2504                 if (!panicstr && (ch_flt->afsr_errs &
2505                     (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_UE &&
2506                     aflt->flt_prot == AFLT_PROT_EC) {
2507                         if (page_retire_check(aflt->flt_addr, NULL) == 0) {
2508                                 /* Zero the address to clear the error */
2509                                 softcall(ecc_page_zero, (void *)aflt->flt_addr);
2510                                 /*
2511                                  * Inform memscrubber - scrubbing induced
2512                                  * UE on a retired page.
2513                                  */
2514                                 memscrub_induced_error();
2515                                 return (0);
2516                         }
2517                 }
2518                 cpu_log_err(aflt);
2519                 break;
2520 
2521         default:
2522                 /*
2523                  * If the us3_common.c code doesn't know the flt_type, it may
2524                  * be an implementation-specific code.  Call into the impldep
2525                  * backend to find out what to do: if it tells us to continue,
2526                  * break and handle as if falling through from a UE; if not,
2527                  * the impldep backend has handled the error and we're done.
2528                  */
2529                 switch (cpu_impl_async_log_err(flt, eqep)) {
2530                 case CH_ASYNC_LOG_DONE:
2531                         return (1);
2532                 case CH_ASYNC_LOG_RECIRC:
2533                         return (0);
2534                 case CH_ASYNC_LOG_CONTINUE:
2535                         break; /* continue on to handle UE-like error */
2536                 default:
2537                         cmn_err(CE_WARN, "discarding error 0x%p with "
2538                             "invalid fault type (0x%x)",
2539                             (void *)aflt, ch_flt->flt_type);
2540                         return (0);
2541                 }
2542         }
2543 
2544         /* ... fall through from the UE case */
2545 
2546         if (aflt->flt_addr != AFLT_INV_ADDR && aflt->flt_in_memory) {
2547                 if (!panicstr) {
2548                         cpu_page_retire(ch_flt);
2549                 } else {
2550                         /*
2551                          * Clear UEs on panic so that we don't
2552                          * get haunted by them during panic or
2553                          * after reboot
2554                          */
2555                         cpu_clearphys(aflt);
2556                         (void) clear_errors(NULL);
2557                 }
2558         }
2559 
2560         return (1);
2561 }
2562 
2563 /*
2564  * Retire the bad page that may contain the flushed error.
2565  */
2566 void
2567 cpu_page_retire(ch_async_flt_t *ch_flt)
2568 {
2569         struct async_flt *aflt = (struct async_flt *)ch_flt;
2570         (void) page_retire(aflt->flt_addr, PR_UE);
2571 }
2572 
2573 /*
2574  * Return true if the error specified in the AFSR indicates
2575  * an E$ data error (L2$ for Cheetah/Cheetah+/Jaguar, L3$
2576  * for Panther, none for Jalapeno/Serrano).
2577  */
2578 /* ARGSUSED */
2579 static int
2580 cpu_error_is_ecache_data(int cpuid, uint64_t t_afsr)
2581 {
2582 #if defined(JALAPENO) || defined(SERRANO)
2583         return (0);
2584 #elif defined(CHEETAH_PLUS)
2585         if (IS_PANTHER(cpunodes[cpuid].implementation))
2586                 return ((t_afsr & C_AFSR_EXT_L3_DATA_ERRS) != 0);
2587         return ((t_afsr & C_AFSR_EC_DATA_ERRS) != 0);
2588 #else   /* CHEETAH_PLUS */
2589         return ((t_afsr & C_AFSR_EC_DATA_ERRS) != 0);
2590 #endif
2591 }
2592 
2593 /*
2594  * The cpu_log_err() function is called by cpu_async_log_err() to perform the
2595  * generic event post-processing for correctable and uncorrectable memory,
2596  * E$, and MTag errors.  Historically this entry point was used to log bits of
2597  * common cmn_err(9F) text; now with FMA it is used to prepare 'flt' to be
2598  * converted into an ereport.  In addition, it transmits the error to any
2599  * platform-specific service-processor FRU logging routines, if available.
2600  */
2601 void
2602 cpu_log_err(struct async_flt *aflt)
2603 {
2604         char unum[UNUM_NAMLEN];
2605         int synd_status, synd_code, afar_status;
2606         ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt;
2607 
2608         if (cpu_error_is_ecache_data(aflt->flt_inst, ch_flt->flt_bit))
2609                 aflt->flt_status |= ECC_ECACHE;
2610         else
2611                 aflt->flt_status &= ~ECC_ECACHE;
2612         /*
2613          * Determine syndrome status.
2614          */
2615         synd_status = afsr_to_synd_status(aflt->flt_inst,
2616             ch_flt->afsr_errs, ch_flt->flt_bit);
2617 
2618         /*
2619          * Determine afar status.
2620          */
2621         if (pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT))
2622                 afar_status = afsr_to_afar_status(ch_flt->afsr_errs,
2623                     ch_flt->flt_bit);
2624         else
2625                 afar_status = AFLT_STAT_INVALID;
2626 
2627         synd_code = synd_to_synd_code(synd_status,
2628             aflt->flt_synd, ch_flt->flt_bit);
2629 
2630         /*
2631          * If afar status is not invalid do a unum lookup.
2632          */
2633         if (afar_status != AFLT_STAT_INVALID) {
2634                 (void) cpu_get_mem_unum_synd(synd_code, aflt, unum);
2635         } else {
2636                 unum[0] = '\0';
2637         }
2638 
2639         /*
2640          * Do not send the fruid message (plat_ecc_error_data_t)
2641          * to the SC if it can handle the enhanced error information
2642          * (plat_ecc_error2_data_t) or when the tunable
2643          * ecc_log_fruid_enable is set to 0.
2644          */
2645 
2646         if (&plat_ecc_capability_sc_get &&
2647             plat_ecc_capability_sc_get(PLAT_ECC_ERROR_MESSAGE)) {
2648                 if (&plat_log_fruid_error)
2649                         plat_log_fruid_error(synd_code, aflt, unum,
2650                             ch_flt->flt_bit);
2651         }
2652 
2653         if (aflt->flt_func != NULL)
2654                 aflt->flt_func(aflt, unum);
2655 
2656         if (afar_status != AFLT_STAT_INVALID)
2657                 cpu_log_diag_info(ch_flt);
2658 
2659         /*
2660          * If we have a CEEN error , we do not reenable CEEN until after
2661          * we exit the trap handler. Otherwise, another error may
2662          * occur causing the handler to be entered recursively.
2663          * We set a timeout to trigger in cpu_ceen_delay_secs seconds,
2664          * to try and ensure that the CPU makes progress in the face
2665          * of a CE storm.
2666          */
2667         if (ch_flt->flt_trapped_ce & CE_CEEN_DEFER) {
2668                 (void) timeout(cpu_delayed_check_ce_errors,
2669                     (void *)(uintptr_t)aflt->flt_inst,
2670                     drv_sectohz((clock_t)cpu_ceen_delay_secs));
2671         }
2672 }
2673 
2674 /*
2675  * Invoked by error_init() early in startup and therefore before
2676  * startup_errorq() is called to drain any error Q -
2677  *
2678  * startup()
2679  *   startup_end()
2680  *     error_init()
2681  *       cpu_error_init()
2682  * errorq_init()
2683  *   errorq_drain()
2684  * start_other_cpus()
2685  *
2686  * The purpose of this routine is to create error-related taskqs.  Taskqs
2687  * are used for this purpose because cpu_lock can't be grabbed from interrupt
2688  * context.
2689  */
2690 void
2691 cpu_error_init(int items)
2692 {
2693         /*
2694          * Create taskq(s) to reenable CE
2695          */
2696         ch_check_ce_tq = taskq_create("cheetah_check_ce", 1, minclsyspri,
2697             items, items, TASKQ_PREPOPULATE);
2698 }
2699 
2700 void
2701 cpu_ce_log_err(struct async_flt *aflt, errorq_elem_t *eqep)
2702 {
2703         char unum[UNUM_NAMLEN];
2704         int len;
2705 
2706         switch (aflt->flt_class) {
2707         case CPU_FAULT:
2708                 cpu_ereport_init(aflt);
2709                 if (cpu_async_log_err(aflt, eqep))
2710                         cpu_ereport_post(aflt);
2711                 break;
2712 
2713         case BUS_FAULT:
2714                 if (aflt->flt_func != NULL) {
2715                         (void) cpu_get_mem_unum_aflt(AFLT_STAT_VALID, aflt,
2716                             unum, UNUM_NAMLEN, &len);
2717                         aflt->flt_func(aflt, unum);
2718                 }
2719                 break;
2720 
2721         case RECIRC_CPU_FAULT:
2722                 aflt->flt_class = CPU_FAULT;
2723                 cpu_log_err(aflt);
2724                 cpu_ereport_post(aflt);
2725                 break;
2726 
2727         case RECIRC_BUS_FAULT:
2728                 ASSERT(aflt->flt_class != RECIRC_BUS_FAULT);
2729                 /*FALLTHRU*/
2730         default:
2731                 cmn_err(CE_WARN, "discarding CE error 0x%p with invalid "
2732                     "fault class (0x%x)", (void *)aflt, aflt->flt_class);
2733                 return;
2734         }
2735 }
2736 
2737 /*
2738  * Scrub and classify a CE.  This function must not modify the
2739  * fault structure passed to it but instead should return the classification
2740  * information.
2741  */
2742 
2743 static uchar_t
2744 cpu_ce_scrub_mem_err_common(struct async_flt *ecc, boolean_t logout_tried)
2745 {
2746         uchar_t disp = CE_XDIAG_EXTALG;
2747         on_trap_data_t otd;
2748         uint64_t orig_err;
2749         ch_cpu_logout_t *clop;
2750 
2751         /*
2752          * Clear CEEN.  CPU CE TL > 0 trap handling will already have done
2753          * this, but our other callers have not.  Disable preemption to
2754          * avoid CPU migration so that we restore CEEN on the correct
2755          * cpu later.
2756          *
2757          * CEEN is cleared so that further CEs that our instruction and
2758          * data footprint induce do not cause use to either creep down
2759          * kernel stack to the point of overflow, or do so much CE
2760          * notification as to make little real forward progress.
2761          *
2762          * NCEEN must not be cleared.  However it is possible that
2763          * our accesses to the flt_addr may provoke a bus error or timeout
2764          * if the offending address has just been unconfigured as part of
2765          * a DR action.  So we must operate under on_trap protection.
2766          */
2767         kpreempt_disable();
2768         orig_err = get_error_enable();
2769         if (orig_err & EN_REG_CEEN)
2770                 set_error_enable(orig_err & ~EN_REG_CEEN);
2771 
2772         /*
2773          * Our classification algorithm includes the line state before
2774          * the scrub; we'd like this captured after the detection and
2775          * before the algorithm below - the earlier the better.
2776          *
2777          * If we've come from a cpu CE trap then this info already exists
2778          * in the cpu logout area.
2779          *
2780          * For a CE detected by memscrub for which there was no trap
2781          * (running with CEEN off) cpu_log_and_clear_ce has called
2782          * cpu_ce_delayed_ec_logout to capture some cache data, and
2783          * marked the fault structure as incomplete as a flag to later
2784          * logging code.
2785          *
2786          * If called directly from an IO detected CE there has been
2787          * no line data capture.  In this case we logout to the cpu logout
2788          * area - that's appropriate since it's the cpu cache data we need
2789          * for classification.  We thus borrow the cpu logout area for a
2790          * short time, and cpu_ce_delayed_ec_logout will mark it as busy in
2791          * this time (we will invalidate it again below).
2792          *
2793          * If called from the partner check xcall handler then this cpu
2794          * (the partner) has not necessarily experienced a CE at this
2795          * address.  But we want to capture line state before its scrub
2796          * attempt since we use that in our classification.
2797          */
2798         if (logout_tried == B_FALSE) {
2799                 if (!cpu_ce_delayed_ec_logout(ecc->flt_addr))
2800                         disp |= CE_XDIAG_NOLOGOUT;
2801         }
2802 
2803         /*
2804          * Scrub memory, then check AFSR for errors.  The AFAR we scrub may
2805          * no longer be valid (if DR'd since the initial event) so we
2806          * perform this scrub under on_trap protection.  If this access is
2807          * ok then further accesses below will also be ok - DR cannot
2808          * proceed while this thread is active (preemption is disabled);
2809          * to be safe we'll nonetheless use on_trap again below.
2810          */
2811         if (!on_trap(&otd, OT_DATA_ACCESS)) {
2812                 cpu_scrubphys(ecc);
2813         } else {
2814                 no_trap();
2815                 if (orig_err & EN_REG_CEEN)
2816                         set_error_enable(orig_err);
2817                 kpreempt_enable();
2818                 return (disp);
2819         }
2820         no_trap();
2821 
2822         /*
2823          * Did the casx read of the scrub log a CE that matches the AFAR?
2824          * Note that it's quite possible that the read sourced the data from
2825          * another cpu.
2826          */
2827         if (clear_ecc(ecc))
2828                 disp |= CE_XDIAG_CE1;
2829 
2830         /*
2831          * Read the data again.  This time the read is very likely to
2832          * come from memory since the scrub induced a writeback to memory.
2833          */
2834         if (!on_trap(&otd, OT_DATA_ACCESS)) {
2835                 (void) lddphys(P2ALIGN(ecc->flt_addr, 8));
2836         } else {
2837                 no_trap();
2838                 if (orig_err & EN_REG_CEEN)
2839                         set_error_enable(orig_err);
2840                 kpreempt_enable();
2841                 return (disp);
2842         }
2843         no_trap();
2844 
2845         /* Did that read induce a CE that matches the AFAR? */
2846         if (clear_ecc(ecc))
2847                 disp |= CE_XDIAG_CE2;
2848 
2849         /*
2850          * Look at the logout information and record whether we found the
2851          * line in l2/l3 cache.  For Panther we are interested in whether
2852          * we found it in either cache (it won't reside in both but
2853          * it is possible to read it that way given the moving target).
2854          */
2855         clop = CPU_PRIVATE(CPU) ? CPU_PRIVATE_PTR(CPU, chpr_cecc_logout) : NULL;
2856         if (!(disp & CE_XDIAG_NOLOGOUT) && clop &&
2857             clop->clo_data.chd_afar != LOGOUT_INVALID) {
2858                 int hit, level;
2859                 int state;
2860                 int totalsize;
2861                 ch_ec_data_t *ecp;
2862 
2863                 /*
2864                  * If hit is nonzero then a match was found and hit will
2865                  * be one greater than the index which hit.  For Panther we
2866                  * also need to pay attention to level to see which of l2$ or
2867                  * l3$ it hit in.
2868                  */
2869                 hit = cpu_matching_ecache_line(ecc->flt_addr, &clop->clo_data,
2870                     0, &level);
2871 
2872                 if (hit) {
2873                         --hit;
2874                         disp |= CE_XDIAG_AFARMATCH;
2875 
2876                         if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) {
2877                                 if (level == 2)
2878                                         ecp = &clop->clo_data.chd_l2_data[hit];
2879                                 else
2880                                         ecp = &clop->clo_data.chd_ec_data[hit];
2881                         } else {
2882                                 ASSERT(level == 2);
2883                                 ecp = &clop->clo_data.chd_ec_data[hit];
2884                         }
2885                         totalsize = cpunodes[CPU->cpu_id].ecache_size;
2886                         state = cpu_ectag_pa_to_subblk_state(totalsize,
2887                             ecc->flt_addr, ecp->ec_tag);
2888 
2889                         /*
2890                          * Cheetah variants use different state encodings -
2891                          * the CH_ECSTATE_* defines vary depending on the
2892                          * module we're compiled for.  Translate into our
2893                          * one true version.  Conflate Owner-Shared state
2894                          * of SSM mode with Owner as victimisation of such
2895                          * lines may cause a writeback.
2896                          */
2897                         switch (state) {
2898                         case CH_ECSTATE_MOD:
2899                                 disp |= EC_STATE_M;
2900                                 break;
2901 
2902                         case CH_ECSTATE_OWN:
2903                         case CH_ECSTATE_OWS:
2904                                 disp |= EC_STATE_O;
2905                                 break;
2906 
2907                         case CH_ECSTATE_EXL:
2908                                 disp |= EC_STATE_E;
2909                                 break;
2910 
2911                         case CH_ECSTATE_SHR:
2912                                 disp |= EC_STATE_S;
2913                                 break;
2914 
2915                         default:
2916                                 disp |= EC_STATE_I;
2917                                 break;
2918                         }
2919                 }
2920 
2921                 /*
2922                  * If we initiated the delayed logout then we are responsible
2923                  * for invalidating the logout area.
2924                  */
2925                 if (logout_tried == B_FALSE) {
2926                         bzero(clop, sizeof (ch_cpu_logout_t));
2927                         clop->clo_data.chd_afar = LOGOUT_INVALID;
2928                 }
2929         }
2930 
2931         /*
2932          * Re-enable CEEN if we turned it off.
2933          */
2934         if (orig_err & EN_REG_CEEN)
2935                 set_error_enable(orig_err);
2936         kpreempt_enable();
2937 
2938         return (disp);
2939 }
2940 
2941 /*
2942  * Scrub a correctable memory error and collect data for classification
2943  * of CE type.  This function is called in the detection path, ie tl0 handling
2944  * of a correctable error trap (cpus) or interrupt (IO) at high PIL.
2945  */
2946 void
2947 cpu_ce_scrub_mem_err(struct async_flt *ecc, boolean_t logout_tried)
2948 {
2949         /*
2950          * Cheetah CE classification does not set any bits in flt_status.
2951          * Instead we will record classification datapoints in flt_disp.
2952          */
2953         ecc->flt_status &= ~(ECC_INTERMITTENT | ECC_PERSISTENT | ECC_STICKY);
2954 
2955         /*
2956          * To check if the error detected by IO is persistent, sticky or
2957          * intermittent.  This is noticed by clear_ecc().
2958          */
2959         if (ecc->flt_status & ECC_IOBUS)
2960                 ecc->flt_stat = C_AFSR_MEMORY;
2961 
2962         /*
2963          * Record information from this first part of the algorithm in
2964          * flt_disp.
2965          */
2966         ecc->flt_disp = cpu_ce_scrub_mem_err_common(ecc, logout_tried);
2967 }
2968 
2969 /*
2970  * Select a partner to perform a further CE classification check from.
2971  * Must be called with kernel preemption disabled (to stop the cpu list
2972  * from changing).  The detecting cpu we are partnering has cpuid
2973  * aflt->flt_inst; we might not be running on the detecting cpu.
2974  *
2975  * Restrict choice to active cpus in the same cpu partition as ourselves in
2976  * an effort to stop bad cpus in one partition causing other partitions to
2977  * perform excessive diagnostic activity.  Actually since the errorq drain
2978  * is run from a softint most of the time and that is a global mechanism
2979  * this isolation is only partial.  Return NULL if we fail to find a
2980  * suitable partner.
2981  *
2982  * We prefer a partner that is in a different latency group to ourselves as
2983  * we will share fewer datapaths.  If such a partner is unavailable then
2984  * choose one in the same lgroup but prefer a different chip and only allow
2985  * a sibling core if flags includes PTNR_SIBLINGOK.  If all else fails and
2986  * flags includes PTNR_SELFOK then permit selection of the original detector.
2987  *
2988  * We keep a cache of the last partner selected for a cpu, and we'll try to
2989  * use that previous partner if no more than cpu_ce_ptnr_cachetime_sec seconds
2990  * have passed since that selection was made.  This provides the benefit
2991  * of the point-of-view of different partners over time but without
2992  * requiring frequent cpu list traversals.
2993  */
2994 
2995 #define PTNR_SIBLINGOK  0x1     /* Allow selection of sibling core */
2996 #define PTNR_SELFOK     0x2     /* Allow selection of cpu to "partner" itself */
2997 
2998 static cpu_t *
2999 ce_ptnr_select(struct async_flt *aflt, int flags, int *typep)
3000 {
3001         cpu_t *sp, *dtcr, *ptnr, *locptnr, *sibptnr;
3002         hrtime_t lasttime, thistime;
3003 
3004         ASSERT(curthread->t_preempt > 0 || getpil() >= DISP_LEVEL);
3005 
3006         dtcr = cpu[aflt->flt_inst];
3007 
3008         /*
3009          * Short-circuit for the following cases:
3010          *      . the dtcr is not flagged active
3011          *      . there is just one cpu present
3012          *      . the detector has disappeared
3013          *      . we were given a bad flt_inst cpuid; this should not happen
3014          *        (eg PCI code now fills flt_inst) but if it does it is no
3015          *        reason to panic.
3016          *      . there is just one cpu left online in the cpu partition
3017          *
3018          * If we return NULL after this point then we do not update the
3019          * chpr_ceptnr_seltime which will cause us to perform a full lookup
3020          * again next time; this is the case where the only other cpu online
3021          * in the detector's partition is on the same chip as the detector
3022          * and since CEEN re-enable is throttled even that case should not
3023          * hurt performance.
3024          */
3025         if (dtcr == NULL || !cpu_flagged_active(dtcr->cpu_flags)) {
3026                 return (NULL);
3027         }
3028         if (ncpus == 1 || dtcr->cpu_part->cp_ncpus == 1) {
3029                 if (flags & PTNR_SELFOK) {
3030                         *typep = CE_XDIAG_PTNR_SELF;
3031                         return (dtcr);
3032                 } else {
3033                         return (NULL);
3034                 }
3035         }
3036 
3037         thistime = gethrtime();
3038         lasttime = CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime);
3039 
3040         /*
3041          * Select a starting point.
3042          */
3043         if (!lasttime) {
3044                 /*
3045                  * We've never selected a partner for this detector before.
3046                  * Start the scan at the next online cpu in the same cpu
3047                  * partition.
3048                  */
3049                 sp = dtcr->cpu_next_part;
3050         } else if (thistime - lasttime < cpu_ce_ptnr_cachetime_sec * NANOSEC) {
3051                 /*
3052                  * Our last selection has not aged yet.  If this partner:
3053                  *      . is still a valid cpu,
3054                  *      . is still in the same partition as the detector
3055                  *      . is still marked active
3056                  *      . satisfies the 'flags' argument criteria
3057                  * then select it again without updating the timestamp.
3058                  */
3059                 sp = cpu[CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id)];
3060                 if (sp == NULL || sp->cpu_part != dtcr->cpu_part ||
3061                     !cpu_flagged_active(sp->cpu_flags) ||
3062                     (sp == dtcr && !(flags & PTNR_SELFOK)) ||
3063                     (pg_plat_cpus_share(sp, dtcr, PGHW_CHIP) &&
3064                     !(flags & PTNR_SIBLINGOK))) {
3065                         sp = dtcr->cpu_next_part;
3066                 } else {
3067                         if (sp->cpu_lpl->lpl_lgrp != dtcr->cpu_lpl->lpl_lgrp) {
3068                                 *typep = CE_XDIAG_PTNR_REMOTE;
3069                         } else if (sp == dtcr) {
3070                                 *typep = CE_XDIAG_PTNR_SELF;
3071                         } else if (pg_plat_cpus_share(sp, dtcr, PGHW_CHIP)) {
3072                                 *typep = CE_XDIAG_PTNR_SIBLING;
3073                         } else {
3074                                 *typep = CE_XDIAG_PTNR_LOCAL;
3075                         }
3076                         return (sp);
3077                 }
3078         } else {
3079                 /*
3080                  * Our last selection has aged.  If it is nonetheless still a
3081                  * valid cpu then start the scan at the next cpu in the
3082                  * partition after our last partner.  If the last selection
3083                  * is no longer a valid cpu then go with our default.  In
3084                  * this way we slowly cycle through possible partners to
3085                  * obtain multiple viewpoints over time.
3086                  */
3087                 sp = cpu[CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id)];
3088                 if (sp == NULL) {
3089                         sp = dtcr->cpu_next_part;
3090                 } else {
3091                         sp = sp->cpu_next_part;              /* may be dtcr */
3092                         if (sp->cpu_part != dtcr->cpu_part)
3093                                 sp = dtcr;
3094                 }
3095         }
3096 
3097         /*
3098          * We have a proposed starting point for our search, but if this
3099          * cpu is offline then its cpu_next_part will point to itself
3100          * so we can't use that to iterate over cpus in this partition in
3101          * the loop below.  We still want to avoid iterating over cpus not
3102          * in our partition, so in the case that our starting point is offline
3103          * we will repoint it to be the detector itself;  and if the detector
3104          * happens to be offline we'll return NULL from the following loop.
3105          */
3106         if (!cpu_flagged_active(sp->cpu_flags)) {
3107                 sp = dtcr;
3108         }
3109 
3110         ptnr = sp;
3111         locptnr = NULL;
3112         sibptnr = NULL;
3113         do {
3114                 if (ptnr == dtcr || !cpu_flagged_active(ptnr->cpu_flags))
3115                         continue;
3116                 if (ptnr->cpu_lpl->lpl_lgrp != dtcr->cpu_lpl->lpl_lgrp) {
3117                         CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = ptnr->cpu_id;
3118                         CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime;
3119                         *typep = CE_XDIAG_PTNR_REMOTE;
3120                         return (ptnr);
3121                 }
3122                 if (pg_plat_cpus_share(ptnr, dtcr, PGHW_CHIP)) {
3123                         if (sibptnr == NULL)
3124                                 sibptnr = ptnr;
3125                         continue;
3126                 }
3127                 if (locptnr == NULL)
3128                         locptnr = ptnr;
3129         } while ((ptnr = ptnr->cpu_next_part) != sp);
3130 
3131         /*
3132          * A foreign partner has already been returned if one was available.
3133          *
3134          * If locptnr is not NULL it is a cpu in the same lgroup as the
3135          * detector, is active, and is not a sibling of the detector.
3136          *
3137          * If sibptnr is not NULL it is a sibling of the detector, and is
3138          * active.
3139          *
3140          * If we have to resort to using the detector itself we have already
3141          * checked that it is active.
3142          */
3143         if (locptnr) {
3144                 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = locptnr->cpu_id;
3145                 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime;
3146                 *typep = CE_XDIAG_PTNR_LOCAL;
3147                 return (locptnr);
3148         } else if (sibptnr && flags & PTNR_SIBLINGOK) {
3149                 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = sibptnr->cpu_id;
3150                 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime;
3151                 *typep = CE_XDIAG_PTNR_SIBLING;
3152                 return (sibptnr);
3153         } else if (flags & PTNR_SELFOK) {
3154                 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = dtcr->cpu_id;
3155                 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime;
3156                 *typep = CE_XDIAG_PTNR_SELF;
3157                 return (dtcr);
3158         }
3159 
3160         return (NULL);
3161 }
3162 
3163 /*
3164  * Cross call handler that is requested to run on the designated partner of
3165  * a cpu that experienced a possibly sticky or possibly persistnet CE.
3166  */
3167 static void
3168 ce_ptnrchk_xc(struct async_flt *aflt, uchar_t *dispp)
3169 {
3170         *dispp = cpu_ce_scrub_mem_err_common(aflt, B_FALSE);
3171 }
3172 
3173 /*
3174  * The associated errorqs are never destroyed so we do not need to deal with
3175  * them disappearing before this timeout fires.  If the affected memory
3176  * has been DR'd out since the original event the scrub algrithm will catch
3177  * any errors and return null disposition info.  If the original detecting
3178  * cpu has been DR'd out then ereport detector info will not be able to
3179  * lookup CPU type;  with a small timeout this is unlikely.
3180  */
3181 static void
3182 ce_lkychk_cb(ce_lkychk_cb_t *cbarg)
3183 {
3184         struct async_flt *aflt = cbarg->lkycb_aflt;
3185         uchar_t disp;
3186         cpu_t *cp;
3187         int ptnrtype;
3188 
3189         kpreempt_disable();
3190         if (cp = ce_ptnr_select(aflt, PTNR_SIBLINGOK | PTNR_SELFOK,
3191             &ptnrtype)) {
3192                 xc_one(cp->cpu_id, (xcfunc_t *)ce_ptnrchk_xc, (uint64_t)aflt,
3193                     (uint64_t)&disp);
3194                 CE_XDIAG_SETLKYINFO(aflt->flt_disp, disp);
3195                 CE_XDIAG_SETPTNRID(aflt->flt_disp, cp->cpu_id);
3196                 CE_XDIAG_SETPTNRTYPE(aflt->flt_disp, ptnrtype);
3197         } else {
3198                 ce_xdiag_lkydrops++;
3199                 if (ncpus > 1)
3200                         CE_XDIAG_SETSKIPCODE(aflt->flt_disp,
3201                             CE_XDIAG_SKIP_NOPTNR);
3202         }
3203         kpreempt_enable();
3204 
3205         errorq_commit(cbarg->lkycb_eqp, cbarg->lkycb_eqep, ERRORQ_ASYNC);
3206         kmem_free(cbarg, sizeof (ce_lkychk_cb_t));
3207 }
3208 
3209 /*
3210  * Called from errorq drain code when processing a CE error, both from
3211  * CPU and PCI drain functions.  Decide what further classification actions,
3212  * if any, we will perform.  Perform immediate actions now, and schedule
3213  * delayed actions as required.  Note that we are no longer necessarily running
3214  * on the detecting cpu, and that the async_flt structure will not persist on
3215  * return from this function.
3216  *
3217  * Calls to this function should aim to be self-throtlling in some way.  With
3218  * the delayed re-enable of CEEN the absolute rate of calls should not
3219  * be excessive.  Callers should also avoid performing in-depth classification
3220  * for events in pages that are already known to be suspect.
3221  *
3222  * We return nonzero to indicate that the event has been copied and
3223  * recirculated for further testing.  The caller should not log the event
3224  * in this case - it will be logged when further test results are available.
3225  *
3226  * Our possible contexts are that of errorq_drain: below lock level or from
3227  * panic context.  We can assume that the cpu we are running on is online.
3228  */
3229 
3230 
3231 #ifdef DEBUG
3232 static int ce_xdiag_forceaction;
3233 #endif
3234 
3235 int
3236 ce_scrub_xdiag_recirc(struct async_flt *aflt, errorq_t *eqp,
3237     errorq_elem_t *eqep, size_t afltoffset)
3238 {
3239         ce_dispact_t dispact, action;
3240         cpu_t *cp;
3241         uchar_t dtcrinfo, disp;
3242         int ptnrtype;
3243 
3244         if (!ce_disp_inited || panicstr || ce_xdiag_off) {
3245                 ce_xdiag_drops++;
3246                 return (0);
3247         } else if (!aflt->flt_in_memory) {
3248                 ce_xdiag_drops++;
3249                 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOTMEM);
3250                 return (0);
3251         }
3252 
3253         dtcrinfo = CE_XDIAG_DTCRINFO(aflt->flt_disp);
3254 
3255         /*
3256          * Some correctable events are not scrubbed/classified, such as those
3257          * noticed at the tail of cpu_deferred_error.  So if there is no
3258          * initial detector classification go no further.
3259          */
3260         if (!CE_XDIAG_EXT_ALG_APPLIED(dtcrinfo)) {
3261                 ce_xdiag_drops++;
3262                 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOSCRUB);
3263                 return (0);
3264         }
3265 
3266         dispact = CE_DISPACT(ce_disp_table,
3267             CE_XDIAG_AFARMATCHED(dtcrinfo),
3268             CE_XDIAG_STATE(dtcrinfo),
3269             CE_XDIAG_CE1SEEN(dtcrinfo),
3270             CE_XDIAG_CE2SEEN(dtcrinfo));
3271 
3272 
3273         action = CE_ACT(dispact);       /* bad lookup caught below */
3274 #ifdef DEBUG
3275         if (ce_xdiag_forceaction != 0)
3276                 action = ce_xdiag_forceaction;
3277 #endif
3278 
3279         switch (action) {
3280         case CE_ACT_LKYCHK: {
3281                 caddr_t ndata;
3282                 errorq_elem_t *neqep;
3283                 struct async_flt *ecc;
3284                 ce_lkychk_cb_t *cbargp;
3285 
3286                 if ((ndata = errorq_elem_dup(eqp, eqep, &neqep)) == NULL) {
3287                         ce_xdiag_lkydrops++;
3288                         CE_XDIAG_SETSKIPCODE(aflt->flt_disp,
3289                             CE_XDIAG_SKIP_DUPFAIL);
3290                         break;
3291                 }
3292                 ecc = (struct async_flt *)(ndata + afltoffset);
3293 
3294                 ASSERT(ecc->flt_class == CPU_FAULT ||
3295                     ecc->flt_class == BUS_FAULT);
3296                 ecc->flt_class = (ecc->flt_class == CPU_FAULT) ?
3297                     RECIRC_CPU_FAULT : RECIRC_BUS_FAULT;
3298 
3299                 cbargp = kmem_alloc(sizeof (ce_lkychk_cb_t), KM_SLEEP);
3300                 cbargp->lkycb_aflt = ecc;
3301                 cbargp->lkycb_eqp = eqp;
3302                 cbargp->lkycb_eqep = neqep;
3303 
3304                 (void) timeout((void (*)(void *))ce_lkychk_cb,
3305                     (void *)cbargp, drv_usectohz(cpu_ce_lkychk_timeout_usec));
3306                 return (1);
3307         }
3308 
3309         case CE_ACT_PTNRCHK:
3310                 kpreempt_disable();     /* stop cpu list changing */
3311                 if ((cp = ce_ptnr_select(aflt, 0, &ptnrtype)) != NULL) {
3312                         xc_one(cp->cpu_id, (xcfunc_t *)ce_ptnrchk_xc,
3313                             (uint64_t)aflt, (uint64_t)&disp);
3314                         CE_XDIAG_SETPTNRINFO(aflt->flt_disp, disp);
3315                         CE_XDIAG_SETPTNRID(aflt->flt_disp, cp->cpu_id);
3316                         CE_XDIAG_SETPTNRTYPE(aflt->flt_disp, ptnrtype);
3317                 } else if (ncpus > 1) {
3318                         ce_xdiag_ptnrdrops++;
3319                         CE_XDIAG_SETSKIPCODE(aflt->flt_disp,
3320                             CE_XDIAG_SKIP_NOPTNR);
3321                 } else {
3322                         ce_xdiag_ptnrdrops++;
3323                         CE_XDIAG_SETSKIPCODE(aflt->flt_disp,
3324                             CE_XDIAG_SKIP_UNIPROC);
3325                 }
3326                 kpreempt_enable();
3327                 break;
3328 
3329         case CE_ACT_DONE:
3330                 break;
3331 
3332         case CE_ACT(CE_DISP_BAD):
3333         default:
3334 #ifdef DEBUG
3335                 cmn_err(CE_PANIC, "ce_scrub_post: Bad action '%d'", action);
3336 #endif
3337                 ce_xdiag_bad++;
3338                 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_ACTBAD);
3339                 break;
3340         }
3341 
3342         return (0);
3343 }
3344 
3345 /*
3346  * We route all errors through a single switch statement.
3347  */
3348 void
3349 cpu_ue_log_err(struct async_flt *aflt)
3350 {
3351         switch (aflt->flt_class) {
3352         case CPU_FAULT:
3353                 cpu_ereport_init(aflt);
3354                 if (cpu_async_log_err(aflt, NULL))
3355                         cpu_ereport_post(aflt);
3356                 break;
3357 
3358         case BUS_FAULT:
3359                 bus_async_log_err(aflt);
3360                 break;
3361 
3362         default:
3363                 cmn_err(CE_WARN, "discarding async error %p with invalid "
3364                     "fault class (0x%x)", (void *)aflt, aflt->flt_class);
3365                 return;
3366         }
3367 }
3368 
3369 /*
3370  * Routine for panic hook callback from panic_idle().
3371  */
3372 void
3373 cpu_async_panic_callb(void)
3374 {
3375         ch_async_flt_t ch_flt;
3376         struct async_flt *aflt;
3377         ch_cpu_errors_t cpu_error_regs;
3378         uint64_t afsr_errs;
3379 
3380         get_cpu_error_state(&cpu_error_regs);
3381 
3382         afsr_errs = (cpu_error_regs.afsr & C_AFSR_ALL_ERRS) |
3383             (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS);
3384 
3385         if (afsr_errs) {
3386 
3387                 bzero(&ch_flt, sizeof (ch_async_flt_t));
3388                 aflt = (struct async_flt *)&ch_flt;
3389                 aflt->flt_id = gethrtime_waitfree();
3390                 aflt->flt_bus_id = getprocessorid();
3391                 aflt->flt_inst = CPU->cpu_id;
3392                 aflt->flt_stat = cpu_error_regs.afsr;
3393                 aflt->flt_addr = cpu_error_regs.afar;
3394                 aflt->flt_prot = AFLT_PROT_NONE;
3395                 aflt->flt_class = CPU_FAULT;
3396                 aflt->flt_priv = ((cpu_error_regs.afsr & C_AFSR_PRIV) != 0);
3397                 aflt->flt_panic = 1;
3398                 ch_flt.afsr_ext = cpu_error_regs.afsr_ext;
3399                 ch_flt.afsr_errs = afsr_errs;
3400 #if defined(SERRANO)
3401                 ch_flt.afar2 = cpu_error_regs.afar2;
3402 #endif  /* SERRANO */
3403                 (void) cpu_queue_events(&ch_flt, NULL, afsr_errs, NULL);
3404         }
3405 }
3406 
3407 /*
3408  * Routine to convert a syndrome into a syndrome code.
3409  */
3410 static int
3411 synd_to_synd_code(int synd_status, ushort_t synd, uint64_t afsr_bit)
3412 {
3413         if (synd_status == AFLT_STAT_INVALID)
3414                 return (-1);
3415 
3416         /*
3417          * Use the syndrome to index the appropriate syndrome table,
3418          * to get the code indicating which bit(s) is(are) bad.
3419          */
3420         if (afsr_bit &
3421             (C_AFSR_MSYND_ERRS | C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) {
3422                 if (afsr_bit & C_AFSR_MSYND_ERRS) {
3423 #if defined(JALAPENO) || defined(SERRANO)
3424                         if ((synd == 0) || (synd >= BSYND_TBL_SIZE))
3425                                 return (-1);
3426                         else
3427                                 return (BPAR0 + synd);
3428 #else /* JALAPENO || SERRANO */
3429                         if ((synd == 0) || (synd >= MSYND_TBL_SIZE))
3430                                 return (-1);
3431                         else
3432                                 return (mtag_syndrome_tab[synd]);
3433 #endif /* JALAPENO || SERRANO */
3434                 } else {
3435                         if ((synd == 0) || (synd >= ESYND_TBL_SIZE))
3436                                 return (-1);
3437                         else
3438                                 return (ecc_syndrome_tab[synd]);
3439                 }
3440         } else {
3441                 return (-1);
3442         }
3443 }
3444 
3445 int
3446 cpu_get_mem_sid(char *unum, char *buf, int buflen, int *lenp)
3447 {
3448         if (&plat_get_mem_sid)
3449                 return (plat_get_mem_sid(unum, buf, buflen, lenp));
3450         else
3451                 return (ENOTSUP);
3452 }
3453 
3454 int
3455 cpu_get_mem_offset(uint64_t flt_addr, uint64_t *offp)
3456 {
3457         if (&plat_get_mem_offset)
3458                 return (plat_get_mem_offset(flt_addr, offp));
3459         else
3460                 return (ENOTSUP);
3461 }
3462 
3463 int
3464 cpu_get_mem_addr(char *unum, char *sid, uint64_t offset, uint64_t *addrp)
3465 {
3466         if (&plat_get_mem_addr)
3467                 return (plat_get_mem_addr(unum, sid, offset, addrp));
3468         else
3469                 return (ENOTSUP);
3470 }
3471 
3472 /*
3473  * Routine to return a string identifying the physical name
3474  * associated with a memory/cache error.
3475  */
3476 int
3477 cpu_get_mem_unum(int synd_status, ushort_t flt_synd, uint64_t flt_stat,
3478     uint64_t flt_addr, int flt_bus_id, int flt_in_memory,
3479     ushort_t flt_status, char *buf, int buflen, int *lenp)
3480 {
3481         int synd_code;
3482         int ret;
3483 
3484         /*
3485          * An AFSR of -1 defaults to a memory syndrome.
3486          */
3487         if (flt_stat == (uint64_t)-1)
3488                 flt_stat = C_AFSR_CE;
3489 
3490         synd_code = synd_to_synd_code(synd_status, flt_synd, flt_stat);
3491 
3492         /*
3493          * Syndrome code must be either a single-bit error code
3494          * (0...143) or -1 for unum lookup.
3495          */
3496         if (synd_code < 0 || synd_code >= M2)
3497                 synd_code = -1;
3498         if (&plat_get_mem_unum) {
3499                 if ((ret = plat_get_mem_unum(synd_code, flt_addr, flt_bus_id,
3500                     flt_in_memory, flt_status, buf, buflen, lenp)) != 0) {
3501                         buf[0] = '\0';
3502                         *lenp = 0;
3503                 }
3504 
3505                 return (ret);
3506         }
3507 
3508         return (ENOTSUP);
3509 }
3510 
3511 /*
3512  * Wrapper for cpu_get_mem_unum() routine that takes an
3513  * async_flt struct rather than explicit arguments.
3514  */
3515 int
3516 cpu_get_mem_unum_aflt(int synd_status, struct async_flt *aflt,
3517     char *buf, int buflen, int *lenp)
3518 {
3519         /*
3520          * If we come thru here for an IO bus error aflt->flt_stat will
3521          * not be the CPU AFSR, and we pass in a -1 to cpu_get_mem_unum()
3522          * so it will interpret this as a memory error.
3523          */
3524         return (cpu_get_mem_unum(synd_status, aflt->flt_synd,
3525             (aflt->flt_class == BUS_FAULT) ?
3526             (uint64_t)-1 : ((ch_async_flt_t *)aflt)->flt_bit,
3527             aflt->flt_addr, aflt->flt_bus_id, aflt->flt_in_memory,
3528             aflt->flt_status, buf, buflen, lenp));
3529 }
3530 
3531 /*
3532  * Return unum string given synd_code and async_flt into
3533  * the buf with size UNUM_NAMLEN
3534  */
3535 static int
3536 cpu_get_mem_unum_synd(int synd_code, struct async_flt *aflt, char *buf)
3537 {
3538         int ret, len;
3539 
3540         /*
3541          * Syndrome code must be either a single-bit error code
3542          * (0...143) or -1 for unum lookup.
3543          */
3544         if (synd_code < 0 || synd_code >= M2)
3545                 synd_code = -1;
3546         if (&plat_get_mem_unum) {
3547                 if ((ret = plat_get_mem_unum(synd_code, aflt->flt_addr,
3548                     aflt->flt_bus_id, aflt->flt_in_memory,
3549                     aflt->flt_status, buf, UNUM_NAMLEN, &len)) != 0) {
3550                         buf[0] = '\0';
3551                 }
3552                 return (ret);
3553         }
3554 
3555         buf[0] = '\0';
3556         return (ENOTSUP);
3557 }
3558 
3559 /*
3560  * This routine is a more generic interface to cpu_get_mem_unum()
3561  * that may be used by other modules (e.g. the 'mm' driver, through
3562  * the 'MEM_NAME' ioctl, which is used by fmd to resolve unum's
3563  * for Jalapeno/Serrano FRC/RCE or FRU/RUE paired events).
3564  */
3565 int
3566 cpu_get_mem_name(uint64_t synd, uint64_t *afsr, uint64_t afar,
3567     char *buf, int buflen, int *lenp)
3568 {
3569         int synd_status, flt_in_memory, ret;
3570         ushort_t flt_status = 0;
3571         char unum[UNUM_NAMLEN];
3572         uint64_t t_afsr_errs;
3573 
3574         /*
3575          * Check for an invalid address.
3576          */
3577         if (afar == (uint64_t)-1)
3578                 return (ENXIO);
3579 
3580         if (synd == (uint64_t)-1)
3581                 synd_status = AFLT_STAT_INVALID;
3582         else
3583                 synd_status = AFLT_STAT_VALID;
3584 
3585         flt_in_memory = (*afsr & C_AFSR_MEMORY) &&
3586             pf_is_memory(afar >> MMU_PAGESHIFT);
3587 
3588         /*
3589          * Get aggregate AFSR for call to cpu_error_is_ecache_data.
3590          */
3591         if (*afsr == (uint64_t)-1)
3592                 t_afsr_errs = C_AFSR_CE;
3593         else {
3594                 t_afsr_errs = (*afsr & C_AFSR_ALL_ERRS);
3595 #if defined(CHEETAH_PLUS)
3596                 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation))
3597                         t_afsr_errs |= (*(afsr + 1) & C_AFSR_EXT_ALL_ERRS);
3598 #endif  /* CHEETAH_PLUS */
3599         }
3600 
3601         /*
3602          * Turn on ECC_ECACHE if error type is E$ Data.
3603          */
3604         if (cpu_error_is_ecache_data(CPU->cpu_id, t_afsr_errs))
3605                 flt_status |= ECC_ECACHE;
3606 
3607         ret = cpu_get_mem_unum(synd_status, (ushort_t)synd, t_afsr_errs, afar,
3608             CPU->cpu_id, flt_in_memory, flt_status, unum, UNUM_NAMLEN, lenp);
3609         if (ret != 0)
3610                 return (ret);
3611 
3612         if (*lenp >= buflen)
3613                 return (ENAMETOOLONG);
3614 
3615         (void) strncpy(buf, unum, buflen);
3616 
3617         return (0);
3618 }
3619 
3620 /*
3621  * Routine to return memory information associated
3622  * with a physical address and syndrome.
3623  */
3624 int
3625 cpu_get_mem_info(uint64_t synd, uint64_t afar,
3626     uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep,
3627     int *segsp, int *banksp, int *mcidp)
3628 {
3629         int synd_status, synd_code;
3630 
3631         if (afar == (uint64_t)-1)
3632                 return (ENXIO);
3633 
3634         if (synd == (uint64_t)-1)
3635                 synd_status = AFLT_STAT_INVALID;
3636         else
3637                 synd_status = AFLT_STAT_VALID;
3638 
3639         synd_code = synd_to_synd_code(synd_status, synd, C_AFSR_CE);
3640 
3641         if (p2get_mem_info != NULL)
3642                 return ((p2get_mem_info)(synd_code, afar,
3643                     mem_sizep, seg_sizep, bank_sizep,
3644                     segsp, banksp, mcidp));
3645         else
3646                 return (ENOTSUP);
3647 }
3648 
3649 /*
3650  * Routine to return a string identifying the physical
3651  * name associated with a cpuid.
3652  */
3653 int
3654 cpu_get_cpu_unum(int cpuid, char *buf, int buflen, int *lenp)
3655 {
3656         int ret;
3657         char unum[UNUM_NAMLEN];
3658 
3659         if (&plat_get_cpu_unum) {
3660                 if ((ret = plat_get_cpu_unum(cpuid, unum, UNUM_NAMLEN, lenp))
3661                     != 0)
3662                         return (ret);
3663         } else {
3664                 return (ENOTSUP);
3665         }
3666 
3667         if (*lenp >= buflen)
3668                 return (ENAMETOOLONG);
3669 
3670         (void) strncpy(buf, unum, buflen);
3671 
3672         return (0);
3673 }
3674 
3675 /*
3676  * This routine exports the name buffer size.
3677  */
3678 size_t
3679 cpu_get_name_bufsize()
3680 {
3681         return (UNUM_NAMLEN);
3682 }
3683 
3684 /*
3685  * Historical function, apparantly not used.
3686  */
3687 /* ARGSUSED */
3688 void
3689 cpu_read_paddr(struct async_flt *ecc, short verbose, short ce_err)
3690 {}
3691 
3692 /*
3693  * Historical function only called for SBus errors in debugging.
3694  */
3695 /*ARGSUSED*/
3696 void
3697 read_ecc_data(struct async_flt *aflt, short verbose, short ce_err)
3698 {}
3699 
3700 /*
3701  * Clear the AFSR sticky bits.  The routine returns a non-zero value if
3702  * any of the AFSR's sticky errors are detected.  If a non-null pointer to
3703  * an async fault structure argument is passed in, the captured error state
3704  * (AFSR, AFAR) info will be returned in the structure.
3705  */
3706 int
3707 clear_errors(ch_async_flt_t *ch_flt)
3708 {
3709         struct async_flt *aflt = (struct async_flt *)ch_flt;
3710         ch_cpu_errors_t cpu_error_regs;
3711 
3712         get_cpu_error_state(&cpu_error_regs);
3713 
3714         if (ch_flt != NULL) {
3715                 aflt->flt_stat = cpu_error_regs.afsr & C_AFSR_MASK;
3716                 aflt->flt_addr = cpu_error_regs.afar;
3717                 ch_flt->afsr_ext = cpu_error_regs.afsr_ext;
3718                 ch_flt->afsr_errs = (cpu_error_regs.afsr & C_AFSR_ALL_ERRS) |
3719                     (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS);
3720 #if defined(SERRANO)
3721                 ch_flt->afar2 = cpu_error_regs.afar2;
3722 #endif  /* SERRANO */
3723         }
3724 
3725         set_cpu_error_state(&cpu_error_regs);
3726 
3727         return (((cpu_error_regs.afsr & C_AFSR_ALL_ERRS) |
3728             (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS)) != 0);
3729 }
3730 
3731 /*
3732  * Clear any AFSR error bits, and check for persistence.
3733  *
3734  * It would be desirable to also insist that syndrome match.  PCI handling
3735  * has already filled flt_synd.  For errors trapped by CPU we only fill
3736  * flt_synd when we queue the event, so we do not have a valid flt_synd
3737  * during initial classification (it is valid if we're called as part of
3738  * subsequent low-pil additional classification attempts).  We could try
3739  * to determine which syndrome to use: we know we're only called for
3740  * CE/RCE (Jalapeno & Serrano) and CE/EMC (others) so the syndrome to use
3741  * would be esynd/none and esynd/msynd, respectively.  If that is
3742  * implemented then what do we do in the case that we do experience an
3743  * error on the same afar but with different syndrome?  At the very least
3744  * we should count such occurences.  Anyway, for now, we'll leave it as
3745  * it has been for ages.
3746  */
3747 static int
3748 clear_ecc(struct async_flt *aflt)
3749 {
3750         ch_cpu_errors_t cpu_error_regs;
3751 
3752         /*
3753          * Snapshot the AFSR and AFAR and clear any errors
3754          */
3755         get_cpu_error_state(&cpu_error_regs);
3756         set_cpu_error_state(&cpu_error_regs);
3757 
3758         /*
3759          * If any of the same memory access error bits are still on and
3760          * the AFAR matches, return that the error is persistent.
3761          */
3762         return ((cpu_error_regs.afsr & (C_AFSR_MEMORY & aflt->flt_stat)) != 0 &&
3763             cpu_error_regs.afar == aflt->flt_addr);
3764 }
3765 
3766 /*
3767  * Turn off all cpu error detection, normally only used for panics.
3768  */
3769 void
3770 cpu_disable_errors(void)
3771 {
3772         xt_all(set_error_enable_tl1, EN_REG_DISABLE, EER_SET_ABSOLUTE);
3773 
3774         /*
3775          * With error detection now turned off, check the other cpus
3776          * logout areas for any unlogged errors.
3777          */
3778         if (enable_check_other_cpus_logout) {
3779                 cpu_check_other_cpus_logout();
3780                 /*
3781                  * Make a second pass over the logout areas, in case
3782                  * there is a failing CPU in an error-trap loop which
3783                  * will write to the logout area once it is emptied.
3784                  */
3785                 cpu_check_other_cpus_logout();
3786         }
3787 }
3788 
3789 /*
3790  * Enable errors.
3791  */
3792 void
3793 cpu_enable_errors(void)
3794 {
3795         xt_all(set_error_enable_tl1, EN_REG_ENABLE, EER_SET_ABSOLUTE);
3796 }
3797 
3798 /*
3799  * Flush the entire ecache using displacement flush by reading through a
3800  * physical address range twice as large as the Ecache.
3801  */
3802 void
3803 cpu_flush_ecache(void)
3804 {
3805         flush_ecache(ecache_flushaddr, cpunodes[CPU->cpu_id].ecache_size,
3806             cpunodes[CPU->cpu_id].ecache_linesize);
3807 }
3808 
3809 /*
3810  * Return CPU E$ set size - E$ size divided by the associativity.
3811  * We use this function in places where the CPU_PRIVATE ptr may not be
3812  * initialized yet.  Note that for send_mondo and in the Ecache scrubber,
3813  * we're guaranteed that CPU_PRIVATE is initialized.  Also, cpunodes is set
3814  * up before the kernel switches from OBP's to the kernel's trap table, so
3815  * we don't have to worry about cpunodes being unitialized.
3816  */
3817 int
3818 cpu_ecache_set_size(struct cpu *cp)
3819 {
3820         if (CPU_PRIVATE(cp))
3821                 return (CPU_PRIVATE_VAL(cp, chpr_ec_set_size));
3822 
3823         return (cpunodes[cp->cpu_id].ecache_size / cpu_ecache_nway());
3824 }
3825 
3826 /*
3827  * Flush Ecache line.
3828  * Uses ASI_EC_DIAG for Cheetah+ and Jalapeno.
3829  * Uses normal displacement flush for Cheetah.
3830  */
3831 static void
3832 cpu_flush_ecache_line(ch_async_flt_t *ch_flt)
3833 {
3834         struct async_flt *aflt = (struct async_flt *)ch_flt;
3835         int ec_set_size = cpu_ecache_set_size(CPU);
3836 
3837         ecache_flush_line(aflt->flt_addr, ec_set_size);
3838 }
3839 
3840 /*
3841  * Scrub physical address.
3842  * Scrub code is different depending upon whether this a Cheetah+ with 2-way
3843  * Ecache or direct-mapped Ecache.
3844  */
3845 static void
3846 cpu_scrubphys(struct async_flt *aflt)
3847 {
3848         int ec_set_size = cpu_ecache_set_size(CPU);
3849 
3850         scrubphys(aflt->flt_addr, ec_set_size);
3851 }
3852 
3853 /*
3854  * Clear physical address.
3855  * Scrub code is different depending upon whether this a Cheetah+ with 2-way
3856  * Ecache or direct-mapped Ecache.
3857  */
3858 void
3859 cpu_clearphys(struct async_flt *aflt)
3860 {
3861         int lsize = cpunodes[CPU->cpu_id].ecache_linesize;
3862         int ec_set_size = cpu_ecache_set_size(CPU);
3863 
3864 
3865         clearphys(aflt->flt_addr, ec_set_size, lsize);
3866 }
3867 
3868 #if defined(CPU_IMP_ECACHE_ASSOC)
3869 /*
3870  * Check for a matching valid line in all the sets.
3871  * If found, return set# + 1. Otherwise return 0.
3872  */
3873 static int
3874 cpu_ecache_line_valid(ch_async_flt_t *ch_flt)
3875 {
3876         struct async_flt *aflt = (struct async_flt *)ch_flt;
3877         int totalsize = cpunodes[CPU->cpu_id].ecache_size;
3878         int ec_set_size = cpu_ecache_set_size(CPU);
3879         ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0];
3880         int nway = cpu_ecache_nway();
3881         int i;
3882 
3883         for (i = 0; i < nway; i++, ecp++) {
3884                 if (!cpu_ectag_line_invalid(totalsize, ecp->ec_tag) &&
3885                     (aflt->flt_addr & P2ALIGN(C_AFAR_PA, ec_set_size)) ==
3886                     cpu_ectag_to_pa(ec_set_size, ecp->ec_tag))
3887                         return (i+1);
3888         }
3889         return (0);
3890 }
3891 #endif /* CPU_IMP_ECACHE_ASSOC */
3892 
3893 /*
3894  * Check whether a line in the given logout info matches the specified
3895  * fault address.  If reqval is set then the line must not be Invalid.
3896  * Returns 0 on failure;  on success (way + 1) is returned an *level is
3897  * set to 2 for l2$ or 3 for l3$.
3898  */
3899 static int
3900 cpu_matching_ecache_line(uint64_t faddr, void *data, int reqval, int *level)
3901 {
3902         ch_diag_data_t *cdp = data;
3903         ch_ec_data_t *ecp;
3904         int totalsize, ec_set_size;
3905         int i, ways;
3906         int match = 0;
3907         int tagvalid;
3908         uint64_t addr, tagpa;
3909         int ispanther = IS_PANTHER(cpunodes[CPU->cpu_id].implementation);
3910 
3911         /*
3912          * Check the l2$ logout data
3913          */
3914         if (ispanther) {
3915                 ecp = &cdp->chd_l2_data[0];
3916                 ec_set_size = PN_L2_SET_SIZE;
3917                 ways = PN_L2_NWAYS;
3918         } else {
3919                 ecp = &cdp->chd_ec_data[0];
3920                 ec_set_size = cpu_ecache_set_size(CPU);
3921                 ways = cpu_ecache_nway();
3922                 totalsize = cpunodes[CPU->cpu_id].ecache_size;
3923         }
3924         /* remove low order PA bits from fault address not used in PA tag */
3925         addr = faddr & P2ALIGN(C_AFAR_PA, ec_set_size);
3926         for (i = 0; i < ways; i++, ecp++) {
3927                 if (ispanther) {
3928                         tagpa = PN_L2TAG_TO_PA(ecp->ec_tag);
3929                         tagvalid = !PN_L2_LINE_INVALID(ecp->ec_tag);
3930                 } else {
3931                         tagpa = cpu_ectag_to_pa(ec_set_size, ecp->ec_tag);
3932                         tagvalid = !cpu_ectag_line_invalid(totalsize,
3933                             ecp->ec_tag);
3934                 }
3935                 if (tagpa == addr && (!reqval || tagvalid)) {
3936                         match = i + 1;
3937                         *level = 2;
3938                         break;
3939                 }
3940         }
3941 
3942         if (match || !ispanther)
3943                 return (match);
3944 
3945         /* For Panther we also check the l3$ */
3946         ecp = &cdp->chd_ec_data[0];
3947         ec_set_size = PN_L3_SET_SIZE;
3948         ways = PN_L3_NWAYS;
3949         addr = faddr & P2ALIGN(C_AFAR_PA, ec_set_size);
3950 
3951         for (i = 0; i < ways; i++, ecp++) {
3952                 if (PN_L3TAG_TO_PA(ecp->ec_tag) == addr && (!reqval ||
3953                     !PN_L3_LINE_INVALID(ecp->ec_tag))) {
3954                         match = i + 1;
3955                         *level = 3;
3956                         break;
3957                 }
3958         }
3959 
3960         return (match);
3961 }
3962 
3963 #if defined(CPU_IMP_L1_CACHE_PARITY)
3964 /*
3965  * Record information related to the source of an Dcache Parity Error.
3966  */
3967 static void
3968 cpu_dcache_parity_info(ch_async_flt_t *ch_flt)
3969 {
3970         int dc_set_size = dcache_size / CH_DCACHE_NWAY;
3971         int index;
3972 
3973         /*
3974          * Since instruction decode cannot be done at high PIL
3975          * just examine the entire Dcache to locate the error.
3976          */
3977         if (ch_flt->parity_data.dpe.cpl_lcnt == 0) {
3978                 ch_flt->parity_data.dpe.cpl_way = -1;
3979                 ch_flt->parity_data.dpe.cpl_off = -1;
3980         }
3981         for (index = 0; index < dc_set_size; index += dcache_linesize)
3982                 cpu_dcache_parity_check(ch_flt, index);
3983 }
3984 
3985 /*
3986  * Check all ways of the Dcache at a specified index for good parity.
3987  */
3988 static void
3989 cpu_dcache_parity_check(ch_async_flt_t *ch_flt, int index)
3990 {
3991         int dc_set_size = dcache_size / CH_DCACHE_NWAY;
3992         uint64_t parity_bits, pbits, data_word;
3993         static int parity_bits_popc[] = { 0, 1, 1, 0 };
3994         int way, word, data_byte;
3995         ch_dc_data_t *dcp = &ch_flt->parity_data.dpe.cpl_dc[0];
3996         ch_dc_data_t tmp_dcp;
3997 
3998         for (way = 0; way < CH_DCACHE_NWAY; way++, dcp++) {
3999                 /*
4000                  * Perform diagnostic read.
4001                  */
4002                 get_dcache_dtag(index + way * dc_set_size,
4003                     (uint64_t *)&tmp_dcp);
4004 
4005                 /*
4006                  * Check tag for even parity.
4007                  * Sum of 1 bits (including parity bit) should be even.
4008                  */
4009                 if (popc64(tmp_dcp.dc_tag & CHP_DCTAG_PARMASK) & 1) {
4010                         /*
4011                          * If this is the first error log detailed information
4012                          * about it and check the snoop tag. Otherwise just
4013                          * record the fact that we found another error.
4014                          */
4015                         if (ch_flt->parity_data.dpe.cpl_lcnt == 0) {
4016                                 ch_flt->parity_data.dpe.cpl_way = way;
4017                                 ch_flt->parity_data.dpe.cpl_cache =
4018                                     CPU_DC_PARITY;
4019                                 ch_flt->parity_data.dpe.cpl_tag |= CHP_DC_TAG;
4020 
4021                                 if (popc64(tmp_dcp.dc_sntag &
4022                                     CHP_DCSNTAG_PARMASK) & 1) {
4023                                         ch_flt->parity_data.dpe.cpl_tag |=
4024                                             CHP_DC_SNTAG;
4025                                         ch_flt->parity_data.dpe.cpl_lcnt++;
4026                                 }
4027 
4028                                 bcopy(&tmp_dcp, dcp, sizeof (ch_dc_data_t));
4029                         }
4030 
4031                         ch_flt->parity_data.dpe.cpl_lcnt++;
4032                 }
4033 
4034                 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) {
4035                         /*
4036                          * Panther has more parity bits than the other
4037                          * processors for covering dcache data and so each
4038                          * byte of data in each word has its own parity bit.
4039                          */
4040                         parity_bits = tmp_dcp.dc_pn_data_parity;
4041                         for (word = 0; word < 4; word++) {
4042                                 data_word = tmp_dcp.dc_data[word];
4043                                 pbits = parity_bits & PN_DC_DATA_PARITY_MASK;
4044                                 for (data_byte = 0; data_byte < 8;
4045                                     data_byte++) {
4046                                         if (((popc64(data_word &
4047                                             PN_DC_DATA_PARITY_MASK)) & 1) ^
4048                                             (pbits & 1)) {
4049                                                 cpu_record_dc_data_parity(
4050                                                     ch_flt, dcp, &tmp_dcp, way,
4051                                                     word);
4052                                         }
4053                                         pbits >>= 1;
4054                                         data_word >>= 8;
4055                                 }
4056                                 parity_bits >>= 8;
4057                         }
4058                 } else {
4059                         /*
4060                          * Check data array for even parity.
4061                          * The 8 parity bits are grouped into 4 pairs each
4062                          * of which covers a 64-bit word.  The endianness is
4063                          * reversed -- the low-order parity bits cover the
4064                          * high-order data words.
4065                          */
4066                         parity_bits = tmp_dcp.dc_utag >> 8;
4067                         for (word = 0; word < 4; word++) {
4068                                 pbits = (parity_bits >> (6 - word * 2)) & 3;
4069                                 if ((popc64(tmp_dcp.dc_data[word]) +
4070                                     parity_bits_popc[pbits]) & 1) {
4071                                         cpu_record_dc_data_parity(ch_flt, dcp,
4072                                             &tmp_dcp, way, word);
4073                                 }
4074                         }
4075                 }
4076         }
4077 }
4078 
4079 static void
4080 cpu_record_dc_data_parity(ch_async_flt_t *ch_flt,
4081     ch_dc_data_t *dest_dcp, ch_dc_data_t *src_dcp, int way, int word)
4082 {
4083         /*
4084          * If this is the first error log detailed information about it.
4085          * Otherwise just record the fact that we found another error.
4086          */
4087         if (ch_flt->parity_data.dpe.cpl_lcnt == 0) {
4088                 ch_flt->parity_data.dpe.cpl_way = way;
4089                 ch_flt->parity_data.dpe.cpl_cache = CPU_DC_PARITY;
4090                 ch_flt->parity_data.dpe.cpl_off = word * 8;
4091                 bcopy(src_dcp, dest_dcp, sizeof (ch_dc_data_t));
4092         }
4093         ch_flt->parity_data.dpe.cpl_lcnt++;
4094 }
4095 
4096 /*
4097  * Record information related to the source of an Icache Parity Error.
4098  *
4099  * Called with the Icache disabled so any diagnostic accesses are safe.
4100  */
4101 static void
4102 cpu_icache_parity_info(ch_async_flt_t *ch_flt)
4103 {
4104         int     ic_set_size;
4105         int     ic_linesize;
4106         int     index;
4107 
4108         if (CPU_PRIVATE(CPU)) {
4109                 ic_set_size = CPU_PRIVATE_VAL(CPU, chpr_icache_size) /
4110                     CH_ICACHE_NWAY;
4111                 ic_linesize = CPU_PRIVATE_VAL(CPU, chpr_icache_linesize);
4112         } else {
4113                 ic_set_size = icache_size / CH_ICACHE_NWAY;
4114                 ic_linesize = icache_linesize;
4115         }
4116 
4117         ch_flt->parity_data.ipe.cpl_way = -1;
4118         ch_flt->parity_data.ipe.cpl_off = -1;
4119 
4120         for (index = 0; index < ic_set_size; index += ic_linesize)
4121                 cpu_icache_parity_check(ch_flt, index);
4122 }
4123 
4124 /*
4125  * Check all ways of the Icache at a specified index for good parity.
4126  */
4127 static void
4128 cpu_icache_parity_check(ch_async_flt_t *ch_flt, int index)
4129 {
4130         uint64_t parmask, pn_inst_parity;
4131         int ic_set_size;
4132         int ic_linesize;
4133         int flt_index, way, instr, num_instr;
4134         struct async_flt *aflt = (struct async_flt *)ch_flt;
4135         ch_ic_data_t *icp = &ch_flt->parity_data.ipe.cpl_ic[0];
4136         ch_ic_data_t tmp_icp;
4137 
4138         if (CPU_PRIVATE(CPU)) {
4139                 ic_set_size = CPU_PRIVATE_VAL(CPU, chpr_icache_size) /
4140                     CH_ICACHE_NWAY;
4141                 ic_linesize = CPU_PRIVATE_VAL(CPU, chpr_icache_linesize);
4142         } else {
4143                 ic_set_size = icache_size / CH_ICACHE_NWAY;
4144                 ic_linesize = icache_linesize;
4145         }
4146 
4147         /*
4148          * Panther has twice as many instructions per icache line and the
4149          * instruction parity bit is in a different location.
4150          */
4151         if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) {
4152                 num_instr = PN_IC_DATA_REG_SIZE / sizeof (uint64_t);
4153                 pn_inst_parity = PN_ICDATA_PARITY_BIT_MASK;
4154         } else {
4155                 num_instr = CH_IC_DATA_REG_SIZE / sizeof (uint64_t);
4156                 pn_inst_parity = 0;
4157         }
4158 
4159         /*
4160          * Index at which we expect to find the parity error.
4161          */
4162         flt_index = P2ALIGN(aflt->flt_addr % ic_set_size, ic_linesize);
4163 
4164         for (way = 0; way < CH_ICACHE_NWAY; way++, icp++) {
4165                 /*
4166                  * Diagnostic reads expect address argument in ASI format.
4167                  */
4168                 get_icache_dtag(2 * (index + way * ic_set_size),
4169                     (uint64_t *)&tmp_icp);
4170 
4171                 /*
4172                  * If this is the index in which we expect to find the
4173                  * error log detailed information about each of the ways.
4174                  * This information will be displayed later if we can't
4175                  * determine the exact way in which the error is located.
4176                  */
4177                 if (flt_index == index)
4178                         bcopy(&tmp_icp, icp, sizeof (ch_ic_data_t));
4179 
4180                 /*
4181                  * Check tag for even parity.
4182                  * Sum of 1 bits (including parity bit) should be even.
4183                  */
4184                 if (popc64(tmp_icp.ic_patag & CHP_ICPATAG_PARMASK) & 1) {
4185                         /*
4186                          * If this way is the one in which we expected
4187                          * to find the error record the way and check the
4188                          * snoop tag. Otherwise just record the fact we
4189                          * found another error.
4190                          */
4191                         if (flt_index == index) {
4192                                 ch_flt->parity_data.ipe.cpl_way = way;
4193                                 ch_flt->parity_data.ipe.cpl_tag |= CHP_IC_TAG;
4194 
4195                                 if (popc64(tmp_icp.ic_sntag &
4196                                     CHP_ICSNTAG_PARMASK) & 1) {
4197                                         ch_flt->parity_data.ipe.cpl_tag |=
4198                                             CHP_IC_SNTAG;
4199                                         ch_flt->parity_data.ipe.cpl_lcnt++;
4200                                 }
4201 
4202                         }
4203                         ch_flt->parity_data.ipe.cpl_lcnt++;
4204                         continue;
4205                 }
4206 
4207                 /*
4208                  * Check instruction data for even parity.
4209                  * Bits participating in parity differ for PC-relative
4210                  * versus non-PC-relative instructions.
4211                  */
4212                 for (instr = 0; instr < num_instr; instr++) {
4213                         parmask = (tmp_icp.ic_data[instr] &
4214                             CH_ICDATA_PRED_ISPCREL) ?
4215                             (CHP_ICDATA_PCREL_PARMASK | pn_inst_parity) :
4216                             (CHP_ICDATA_NPCREL_PARMASK | pn_inst_parity);
4217                         if (popc64(tmp_icp.ic_data[instr] & parmask) & 1) {
4218                                 /*
4219                                  * If this way is the one in which we expected
4220                                  * to find the error record the way and offset.
4221                                  * Otherwise just log the fact we found another
4222                                  * error.
4223                                  */
4224                                 if (flt_index == index) {
4225                                         ch_flt->parity_data.ipe.cpl_way = way;
4226                                         ch_flt->parity_data.ipe.cpl_off =
4227                                             instr * 4;
4228                                 }
4229                                 ch_flt->parity_data.ipe.cpl_lcnt++;
4230                                 continue;
4231                         }
4232                 }
4233         }
4234 }
4235 
4236 /*
4237  * Record information related to the source of an Pcache Parity Error.
4238  */
4239 static void
4240 cpu_pcache_parity_info(ch_async_flt_t *ch_flt)
4241 {
4242         int pc_set_size = CH_PCACHE_SIZE / CH_PCACHE_NWAY;
4243         int index;
4244 
4245         /*
4246          * Since instruction decode cannot be done at high PIL just
4247          * examine the entire Pcache to check for any parity errors.
4248          */
4249         if (ch_flt->parity_data.dpe.cpl_lcnt == 0) {
4250                 ch_flt->parity_data.dpe.cpl_way = -1;
4251                 ch_flt->parity_data.dpe.cpl_off = -1;
4252         }
4253         for (index = 0; index < pc_set_size; index += CH_PCACHE_LSIZE)
4254                 cpu_pcache_parity_check(ch_flt, index);
4255 }
4256 
4257 /*
4258  * Check all ways of the Pcache at a specified index for good parity.
4259  */
4260 static void
4261 cpu_pcache_parity_check(ch_async_flt_t *ch_flt, int index)
4262 {
4263         int pc_set_size = CH_PCACHE_SIZE / CH_PCACHE_NWAY;
4264         int pc_data_words = CH_PC_DATA_REG_SIZE / sizeof (uint64_t);
4265         int way, word, pbit, parity_bits;
4266         ch_pc_data_t *pcp = &ch_flt->parity_data.dpe.cpl_pc[0];
4267         ch_pc_data_t tmp_pcp;
4268 
4269         for (way = 0; way < CH_PCACHE_NWAY; way++, pcp++) {
4270                 /*
4271                  * Perform diagnostic read.
4272                  */
4273                 get_pcache_dtag(index + way * pc_set_size,
4274                     (uint64_t *)&tmp_pcp);
4275                 /*
4276                  * Check data array for odd parity. There are 8 parity
4277                  * bits (bits 57:50 of ASI_PCACHE_STATUS_DATA) and each
4278                  * of those bits covers exactly 8 bytes of the data
4279                  * array:
4280                  *
4281                  *      parity bit      P$ data bytes covered
4282                  *      ----------      ---------------------
4283                  *      50              63:56
4284                  *      51              55:48
4285                  *      52              47:40
4286                  *      53              39:32
4287                  *      54              31:24
4288                  *      55              23:16
4289                  *      56              15:8
4290                  *      57              7:0
4291                  */
4292                 parity_bits = PN_PC_PARITY_BITS(tmp_pcp.pc_status);
4293                 for (word = 0; word < pc_data_words; word++) {
4294                         pbit = (parity_bits >> (pc_data_words - word - 1)) & 1;
4295                         if ((popc64(tmp_pcp.pc_data[word]) & 1) ^ pbit) {
4296                                 /*
4297                                  * If this is the first error log detailed
4298                                  * information about it. Otherwise just record
4299                                  * the fact that we found another error.
4300                                  */
4301                                 if (ch_flt->parity_data.dpe.cpl_lcnt == 0) {
4302                                         ch_flt->parity_data.dpe.cpl_way = way;
4303                                         ch_flt->parity_data.dpe.cpl_cache =
4304                                             CPU_PC_PARITY;
4305                                         ch_flt->parity_data.dpe.cpl_off =
4306                                             word * sizeof (uint64_t);
4307                                         bcopy(&tmp_pcp, pcp,
4308                                             sizeof (ch_pc_data_t));
4309                                 }
4310                                 ch_flt->parity_data.dpe.cpl_lcnt++;
4311                         }
4312                 }
4313         }
4314 }
4315 
4316 
4317 /*
4318  * Add L1 Data cache data to the ereport payload.
4319  */
4320 static void
4321 cpu_payload_add_dcache(struct async_flt *aflt, nvlist_t *nvl)
4322 {
4323         ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt;
4324         ch_dc_data_t *dcp;
4325         ch_dc_data_t dcdata[CH_DCACHE_NWAY];
4326         uint_t nelem;
4327         int i, ways_to_check, ways_logged = 0;
4328 
4329         /*
4330          * If this is an D$ fault then there may be multiple
4331          * ways captured in the ch_parity_log_t structure.
4332          * Otherwise, there will be at most one way captured
4333          * in the ch_diag_data_t struct.
4334          * Check each way to see if it should be encoded.
4335          */
4336         if (ch_flt->flt_type == CPU_DC_PARITY)
4337                 ways_to_check = CH_DCACHE_NWAY;
4338         else
4339                 ways_to_check = 1;
4340         for (i = 0; i < ways_to_check; i++) {
4341                 if (ch_flt->flt_type == CPU_DC_PARITY)
4342                         dcp = &ch_flt->parity_data.dpe.cpl_dc[i];
4343                 else
4344                         dcp = &ch_flt->flt_diag_data.chd_dc_data;
4345                 if (dcp->dc_logflag == DC_LOGFLAG_MAGIC) {
4346                         bcopy(dcp, &dcdata[ways_logged],
4347                             sizeof (ch_dc_data_t));
4348                         ways_logged++;
4349                 }
4350         }
4351 
4352         /*
4353          * Add the dcache data to the payload.
4354          */
4355         fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1D_WAYS,
4356             DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL);
4357         if (ways_logged != 0) {
4358                 nelem = sizeof (ch_dc_data_t) / sizeof (uint64_t) * ways_logged;
4359                 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1D_DATA,
4360                     DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)dcdata, NULL);
4361         }
4362 }
4363 
4364 /*
4365  * Add L1 Instruction cache data to the ereport payload.
4366  */
4367 static void
4368 cpu_payload_add_icache(struct async_flt *aflt, nvlist_t *nvl)
4369 {
4370         ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt;
4371         ch_ic_data_t *icp;
4372         ch_ic_data_t icdata[CH_ICACHE_NWAY];
4373         uint_t nelem;
4374         int i, ways_to_check, ways_logged = 0;
4375 
4376         /*
4377          * If this is an I$ fault then there may be multiple
4378          * ways captured in the ch_parity_log_t structure.
4379          * Otherwise, there will be at most one way captured
4380          * in the ch_diag_data_t struct.
4381          * Check each way to see if it should be encoded.
4382          */
4383         if (ch_flt->flt_type == CPU_IC_PARITY)
4384                 ways_to_check = CH_ICACHE_NWAY;
4385         else
4386                 ways_to_check = 1;
4387         for (i = 0; i < ways_to_check; i++) {
4388                 if (ch_flt->flt_type == CPU_IC_PARITY)
4389                         icp = &ch_flt->parity_data.ipe.cpl_ic[i];
4390                 else
4391                         icp = &ch_flt->flt_diag_data.chd_ic_data;
4392                 if (icp->ic_logflag == IC_LOGFLAG_MAGIC) {
4393                         bcopy(icp, &icdata[ways_logged],
4394                             sizeof (ch_ic_data_t));
4395                         ways_logged++;
4396                 }
4397         }
4398 
4399         /*
4400          * Add the icache data to the payload.
4401          */
4402         fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1I_WAYS,
4403             DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL);
4404         if (ways_logged != 0) {
4405                 nelem = sizeof (ch_ic_data_t) / sizeof (uint64_t) * ways_logged;
4406                 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1I_DATA,
4407                     DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)icdata, NULL);
4408         }
4409 }
4410 
4411 #endif  /* CPU_IMP_L1_CACHE_PARITY */
4412 
4413 /*
4414  * Add ecache data to payload.
4415  */
4416 static void
4417 cpu_payload_add_ecache(struct async_flt *aflt, nvlist_t *nvl)
4418 {
4419         ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt;
4420         ch_ec_data_t *ecp;
4421         ch_ec_data_t ecdata[CHD_EC_DATA_SETS];
4422         uint_t nelem;
4423         int i, ways_logged = 0;
4424 
4425         /*
4426          * Check each way to see if it should be encoded
4427          * and concatinate it into a temporary buffer.
4428          */
4429         for (i = 0; i < CHD_EC_DATA_SETS; i++) {
4430                 ecp = &ch_flt->flt_diag_data.chd_ec_data[i];
4431                 if (ecp->ec_logflag == EC_LOGFLAG_MAGIC) {
4432                         bcopy(ecp, &ecdata[ways_logged],
4433                             sizeof (ch_ec_data_t));
4434                         ways_logged++;
4435                 }
4436         }
4437 
4438         /*
4439          * Panther CPUs have an additional level of cache and so
4440          * what we just collected was the L3 (ecache) and not the
4441          * L2 cache.
4442          */
4443         if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) {
4444                 /*
4445                  * Add the L3 (ecache) data to the payload.
4446                  */
4447                 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L3_WAYS,
4448                     DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL);
4449                 if (ways_logged != 0) {
4450                         nelem = sizeof (ch_ec_data_t) /
4451                             sizeof (uint64_t) * ways_logged;
4452                         fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L3_DATA,
4453                             DATA_TYPE_UINT64_ARRAY, nelem,
4454                             (uint64_t *)ecdata, NULL);
4455                 }
4456 
4457                 /*
4458                  * Now collect the L2 cache.
4459                  */
4460                 ways_logged = 0;
4461                 for (i = 0; i < PN_L2_NWAYS; i++) {
4462                         ecp = &ch_flt->flt_diag_data.chd_l2_data[i];
4463                         if (ecp->ec_logflag == EC_LOGFLAG_MAGIC) {
4464                                 bcopy(ecp, &ecdata[ways_logged],
4465                                     sizeof (ch_ec_data_t));
4466                                 ways_logged++;
4467                         }
4468                 }
4469         }
4470 
4471         /*
4472          * Add the L2 cache data to the payload.
4473          */
4474         fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L2_WAYS,
4475             DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL);
4476         if (ways_logged != 0) {
4477                 nelem = sizeof (ch_ec_data_t) /
4478                     sizeof (uint64_t) * ways_logged;
4479                 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L2_DATA,
4480                     DATA_TYPE_UINT64_ARRAY, nelem,  (uint64_t *)ecdata, NULL);
4481         }
4482 }
4483 
4484 /*
4485  * Initialize cpu scheme for specified cpu.
4486  */
4487 static void
4488 cpu_fmri_cpu_set(nvlist_t *cpu_fmri, int cpuid)
4489 {
4490         char sbuf[21]; /* sizeof (UINT64_MAX) + '\0' */
4491         uint8_t mask;
4492 
4493         mask = cpunodes[cpuid].version;
4494         (void) snprintf(sbuf, sizeof (sbuf), "%llX",
4495             (u_longlong_t)cpunodes[cpuid].device_id);
4496         (void) fm_fmri_cpu_set(cpu_fmri, FM_CPU_SCHEME_VERSION, NULL,
4497             cpuid, &mask, (const char *)sbuf);
4498 }
4499 
4500 /*
4501  * Returns ereport resource type.
4502  */
4503 static int
4504 cpu_error_to_resource_type(struct async_flt *aflt)
4505 {
4506         ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt;
4507 
4508         switch (ch_flt->flt_type) {
4509 
4510         case CPU_CE_ECACHE:
4511         case CPU_UE_ECACHE:
4512         case CPU_UE_ECACHE_RETIRE:
4513         case CPU_ORPH:
4514                 /*
4515                  * If AFSR error bit indicates L2$ Data for Cheetah,
4516                  * Cheetah+ or Jaguar, or L3$ Data for Panther, return
4517                  * E$ Data type, otherwise, return CPU type.
4518                  */
4519                 if (cpu_error_is_ecache_data(aflt->flt_inst,
4520                     ch_flt->flt_bit))
4521                         return (ERRTYPE_ECACHE_DATA);
4522                 return (ERRTYPE_CPU);
4523 
4524         case CPU_CE:
4525         case CPU_UE:
4526         case CPU_EMC:
4527         case CPU_DUE:
4528         case CPU_RCE:
4529         case CPU_RUE:
4530         case CPU_FRC:
4531         case CPU_FRU:
4532                 return (ERRTYPE_MEMORY);
4533 
4534         case CPU_IC_PARITY:
4535         case CPU_DC_PARITY:
4536         case CPU_FPUERR:
4537         case CPU_PC_PARITY:
4538         case CPU_ITLB_PARITY:
4539         case CPU_DTLB_PARITY:
4540                 return (ERRTYPE_CPU);
4541         }
4542         return (ERRTYPE_UNKNOWN);
4543 }
4544 
4545 /*
4546  * Encode the data saved in the ch_async_flt_t struct into
4547  * the FM ereport payload.
4548  */
4549 static void
4550 cpu_payload_add_aflt(struct async_flt *aflt, nvlist_t *payload,
4551         nvlist_t *resource, int *afar_status, int *synd_status)
4552 {
4553         ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt;
4554         *synd_status = AFLT_STAT_INVALID;
4555         *afar_status = AFLT_STAT_INVALID;
4556 
4557         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFSR) {
4558                 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFSR,
4559                     DATA_TYPE_UINT64, aflt->flt_stat, NULL);
4560         }
4561 
4562         if ((aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFSR_EXT) &&
4563             IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) {
4564                 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFSR_EXT,
4565                     DATA_TYPE_UINT64, ch_flt->afsr_ext, NULL);
4566         }
4567 
4568         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFAR_STATUS) {
4569                 *afar_status = afsr_to_afar_status(ch_flt->afsr_errs,
4570                     ch_flt->flt_bit);
4571                 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFAR_STATUS,
4572                     DATA_TYPE_UINT8, (uint8_t)*afar_status, NULL);
4573         }
4574 
4575         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFAR) {
4576                 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFAR,
4577                     DATA_TYPE_UINT64, aflt->flt_addr, NULL);
4578         }
4579 
4580         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_PC) {
4581                 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_PC,
4582                     DATA_TYPE_UINT64, (uint64_t)aflt->flt_pc, NULL);
4583         }
4584 
4585         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_TL) {
4586                 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_TL,
4587                     DATA_TYPE_UINT8, (uint8_t)aflt->flt_tl, NULL);
4588         }
4589 
4590         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_TT) {
4591                 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_TT,
4592                     DATA_TYPE_UINT8, flt_to_trap_type(aflt), NULL);
4593         }
4594 
4595         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_PRIV) {
4596                 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_PRIV,
4597                     DATA_TYPE_BOOLEAN_VALUE,
4598                     (aflt->flt_priv ? B_TRUE : B_FALSE), NULL);
4599         }
4600 
4601         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ME) {
4602                 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ME,
4603                     DATA_TYPE_BOOLEAN_VALUE,
4604                     (aflt->flt_stat & C_AFSR_ME) ? B_TRUE : B_FALSE, NULL);
4605         }
4606 
4607         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_SYND_STATUS) {
4608                 *synd_status = afsr_to_synd_status(aflt->flt_inst,
4609                     ch_flt->afsr_errs, ch_flt->flt_bit);
4610                 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_SYND_STATUS,
4611                     DATA_TYPE_UINT8, (uint8_t)*synd_status, NULL);
4612         }
4613 
4614         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_SYND) {
4615                 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_SYND,
4616                     DATA_TYPE_UINT16, (uint16_t)aflt->flt_synd, NULL);
4617         }
4618 
4619         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ERR_TYPE) {
4620                 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ERR_TYPE,
4621                     DATA_TYPE_STRING, flt_to_error_type(aflt), NULL);
4622         }
4623 
4624         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ERR_DISP) {
4625                 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ERR_DISP,
4626                     DATA_TYPE_UINT64, aflt->flt_disp, NULL);
4627         }
4628 
4629         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L2)
4630                 cpu_payload_add_ecache(aflt, payload);
4631 
4632         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_COPYFUNCTION) {
4633                 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_COPYFUNCTION,
4634                     DATA_TYPE_UINT8, (uint8_t)aflt->flt_status & 0xff, NULL);
4635         }
4636 
4637         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_HOWDETECTED) {
4638                 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_HOWDETECTED,
4639                     DATA_TYPE_UINT8, (uint8_t)(aflt->flt_status >> 8), NULL);
4640         }
4641 
4642         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_INSTRBLOCK) {
4643                 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_INSTRBLOCK,
4644                     DATA_TYPE_UINT32_ARRAY, 16,
4645                     (uint32_t *)&ch_flt->flt_fpdata, NULL);
4646         }
4647 
4648 #if defined(CPU_IMP_L1_CACHE_PARITY)
4649         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1D)
4650                 cpu_payload_add_dcache(aflt, payload);
4651         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1I)
4652                 cpu_payload_add_icache(aflt, payload);
4653 #endif  /* CPU_IMP_L1_CACHE_PARITY */
4654 
4655 #if defined(CHEETAH_PLUS)
4656         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1P)
4657                 cpu_payload_add_pcache(aflt, payload);
4658         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_TLB)
4659                 cpu_payload_add_tlb(aflt, payload);
4660 #endif  /* CHEETAH_PLUS */
4661         /*
4662          * Create the FMRI that goes into the payload
4663          * and contains the unum info if necessary.
4664          */
4665         if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_RESOURCE) {
4666                 char unum[UNUM_NAMLEN] = "";
4667                 char sid[DIMM_SERIAL_ID_LEN] = "";
4668                 int len, ret, rtype, synd_code;
4669                 uint64_t offset = (uint64_t)-1;
4670 
4671                 rtype = cpu_error_to_resource_type(aflt);
4672                 switch (rtype) {
4673 
4674                 case ERRTYPE_MEMORY:
4675                 case ERRTYPE_ECACHE_DATA:
4676 
4677                         /*
4678                          * Memory errors, do unum lookup
4679                          */
4680                         if (*afar_status == AFLT_STAT_INVALID)
4681                                 break;
4682 
4683                         if (rtype == ERRTYPE_ECACHE_DATA)
4684                                 aflt->flt_status |= ECC_ECACHE;
4685                         else
4686                                 aflt->flt_status &= ~ECC_ECACHE;
4687 
4688                         synd_code = synd_to_synd_code(*synd_status,
4689                             aflt->flt_synd, ch_flt->flt_bit);
4690 
4691                         if (cpu_get_mem_unum_synd(synd_code, aflt, unum) != 0)
4692                                 break;
4693 
4694                         ret = cpu_get_mem_sid(unum, sid, DIMM_SERIAL_ID_LEN,
4695                             &len);
4696 
4697                         if (ret == 0) {
4698                                 (void) cpu_get_mem_offset(aflt->flt_addr,
4699                                     &offset);
4700                         }
4701 
4702                         fm_fmri_mem_set(resource, FM_MEM_SCHEME_VERSION,
4703                             NULL, unum, (ret == 0) ? sid : NULL, offset);
4704                         fm_payload_set(payload,
4705                             FM_EREPORT_PAYLOAD_NAME_RESOURCE,
4706                             DATA_TYPE_NVLIST, resource, NULL);
4707                         break;
4708 
4709                 case ERRTYPE_CPU:
4710                         /*
4711                          * On-board processor array error, add cpu resource.
4712                          */
4713                         cpu_fmri_cpu_set(resource, aflt->flt_inst);
4714                         fm_payload_set(payload,
4715                             FM_EREPORT_PAYLOAD_NAME_RESOURCE,
4716                             DATA_TYPE_NVLIST, resource, NULL);
4717                         break;
4718                 }
4719         }
4720 }
4721 
4722 /*
4723  * Initialize the way info if necessary.
4724  */
4725 void
4726 cpu_ereport_init(struct async_flt *aflt)
4727 {
4728         ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt;
4729         ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0];
4730         ch_ec_data_t *l2p = &ch_flt->flt_diag_data.chd_l2_data[0];
4731         int i;
4732 
4733         /*
4734          * Initialize the info in the CPU logout structure.
4735          * The I$/D$ way information is not initialized here
4736          * since it is captured in the logout assembly code.
4737          */
4738         for (i = 0; i < CHD_EC_DATA_SETS; i++)
4739                 (ecp + i)->ec_way = i;
4740 
4741         for (i = 0; i < PN_L2_NWAYS; i++)
4742                 (l2p + i)->ec_way = i;
4743 }
4744 
4745 /*
4746  * Returns whether fault address is valid for this error bit and
4747  * whether the address is "in memory" (i.e. pf_is_memory returns 1).
4748  */
4749 int
4750 cpu_flt_in_memory(ch_async_flt_t *ch_flt, uint64_t t_afsr_bit)
4751 {
4752         struct async_flt *aflt = (struct async_flt *)ch_flt;
4753 
4754         return ((t_afsr_bit & C_AFSR_MEMORY) &&
4755             afsr_to_afar_status(ch_flt->afsr_errs, t_afsr_bit) ==
4756             AFLT_STAT_VALID &&
4757             pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT));
4758 }
4759 
4760 /*
4761  * Returns whether fault address is valid based on the error bit for the
4762  * one event being queued and whether the address is "in memory".
4763  */
4764 static int
4765 cpu_flt_in_memory_one_event(ch_async_flt_t *ch_flt, uint64_t t_afsr_bit)
4766 {
4767         struct async_flt *aflt = (struct async_flt *)ch_flt;
4768         int afar_status;
4769         uint64_t afsr_errs, afsr_ow, *ow_bits;
4770 
4771         if (!(t_afsr_bit & C_AFSR_MEMORY) ||
4772             !pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT))
4773                 return (0);
4774 
4775         afsr_errs = ch_flt->afsr_errs;
4776         afar_status = afsr_to_afar_status(afsr_errs, t_afsr_bit);
4777 
4778         switch (afar_status) {
4779         case AFLT_STAT_VALID:
4780                 return (1);
4781 
4782         case AFLT_STAT_AMBIGUOUS:
4783                 /*
4784                  * Status is ambiguous since another error bit (or bits)
4785                  * of equal priority to the specified bit on in the afsr,
4786                  * so check those bits. Return 1 only if the bits on in the
4787                  * same class as the t_afsr_bit are also C_AFSR_MEMORY bits.
4788                  * Otherwise not all the equal priority bits are for memory
4789                  * errors, so return 0.
4790                  */
4791                 ow_bits = afar_overwrite;
4792                 while ((afsr_ow = *ow_bits++) != 0) {
4793                         /*
4794                          * Get other bits that are on in t_afsr_bit's priority
4795                          * class to check for Memory Error bits only.
4796                          */
4797                         if (afsr_ow & t_afsr_bit) {
4798                                 if ((afsr_errs & afsr_ow) & ~C_AFSR_MEMORY)
4799                                         return (0);
4800                                 else
4801                                         return (1);
4802                         }
4803                 }
4804                 /*FALLTHRU*/
4805 
4806         default:
4807                 return (0);
4808         }
4809 }
4810 
4811 static void
4812 cpu_log_diag_info(ch_async_flt_t *ch_flt)
4813 {
4814         struct async_flt *aflt = (struct async_flt *)ch_flt;
4815         ch_dc_data_t *dcp = &ch_flt->flt_diag_data.chd_dc_data;
4816         ch_ic_data_t *icp = &ch_flt->flt_diag_data.chd_ic_data;
4817         ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0];
4818 #if defined(CPU_IMP_ECACHE_ASSOC)
4819         int i, nway;
4820 #endif /* CPU_IMP_ECACHE_ASSOC */
4821 
4822         /*
4823          * Check if the CPU log out captured was valid.
4824          */
4825         if (ch_flt->flt_diag_data.chd_afar == LOGOUT_INVALID ||
4826             ch_flt->flt_data_incomplete)
4827                 return;
4828 
4829 #if defined(CPU_IMP_ECACHE_ASSOC)
4830         nway = cpu_ecache_nway();
4831         i =  cpu_ecache_line_valid(ch_flt);
4832         if (i == 0 || i > nway) {
4833                 for (i = 0; i < nway; i++)
4834                         ecp[i].ec_logflag = EC_LOGFLAG_MAGIC;
4835         } else
4836                 ecp[i - 1].ec_logflag = EC_LOGFLAG_MAGIC;
4837 #else /* CPU_IMP_ECACHE_ASSOC */
4838         ecp->ec_logflag = EC_LOGFLAG_MAGIC;
4839 #endif /* CPU_IMP_ECACHE_ASSOC */
4840 
4841 #if defined(CHEETAH_PLUS)
4842         pn_cpu_log_diag_l2_info(ch_flt);
4843 #endif /* CHEETAH_PLUS */
4844 
4845         if (CH_DCTAG_MATCH(dcp->dc_tag, aflt->flt_addr)) {
4846                 dcp->dc_way = CH_DCIDX_TO_WAY(dcp->dc_idx);
4847                 dcp->dc_logflag = DC_LOGFLAG_MAGIC;
4848         }
4849 
4850         if (CH_ICTAG_MATCH(icp, aflt->flt_addr)) {
4851                 if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation))
4852                         icp->ic_way = PN_ICIDX_TO_WAY(icp->ic_idx);
4853                 else
4854                         icp->ic_way = CH_ICIDX_TO_WAY(icp->ic_idx);
4855                 icp->ic_logflag = IC_LOGFLAG_MAGIC;
4856         }
4857 }
4858 
4859 /*
4860  * Cheetah ECC calculation.
4861  *
4862  * We only need to do the calculation on the data bits and can ignore check
4863  * bit and Mtag bit terms in the calculation.
4864  */
4865 static uint64_t ch_ecc_table[9][2] = {
4866         /*
4867          * low order 64-bits   high-order 64-bits
4868          */
4869         { 0x46bffffeccd1177f, 0x488800022100014c },
4870         { 0x42fccc81331ff77f, 0x14424f1010249184 },
4871         { 0x8898827c222f1ffe, 0x22c1222808184aaf },
4872         { 0xf7632203e131ccf1, 0xe1241121848292b8 },
4873         { 0x7f5511421b113809, 0x901c88d84288aafe },
4874         { 0x1d49412184882487, 0x8f338c87c044c6ef },
4875         { 0xf552181014448344, 0x7ff8f4443e411911 },
4876         { 0x2189240808f24228, 0xfeeff8cc81333f42 },
4877         { 0x3280008440001112, 0xfee88b337ffffd62 },
4878 };
4879 
4880 /*
4881  * 64-bit population count, use well-known popcnt trick.
4882  * We could use the UltraSPARC V9 POPC instruction, but some
4883  * CPUs including Cheetahplus and Jaguar do not support that
4884  * instruction.
4885  */
4886 int
4887 popc64(uint64_t val)
4888 {
4889         int cnt;
4890 
4891         for (cnt = 0; val != 0; val &= val - 1)
4892                 cnt++;
4893         return (cnt);
4894 }
4895 
4896 /*
4897  * Generate the 9 ECC bits for the 128-bit chunk based on the table above.
4898  * Note that xor'ing an odd number of 1 bits == 1 and xor'ing an even number
4899  * of 1 bits == 0, so we can just use the least significant bit of the popcnt
4900  * instead of doing all the xor's.
4901  */
4902 uint32_t
4903 us3_gen_ecc(uint64_t data_low, uint64_t data_high)
4904 {
4905         int bitno, s;
4906         int synd = 0;
4907 
4908         for (bitno = 0; bitno < 9; bitno++) {
4909                 s = (popc64(data_low & ch_ecc_table[bitno][0]) +
4910                     popc64(data_high & ch_ecc_table[bitno][1])) & 1;
4911                 synd |= (s << bitno);
4912         }
4913         return (synd);
4914 
4915 }
4916 
4917 /*
4918  * Queue one event based on ecc_type_to_info entry.  If the event has an AFT1
4919  * tag associated with it or is a fatal event (aflt_panic set), it is sent to
4920  * the UE event queue.  Otherwise it is dispatched to the CE event queue.
4921  */
4922 static void
4923 cpu_queue_one_event(ch_async_flt_t *ch_flt, char *reason,
4924     ecc_type_to_info_t *eccp, ch_diag_data_t *cdp)
4925 {
4926         struct async_flt *aflt = (struct async_flt *)ch_flt;
4927 
4928         if (reason &&
4929             strlen(reason) + strlen(eccp->ec_reason) < MAX_REASON_STRING) {
4930                 (void) strcat(reason, eccp->ec_reason);
4931         }
4932 
4933         ch_flt->flt_bit = eccp->ec_afsr_bit;
4934         ch_flt->flt_type = eccp->ec_flt_type;
4935         if (cdp != NULL && cdp->chd_afar != LOGOUT_INVALID)
4936                 ch_flt->flt_diag_data = *cdp;
4937         else
4938                 ch_flt->flt_diag_data.chd_afar = LOGOUT_INVALID;
4939         aflt->flt_in_memory =
4940             cpu_flt_in_memory_one_event(ch_flt, ch_flt->flt_bit);
4941 
4942         if (ch_flt->flt_bit & C_AFSR_MSYND_ERRS)
4943                 aflt->flt_synd = GET_M_SYND(aflt->flt_stat);
4944         else if (ch_flt->flt_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS))
4945                 aflt->flt_synd = GET_E_SYND(aflt->flt_stat);
4946         else
4947                 aflt->flt_synd = 0;
4948 
4949         aflt->flt_payload = eccp->ec_err_payload;
4950 
4951         if (aflt->flt_panic || (eccp->ec_afsr_bit &
4952             (C_AFSR_LEVEL1 | C_AFSR_EXT_LEVEL1)))
4953                 cpu_errorq_dispatch(eccp->ec_err_class,
4954                     (void *)ch_flt, sizeof (ch_async_flt_t), ue_queue,
4955                     aflt->flt_panic);
4956         else
4957                 cpu_errorq_dispatch(eccp->ec_err_class,
4958                     (void *)ch_flt, sizeof (ch_async_flt_t), ce_queue,
4959                     aflt->flt_panic);
4960 }
4961 
4962 /*
4963  * Queue events on async event queue one event per error bit.  First we
4964  * queue the events that we "expect" for the given trap, then we queue events
4965  * that we may not expect.  Return number of events queued.
4966  */
4967 int
4968 cpu_queue_events(ch_async_flt_t *ch_flt, char *reason, uint64_t t_afsr_errs,
4969     ch_cpu_logout_t *clop)
4970 {
4971         struct async_flt *aflt = (struct async_flt *)ch_flt;
4972         ecc_type_to_info_t *eccp;
4973         int nevents = 0;
4974         uint64_t primary_afar = aflt->flt_addr, primary_afsr = aflt->flt_stat;
4975 #if defined(CHEETAH_PLUS)
4976         uint64_t orig_t_afsr_errs;
4977 #endif
4978         uint64_t primary_afsr_ext = ch_flt->afsr_ext;
4979         uint64_t primary_afsr_errs = ch_flt->afsr_errs;
4980         ch_diag_data_t *cdp = NULL;
4981 
4982         t_afsr_errs &= ((C_AFSR_ALL_ERRS & ~C_AFSR_ME) | C_AFSR_EXT_ALL_ERRS);
4983 
4984 #if defined(CHEETAH_PLUS)
4985         orig_t_afsr_errs = t_afsr_errs;
4986 
4987         /*
4988          * For Cheetah+, log the shadow AFSR/AFAR bits first.
4989          */
4990         if (clop != NULL) {
4991                 /*
4992                  * Set the AFSR and AFAR fields to the shadow registers.  The
4993                  * flt_addr and flt_stat fields will be reset to the primaries
4994                  * below, but the sdw_addr and sdw_stat will stay as the
4995                  * secondaries.
4996                  */
4997                 cdp = &clop->clo_sdw_data;
4998                 aflt->flt_addr = ch_flt->flt_sdw_afar = cdp->chd_afar;
4999                 aflt->flt_stat = ch_flt->flt_sdw_afsr = cdp->chd_afsr;
5000                 ch_flt->afsr_ext = ch_flt->flt_sdw_afsr_ext = cdp->chd_afsr_ext;
5001                 ch_flt->afsr_errs = (cdp->chd_afsr_ext & C_AFSR_EXT_ALL_ERRS) |
5002                     (cdp->chd_afsr & C_AFSR_ALL_ERRS);
5003 
5004                 /*
5005                  * If the primary and shadow AFSR differ, tag the shadow as
5006                  * the first fault.
5007                  */
5008                 if ((primary_afar != cdp->chd_afar) ||
5009                     (primary_afsr_errs != ch_flt->afsr_errs)) {
5010                         aflt->flt_stat |= (1ull << C_AFSR_FIRSTFLT_SHIFT);
5011                 }
5012 
5013                 /*
5014                  * Check AFSR bits as well as AFSR_EXT bits in order of
5015                  * the AFAR overwrite priority. Our stored AFSR_EXT value
5016                  * is expected to be zero for those CPUs which do not have
5017                  * an AFSR_EXT register.
5018                  */
5019                 for (eccp = ecc_type_to_info; eccp->ec_desc != NULL; eccp++) {
5020                         if ((eccp->ec_afsr_bit &
5021                             (ch_flt->afsr_errs & t_afsr_errs)) &&
5022                             ((eccp->ec_flags & aflt->flt_status) != 0)) {
5023                                 cpu_queue_one_event(ch_flt, reason, eccp, cdp);
5024                                 cdp = NULL;
5025                                 t_afsr_errs &= ~eccp->ec_afsr_bit;
5026                                 nevents++;
5027                         }
5028                 }
5029 
5030                 /*
5031                  * If the ME bit is on in the primary AFSR turn all the
5032                  * error bits on again that may set the ME bit to make
5033                  * sure we see the ME AFSR error logs.
5034                  */
5035                 if ((primary_afsr & C_AFSR_ME) != 0)
5036                         t_afsr_errs = (orig_t_afsr_errs & C_AFSR_ALL_ME_ERRS);
5037         }
5038 #endif  /* CHEETAH_PLUS */
5039 
5040         if (clop != NULL)
5041                 cdp = &clop->clo_data;
5042 
5043         /*
5044          * Queue expected errors, error bit and fault type must match
5045          * in the ecc_type_to_info table.
5046          */
5047         for (eccp = ecc_type_to_info; t_afsr_errs != 0 && eccp->ec_desc != NULL;
5048             eccp++) {
5049                 if ((eccp->ec_afsr_bit & t_afsr_errs) != 0 &&
5050                     (eccp->ec_flags & aflt->flt_status) != 0) {
5051 #if defined(SERRANO)
5052                         /*
5053                          * For FRC/FRU errors on Serrano the afar2 captures
5054                          * the address and the associated data is
5055                          * in the shadow logout area.
5056                          */
5057                         if (eccp->ec_afsr_bit  & (C_AFSR_FRC | C_AFSR_FRU)) {
5058                                 if (clop != NULL)
5059                                         cdp = &clop->clo_sdw_data;
5060                                 aflt->flt_addr = ch_flt->afar2;
5061                         } else {
5062                                 if (clop != NULL)
5063                                         cdp = &clop->clo_data;
5064                                 aflt->flt_addr = primary_afar;
5065                         }
5066 #else   /* SERRANO */
5067                         aflt->flt_addr = primary_afar;
5068 #endif  /* SERRANO */
5069                         aflt->flt_stat = primary_afsr;
5070                         ch_flt->afsr_ext = primary_afsr_ext;
5071                         ch_flt->afsr_errs = primary_afsr_errs;
5072                         cpu_queue_one_event(ch_flt, reason, eccp, cdp);
5073                         cdp = NULL;
5074                         t_afsr_errs &= ~eccp->ec_afsr_bit;
5075                         nevents++;
5076                 }
5077         }
5078 
5079         /*
5080          * Queue unexpected errors, error bit only match.
5081          */
5082         for (eccp = ecc_type_to_info; t_afsr_errs != 0 && eccp->ec_desc != NULL;
5083             eccp++) {
5084                 if (eccp->ec_afsr_bit & t_afsr_errs) {
5085 #if defined(SERRANO)
5086                         /*
5087                          * For FRC/FRU errors on Serrano the afar2 captures
5088                          * the address and the associated data is
5089                          * in the shadow logout area.
5090                          */
5091                         if (eccp->ec_afsr_bit  & (C_AFSR_FRC | C_AFSR_FRU)) {
5092                                 if (clop != NULL)
5093                                         cdp = &clop->clo_sdw_data;
5094                                 aflt->flt_addr = ch_flt->afar2;
5095                         } else {
5096                                 if (clop != NULL)
5097                                         cdp = &clop->clo_data;
5098                                 aflt->flt_addr = primary_afar;
5099                         }
5100 #else   /* SERRANO */
5101                         aflt->flt_addr = primary_afar;
5102 #endif  /* SERRANO */
5103                         aflt->flt_stat = primary_afsr;
5104                         ch_flt->afsr_ext = primary_afsr_ext;
5105                         ch_flt->afsr_errs = primary_afsr_errs;
5106                         cpu_queue_one_event(ch_flt, reason, eccp, cdp);
5107                         cdp = NULL;
5108                         t_afsr_errs &= ~eccp->ec_afsr_bit;
5109                         nevents++;
5110                 }
5111         }
5112         return (nevents);
5113 }
5114 
5115 /*
5116  * Return trap type number.
5117  */
5118 uint8_t
5119 flt_to_trap_type(struct async_flt *aflt)
5120 {
5121         if (aflt->flt_status & ECC_I_TRAP)
5122                 return (TRAP_TYPE_ECC_I);
5123         if (aflt->flt_status & ECC_D_TRAP)
5124                 return (TRAP_TYPE_ECC_D);
5125         if (aflt->flt_status & ECC_F_TRAP)
5126                 return (TRAP_TYPE_ECC_F);
5127         if (aflt->flt_status & ECC_C_TRAP)
5128                 return (TRAP_TYPE_ECC_C);
5129         if (aflt->flt_status & ECC_DP_TRAP)
5130                 return (TRAP_TYPE_ECC_DP);
5131         if (aflt->flt_status & ECC_IP_TRAP)
5132                 return (TRAP_TYPE_ECC_IP);
5133         if (aflt->flt_status & ECC_ITLB_TRAP)
5134                 return (TRAP_TYPE_ECC_ITLB);
5135         if (aflt->flt_status & ECC_DTLB_TRAP)
5136                 return (TRAP_TYPE_ECC_DTLB);
5137         return (TRAP_TYPE_UNKNOWN);
5138 }
5139 
5140 /*
5141  * Decide an error type based on detector and leaky/partner tests.
5142  * The following array is used for quick translation - it must
5143  * stay in sync with ce_dispact_t.
5144  */
5145 
5146 static char *cetypes[] = {
5147         CE_DISP_DESC_U,
5148         CE_DISP_DESC_I,
5149         CE_DISP_DESC_PP,
5150         CE_DISP_DESC_P,
5151         CE_DISP_DESC_L,
5152         CE_DISP_DESC_PS,
5153         CE_DISP_DESC_S
5154 };
5155 
5156 char *
5157 flt_to_error_type(struct async_flt *aflt)
5158 {
5159         ce_dispact_t dispact, disp;
5160         uchar_t dtcrinfo, ptnrinfo, lkyinfo;
5161 
5162         /*
5163          * The memory payload bundle is shared by some events that do
5164          * not perform any classification.  For those flt_disp will be
5165          * 0 and we will return "unknown".
5166          */
5167         if (!ce_disp_inited || !aflt->flt_in_memory || aflt->flt_disp == 0)
5168                 return (cetypes[CE_DISP_UNKNOWN]);
5169 
5170         dtcrinfo = CE_XDIAG_DTCRINFO(aflt->flt_disp);
5171 
5172         /*
5173          * It is also possible that no scrub/classification was performed
5174          * by the detector, for instance where a disrupting error logged
5175          * in the AFSR while CEEN was off in cpu_deferred_error.
5176          */
5177         if (!CE_XDIAG_EXT_ALG_APPLIED(dtcrinfo))
5178                 return (cetypes[CE_DISP_UNKNOWN]);
5179 
5180         /*
5181          * Lookup type in initial classification/action table
5182          */
5183         dispact = CE_DISPACT(ce_disp_table,
5184             CE_XDIAG_AFARMATCHED(dtcrinfo),
5185             CE_XDIAG_STATE(dtcrinfo),
5186             CE_XDIAG_CE1SEEN(dtcrinfo),
5187             CE_XDIAG_CE2SEEN(dtcrinfo));
5188 
5189         /*
5190          * A bad lookup is not something to panic production systems for.
5191          */
5192         ASSERT(dispact != CE_DISP_BAD);
5193         if (dispact == CE_DISP_BAD)
5194                 return (cetypes[CE_DISP_UNKNOWN]);
5195 
5196         disp = CE_DISP(dispact);
5197 
5198         switch (disp) {
5199         case CE_DISP_UNKNOWN:
5200         case CE_DISP_INTERMITTENT:
5201                 break;
5202 
5203         case CE_DISP_POSS_PERS:
5204                 /*
5205                  * "Possible persistent" errors to which we have applied a valid
5206                  * leaky test can be separated into "persistent" or "leaky".
5207                  */
5208                 lkyinfo = CE_XDIAG_LKYINFO(aflt->flt_disp);
5209                 if (CE_XDIAG_TESTVALID(lkyinfo)) {
5210                         if (CE_XDIAG_CE1SEEN(lkyinfo) ||
5211                             CE_XDIAG_CE2SEEN(lkyinfo))
5212                                 disp = CE_DISP_LEAKY;
5213                         else
5214                                 disp = CE_DISP_PERS;
5215                 }
5216                 break;
5217 
5218         case CE_DISP_POSS_STICKY:
5219                 /*
5220                  * Promote "possible sticky" results that have been
5221                  * confirmed by a partner test to "sticky".  Unconfirmed
5222                  * "possible sticky" events are left at that status - we do not
5223                  * guess at any bad reader/writer etc status here.
5224                  */
5225                 ptnrinfo = CE_XDIAG_PTNRINFO(aflt->flt_disp);
5226                 if (CE_XDIAG_TESTVALID(ptnrinfo) &&
5227                     CE_XDIAG_CE1SEEN(ptnrinfo) && CE_XDIAG_CE2SEEN(ptnrinfo))
5228                         disp = CE_DISP_STICKY;
5229 
5230                 /*
5231                  * Promote "possible sticky" results on a uniprocessor
5232                  * to "sticky"
5233                  */
5234                 if (disp == CE_DISP_POSS_STICKY &&
5235                     CE_XDIAG_SKIPCODE(disp) == CE_XDIAG_SKIP_UNIPROC)
5236                         disp = CE_DISP_STICKY;
5237                 break;
5238 
5239         default:
5240                 disp = CE_DISP_UNKNOWN;
5241                 break;
5242         }
5243 
5244         return (cetypes[disp]);
5245 }
5246 
5247 /*
5248  * Given the entire afsr, the specific bit to check and a prioritized list of
5249  * error bits, determine the validity of the various overwrite priority
5250  * features of the AFSR/AFAR: AFAR, ESYND and MSYND, each of which have
5251  * different overwrite priorities.
5252  *
5253  * Given a specific afsr error bit and the entire afsr, there are three cases:
5254  *   INVALID:   The specified bit is lower overwrite priority than some other
5255  *              error bit which is on in the afsr (or IVU/IVC).
5256  *   VALID:     The specified bit is higher priority than all other error bits
5257  *              which are on in the afsr.
5258  *   AMBIGUOUS: Another error bit (or bits) of equal priority to the specified
5259  *              bit is on in the afsr.
5260  */
5261 int
5262 afsr_to_overw_status(uint64_t afsr, uint64_t afsr_bit, uint64_t *ow_bits)
5263 {
5264         uint64_t afsr_ow;
5265 
5266         while ((afsr_ow = *ow_bits++) != 0) {
5267                 /*
5268                  * If bit is in the priority class, check to see if another
5269                  * bit in the same class is on => ambiguous.  Otherwise,
5270                  * the value is valid.  If the bit is not on at this priority
5271                  * class, but a higher priority bit is on, then the value is
5272                  * invalid.
5273                  */
5274                 if (afsr_ow & afsr_bit) {
5275                         /*
5276                          * If equal pri bit is on, ambiguous.
5277                          */
5278                         if (afsr & (afsr_ow & ~afsr_bit))
5279                                 return (AFLT_STAT_AMBIGUOUS);
5280                         return (AFLT_STAT_VALID);
5281                 } else if (afsr & afsr_ow)
5282                         break;
5283         }
5284 
5285         /*
5286          * We didn't find a match or a higher priority bit was on.  Not
5287          * finding a match handles the case of invalid AFAR for IVC, IVU.
5288          */
5289         return (AFLT_STAT_INVALID);
5290 }
5291 
5292 static int
5293 afsr_to_afar_status(uint64_t afsr, uint64_t afsr_bit)
5294 {
5295 #if defined(SERRANO)
5296         if (afsr_bit & (C_AFSR_FRC | C_AFSR_FRU))
5297                 return (afsr_to_overw_status(afsr, afsr_bit, afar2_overwrite));
5298         else
5299 #endif  /* SERRANO */
5300                 return (afsr_to_overw_status(afsr, afsr_bit, afar_overwrite));
5301 }
5302 
5303 static int
5304 afsr_to_esynd_status(uint64_t afsr, uint64_t afsr_bit)
5305 {
5306         return (afsr_to_overw_status(afsr, afsr_bit, esynd_overwrite));
5307 }
5308 
5309 static int
5310 afsr_to_msynd_status(uint64_t afsr, uint64_t afsr_bit)
5311 {
5312         return (afsr_to_overw_status(afsr, afsr_bit, msynd_overwrite));
5313 }
5314 
5315 static int
5316 afsr_to_synd_status(uint_t cpuid, uint64_t afsr, uint64_t afsr_bit)
5317 {
5318 #ifdef lint
5319         cpuid = cpuid;
5320 #endif
5321 #if defined(CHEETAH_PLUS)
5322         /*
5323          * The M_SYND overwrite policy is combined with the E_SYND overwrite
5324          * policy for Cheetah+ and separate for Panther CPUs.
5325          */
5326         if (afsr_bit & C_AFSR_MSYND_ERRS) {
5327                 if (IS_PANTHER(cpunodes[cpuid].implementation))
5328                         return (afsr_to_msynd_status(afsr, afsr_bit));
5329                 else
5330                         return (afsr_to_esynd_status(afsr, afsr_bit));
5331         } else if (afsr_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) {
5332                 if (IS_PANTHER(cpunodes[cpuid].implementation))
5333                         return (afsr_to_pn_esynd_status(afsr, afsr_bit));
5334                 else
5335                         return (afsr_to_esynd_status(afsr, afsr_bit));
5336 #else /* CHEETAH_PLUS */
5337         if (afsr_bit & C_AFSR_MSYND_ERRS) {
5338                 return (afsr_to_msynd_status(afsr, afsr_bit));
5339         } else if (afsr_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) {
5340                 return (afsr_to_esynd_status(afsr, afsr_bit));
5341 #endif /* CHEETAH_PLUS */
5342         } else {
5343                 return (AFLT_STAT_INVALID);
5344         }
5345 }
5346 
5347 /*
5348  * Slave CPU stick synchronization.
5349  */
5350 void
5351 sticksync_slave(void)
5352 {
5353         int             i;
5354         int             tries = 0;
5355         int64_t         tskew;
5356         int64_t         av_tskew;
5357 
5358         kpreempt_disable();
5359         /* wait for the master side */
5360         while (stick_sync_cmd != SLAVE_START)
5361                 ;
5362         /*
5363          * Synchronization should only take a few tries at most. But in the
5364          * odd case where the cpu isn't cooperating we'll keep trying. A cpu
5365          * without it's stick synchronized wouldn't be a good citizen.
5366          */
5367         while (slave_done == 0) {
5368                 /*
5369                  * Time skew calculation.
5370                  */
5371                 av_tskew = tskew = 0;
5372 
5373                 for (i = 0; i < stick_iter; i++) {
5374                         /* make location hot */
5375                         timestamp[EV_A_START] = 0;
5376                         stick_timestamp(&timestamp[EV_A_START]);
5377 
5378                         /* tell the master we're ready */
5379                         stick_sync_cmd = MASTER_START;
5380 
5381                         /* and wait */
5382                         while (stick_sync_cmd != SLAVE_CONT)
5383                                 ;
5384                         /* Event B end */
5385                         stick_timestamp(&timestamp[EV_B_END]);
5386 
5387                         /* calculate time skew */
5388                         tskew = ((timestamp[EV_B_END] - timestamp[EV_B_START])
5389                             - (timestamp[EV_A_END] - timestamp[EV_A_START]))
5390                             / 2;
5391 
5392                         /* keep running count */
5393                         av_tskew += tskew;
5394                 } /* for */
5395 
5396                 /*
5397                  * Adjust stick for time skew if not within the max allowed;
5398                  * otherwise we're all done.
5399                  */
5400                 if (stick_iter != 0)
5401                         av_tskew = av_tskew/stick_iter;
5402                 if (ABS(av_tskew) > stick_tsk) {
5403                         /*
5404                          * If the skew is 1 (the slave's STICK register
5405                          * is 1 STICK ahead of the master's), stick_adj
5406                          * could fail to adjust the slave's STICK register
5407                          * if the STICK read on the slave happens to
5408                          * align with the increment of the STICK.
5409                          * Therefore, we increment the skew to 2.
5410                          */
5411                         if (av_tskew == 1)
5412                                 av_tskew++;
5413                         stick_adj(-av_tskew);
5414                 } else
5415                         slave_done = 1;
5416 #ifdef DEBUG
5417                 if (tries < DSYNC_ATTEMPTS)
5418                         stick_sync_stats[CPU->cpu_id].skew_val[tries] =
5419                             av_tskew;
5420                 ++tries;
5421 #endif /* DEBUG */
5422 #ifdef lint
5423                 tries = tries;
5424 #endif
5425 
5426         } /* while */
5427 
5428         /* allow the master to finish */
5429         stick_sync_cmd = EVENT_NULL;
5430         kpreempt_enable();
5431 }
5432 
5433 /*
5434  * Master CPU side of stick synchronization.
5435  *  - timestamp end of Event A
5436  *  - timestamp beginning of Event B
5437  */
5438 void
5439 sticksync_master(void)
5440 {
5441         int             i;
5442 
5443         kpreempt_disable();
5444         /* tell the slave we've started */
5445         slave_done = 0;
5446         stick_sync_cmd = SLAVE_START;
5447 
5448         while (slave_done == 0) {
5449                 for (i = 0; i < stick_iter; i++) {
5450                         /* wait for the slave */
5451                         while (stick_sync_cmd != MASTER_START)
5452                                 ;
5453                         /* Event A end */
5454                         stick_timestamp(&timestamp[EV_A_END]);
5455 
5456                         /* make location hot */
5457                         timestamp[EV_B_START] = 0;
5458                         stick_timestamp(&timestamp[EV_B_START]);
5459 
5460                         /* tell the slave to continue */
5461                         stick_sync_cmd = SLAVE_CONT;
5462                 } /* for */
5463 
5464                 /* wait while slave calculates time skew */
5465                 while (stick_sync_cmd == SLAVE_CONT)
5466                         ;
5467         } /* while */
5468         kpreempt_enable();
5469 }
5470 
5471 /*
5472  * Cheetah/Cheetah+ have disrupting error for copyback's, so we don't need to
5473  * do Spitfire hack of xcall'ing all the cpus to ask to check for them.  Also,
5474  * in cpu_async_panic_callb, each cpu checks for CPU events on its way to
5475  * panic idle.
5476  */
5477 /*ARGSUSED*/
5478 void
5479 cpu_check_allcpus(struct async_flt *aflt)
5480 {}
5481 
5482 struct kmem_cache *ch_private_cache;
5483 
5484 /*
5485  * Cpu private unitialization.  Uninitialize the Ecache scrubber and
5486  * deallocate the scrubber data structures and cpu_private data structure.
5487  */
5488 void
5489 cpu_uninit_private(struct cpu *cp)
5490 {
5491         cheetah_private_t *chprp = CPU_PRIVATE(cp);
5492 
5493         ASSERT(chprp);
5494         cpu_uninit_ecache_scrub_dr(cp);
5495         CPU_PRIVATE(cp) = NULL;
5496         ch_err_tl1_paddrs[cp->cpu_id] = NULL;
5497         kmem_cache_free(ch_private_cache, chprp);
5498         cmp_delete_cpu(cp->cpu_id);
5499 
5500 }
5501 
5502 /*
5503  * Cheetah Cache Scrubbing
5504  *
5505  * The primary purpose of Cheetah cache scrubbing is to reduce the exposure
5506  * of E$ tags, D$ data, and I$ data to cosmic ray events since they are not
5507  * protected by either parity or ECC.
5508  *
5509  * We currently default the E$ and D$ scan rate to 100 (scan 10% of the
5510  * cache per second). Due to the the specifics of how the I$ control
5511  * logic works with respect to the ASI used to scrub I$ lines, the entire
5512  * I$ is scanned at once.
5513  */
5514 
5515 /*
5516  * Tuneables to enable and disable the scrubbing of the caches, and to tune
5517  * scrubbing behavior.  These may be changed via /etc/system or using mdb
5518  * on a running system.
5519  */
5520 int dcache_scrub_enable = 1;            /* D$ scrubbing is on by default */
5521 
5522 /*
5523  * The following are the PIL levels that the softints/cross traps will fire at.
5524  */
5525 uint_t ecache_scrub_pil = PIL_9;        /* E$ scrub PIL for cross traps */
5526 uint_t dcache_scrub_pil = PIL_9;        /* D$ scrub PIL for cross traps */
5527 uint_t icache_scrub_pil = PIL_9;        /* I$ scrub PIL for cross traps */
5528 
5529 #if defined(JALAPENO)
5530 
5531 /*
5532  * Due to several errata (82, 85, 86), we don't enable the L2$ scrubber
5533  * on Jalapeno.
5534  */
5535 int ecache_scrub_enable = 0;
5536 
5537 #else   /* JALAPENO */
5538 
5539 /*
5540  * With all other cpu types, E$ scrubbing is on by default
5541  */
5542 int ecache_scrub_enable = 1;
5543 
5544 #endif  /* JALAPENO */
5545 
5546 
5547 #if defined(CHEETAH_PLUS) || defined(JALAPENO) || defined(SERRANO)
5548 
5549 /*
5550  * The I$ scrubber tends to cause latency problems for real-time SW, so it
5551  * is disabled by default on non-Cheetah systems
5552  */
5553 int icache_scrub_enable = 0;
5554 
5555 /*
5556  * Tuneables specifying the scrub calls per second and the scan rate
5557  * for each cache
5558  *
5559  * The cyclic times are set during boot based on the following values.
5560  * Changing these values in mdb after this time will have no effect.  If
5561  * a different value is desired, it must be set in /etc/system before a
5562  * reboot.
5563  */
5564 int ecache_calls_a_sec = 1;
5565 int dcache_calls_a_sec = 2;
5566 int icache_calls_a_sec = 2;
5567 
5568 int ecache_scan_rate_idle = 1;
5569 int ecache_scan_rate_busy = 1;
5570 int dcache_scan_rate_idle = 1;
5571 int dcache_scan_rate_busy = 1;
5572 int icache_scan_rate_idle = 1;
5573 int icache_scan_rate_busy = 1;
5574 
5575 #else   /* CHEETAH_PLUS || JALAPENO || SERRANO */
5576 
5577 int icache_scrub_enable = 1;            /* I$ scrubbing is on by default */
5578 
5579 int ecache_calls_a_sec = 100;           /* E$ scrub calls per seconds */
5580 int dcache_calls_a_sec = 100;           /* D$ scrub calls per seconds */
5581 int icache_calls_a_sec = 100;           /* I$ scrub calls per seconds */
5582 
5583 int ecache_scan_rate_idle = 100;        /* E$ scan rate (in tenths of a %) */
5584 int ecache_scan_rate_busy = 100;        /* E$ scan rate (in tenths of a %) */
5585 int dcache_scan_rate_idle = 100;        /* D$ scan rate (in tenths of a %) */
5586 int dcache_scan_rate_busy = 100;        /* D$ scan rate (in tenths of a %) */
5587 int icache_scan_rate_idle = 100;        /* I$ scan rate (in tenths of a %) */
5588 int icache_scan_rate_busy = 100;        /* I$ scan rate (in tenths of a %) */
5589 
5590 #endif  /* CHEETAH_PLUS || JALAPENO || SERRANO */
5591 
5592 /*
5593  * In order to scrub on offline cpus, a cross trap is sent.  The handler will
5594  * increment the outstanding request counter and schedule a softint to run
5595  * the scrubber.
5596  */
5597 extern xcfunc_t cache_scrubreq_tl1;
5598 
5599 /*
5600  * These are the softint functions for each cache scrubber
5601  */
5602 static uint_t scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2);
5603 static uint_t scrub_dcache_line_intr(caddr_t arg1, caddr_t arg2);
5604 static uint_t scrub_icache_line_intr(caddr_t arg1, caddr_t arg2);
5605 
5606 /*
5607  * The cache scrub info table contains cache specific information
5608  * and allows for some of the scrub code to be table driven, reducing
5609  * duplication of cache similar code.
5610  *
5611  * This table keeps a copy of the value in the calls per second variable
5612  * (?cache_calls_a_sec).  This makes it much more difficult for someone
5613  * to cause us problems (for example, by setting ecache_calls_a_sec to 0 in
5614  * mdb in a misguided attempt to disable the scrubber).
5615  */
5616 struct scrub_info {
5617         int             *csi_enable;    /* scrubber enable flag */
5618         int             csi_freq;       /* scrubber calls per second */
5619         int             csi_index;      /* index to chsm_outstanding[] */
5620         uint64_t        csi_inum;       /* scrubber interrupt number */
5621         cyclic_id_t     csi_omni_cyc_id;        /* omni cyclic ID */
5622         cyclic_id_t     csi_offline_cyc_id;     /* offline cyclic ID */
5623         char            csi_name[3];    /* cache name for this scrub entry */
5624 } cache_scrub_info[] = {
5625 { &ecache_scrub_enable, 0, CACHE_SCRUBBER_INFO_E, 0, 0, 0, "E$"},
5626 { &dcache_scrub_enable, 0, CACHE_SCRUBBER_INFO_D, 0, 0, 0, "D$"},
5627 { &icache_scrub_enable, 0, CACHE_SCRUBBER_INFO_I, 0, 0, 0, "I$"}
5628 };
5629 
5630 /*
5631  * If scrubbing is enabled, increment the outstanding request counter.  If it
5632  * is 1 (meaning there were no previous requests outstanding), call
5633  * setsoftint_tl1 through xt_one_unchecked, which eventually ends up doing
5634  * a self trap.
5635  */
5636 static void
5637 do_scrub(struct scrub_info *csi)
5638 {
5639         ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc);
5640         int index = csi->csi_index;
5641         uint32_t *outstanding = &csmp->chsm_outstanding[index];
5642 
5643         if (*(csi->csi_enable) && (csmp->chsm_enable[index])) {
5644                 if (atomic_inc_32_nv(outstanding) == 1) {
5645                         xt_one_unchecked(CPU->cpu_id, setsoftint_tl1,
5646                             csi->csi_inum, 0);
5647                 }
5648         }
5649 }
5650 
5651 /*
5652  * Omni cyclics don't fire on offline cpus, so we use another cyclic to
5653  * cross-trap the offline cpus.
5654  */
5655 static void
5656 do_scrub_offline(struct scrub_info *csi)
5657 {
5658         ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc);
5659 
5660         if (CPUSET_ISNULL(cpu_offline_set)) {
5661                 /*
5662                  * No offline cpus - nothing to do
5663                  */
5664                 return;
5665         }
5666 
5667         if (*(csi->csi_enable) && (csmp->chsm_enable[csi->csi_index])) {
5668                 xt_some(cpu_offline_set, cache_scrubreq_tl1, csi->csi_inum,
5669                     csi->csi_index);
5670         }
5671 }
5672 
5673 /*
5674  * This is the initial setup for the scrubber cyclics - it sets the
5675  * interrupt level, frequency, and function to call.
5676  */
5677 /*ARGSUSED*/
5678 static void
5679 cpu_scrub_cyclic_setup(void *arg, cpu_t *cpu, cyc_handler_t *hdlr,
5680     cyc_time_t *when)
5681 {
5682         struct scrub_info *csi = (struct scrub_info *)arg;
5683 
5684         ASSERT(csi != NULL);
5685         hdlr->cyh_func = (cyc_func_t)do_scrub;
5686         hdlr->cyh_level = CY_LOW_LEVEL;
5687         hdlr->cyh_arg = arg;
5688 
5689         when->cyt_when = 0;  /* Start immediately */
5690         when->cyt_interval = NANOSEC / csi->csi_freq;
5691 }
5692 
5693 /*
5694  * Initialization for cache scrubbing.
5695  * This routine is called AFTER all cpus have had cpu_init_private called
5696  * to initialize their private data areas.
5697  */
5698 void
5699 cpu_init_cache_scrub(void)
5700 {
5701         int i;
5702         struct scrub_info *csi;
5703         cyc_omni_handler_t omni_hdlr;
5704         cyc_handler_t offline_hdlr;
5705         cyc_time_t when;
5706 
5707         /*
5708          * save away the maximum number of lines for the D$
5709          */
5710         dcache_nlines = dcache_size / dcache_linesize;
5711 
5712         /*
5713          * register the softints for the cache scrubbing
5714          */
5715         cache_scrub_info[CACHE_SCRUBBER_INFO_E].csi_inum =
5716             add_softintr(ecache_scrub_pil, scrub_ecache_line_intr,
5717             (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_E], SOFTINT_MT);
5718         cache_scrub_info[CACHE_SCRUBBER_INFO_E].csi_freq = ecache_calls_a_sec;
5719 
5720         cache_scrub_info[CACHE_SCRUBBER_INFO_D].csi_inum =
5721             add_softintr(dcache_scrub_pil, scrub_dcache_line_intr,
5722             (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_D], SOFTINT_MT);
5723         cache_scrub_info[CACHE_SCRUBBER_INFO_D].csi_freq = dcache_calls_a_sec;
5724 
5725         cache_scrub_info[CACHE_SCRUBBER_INFO_I].csi_inum =
5726             add_softintr(icache_scrub_pil, scrub_icache_line_intr,
5727             (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_I], SOFTINT_MT);
5728         cache_scrub_info[CACHE_SCRUBBER_INFO_I].csi_freq = icache_calls_a_sec;
5729 
5730         /*
5731          * start the scrubbing for all the caches
5732          */
5733         mutex_enter(&cpu_lock);
5734         for (i = 0; i < CACHE_SCRUBBER_COUNT; i++) {
5735 
5736                 csi = &cache_scrub_info[i];
5737 
5738                 if (!(*csi->csi_enable))
5739                         continue;
5740 
5741                 /*
5742                  * force the following to be true:
5743                  *      1 <= calls_a_sec <= hz
5744                  */
5745                 if (csi->csi_freq > hz) {
5746                         cmn_err(CE_NOTE, "%s scrub calls_a_sec set too high "
5747                             "(%d); resetting to hz (%d)", csi->csi_name,
5748                             csi->csi_freq, hz);
5749                         csi->csi_freq = hz;
5750                 } else if (csi->csi_freq < 1) {
5751                         cmn_err(CE_NOTE, "%s scrub calls_a_sec set too low "
5752                             "(%d); resetting to 1", csi->csi_name,
5753                             csi->csi_freq);
5754                         csi->csi_freq = 1;
5755                 }
5756 
5757                 omni_hdlr.cyo_online = cpu_scrub_cyclic_setup;
5758                 omni_hdlr.cyo_offline = NULL;
5759                 omni_hdlr.cyo_arg = (void *)csi;
5760 
5761                 offline_hdlr.cyh_func = (cyc_func_t)do_scrub_offline;
5762                 offline_hdlr.cyh_arg = (void *)csi;
5763                 offline_hdlr.cyh_level = CY_LOW_LEVEL;
5764 
5765                 when.cyt_when = 0;      /* Start immediately */
5766                 when.cyt_interval = NANOSEC / csi->csi_freq;
5767 
5768                 csi->csi_omni_cyc_id = cyclic_add_omni(&omni_hdlr);
5769                 csi->csi_offline_cyc_id = cyclic_add(&offline_hdlr, &when);
5770         }
5771         register_cpu_setup_func(cpu_scrub_cpu_setup, NULL);
5772         mutex_exit(&cpu_lock);
5773 }
5774 
5775 /*
5776  * Indicate that the specified cpu is idle.
5777  */
5778 void
5779 cpu_idle_ecache_scrub(struct cpu *cp)
5780 {
5781         if (CPU_PRIVATE(cp) != NULL) {
5782                 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc);
5783                 csmp->chsm_ecache_busy = ECACHE_CPU_IDLE;
5784         }
5785 }
5786 
5787 /*
5788  * Indicate that the specified cpu is busy.
5789  */
5790 void
5791 cpu_busy_ecache_scrub(struct cpu *cp)
5792 {
5793         if (CPU_PRIVATE(cp) != NULL) {
5794                 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc);
5795                 csmp->chsm_ecache_busy = ECACHE_CPU_BUSY;
5796         }
5797 }
5798 
5799 /*
5800  * Initialization for cache scrubbing for the specified cpu.
5801  */
5802 void
5803 cpu_init_ecache_scrub_dr(struct cpu *cp)
5804 {
5805         ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc);
5806         int cpuid = cp->cpu_id;
5807 
5808         /* initialize the number of lines in the caches */
5809         csmp->chsm_ecache_nlines = cpunodes[cpuid].ecache_size /
5810             cpunodes[cpuid].ecache_linesize;
5811         csmp->chsm_icache_nlines = CPU_PRIVATE_VAL(cp, chpr_icache_size) /
5812             CPU_PRIVATE_VAL(cp, chpr_icache_linesize);
5813 
5814         /*
5815          * do_scrub() and do_scrub_offline() check both the global
5816          * ?cache_scrub_enable and this per-cpu enable variable.  All scrubbers
5817          * check this value before scrubbing.  Currently, we use it to
5818          * disable the E$ scrubber on multi-core cpus or while running at
5819          * slowed speed.  For now, just turn everything on and allow
5820          * cpu_init_private() to change it if necessary.
5821          */
5822         csmp->chsm_enable[CACHE_SCRUBBER_INFO_E] = 1;
5823         csmp->chsm_enable[CACHE_SCRUBBER_INFO_D] = 1;
5824         csmp->chsm_enable[CACHE_SCRUBBER_INFO_I] = 1;
5825 
5826         cpu_busy_ecache_scrub(cp);
5827 }
5828 
5829 /*
5830  * Un-initialization for cache scrubbing for the specified cpu.
5831  */
5832 static void
5833 cpu_uninit_ecache_scrub_dr(struct cpu *cp)
5834 {
5835         ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc);
5836 
5837         /*
5838          * un-initialize bookkeeping for cache scrubbing
5839          */
5840         bzero(csmp, sizeof (ch_scrub_misc_t));
5841 
5842         cpu_idle_ecache_scrub(cp);
5843 }
5844 
5845 /*
5846  * Called periodically on each CPU to scrub the D$.
5847  */
5848 static void
5849 scrub_dcache(int how_many)
5850 {
5851         int i;
5852         ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc);
5853         int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_D];
5854 
5855         /*
5856          * scrub the desired number of lines
5857          */
5858         for (i = 0; i < how_many; i++) {
5859                 /*
5860                  * scrub a D$ line
5861                  */
5862                 dcache_inval_line(index);
5863 
5864                 /*
5865                  * calculate the next D$ line to scrub, assumes
5866                  * that dcache_nlines is a power of 2
5867                  */
5868                 index = (index + 1) & (dcache_nlines - 1);
5869         }
5870 
5871         /*
5872          * set the scrub index for the next visit
5873          */
5874         csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_D] = index;
5875 }
5876 
5877 /*
5878  * Handler for D$ scrub inum softint. Call scrub_dcache until
5879  * we decrement the outstanding request count to zero.
5880  */
5881 /*ARGSUSED*/
5882 static uint_t
5883 scrub_dcache_line_intr(caddr_t arg1, caddr_t arg2)
5884 {
5885         int i;
5886         int how_many;
5887         int outstanding;
5888         ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc);
5889         uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_D];
5890         struct scrub_info *csi = (struct scrub_info *)arg1;
5891         int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ?
5892             dcache_scan_rate_idle : dcache_scan_rate_busy;
5893 
5894         /*
5895          * The scan rates are expressed in units of tenths of a
5896          * percent.  A scan rate of 1000 (100%) means the whole
5897          * cache is scanned every second.
5898          */
5899         how_many = (dcache_nlines * scan_rate) / (1000 * csi->csi_freq);
5900 
5901         do {
5902                 outstanding = *countp;
5903                 for (i = 0; i < outstanding; i++) {
5904                         scrub_dcache(how_many);
5905                 }
5906         } while (atomic_add_32_nv(countp, -outstanding));
5907 
5908         return (DDI_INTR_CLAIMED);
5909 }
5910 
5911 /*
5912  * Called periodically on each CPU to scrub the I$. The I$ is scrubbed
5913  * by invalidating lines. Due to the characteristics of the ASI which
5914  * is used to invalidate an I$ line, the entire I$ must be invalidated
5915  * vs. an individual I$ line.
5916  */
5917 static void
5918 scrub_icache(int how_many)
5919 {
5920         int i;
5921         ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc);
5922         int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_I];
5923         int icache_nlines = csmp->chsm_icache_nlines;
5924 
5925         /*
5926          * scrub the desired number of lines
5927          */
5928         for (i = 0; i < how_many; i++) {
5929                 /*
5930                  * since the entire I$ must be scrubbed at once,
5931                  * wait until the index wraps to zero to invalidate
5932                  * the entire I$
5933                  */
5934                 if (index == 0) {
5935                         icache_inval_all();
5936                 }
5937 
5938                 /*
5939                  * calculate the next I$ line to scrub, assumes
5940                  * that chsm_icache_nlines is a power of 2
5941                  */
5942                 index = (index + 1) & (icache_nlines - 1);
5943         }
5944 
5945         /*
5946          * set the scrub index for the next visit
5947          */
5948         csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_I] = index;
5949 }
5950 
5951 /*
5952  * Handler for I$ scrub inum softint. Call scrub_icache until
5953  * we decrement the outstanding request count to zero.
5954  */
5955 /*ARGSUSED*/
5956 static uint_t
5957 scrub_icache_line_intr(caddr_t arg1, caddr_t arg2)
5958 {
5959         int i;
5960         int how_many;
5961         int outstanding;
5962         ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc);
5963         uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_I];
5964         struct scrub_info *csi = (struct scrub_info *)arg1;
5965         int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ?
5966             icache_scan_rate_idle : icache_scan_rate_busy;
5967         int icache_nlines = csmp->chsm_icache_nlines;
5968 
5969         /*
5970          * The scan rates are expressed in units of tenths of a
5971          * percent.  A scan rate of 1000 (100%) means the whole
5972          * cache is scanned every second.
5973          */
5974         how_many = (icache_nlines * scan_rate) / (1000 * csi->csi_freq);
5975 
5976         do {
5977                 outstanding = *countp;
5978                 for (i = 0; i < outstanding; i++) {
5979                         scrub_icache(how_many);
5980                 }
5981         } while (atomic_add_32_nv(countp, -outstanding));
5982 
5983         return (DDI_INTR_CLAIMED);
5984 }
5985 
5986 /*
5987  * Called periodically on each CPU to scrub the E$.
5988  */
5989 static void
5990 scrub_ecache(int how_many)
5991 {
5992         ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc);
5993         int i;
5994         int cpuid = CPU->cpu_id;
5995         int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_E];
5996         int nlines = csmp->chsm_ecache_nlines;
5997         int linesize = cpunodes[cpuid].ecache_linesize;
5998         int ec_set_size = cpu_ecache_set_size(CPU);
5999 
6000         /*
6001          * scrub the desired number of lines
6002          */
6003         for (i = 0; i < how_many; i++) {
6004                 /*
6005                  * scrub the E$ line
6006                  */
6007                 ecache_flush_line(ecache_flushaddr + (index * linesize),
6008                     ec_set_size);
6009 
6010                 /*
6011                  * calculate the next E$ line to scrub based on twice
6012                  * the number of E$ lines (to displace lines containing
6013                  * flush area data), assumes that the number of lines
6014                  * is a power of 2
6015                  */
6016                 index = (index + 1) & ((nlines << 1) - 1);
6017         }
6018 
6019         /*
6020          * set the ecache scrub index for the next visit
6021          */
6022         csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_E] = index;
6023 }
6024 
6025 /*
6026  * Handler for E$ scrub inum softint. Call the E$ scrubber until
6027  * we decrement the outstanding request count to zero.
6028  *
6029  * Due to interactions with cpu_scrub_cpu_setup(), the outstanding count may
6030  * become negative after the atomic_add_32_nv().  This is not a problem, as
6031  * the next trip around the loop won't scrub anything, and the next add will
6032  * reset the count back to zero.
6033  */
6034 /*ARGSUSED*/
6035 static uint_t
6036 scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2)
6037 {
6038         int i;
6039         int how_many;
6040         int outstanding;
6041         ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc);
6042         uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_E];
6043         struct scrub_info *csi = (struct scrub_info *)arg1;
6044         int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ?
6045             ecache_scan_rate_idle : ecache_scan_rate_busy;
6046         int ecache_nlines = csmp->chsm_ecache_nlines;
6047 
6048         /*
6049          * The scan rates are expressed in units of tenths of a
6050          * percent.  A scan rate of 1000 (100%) means the whole
6051          * cache is scanned every second.
6052          */
6053         how_many = (ecache_nlines * scan_rate) / (1000 * csi->csi_freq);
6054 
6055         do {
6056                 outstanding = *countp;
6057                 for (i = 0; i < outstanding; i++) {
6058                         scrub_ecache(how_many);
6059                 }
6060         } while (atomic_add_32_nv(countp, -outstanding));
6061 
6062         return (DDI_INTR_CLAIMED);
6063 }
6064 
6065 /*
6066  * Timeout function to reenable CE
6067  */
6068 static void
6069 cpu_delayed_check_ce_errors(void *arg)
6070 {
6071         if (!taskq_dispatch(ch_check_ce_tq, cpu_check_ce_errors, arg,
6072             TQ_NOSLEEP)) {
6073                 (void) timeout(cpu_delayed_check_ce_errors, arg,
6074                     drv_sectohz((clock_t)cpu_ceen_delay_secs));
6075         }
6076 }
6077 
6078 /*
6079  * CE Deferred Re-enable after trap.
6080  *
6081  * When the CPU gets a disrupting trap for any of the errors
6082  * controlled by the CEEN bit, CEEN is disabled in the trap handler
6083  * immediately. To eliminate the possibility of multiple CEs causing
6084  * recursive stack overflow in the trap handler, we cannot
6085  * reenable CEEN while still running in the trap handler. Instead,
6086  * after a CE is logged on a CPU, we schedule a timeout function,
6087  * cpu_check_ce_errors(), to trigger after cpu_ceen_delay_secs
6088  * seconds. This function will check whether any further CEs
6089  * have occurred on that CPU, and if none have, will reenable CEEN.
6090  *
6091  * If further CEs have occurred while CEEN is disabled, another
6092  * timeout will be scheduled. This is to ensure that the CPU can
6093  * make progress in the face of CE 'storms', and that it does not
6094  * spend all its time logging CE errors.
6095  */
6096 static void
6097 cpu_check_ce_errors(void *arg)
6098 {
6099         int     cpuid = (int)(uintptr_t)arg;
6100         cpu_t   *cp;
6101 
6102         /*
6103          * We acquire cpu_lock.
6104          */
6105         ASSERT(curthread->t_pil == 0);
6106 
6107         /*
6108          * verify that the cpu is still around, DR
6109          * could have got there first ...
6110          */
6111         mutex_enter(&cpu_lock);
6112         cp = cpu_get(cpuid);
6113         if (cp == NULL) {
6114                 mutex_exit(&cpu_lock);
6115                 return;
6116         }
6117         /*
6118          * make sure we don't migrate across CPUs
6119          * while checking our CE status.
6120          */
6121         kpreempt_disable();
6122 
6123         /*
6124          * If we are running on the CPU that got the
6125          * CE, we can do the checks directly.
6126          */
6127         if (cp->cpu_id == CPU->cpu_id) {
6128                 mutex_exit(&cpu_lock);
6129                 cpu_check_ce(TIMEOUT_CEEN_CHECK, 0, 0, 0);
6130                 kpreempt_enable();
6131                 return;
6132         }
6133         kpreempt_enable();
6134 
6135         /*
6136          * send an x-call to get the CPU that originally
6137          * got the CE to do the necessary checks. If we can't
6138          * send the x-call, reschedule the timeout, otherwise we
6139          * lose CEEN forever on that CPU.
6140          */
6141         if (CPU_XCALL_READY(cp->cpu_id) && (!(cp->cpu_flags & CPU_QUIESCED))) {
6142                 xc_one(cp->cpu_id, (xcfunc_t *)cpu_check_ce,
6143                     TIMEOUT_CEEN_CHECK, 0);
6144                 mutex_exit(&cpu_lock);
6145         } else {
6146                 /*
6147                  * When the CPU is not accepting xcalls, or
6148                  * the processor is offlined, we don't want to
6149                  * incur the extra overhead of trying to schedule the
6150                  * CE timeout indefinitely. However, we don't want to lose
6151                  * CE checking forever.
6152                  *
6153                  * Keep rescheduling the timeout, accepting the additional
6154                  * overhead as the cost of correctness in the case where we get
6155                  * a CE, disable CEEN, offline the CPU during the
6156                  * the timeout interval, and then online it at some
6157                  * point in the future. This is unlikely given the short
6158                  * cpu_ceen_delay_secs.
6159                  */
6160                 mutex_exit(&cpu_lock);
6161                 (void) timeout(cpu_delayed_check_ce_errors,
6162                     (void *)(uintptr_t)cp->cpu_id,
6163                     drv_sectohz((clock_t)cpu_ceen_delay_secs));
6164         }
6165 }
6166 
6167 /*
6168  * This routine will check whether CEs have occurred while
6169  * CEEN is disabled. Any CEs detected will be logged and, if
6170  * possible, scrubbed.
6171  *
6172  * The memscrubber will also use this routine to clear any errors
6173  * caused by its scrubbing with CEEN disabled.
6174  *
6175  * flag == SCRUBBER_CEEN_CHECK
6176  *              called from memscrubber, just check/scrub, no reset
6177  *              paddr   physical addr. for start of scrub pages
6178  *              vaddr   virtual addr. for scrub area
6179  *              psz     page size of area to be scrubbed
6180  *
6181  * flag == TIMEOUT_CEEN_CHECK
6182  *              timeout function has triggered, reset timeout or CEEN
6183  *
6184  * Note: We must not migrate cpus during this function.  This can be
6185  * achieved by one of:
6186  *    - invoking as target of an x-call in which case we're at XCALL_PIL
6187  *      The flag value must be first xcall argument.
6188  *    - disabling kernel preemption.  This should be done for very short
6189  *      periods so is not suitable for SCRUBBER_CEEN_CHECK where we might
6190  *      scrub an extended area with cpu_check_block.  The call for
6191  *      TIMEOUT_CEEN_CHECK uses this so cpu_check_ce must be kept
6192  *      brief for this case.
6193  *    - binding to a cpu, eg with thread_affinity_set().  This is used
6194  *      in the SCRUBBER_CEEN_CHECK case, but is not practical for
6195  *      the TIMEOUT_CEEN_CHECK because both need cpu_lock.
6196  */
6197 void
6198 cpu_check_ce(int flag, uint64_t pa, caddr_t va, uint_t psz)
6199 {
6200         ch_cpu_errors_t cpu_error_regs;
6201         uint64_t        ec_err_enable;
6202         uint64_t        page_offset;
6203 
6204         /* Read AFSR */
6205         get_cpu_error_state(&cpu_error_regs);
6206 
6207         /*
6208          * If no CEEN errors have occurred during the timeout
6209          * interval, it is safe to re-enable CEEN and exit.
6210          */
6211         if (((cpu_error_regs.afsr & C_AFSR_CECC_ERRS) |
6212             (cpu_error_regs.afsr_ext & C_AFSR_EXT_CECC_ERRS)) == 0) {
6213                 if (flag == TIMEOUT_CEEN_CHECK &&
6214                     !((ec_err_enable = get_error_enable()) & EN_REG_CEEN))
6215                         set_error_enable(ec_err_enable | EN_REG_CEEN);
6216                 return;
6217         }
6218 
6219         /*
6220          * Ensure that CEEN was not reenabled (maybe by DR) before
6221          * we log/clear the error.
6222          */
6223         if ((ec_err_enable = get_error_enable()) & EN_REG_CEEN)
6224                 set_error_enable(ec_err_enable & ~EN_REG_CEEN);
6225 
6226         /*
6227          * log/clear the CE. If CE_CEEN_DEFER is passed, the
6228          * timeout will be rescheduled when the error is logged.
6229          */
6230         if (!((cpu_error_regs.afsr & cpu_ce_not_deferred) |
6231             (cpu_error_regs.afsr_ext & cpu_ce_not_deferred_ext)))
6232                 cpu_ce_detected(&cpu_error_regs,
6233                     CE_CEEN_DEFER | CE_CEEN_TIMEOUT);
6234         else
6235                 cpu_ce_detected(&cpu_error_regs, CE_CEEN_TIMEOUT);
6236 
6237         /*
6238          * If the memory scrubber runs while CEEN is
6239          * disabled, (or if CEEN is disabled during the
6240          * scrub as a result of a CE being triggered by
6241          * it), the range being scrubbed will not be
6242          * completely cleaned. If there are multiple CEs
6243          * in the range at most two of these will be dealt
6244          * with, (one by the trap handler and one by the
6245          * timeout). It is also possible that none are dealt
6246          * with, (CEEN disabled and another CE occurs before
6247          * the timeout triggers). So to ensure that the
6248          * memory is actually scrubbed, we have to access each
6249          * memory location in the range and then check whether
6250          * that access causes a CE.
6251          */
6252         if (flag == SCRUBBER_CEEN_CHECK && va) {
6253                 if ((cpu_error_regs.afar >= pa) &&
6254                     (cpu_error_regs.afar < (pa + psz))) {
6255                         /*
6256                          * Force a load from physical memory for each
6257                          * 64-byte block, then check AFSR to determine
6258                          * whether this access caused an error.
6259                          *
6260                          * This is a slow way to do a scrub, but as it will
6261                          * only be invoked when the memory scrubber actually
6262                          * triggered a CE, it should not happen too
6263                          * frequently.
6264                          *
6265                          * cut down what we need to check as the scrubber
6266                          * has verified up to AFAR, so get it's offset
6267                          * into the page and start there.
6268                          */
6269                         page_offset = (uint64_t)(cpu_error_regs.afar &
6270                             (psz - 1));
6271                         va = (caddr_t)(va + (P2ALIGN(page_offset, 64)));
6272                         psz -= (uint_t)(P2ALIGN(page_offset, 64));
6273                         cpu_check_block((caddr_t)(P2ALIGN((uint64_t)va, 64)),
6274                             psz);
6275                 }
6276         }
6277 
6278         /*
6279          * Reset error enable if this CE is not masked.
6280          */
6281         if ((flag == TIMEOUT_CEEN_CHECK) &&
6282             (cpu_error_regs.afsr & cpu_ce_not_deferred))
6283                 set_error_enable(ec_err_enable | EN_REG_CEEN);
6284 
6285 }
6286 
6287 /*
6288  * Attempt a cpu logout for an error that we did not trap for, such
6289  * as a CE noticed with CEEN off.  It is assumed that we are still running
6290  * on the cpu that took the error and that we cannot migrate.  Returns
6291  * 0 on success, otherwise nonzero.
6292  */
6293 static int
6294 cpu_ce_delayed_ec_logout(uint64_t afar)
6295 {
6296         ch_cpu_logout_t *clop;
6297 
6298         if (CPU_PRIVATE(CPU) == NULL)
6299                 return (0);
6300 
6301         clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout);
6302         if (atomic_cas_64(&clop->clo_data.chd_afar, LOGOUT_INVALID, afar) !=
6303             LOGOUT_INVALID)
6304                 return (0);
6305 
6306         cpu_delayed_logout(afar, clop);
6307         return (1);
6308 }
6309 
6310 /*
6311  * We got an error while CEEN was disabled. We
6312  * need to clean up after it and log whatever
6313  * information we have on the CE.
6314  */
6315 void
6316 cpu_ce_detected(ch_cpu_errors_t *cpu_error_regs, int flag)
6317 {
6318         ch_async_flt_t  ch_flt;
6319         struct async_flt *aflt;
6320         char            pr_reason[MAX_REASON_STRING];
6321 
6322         bzero(&ch_flt, sizeof (ch_async_flt_t));
6323         ch_flt.flt_trapped_ce = flag;
6324         aflt = (struct async_flt *)&ch_flt;
6325         aflt->flt_stat = cpu_error_regs->afsr & C_AFSR_MASK;
6326         ch_flt.afsr_ext = cpu_error_regs->afsr_ext;
6327         ch_flt.afsr_errs = (cpu_error_regs->afsr_ext & C_AFSR_EXT_ALL_ERRS) |
6328             (cpu_error_regs->afsr & C_AFSR_ALL_ERRS);
6329         aflt->flt_addr = cpu_error_regs->afar;
6330 #if defined(SERRANO)
6331         ch_flt.afar2 = cpu_error_regs->afar2;
6332 #endif  /* SERRANO */
6333         aflt->flt_pc = NULL;
6334         aflt->flt_priv = ((cpu_error_regs->afsr & C_AFSR_PRIV) != 0);
6335         aflt->flt_tl = 0;
6336         aflt->flt_panic = 0;
6337         cpu_log_and_clear_ce(&ch_flt);
6338 
6339         /*
6340          * check if we caused any errors during cleanup
6341          */
6342         if (clear_errors(&ch_flt)) {
6343                 pr_reason[0] = '\0';
6344                 (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs,
6345                     NULL);
6346         }
6347 }
6348 
6349 /*
6350  * Log/clear CEEN-controlled disrupting errors
6351  */
6352 static void
6353 cpu_log_and_clear_ce(ch_async_flt_t *ch_flt)
6354 {
6355         struct async_flt *aflt;
6356         uint64_t afsr, afsr_errs;
6357         ch_cpu_logout_t *clop;
6358         char            pr_reason[MAX_REASON_STRING];
6359         on_trap_data_t  *otp = curthread->t_ontrap;
6360 
6361         aflt = (struct async_flt *)ch_flt;
6362         afsr = aflt->flt_stat;
6363         afsr_errs = ch_flt->afsr_errs;
6364         aflt->flt_id = gethrtime_waitfree();
6365         aflt->flt_bus_id = getprocessorid();
6366         aflt->flt_inst = CPU->cpu_id;
6367         aflt->flt_prot = AFLT_PROT_NONE;
6368         aflt->flt_class = CPU_FAULT;
6369         aflt->flt_status = ECC_C_TRAP;
6370 
6371         pr_reason[0] = '\0';
6372         /*
6373          * Get the CPU log out info for Disrupting Trap.
6374          */
6375         if (CPU_PRIVATE(CPU) == NULL) {
6376                 clop = NULL;
6377                 ch_flt->flt_diag_data.chd_afar = LOGOUT_INVALID;
6378         } else {
6379                 clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout);
6380         }
6381 
6382         if (clop && ch_flt->flt_trapped_ce & CE_CEEN_TIMEOUT) {
6383                 ch_cpu_errors_t cpu_error_regs;
6384 
6385                 get_cpu_error_state(&cpu_error_regs);
6386                 (void) cpu_ce_delayed_ec_logout(cpu_error_regs.afar);
6387                 clop->clo_data.chd_afsr = cpu_error_regs.afsr;
6388                 clop->clo_data.chd_afar = cpu_error_regs.afar;
6389                 clop->clo_data.chd_afsr_ext = cpu_error_regs.afsr_ext;
6390                 clop->clo_sdw_data.chd_afsr = cpu_error_regs.shadow_afsr;
6391                 clop->clo_sdw_data.chd_afar = cpu_error_regs.shadow_afar;
6392                 clop->clo_sdw_data.chd_afsr_ext =
6393                     cpu_error_regs.shadow_afsr_ext;
6394 #if defined(SERRANO)
6395                 clop->clo_data.chd_afar2 = cpu_error_regs.afar2;
6396 #endif  /* SERRANO */
6397                 ch_flt->flt_data_incomplete = 1;
6398 
6399                 /*
6400                  * The logging/clear code expects AFSR/AFAR to be cleared.
6401                  * The trap handler does it for CEEN enabled errors
6402                  * so we need to do it here.
6403                  */
6404                 set_cpu_error_state(&cpu_error_regs);
6405         }
6406 
6407 #if defined(JALAPENO) || defined(SERRANO)
6408         /*
6409          * FRC: Can't scrub memory as we don't have AFAR for Jalapeno.
6410          * For Serrano, even thou we do have the AFAR, we still do the
6411          * scrub on the RCE side since that's where the error type can
6412          * be properly classified as intermittent, persistent, etc.
6413          *
6414          * CE/RCE:  If error is in memory and AFAR is valid, scrub the memory.
6415          * Must scrub memory before cpu_queue_events, as scrubbing memory sets
6416          * the flt_status bits.
6417          */
6418         if ((afsr & (C_AFSR_CE|C_AFSR_RCE)) &&
6419             (cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_CE)) ||
6420             cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_RCE)))) {
6421                 cpu_ce_scrub_mem_err(aflt, B_TRUE);
6422         }
6423 #else /* JALAPENO || SERRANO */
6424         /*
6425          * CE/EMC:  If error is in memory and AFAR is valid, scrub the memory.
6426          * Must scrub memory before cpu_queue_events, as scrubbing memory sets
6427          * the flt_status bits.
6428          */
6429         if (afsr & (C_AFSR_CE|C_AFSR_EMC)) {
6430                 if (cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_CE)) ||
6431                     cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_EMC))) {
6432                         cpu_ce_scrub_mem_err(aflt, B_TRUE);
6433                 }
6434         }
6435 
6436 #endif /* JALAPENO || SERRANO */
6437 
6438         /*
6439          * Update flt_prot if this error occurred under on_trap protection.
6440          */
6441         if (otp != NULL && (otp->ot_prot & OT_DATA_EC))
6442                 aflt->flt_prot = AFLT_PROT_EC;
6443 
6444         /*
6445          * Queue events on the async event queue, one event per error bit.
6446          */
6447         if (cpu_queue_events(ch_flt, pr_reason, afsr_errs, clop) == 0 ||
6448             (afsr_errs & (C_AFSR_CECC_ERRS | C_AFSR_EXT_CECC_ERRS)) == 0) {
6449                 ch_flt->flt_type = CPU_INV_AFSR;
6450                 cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR,
6451                     (void *)ch_flt, sizeof (ch_async_flt_t), ue_queue,
6452                     aflt->flt_panic);
6453         }
6454 
6455         /*
6456          * Zero out + invalidate CPU logout.
6457          */
6458         if (clop) {
6459                 bzero(clop, sizeof (ch_cpu_logout_t));
6460                 clop->clo_data.chd_afar = LOGOUT_INVALID;
6461         }
6462 
6463         /*
6464          * If either a CPC, WDC or EDC error has occurred while CEEN
6465          * was disabled, we need to flush either the entire
6466          * E$ or an E$ line.
6467          */
6468 #if defined(JALAPENO) || defined(SERRANO)
6469         if (afsr & (C_AFSR_EDC | C_AFSR_CPC | C_AFSR_CPU | C_AFSR_WDC))
6470 #else   /* JALAPENO || SERRANO */
6471         if (afsr_errs & (C_AFSR_EDC | C_AFSR_CPC | C_AFSR_WDC | C_AFSR_L3_EDC |
6472             C_AFSR_L3_CPC | C_AFSR_L3_WDC))
6473 #endif  /* JALAPENO || SERRANO */
6474                 cpu_error_ecache_flush(ch_flt);
6475 
6476 }
6477 
6478 /*
6479  * depending on the error type, we determine whether we
6480  * need to flush the entire ecache or just a line.
6481  */
6482 static int
6483 cpu_error_ecache_flush_required(ch_async_flt_t *ch_flt)
6484 {
6485         struct async_flt *aflt;
6486         uint64_t        afsr;
6487         uint64_t        afsr_errs = ch_flt->afsr_errs;
6488 
6489         aflt = (struct async_flt *)ch_flt;
6490         afsr = aflt->flt_stat;
6491 
6492         /*
6493          * If we got multiple errors, no point in trying
6494          * the individual cases, just flush the whole cache
6495          */
6496         if (afsr & C_AFSR_ME) {
6497                 return (ECACHE_FLUSH_ALL);
6498         }
6499 
6500         /*
6501          * If either a CPC, WDC or EDC error has occurred while CEEN
6502          * was disabled, we need to flush entire E$. We can't just
6503          * flush the cache line affected as the ME bit
6504          * is not set when multiple correctable errors of the same
6505          * type occur, so we might have multiple CPC or EDC errors,
6506          * with only the first recorded.
6507          */
6508 #if defined(JALAPENO) || defined(SERRANO)
6509         if (afsr & (C_AFSR_CPC | C_AFSR_CPU | C_AFSR_EDC | C_AFSR_WDC)) {
6510 #else   /* JALAPENO || SERRANO */
6511         if (afsr_errs & (C_AFSR_CPC | C_AFSR_EDC | C_AFSR_WDC | C_AFSR_L3_CPC |
6512             C_AFSR_L3_EDC | C_AFSR_L3_WDC)) {
6513 #endif  /* JALAPENO || SERRANO */
6514                 return (ECACHE_FLUSH_ALL);
6515         }
6516 
6517 #if defined(JALAPENO) || defined(SERRANO)
6518         /*
6519          * If only UE or RUE is set, flush the Ecache line, otherwise
6520          * flush the entire Ecache.
6521          */
6522         if (afsr & (C_AFSR_UE|C_AFSR_RUE)) {
6523                 if ((afsr & C_AFSR_ALL_ERRS) == C_AFSR_UE ||
6524                     (afsr & C_AFSR_ALL_ERRS) == C_AFSR_RUE) {
6525                         return (ECACHE_FLUSH_LINE);
6526                 } else {
6527                         return (ECACHE_FLUSH_ALL);
6528                 }
6529         }
6530 #else /* JALAPENO || SERRANO */
6531         /*
6532          * If UE only is set, flush the Ecache line, otherwise
6533          * flush the entire Ecache.
6534          */
6535         if (afsr_errs & C_AFSR_UE) {
6536                 if ((afsr_errs & (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) ==
6537                     C_AFSR_UE) {
6538                         return (ECACHE_FLUSH_LINE);
6539                 } else {
6540                         return (ECACHE_FLUSH_ALL);
6541                 }
6542         }
6543 #endif /* JALAPENO || SERRANO */
6544 
6545         /*
6546          * EDU: If EDU only is set, flush the ecache line, otherwise
6547          * flush the entire Ecache.
6548          */
6549         if (afsr_errs & (C_AFSR_EDU | C_AFSR_L3_EDU)) {
6550                 if (((afsr_errs & ~C_AFSR_EDU) == 0) ||
6551                     ((afsr_errs & ~C_AFSR_L3_EDU) == 0)) {
6552                         return (ECACHE_FLUSH_LINE);
6553                 } else {
6554                         return (ECACHE_FLUSH_ALL);
6555                 }
6556         }
6557 
6558         /*
6559          * BERR: If BERR only is set, flush the Ecache line, otherwise
6560          * flush the entire Ecache.
6561          */
6562         if (afsr_errs & C_AFSR_BERR) {
6563                 if ((afsr_errs & ~C_AFSR_BERR) == 0) {
6564                         return (ECACHE_FLUSH_LINE);
6565                 } else {
6566                         return (ECACHE_FLUSH_ALL);
6567                 }
6568         }
6569 
6570         return (0);
6571 }
6572 
6573 void
6574 cpu_error_ecache_flush(ch_async_flt_t *ch_flt)
6575 {
6576         int     ecache_flush_flag =
6577             cpu_error_ecache_flush_required(ch_flt);
6578 
6579         /*
6580          * Flush Ecache line or entire Ecache based on above checks.
6581          */
6582         if (ecache_flush_flag == ECACHE_FLUSH_ALL)
6583                 cpu_flush_ecache();
6584         else if (ecache_flush_flag == ECACHE_FLUSH_LINE) {
6585                 cpu_flush_ecache_line(ch_flt);
6586         }
6587 
6588 }
6589 
6590 /*
6591  * Extract the PA portion from the E$ tag.
6592  */
6593 uint64_t
6594 cpu_ectag_to_pa(int setsize, uint64_t tag)
6595 {
6596         if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation))
6597                 return (JG_ECTAG_TO_PA(setsize, tag));
6598         else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation))
6599                 return (PN_L3TAG_TO_PA(tag));
6600         else
6601                 return (CH_ECTAG_TO_PA(setsize, tag));
6602 }
6603 
6604 /*
6605  * Convert the E$ tag PA into an E$ subblock index.
6606  */
6607 int
6608 cpu_ectag_pa_to_subblk(int cachesize, uint64_t subaddr)
6609 {
6610         if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation))
6611                 return (JG_ECTAG_PA_TO_SUBBLK(cachesize, subaddr));
6612         else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation))
6613                 /* Panther has only one subblock per line */
6614                 return (0);
6615         else
6616                 return (CH_ECTAG_PA_TO_SUBBLK(cachesize, subaddr));
6617 }
6618 
6619 /*
6620  * All subblocks in an E$ line must be invalid for
6621  * the line to be invalid.
6622  */
6623 int
6624 cpu_ectag_line_invalid(int cachesize, uint64_t tag)
6625 {
6626         if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation))
6627                 return (JG_ECTAG_LINE_INVALID(cachesize, tag));
6628         else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation))
6629                 return (PN_L3_LINE_INVALID(tag));
6630         else
6631                 return (CH_ECTAG_LINE_INVALID(cachesize, tag));
6632 }
6633 
6634 /*
6635  * Extract state bits for a subblock given the tag.  Note that for Panther
6636  * this works on both l2 and l3 tags.
6637  */
6638 int
6639 cpu_ectag_pa_to_subblk_state(int cachesize, uint64_t subaddr, uint64_t tag)
6640 {
6641         if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation))
6642                 return (JG_ECTAG_PA_TO_SUBBLK_STATE(cachesize, subaddr, tag));
6643         else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation))
6644                 return (tag & CH_ECSTATE_MASK);
6645         else
6646                 return (CH_ECTAG_PA_TO_SUBBLK_STATE(cachesize, subaddr, tag));
6647 }
6648 
6649 /*
6650  * Cpu specific initialization.
6651  */
6652 void
6653 cpu_mp_init(void)
6654 {
6655 #ifdef  CHEETAHPLUS_ERRATUM_25
6656         if (cheetah_sendmondo_recover) {
6657                 cheetah_nudge_init();
6658         }
6659 #endif
6660 }
6661 
6662 void
6663 cpu_ereport_post(struct async_flt *aflt)
6664 {
6665         char *cpu_type, buf[FM_MAX_CLASS];
6666         nv_alloc_t *nva = NULL;
6667         nvlist_t *ereport, *detector, *resource;
6668         errorq_elem_t *eqep;
6669         ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt;
6670         char unum[UNUM_NAMLEN];
6671         int synd_code;
6672         uint8_t msg_type;
6673         plat_ecc_ch_async_flt_t plat_ecc_ch_flt;
6674 
6675         if (aflt->flt_panic || panicstr) {
6676                 eqep = errorq_reserve(ereport_errorq);
6677                 if (eqep == NULL)
6678                         return;
6679                 ereport = errorq_elem_nvl(ereport_errorq, eqep);
6680                 nva = errorq_elem_nva(ereport_errorq, eqep);
6681         } else {
6682                 ereport = fm_nvlist_create(nva);
6683         }
6684 
6685         /*
6686          * Create the scheme "cpu" FMRI.
6687          */
6688         detector = fm_nvlist_create(nva);
6689         resource = fm_nvlist_create(nva);
6690         switch (cpunodes[aflt->flt_inst].implementation) {
6691         case CHEETAH_IMPL:
6692                 cpu_type = FM_EREPORT_CPU_USIII;
6693                 break;
6694         case CHEETAH_PLUS_IMPL:
6695                 cpu_type = FM_EREPORT_CPU_USIIIplus;
6696                 break;
6697         case JALAPENO_IMPL:
6698                 cpu_type = FM_EREPORT_CPU_USIIIi;
6699                 break;
6700         case SERRANO_IMPL:
6701                 cpu_type = FM_EREPORT_CPU_USIIIiplus;
6702                 break;
6703         case JAGUAR_IMPL:
6704                 cpu_type = FM_EREPORT_CPU_USIV;
6705                 break;
6706         case PANTHER_IMPL:
6707                 cpu_type = FM_EREPORT_CPU_USIVplus;
6708                 break;
6709         default:
6710                 cpu_type = FM_EREPORT_CPU_UNSUPPORTED;
6711                 break;
6712         }
6713 
6714         cpu_fmri_cpu_set(detector, aflt->flt_inst);
6715 
6716         /*
6717          * Encode all the common data into the ereport.
6718          */
6719         (void) snprintf(buf, FM_MAX_CLASS, "%s.%s.%s",
6720             FM_ERROR_CPU, cpu_type, aflt->flt_erpt_class);
6721 
6722         fm_ereport_set(ereport, FM_EREPORT_VERSION, buf,
6723             fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst, FM_ENA_FMT1),
6724             detector, NULL);
6725 
6726         /*
6727          * Encode the error specific data that was saved in
6728          * the async_flt structure into the ereport.
6729          */
6730         cpu_payload_add_aflt(aflt, ereport, resource,
6731             &plat_ecc_ch_flt.ecaf_afar_status,
6732             &plat_ecc_ch_flt.ecaf_synd_status);
6733 
6734         if (aflt->flt_panic || panicstr) {
6735                 errorq_commit(ereport_errorq, eqep, ERRORQ_SYNC);
6736         } else {
6737                 (void) fm_ereport_post(ereport, EVCH_TRYHARD);
6738                 fm_nvlist_destroy(ereport, FM_NVA_FREE);
6739                 fm_nvlist_destroy(detector, FM_NVA_FREE);
6740                 fm_nvlist_destroy(resource, FM_NVA_FREE);
6741         }
6742         /*
6743          * Send the enhanced error information (plat_ecc_error2_data_t)
6744          * to the SC olny if it can process it.
6745          */
6746 
6747         if (&plat_ecc_capability_sc_get &&
6748             plat_ecc_capability_sc_get(PLAT_ECC_ERROR2_MESSAGE)) {
6749                 msg_type = cpu_flt_bit_to_plat_error(aflt);
6750                 if (msg_type != PLAT_ECC_ERROR2_NONE) {
6751                         /*
6752                          * If afar status is not invalid do a unum lookup.
6753                          */
6754                         if (plat_ecc_ch_flt.ecaf_afar_status !=
6755                             AFLT_STAT_INVALID) {
6756                                 synd_code = synd_to_synd_code(
6757                                     plat_ecc_ch_flt.ecaf_synd_status,
6758                                     aflt->flt_synd, ch_flt->flt_bit);
6759                                 (void) cpu_get_mem_unum_synd(synd_code,
6760                                     aflt, unum);
6761                         } else {
6762                                 unum[0] = '\0';
6763                         }
6764                         plat_ecc_ch_flt.ecaf_sdw_afar = ch_flt->flt_sdw_afar;
6765                         plat_ecc_ch_flt.ecaf_sdw_afsr = ch_flt->flt_sdw_afsr;
6766                         plat_ecc_ch_flt.ecaf_afsr_ext = ch_flt->afsr_ext;
6767                         plat_ecc_ch_flt.ecaf_sdw_afsr_ext =
6768                             ch_flt->flt_sdw_afsr_ext;
6769 
6770                         if (&plat_log_fruid_error2)
6771                                 plat_log_fruid_error2(msg_type, unum, aflt,
6772                                     &plat_ecc_ch_flt);
6773                 }
6774         }
6775 }
6776 
6777 void
6778 cpu_run_bus_error_handlers(struct async_flt *aflt, int expected)
6779 {
6780         int status;
6781         ddi_fm_error_t de;
6782 
6783         bzero(&de, sizeof (ddi_fm_error_t));
6784 
6785         de.fme_version = DDI_FME_VERSION;
6786         de.fme_ena = fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst,
6787             FM_ENA_FMT1);
6788         de.fme_flag = expected;
6789         de.fme_bus_specific = (void *)aflt->flt_addr;
6790         status = ndi_fm_handler_dispatch(ddi_root_node(), NULL, &de);
6791         if ((aflt->flt_prot == AFLT_PROT_NONE) && (status == DDI_FM_FATAL))
6792                 aflt->flt_panic = 1;
6793 }
6794 
6795 void
6796 cpu_errorq_dispatch(char *error_class, void *payload, size_t payload_sz,
6797     errorq_t *eqp, uint_t flag)
6798 {
6799         struct async_flt *aflt = (struct async_flt *)payload;
6800 
6801         aflt->flt_erpt_class = error_class;
6802         errorq_dispatch(eqp, payload, payload_sz, flag);
6803 }
6804 
6805 /*
6806  * This routine may be called by the IO module, but does not do
6807  * anything in this cpu module. The SERD algorithm is handled by
6808  * cpumem-diagnosis engine instead.
6809  */
6810 /*ARGSUSED*/
6811 void
6812 cpu_ce_count_unum(struct async_flt *ecc, int len, char *unum)
6813 {}
6814 
6815 void
6816 adjust_hw_copy_limits(int ecache_size)
6817 {
6818         /*
6819          * Set hw copy limits.
6820          *
6821          * /etc/system will be parsed later and can override one or more
6822          * of these settings.
6823          *
6824          * At this time, ecache size seems only mildly relevant.
6825          * We seem to run into issues with the d-cache and stalls
6826          * we see on misses.
6827          *
6828          * Cycle measurement indicates that 2 byte aligned copies fare
6829          * little better than doing things with VIS at around 512 bytes.
6830          * 4 byte aligned shows promise until around 1024 bytes. 8 Byte
6831          * aligned is faster whenever the source and destination data
6832          * in cache and the total size is less than 2 Kbytes.  The 2K
6833          * limit seems to be driven by the 2K write cache.
6834          * When more than 2K of copies are done in non-VIS mode, stores
6835          * backup in the write cache.  In VIS mode, the write cache is
6836          * bypassed, allowing faster cache-line writes aligned on cache
6837          * boundaries.
6838          *
6839          * In addition, in non-VIS mode, there is no prefetching, so
6840          * for larger copies, the advantage of prefetching to avoid even
6841          * occasional cache misses is enough to justify using the VIS code.
6842          *
6843          * During testing, it was discovered that netbench ran 3% slower
6844          * when hw_copy_limit_8 was 2K or larger.  Apparently for server
6845          * applications, data is only used once (copied to the output
6846          * buffer, then copied by the network device off the system).  Using
6847          * the VIS copy saves more L2 cache state.  Network copies are
6848          * around 1.3K to 1.5K in size for historical reasons.
6849          *
6850          * Therefore, a limit of 1K bytes will be used for the 8 byte
6851          * aligned copy even for large caches and 8 MB ecache.  The
6852          * infrastructure to allow different limits for different sized
6853          * caches is kept to allow further tuning in later releases.
6854          */
6855 
6856         if (min_ecache_size == 0 && use_hw_bcopy) {
6857                 /*
6858                  * First time through - should be before /etc/system
6859                  * is read.
6860                  * Could skip the checks for zero but this lets us
6861                  * preserve any debugger rewrites.
6862                  */
6863                 if (hw_copy_limit_1 == 0) {
6864                         hw_copy_limit_1 = VIS_COPY_THRESHOLD;
6865                         priv_hcl_1 = hw_copy_limit_1;
6866                 }
6867                 if (hw_copy_limit_2 == 0) {
6868                         hw_copy_limit_2 = 2 * VIS_COPY_THRESHOLD;
6869                         priv_hcl_2 = hw_copy_limit_2;
6870                 }
6871                 if (hw_copy_limit_4 == 0) {
6872                         hw_copy_limit_4 = 4 * VIS_COPY_THRESHOLD;
6873                         priv_hcl_4 = hw_copy_limit_4;
6874                 }
6875                 if (hw_copy_limit_8 == 0) {
6876                         hw_copy_limit_8 = 4 * VIS_COPY_THRESHOLD;
6877                         priv_hcl_8 = hw_copy_limit_8;
6878                 }
6879                 min_ecache_size = ecache_size;
6880         } else {
6881                 /*
6882                  * MP initialization. Called *after* /etc/system has
6883                  * been parsed. One CPU has already been initialized.
6884                  * Need to cater for /etc/system having scragged one
6885                  * of our values.
6886                  */
6887                 if (ecache_size == min_ecache_size) {
6888                         /*
6889                          * Same size ecache. We do nothing unless we
6890                          * have a pessimistic ecache setting. In that
6891                          * case we become more optimistic (if the cache is
6892                          * large enough).
6893                          */
6894                         if (hw_copy_limit_8 == 4 * VIS_COPY_THRESHOLD) {
6895                                 /*
6896                                  * Need to adjust hw_copy_limit* from our
6897                                  * pessimistic uniprocessor value to a more
6898                                  * optimistic UP value *iff* it hasn't been
6899                                  * reset.
6900                                  */
6901                                 if ((ecache_size > 1048576) &&
6902                                     (priv_hcl_8 == hw_copy_limit_8)) {
6903                                         if (ecache_size <= 2097152)
6904                                                 hw_copy_limit_8 = 4 *
6905                                                     VIS_COPY_THRESHOLD;
6906                                         else if (ecache_size <= 4194304)
6907                                                 hw_copy_limit_8 = 4 *
6908                                                     VIS_COPY_THRESHOLD;
6909                                         else
6910                                                 hw_copy_limit_8 = 4 *
6911                                                     VIS_COPY_THRESHOLD;
6912                                         priv_hcl_8 = hw_copy_limit_8;
6913                                 }
6914                         }
6915                 } else if (ecache_size < min_ecache_size) {
6916                         /*
6917                          * A different ecache size. Can this even happen?
6918                          */
6919                         if (priv_hcl_8 == hw_copy_limit_8) {
6920                                 /*
6921                                  * The previous value that we set
6922                                  * is unchanged (i.e., it hasn't been
6923                                  * scragged by /etc/system). Rewrite it.
6924                                  */
6925                                 if (ecache_size <= 1048576)
6926                                         hw_copy_limit_8 = 8 *
6927                                             VIS_COPY_THRESHOLD;
6928                                 else if (ecache_size <= 2097152)
6929                                         hw_copy_limit_8 = 8 *
6930                                             VIS_COPY_THRESHOLD;
6931                                 else if (ecache_size <= 4194304)
6932                                         hw_copy_limit_8 = 8 *
6933                                             VIS_COPY_THRESHOLD;
6934                                 else
6935                                         hw_copy_limit_8 = 10 *
6936                                             VIS_COPY_THRESHOLD;
6937                                 priv_hcl_8 = hw_copy_limit_8;
6938                                 min_ecache_size = ecache_size;
6939                         }
6940                 }
6941         }
6942 }
6943 
6944 /*
6945  * Called from illegal instruction trap handler to see if we can attribute
6946  * the trap to a fpras check.
6947  */
6948 int
6949 fpras_chktrap(struct regs *rp)
6950 {
6951         int op;
6952         struct fpras_chkfngrp *cgp;
6953         uintptr_t tpc = (uintptr_t)rp->r_pc;
6954 
6955         if (fpras_chkfngrps == NULL)
6956                 return (0);
6957 
6958         cgp = &fpras_chkfngrps[CPU->cpu_id];
6959         for (op = 0; op < FPRAS_NCOPYOPS; ++op) {
6960                 if (tpc >= (uintptr_t)&cgp->fpras_fn[op].fpras_blk0 &&
6961                     tpc < (uintptr_t)&cgp->fpras_fn[op].fpras_chkresult)
6962                         break;
6963         }
6964         if (op == FPRAS_NCOPYOPS)
6965                 return (0);
6966 
6967         /*
6968          * This is an fpRAS failure caught through an illegal
6969          * instruction - trampoline.
6970          */
6971         rp->r_pc = (uintptr_t)&cgp->fpras_fn[op].fpras_trampoline;
6972         rp->r_npc = rp->r_pc + 4;
6973         return (1);
6974 }
6975 
6976 /*
6977  * fpras_failure is called when a fpras check detects a bad calculation
6978  * result or an illegal instruction trap is attributed to an fpras
6979  * check.  In all cases we are still bound to CPU.
6980  */
6981 int
6982 fpras_failure(int op, int how)
6983 {
6984         int use_hw_bcopy_orig, use_hw_bzero_orig;
6985         uint_t hcl1_orig, hcl2_orig, hcl4_orig, hcl8_orig;
6986         ch_async_flt_t ch_flt;
6987         struct async_flt *aflt = (struct async_flt *)&ch_flt;
6988         struct fpras_chkfn *sfp, *cfp;
6989         uint32_t *sip, *cip;
6990         int i;
6991 
6992         /*
6993          * We're running on a sick CPU.  Avoid further FPU use at least for
6994          * the time in which we dispatch an ereport and (if applicable) panic.
6995          */
6996         use_hw_bcopy_orig = use_hw_bcopy;
6997         use_hw_bzero_orig = use_hw_bzero;
6998         hcl1_orig = hw_copy_limit_1;
6999         hcl2_orig = hw_copy_limit_2;
7000         hcl4_orig = hw_copy_limit_4;
7001         hcl8_orig = hw_copy_limit_8;
7002         use_hw_bcopy = use_hw_bzero = 0;
7003         hw_copy_limit_1 = hw_copy_limit_2 = hw_copy_limit_4 =
7004             hw_copy_limit_8 = 0;
7005 
7006         bzero(&ch_flt, sizeof (ch_async_flt_t));
7007         aflt->flt_id = gethrtime_waitfree();
7008         aflt->flt_class = CPU_FAULT;
7009         aflt->flt_inst = CPU->cpu_id;
7010         aflt->flt_status = (how << 8) | op;
7011         aflt->flt_payload = FM_EREPORT_PAYLOAD_FPU_HWCOPY;
7012         ch_flt.flt_type = CPU_FPUERR;
7013 
7014         /*
7015          * We must panic if the copy operation had no lofault protection -
7016          * ie, don't panic for copyin, copyout, kcopy and bcopy called
7017          * under on_fault and do panic for unprotected bcopy and hwblkpagecopy.
7018          */
7019         aflt->flt_panic = (curthread->t_lofault == NULL);
7020 
7021         /*
7022          * XOR the source instruction block with the copied instruction
7023          * block - this will show us which bit(s) are corrupted.
7024          */
7025         sfp = (struct fpras_chkfn *)fpras_chkfn_type1;
7026         cfp = &fpras_chkfngrps[CPU->cpu_id].fpras_fn[op];
7027         if (op == FPRAS_BCOPY || op == FPRAS_COPYOUT) {
7028                 sip = &sfp->fpras_blk0[0];
7029                 cip = &cfp->fpras_blk0[0];
7030         } else {
7031                 sip = &sfp->fpras_blk1[0];
7032                 cip = &cfp->fpras_blk1[0];
7033         }
7034         for (i = 0; i < 16; ++i, ++sip, ++cip)
7035                 ch_flt.flt_fpdata[i] = *sip ^ *cip;
7036 
7037         cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_FPU_HWCOPY, (void *)&ch_flt,
7038             sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic);
7039 
7040         if (aflt->flt_panic)
7041                 fm_panic("FPU failure on CPU %d", CPU->cpu_id);
7042 
7043         /*
7044          * We get here for copyin/copyout and kcopy or bcopy where the
7045          * caller has used on_fault.  We will flag the error so that
7046          * the process may be killed  The trap_async_hwerr mechanism will
7047          * take appropriate further action (such as a reboot, contract
7048          * notification etc).  Since we may be continuing we will
7049          * restore the global hardware copy acceleration switches.
7050          *
7051          * When we return from this function to the copy function we want to
7052          * avoid potentially bad data being used, ie we want the affected
7053          * copy function to return an error.  The caller should therefore
7054          * invoke its lofault handler (which always exists for these functions)
7055          * which will return the appropriate error.
7056          */
7057         ttolwp(curthread)->lwp_pcb.pcb_flags |= ASYNC_HWERR;
7058         aston(curthread);
7059 
7060         use_hw_bcopy = use_hw_bcopy_orig;
7061         use_hw_bzero = use_hw_bzero_orig;
7062         hw_copy_limit_1 = hcl1_orig;
7063         hw_copy_limit_2 = hcl2_orig;
7064         hw_copy_limit_4 = hcl4_orig;
7065         hw_copy_limit_8 = hcl8_orig;
7066 
7067         return (1);
7068 }
7069 
7070 #define VIS_BLOCKSIZE           64
7071 
7072 int
7073 dtrace_blksuword32_err(uintptr_t addr, uint32_t *data)
7074 {
7075         int ret, watched;
7076 
7077         watched = watch_disable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE);
7078         ret = dtrace_blksuword32(addr, data, 0);
7079         if (watched)
7080                 watch_enable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE);
7081 
7082         return (ret);
7083 }
7084 
7085 /*
7086  * Called when a cpu enters the CPU_FAULTED state (by the cpu placing the
7087  * faulted cpu into that state).  Cross-trap to the faulted cpu to clear
7088  * CEEN from the EER to disable traps for further disrupting error types
7089  * on that cpu.  We could cross-call instead, but that has a larger
7090  * instruction and data footprint than cross-trapping, and the cpu is known
7091  * to be faulted.
7092  */
7093 
7094 void
7095 cpu_faulted_enter(struct cpu *cp)
7096 {
7097         xt_one(cp->cpu_id, set_error_enable_tl1, EN_REG_CEEN, EER_SET_CLRBITS);
7098 }
7099 
7100 /*
7101  * Called when a cpu leaves the CPU_FAULTED state to return to one of
7102  * offline, spare, or online (by the cpu requesting this state change).
7103  * First we cross-call to clear the AFSR (and AFSR_EXT on Panther) of
7104  * disrupting error bits that have accumulated without trapping, then
7105  * we cross-trap to re-enable CEEN controlled traps.
7106  */
7107 void
7108 cpu_faulted_exit(struct cpu *cp)
7109 {
7110         ch_cpu_errors_t cpu_error_regs;
7111 
7112         cpu_error_regs.afsr = C_AFSR_CECC_ERRS;
7113         if (IS_PANTHER(cpunodes[cp->cpu_id].implementation))
7114                 cpu_error_regs.afsr_ext &= C_AFSR_EXT_CECC_ERRS;
7115         xc_one(cp->cpu_id, (xcfunc_t *)set_cpu_error_state,
7116             (uint64_t)&cpu_error_regs, 0);
7117 
7118         xt_one(cp->cpu_id, set_error_enable_tl1, EN_REG_CEEN, EER_SET_SETBITS);
7119 }
7120 
7121 /*
7122  * Return 1 if the errors in ch_flt's AFSR are secondary errors caused by
7123  * the errors in the original AFSR, 0 otherwise.
7124  *
7125  * For all procs if the initial error was a BERR or TO, then it is possible
7126  * that we may have caused a secondary BERR or TO in the process of logging the
7127  * inital error via cpu_run_bus_error_handlers().  If this is the case then
7128  * if the request was protected then a panic is still not necessary, if not
7129  * protected then aft_panic is already set - so either way there's no need
7130  * to set aft_panic for the secondary error.
7131  *
7132  * For Cheetah and Jalapeno, if the original error was a UE which occurred on
7133  * a store merge, then the error handling code will call cpu_deferred_error().
7134  * When clear_errors() is called, it will determine that secondary errors have
7135  * occurred - in particular, the store merge also caused a EDU and WDU that
7136  * weren't discovered until this point.
7137  *
7138  * We do three checks to verify that we are in this case.  If we pass all three
7139  * checks, we return 1 to indicate that we should not panic.  If any unexpected
7140  * errors occur, we return 0.
7141  *
7142  * For Cheetah+ and derivative procs, the store merge causes a DUE, which is
7143  * handled in cpu_disrupting_errors().  Since this function is not even called
7144  * in the case we are interested in, we just return 0 for these processors.
7145  */
7146 /*ARGSUSED*/
7147 static int
7148 cpu_check_secondary_errors(ch_async_flt_t *ch_flt, uint64_t t_afsr_errs,
7149     uint64_t t_afar)
7150 {
7151 #if defined(CHEETAH_PLUS)
7152 #else   /* CHEETAH_PLUS */
7153         struct async_flt *aflt = (struct async_flt *)ch_flt;
7154 #endif  /* CHEETAH_PLUS */
7155 
7156         /*
7157          * Was the original error a BERR or TO and only a BERR or TO
7158          * (multiple errors are also OK)
7159          */
7160         if ((t_afsr_errs & ~(C_AFSR_BERR | C_AFSR_TO | C_AFSR_ME)) == 0) {
7161                 /*
7162                  * Is the new error a BERR or TO and only a BERR or TO
7163                  * (multiple errors are also OK)
7164                  */
7165                 if ((ch_flt->afsr_errs &
7166                     ~(C_AFSR_BERR | C_AFSR_TO | C_AFSR_ME)) == 0)
7167                         return (1);
7168         }
7169 
7170 #if defined(CHEETAH_PLUS)
7171         return (0);
7172 #else   /* CHEETAH_PLUS */
7173         /*
7174          * Now look for secondary effects of a UE on cheetah/jalapeno
7175          *
7176          * Check the original error was a UE, and only a UE.  Note that
7177          * the ME bit will cause us to fail this check.
7178          */
7179         if (t_afsr_errs != C_AFSR_UE)
7180                 return (0);
7181 
7182         /*
7183          * Check the secondary errors were exclusively an EDU and/or WDU.
7184          */
7185         if ((ch_flt->afsr_errs & ~(C_AFSR_EDU|C_AFSR_WDU)) != 0)
7186                 return (0);
7187 
7188         /*
7189          * Check the AFAR of the original error and secondary errors
7190          * match to the 64-byte boundary
7191          */
7192         if (P2ALIGN(aflt->flt_addr, 64) != P2ALIGN(t_afar, 64))
7193                 return (0);
7194 
7195         /*
7196          * We've passed all the checks, so it's a secondary error!
7197          */
7198         return (1);
7199 #endif  /* CHEETAH_PLUS */
7200 }
7201 
7202 /*
7203  * Translate the flt_bit or flt_type into an error type.  First, flt_bit
7204  * is checked for any valid errors.  If found, the error type is
7205  * returned. If not found, the flt_type is checked for L1$ parity errors.
7206  */
7207 /*ARGSUSED*/
7208 static uint8_t
7209 cpu_flt_bit_to_plat_error(struct async_flt *aflt)
7210 {
7211 #if defined(JALAPENO)
7212         /*
7213          * Currently, logging errors to the SC is not supported on Jalapeno
7214          */
7215         return (PLAT_ECC_ERROR2_NONE);
7216 #else
7217         ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt;
7218 
7219         switch (ch_flt->flt_bit) {
7220         case C_AFSR_CE:
7221                 return (PLAT_ECC_ERROR2_CE);
7222         case C_AFSR_UCC:
7223         case C_AFSR_EDC:
7224         case C_AFSR_WDC:
7225         case C_AFSR_CPC:
7226                 return (PLAT_ECC_ERROR2_L2_CE);
7227         case C_AFSR_EMC:
7228                 return (PLAT_ECC_ERROR2_EMC);
7229         case C_AFSR_IVC:
7230                 return (PLAT_ECC_ERROR2_IVC);
7231         case C_AFSR_UE:
7232                 return (PLAT_ECC_ERROR2_UE);
7233         case C_AFSR_UCU:
7234         case C_AFSR_EDU:
7235         case C_AFSR_WDU:
7236         case C_AFSR_CPU:
7237                 return (PLAT_ECC_ERROR2_L2_UE);
7238         case C_AFSR_IVU:
7239                 return (PLAT_ECC_ERROR2_IVU);
7240         case C_AFSR_TO:
7241                 return (PLAT_ECC_ERROR2_TO);
7242         case C_AFSR_BERR:
7243                 return (PLAT_ECC_ERROR2_BERR);
7244 #if defined(CHEETAH_PLUS)
7245         case C_AFSR_L3_EDC:
7246         case C_AFSR_L3_UCC:
7247         case C_AFSR_L3_CPC:
7248         case C_AFSR_L3_WDC:
7249                 return (PLAT_ECC_ERROR2_L3_CE);
7250         case C_AFSR_IMC:
7251                 return (PLAT_ECC_ERROR2_IMC);
7252         case C_AFSR_TSCE:
7253                 return (PLAT_ECC_ERROR2_L2_TSCE);
7254         case C_AFSR_THCE:
7255                 return (PLAT_ECC_ERROR2_L2_THCE);
7256         case C_AFSR_L3_MECC:
7257                 return (PLAT_ECC_ERROR2_L3_MECC);
7258         case C_AFSR_L3_THCE:
7259                 return (PLAT_ECC_ERROR2_L3_THCE);
7260         case C_AFSR_L3_CPU:
7261         case C_AFSR_L3_EDU:
7262         case C_AFSR_L3_UCU:
7263         case C_AFSR_L3_WDU:
7264                 return (PLAT_ECC_ERROR2_L3_UE);
7265         case C_AFSR_DUE:
7266                 return (PLAT_ECC_ERROR2_DUE);
7267         case C_AFSR_DTO:
7268                 return (PLAT_ECC_ERROR2_DTO);
7269         case C_AFSR_DBERR:
7270                 return (PLAT_ECC_ERROR2_DBERR);
7271 #endif  /* CHEETAH_PLUS */
7272         default:
7273                 switch (ch_flt->flt_type) {
7274 #if defined(CPU_IMP_L1_CACHE_PARITY)
7275                 case CPU_IC_PARITY:
7276                         return (PLAT_ECC_ERROR2_IPE);
7277                 case CPU_DC_PARITY:
7278                         if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) {
7279                                 if (ch_flt->parity_data.dpe.cpl_cache ==
7280                                     CPU_PC_PARITY) {
7281                                         return (PLAT_ECC_ERROR2_PCACHE);
7282                                 }
7283                         }
7284                         return (PLAT_ECC_ERROR2_DPE);
7285 #endif /* CPU_IMP_L1_CACHE_PARITY */
7286                 case CPU_ITLB_PARITY:
7287                         return (PLAT_ECC_ERROR2_ITLB);
7288                 case CPU_DTLB_PARITY:
7289                         return (PLAT_ECC_ERROR2_DTLB);
7290                 default:
7291                         return (PLAT_ECC_ERROR2_NONE);
7292                 }
7293         }
7294 #endif  /* JALAPENO */
7295 }