Print this page
patch tsoome-feedback
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/cmd/fm/modules/sun4v/generic-mem/gmem_memerr.c
+++ new/usr/src/cmd/fm/modules/sun4v/generic-mem/gmem_memerr.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 */
24 24
25 25
26 26 /*
27 27 * Ereport-handling routines for memory errors
28 28 */
29 29
30 30 #include <gmem_mem.h>
31 31 #include <gmem_dimm.h>
32 32 #include <gmem_page.h>
33 33 #include <gmem.h>
34 34
35 35 #include <strings.h>
36 36 #include <string.h>
37 37 #include <errno.h>
38 38 #include <assert.h>
39 39 #include <fm/fmd_api.h>
40 40 #include <fm/libtopo.h>
41 41 #include <sys/fm/protocol.h>
42 42 #include <sys/async.h>
43 43 #include <sys/errclassify.h>
44 44
45 45 #define OFFBIT 0xFFFFFFFFFFFC07FFULL
46 46 #define BIT28_32 0x00000001F0000000ULL
47 47 #define BIT13_17 0x000000000003E000ULL
48 48 #define BIT18_19 0x00000000000C0000ULL
49 49 #define BIT11_12 0x0000000000001800ULL
50 50
51 51 struct ce_name2type {
52 52 const char *name;
53 53 ce_dispact_t type;
54 54 };
55 55
56 56 nvlist_t *fru_nvl;
57 57
58 58 static ce_dispact_t
59 59 gmem_mem_name2type(const char *name)
60 60 {
61 61 static const struct ce_name2type new[] = {
62 62 { "mem-unk", CE_DISP_UNKNOWN },
63 63 { "mem-is", CE_DISP_INTERMITTENT },
64 64 { "mem-cs", CE_DISP_PERS },
65 65 { "mem-ss", CE_DISP_STICKY },
66 66 { NULL }
67 67 };
68 68 const struct ce_name2type *names = &new[0];
69 69 const struct ce_name2type *tp;
70 70
71 71 for (tp = names; tp->name != NULL; tp++) {
72 72 if (strcasecmp(name, tp->name) == 0)
73 73 return (tp->type);
74 74 }
75 75
76 76 return (CE_DISP_UNKNOWN);
77 77 }
78 78
79 79 /*ARGSUSED*/
80 80 static int
81 81 find_fault_fru(topo_hdl_t *thp, tnode_t *node, void *arg)
82 82 {
83 83 nvlist_t *nvl = (nvlist_t *)arg;
84 84 nvlist_t *rsc = NULL, *fru = NULL;
85 85 nvlist_t **hcl, **topo_hcl;
86 86 uint_t n1, n2;
87 87 char *name, *name1, *name2;
88 88 char *id1, *id2;
89 89 int err, i;
90 90
91 91 if (topo_node_resource(node, &rsc, &err) < 0)
92 92 return (TOPO_WALK_NEXT);
93 93
94 94 err = nvlist_lookup_nvlist_array(rsc, FM_FMRI_HC_LIST, &topo_hcl, &n1);
95 95
96 96 if (err != 0) {
97 97 nvlist_free(rsc);
98 98 return (TOPO_WALK_NEXT);
99 99 }
100 100
101 101 (void) nvlist_lookup_string(topo_hcl[n1 - 1], FM_FMRI_HC_NAME, &name);
102 102 if (strcmp(name, "chip") != 0) {
103 103 nvlist_free(rsc);
104 104 return (TOPO_WALK_NEXT);
105 105 }
106 106
107 107 (void) nvlist_lookup_nvlist_array(nvl, FM_FMRI_HC_LIST, &hcl, &n2);
108 108
109 109 if (n1 != n2) {
110 110 nvlist_free(rsc);
111 111 return (TOPO_WALK_NEXT);
112 112 }
113 113
114 114 for (i = 0; i < n1; i++) {
115 115 (void) nvlist_lookup_string(topo_hcl[i], FM_FMRI_HC_NAME,
116 116 &name1);
117 117 (void) nvlist_lookup_string(topo_hcl[i], FM_FMRI_HC_ID, &id1);
118 118 (void) nvlist_lookup_string(hcl[i], FM_FMRI_HC_NAME, &name2);
119 119 (void) nvlist_lookup_string(hcl[i], FM_FMRI_HC_ID, &id2);
120 120 if (strcmp(name1, name2) != 0 || strcmp(id1, id2) != 0) {
121 121 nvlist_free(rsc);
122 122 return (TOPO_WALK_NEXT);
123 123 }
124 124 }
125 125
126 126 (void) topo_node_fru(node, &fru, NULL, &err);
127 127 if (fru != NULL) {
128 128 (void) nvlist_dup(fru, &fru_nvl, NV_UNIQUE_NAME);
129 129 nvlist_free(fru);
130 130 }
131 131 nvlist_free(rsc);
132 132 return (TOPO_WALK_TERMINATE);
133 133 }
134 134
135 135 nvlist_t *
136 136 gmem_find_fault_fru(fmd_hdl_t *hdl, nvlist_t *nvl) {
137 137 topo_hdl_t *thp;
138 138 topo_walk_t *twp;
139 139 int err;
140 140 fru_nvl = NULL;
141 141
142 142 if ((thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION)) == NULL)
143 143 return (NULL);
144 144
145 145 if ((twp = topo_walk_init(thp, FM_FMRI_SCHEME_HC,
146 146 find_fault_fru, nvl, &err)) == NULL) {
147 147 fmd_hdl_topo_rele(hdl, thp);
148 148 return (NULL);
149 149 }
150 150
151 151 (void) topo_walk_step(twp, TOPO_WALK_CHILD);
152 152 topo_walk_fini(twp);
153 153 fmd_hdl_topo_rele(hdl, thp);
154 154 return (fru_nvl);
155 155 }
156 156
157 157 /*
158 158 * fault the FRU of the common detector between two DIMMs
159 159 */
160 160 void
161 161 gmem_gen_datapath_fault(fmd_hdl_t *hdl, nvlist_t *det)
162 162 {
163 163 char *name, *id;
164 164 nvlist_t **hcl1, **hcl;
165 165 uint_t n;
166 166 int i, j;
167 167 fmd_case_t *cp;
168 168 nvlist_t *fltlist, *rsrc;
169 169 nvlist_t *fru = NULL;
170 170
171 171 if (nvlist_lookup_nvlist_array(det, FM_FMRI_HC_LIST, &hcl1, &n) < 0)
172 172 return;
173 173
174 174 for (i = 0; i < n; i++) {
175 175 (void) nvlist_lookup_string(hcl1[i], FM_FMRI_HC_NAME, &name);
176 176 if (strcmp(name, "chip") == 0)
177 177 break;
178 178 }
179 179
180 180 n = i + 1;
181 181 hcl = fmd_hdl_zalloc(hdl, sizeof (nvlist_t *) * n, FMD_SLEEP);
182 182 if (hcl == NULL)
183 183 return;
184 184
185 185 for (i = 0; i < n; i++) {
186 186 (void) nvlist_alloc(&hcl[i],
187 187 NV_UNIQUE_NAME|NV_UNIQUE_NAME_TYPE, 0);
188 188 }
189 189
190 190 for (i = 0, j = 0; i < n; i++) {
191 191 (void) nvlist_lookup_string(hcl1[i], FM_FMRI_HC_NAME, &name);
|
↓ open down ↓ |
191 lines elided |
↑ open up ↑ |
192 192 (void) nvlist_lookup_string(hcl1[i], FM_FMRI_HC_ID, &id);
193 193 (void) nvlist_add_string(hcl[j], FM_FMRI_HC_NAME, name);
194 194 (void) nvlist_add_string(hcl[j], FM_FMRI_HC_ID, id);
195 195 j++;
196 196 if (strcmp(name, "chip") == 0)
197 197 break;
198 198 }
199 199
200 200 if (nvlist_alloc(&rsrc, NV_UNIQUE_NAME|NV_UNIQUE_NAME_TYPE, 0) != 0) {
201 201 for (i = 0; i < n; i++) {
202 - if (hcl[i] != NULL)
203 - nvlist_free(hcl[i]);
202 + nvlist_free(hcl[i]);
204 203 }
205 204 fmd_hdl_free(hdl, hcl, sizeof (nvlist_t *) * n);
206 205 }
207 206
208 207 if (nvlist_add_uint8(rsrc, FM_VERSION, FM_HC_SCHEME_VERSION) != 0 ||
209 208 nvlist_add_string(rsrc, FM_FMRI_SCHEME, FM_FMRI_SCHEME_HC) != 0 ||
210 209 nvlist_add_string(rsrc, FM_FMRI_HC_ROOT, "") != 0 ||
211 210 nvlist_add_uint32(rsrc, FM_FMRI_HC_LIST_SZ, n) != 0 ||
212 211 nvlist_add_nvlist_array(rsrc, FM_FMRI_HC_LIST, hcl, n) != 0) {
213 212 for (i = 0; i < n; i++) {
214 - if (hcl[i] != NULL)
215 - nvlist_free(hcl[i]);
213 + nvlist_free(hcl[i]);
216 214 }
217 215 fmd_hdl_free(hdl, hcl, sizeof (nvlist_t *) * n);
218 216 nvlist_free(rsrc);
219 217 }
220 218
221 219 fru = gmem_find_fault_fru(hdl, rsrc);
222 220 if (fru != NULL) {
223 221 cp = fmd_case_open(hdl, NULL);
224 222 fltlist = fmd_nvl_create_fault(hdl, "fault.memory.datapath",
225 223 100, fru, fru, fru);
226 224 fmd_case_add_suspect(hdl, cp, fltlist);
227 225 fmd_case_solve(hdl, cp);
228 226 nvlist_free(fru);
229 227 }
230 228
231 229 for (i = 0; i < n; i++) {
232 - if (hcl[i] != NULL)
233 - nvlist_free(hcl[i]);
230 + nvlist_free(hcl[i]);
234 231 }
235 232
236 233 fmd_hdl_free(hdl, hcl, sizeof (nvlist_t *) * n);
237 234 nvlist_free(rsrc);
238 235 }
239 236
240 237 /*
241 238 * formula to conver an unhashed address to hashed address
242 239 * PA[17:11] = (PA[32:28] xor PA[17:13]) :: ((PA[19:18] xor PA[12:11])
243 240 */
244 241 static void
245 242 gmem_to_hashed_addr(uint64_t *addr, uint64_t afar)
246 243 {
247 244
248 245 *addr = (afar & OFFBIT) | ((afar & BIT28_32) >> 15) ^ (afar & BIT13_17)
249 246 | ((afar & BIT18_19) >> 7) ^ (afar & BIT11_12);
250 247 }
251 248
252 249 /*
253 250 * check if a dimm has n CEs that have the same symbol-in-error
254 251 */
255 252 int
256 253 upos_thresh_check(gmem_dimm_t *dimm, uint16_t upos, uint32_t threshold)
257 254 {
258 255 int i;
259 256 gmem_mq_t *ip, *next;
260 257 int count = 0;
261 258
262 259 for (i = 0; i < GMEM_MAX_CKWDS; i++) {
263 260 for (ip = gmem_list_next(&dimm->mq_root[i]); ip != NULL;
264 261 ip = next) {
265 262 next = gmem_list_next(ip);
266 263 if (ip->mq_unit_position == upos) {
267 264 count++;
268 265 if (count >= threshold)
269 266 return (1);
270 267 }
271 268 }
272 269 }
273 270 return (0);
274 271 }
275 272
276 273 /*
277 274 * check if smaller number of retired pages > 1/16 of larger number of
278 275 * retired pages
279 276 */
280 277 int
281 278 check_bad_rw_retired_pages(fmd_hdl_t *hdl, gmem_dimm_t *d1, gmem_dimm_t *d2)
282 279 {
283 280 uint_t sret, lret;
284 281 double ratio;
285 282
286 283 sret = lret = 0;
287 284
288 285 if (d2->dimm_nretired < d1->dimm_nretired) {
289 286 sret = d2->dimm_nretired;
290 287 lret = d1->dimm_nretired;
291 288 } else if (d2->dimm_nretired > d1->dimm_nretired) {
292 289 sret = d1->dimm_nretired;
293 290 lret = d2->dimm_nretired;
294 291 } else
295 292 return (0);
296 293
297 294 ratio = lret * GMEM_MQ_RATIO;
298 295
299 296 if (sret > ratio) {
300 297 fmd_hdl_debug(hdl, "sret=%d lret=%d ratio=%.3f",
301 298 sret, lret, ratio);
302 299 return (1);
303 300 }
304 301 return (0);
305 302 }
306 303
307 304 /*
308 305 * check bad rw on any two DIMMs. The check succeeds if
309 306 * - each DIMM has a n CEs which have the same symbol-in-error,
310 307 * - the smaller number of retired pages > 1/16 larger number of retired pages
311 308 */
312 309 static int
313 310 check_bad_rw_between_dimms(fmd_hdl_t *hdl, gmem_dimm_t *d1, gmem_dimm_t *d2,
314 311 uint16_t *rupos)
315 312 {
316 313 int i;
317 314 gmem_mq_t *ip, *next;
318 315 uint16_t upos;
319 316
320 317 for (i = 0; i < GMEM_MAX_CKWDS; i++) {
321 318 for (ip = gmem_list_next(&d1->mq_root[i]); ip != NULL;
322 319 ip = next) {
323 320 next = gmem_list_next(ip);
324 321 upos = ip->mq_unit_position;
325 322 if (upos_thresh_check(d1, upos, gmem.gm_nupos)) {
326 323 if (upos_thresh_check(d2, upos,
327 324 gmem.gm_nupos)) {
328 325 if (check_bad_rw_retired_pages(hdl,
329 326 d1, d2)) {
330 327 *rupos = upos;
331 328 return (1);
332 329 }
333 330 }
334 331 }
335 332 }
336 333 }
337 334
338 335 return (0);
339 336 }
340 337
341 338 static void
342 339 bad_reader_writer_check(fmd_hdl_t *hdl, nvlist_t *det, gmem_dimm_t *ce_dimm)
343 340 {
344 341 gmem_dimm_t *d, *next;
345 342 uint16_t upos;
346 343
347 344 for (d = gmem_list_next(&gmem.gm_dimms); d != NULL; d = next) {
348 345 next = gmem_list_next(d);
349 346 if (d == ce_dimm)
350 347 continue;
351 348 if (!gmem_same_datapath_dimms(hdl, ce_dimm, d))
352 349 continue;
353 350 if (check_bad_rw_between_dimms(hdl, ce_dimm, d, &upos)) {
354 351 gmem_gen_datapath_fault(hdl, det);
355 352 gmem_save_symbol_error(hdl, ce_dimm, upos);
356 353 fmd_hdl_debug(hdl,
357 354 "check_bad_rw_dimms succeeded: %s %s\n",
358 355 ce_dimm->dimm_serial, d->dimm_serial);
359 356 return;
360 357 }
361 358 }
362 359 }
363 360
364 361 /*
365 362 * rule 5a checking. The check succeeds if
366 363 * - nretired >= 512
367 364 * - nretired >= 128 and (addr_hi - addr_low) / (nretired -1 ) > 512KB
368 365 */
369 366 static void
370 367 ce_thresh_check(fmd_hdl_t *hdl, gmem_dimm_t *dimm)
371 368 {
372 369 nvlist_t *flt, *rsrc;
373 370 fmd_case_t *cp;
374 371 uint_t nret;
375 372 uint64_t delta_addr = 0;
376 373
377 374 if (dimm->dimm_flags & GMEM_F_FAULTING)
378 375 return;
379 376
380 377 nret = dimm->dimm_nretired;
381 378
382 379 if (nret < gmem.gm_low_ce_thresh)
383 380 return;
384 381
385 382 if (dimm->dimm_phys_addr_hi >= dimm->dimm_phys_addr_low)
386 383 delta_addr =
387 384 (dimm->dimm_phys_addr_hi - dimm->dimm_phys_addr_low) /
388 385 (nret - 1);
389 386
390 387 if (nret >= gmem.gm_max_retired_pages || delta_addr > GMEM_MQ_512KB) {
391 388
|
↓ open down ↓ |
148 lines elided |
↑ open up ↑ |
392 389 fmd_hdl_debug(hdl, "ce_thresh_check succeeded nret=%d", nret);
393 390 dimm->dimm_flags |= GMEM_F_FAULTING;
394 391 gmem_dimm_dirty(hdl, dimm);
395 392
396 393 cp = fmd_case_open(hdl, NULL);
397 394 rsrc = gmem_find_dimm_rsc(hdl, dimm->dimm_serial);
398 395 flt = fmd_nvl_create_fault(hdl, GMEM_FAULT_DIMM_PAGES,
399 396 GMEM_FLTMAXCONF, NULL, gmem_dimm_fru(dimm), rsrc);
400 397 fmd_case_add_suspect(hdl, cp, flt);
401 398 fmd_case_solve(hdl, cp);
402 - if (rsrc != NULL)
403 - nvlist_free(rsrc);
399 + nvlist_free(rsrc);
404 400 }
405 401 }
406 402
407 403 /*
408 404 * rule 5b checking. The check succeeds if more than 120
409 405 * non-intermittent CEs are reported against one symbol
410 406 * position of one afar in 72 hours
411 407 */
412 408 static void
413 409 mq_5b_check(fmd_hdl_t *hdl, gmem_dimm_t *dimm)
414 410 {
415 411 nvlist_t *flt, *rsrc;
416 412 fmd_case_t *cp;
417 413 gmem_mq_t *ip, *next;
418 414 int cw;
419 415
420 416 for (cw = 0; cw < GMEM_MAX_CKWDS; cw++) {
421 417 for (ip = gmem_list_next(&dimm->mq_root[cw]);
422 418 ip != NULL; ip = next) {
423 419 next = gmem_list_next(ip);
424 420 if (ip->mq_dupce_count >= gmem.gm_dupce) {
425 421 fmd_hdl_debug(hdl,
426 422 "mq_5b_check succeeded: duplicate CE=%d",
427 423 ip->mq_dupce_count);
|
↓ open down ↓ |
14 lines elided |
↑ open up ↑ |
428 424 cp = fmd_case_open(hdl, NULL);
429 425 rsrc = gmem_find_dimm_rsc(hdl,
430 426 dimm->dimm_serial);
431 427 flt = fmd_nvl_create_fault(hdl,
432 428 GMEM_FAULT_DIMM_PAGES, GMEM_FLTMAXCONF,
433 429 NULL, gmem_dimm_fru(dimm), rsrc);
434 430 dimm->dimm_flags |= GMEM_F_FAULTING;
435 431 gmem_dimm_dirty(hdl, dimm);
436 432 fmd_case_add_suspect(hdl, cp, flt);
437 433 fmd_case_solve(hdl, cp);
438 - if (rsrc != NULL)
439 - nvlist_free(rsrc);
434 + nvlist_free(rsrc);
440 435 return;
441 436 }
442 437 }
443 438 }
444 439 }
445 440
446 441 /*
447 442 * delete the expired duplicate CE time stamps
448 443 */
449 444 static void
450 445 mq_prune_dup(fmd_hdl_t *hdl, gmem_mq_t *ip, uint64_t now)
451 446 {
452 447 tstamp_t *tsp, *next;
453 448
454 449 for (tsp = gmem_list_next(&ip->mq_dupce_tstamp); tsp != NULL;
455 450 tsp = next) {
456 451 next = gmem_list_next(tsp);
457 452 if (tsp->tstamp < now - GMEM_MQ_TIMELIM) {
458 453 gmem_list_delete(&ip->mq_dupce_tstamp, &tsp->ts_l);
459 454 fmd_hdl_free(hdl, tsp, sizeof (tstamp_t));
460 455 ip->mq_dupce_count--;
461 456 }
462 457 }
463 458 }
464 459
465 460 static void
466 461 mq_update(fmd_hdl_t *hdl, fmd_event_t *ep, gmem_mq_t *ip, uint64_t now)
467 462 {
468 463 tstamp_t *tsp;
469 464
470 465 ip->mq_tstamp = now;
471 466 ip->mq_ep = ep;
472 467 if (fmd_serd_exists(hdl, ip->mq_serdnm))
473 468 fmd_serd_destroy(hdl, ip->mq_serdnm);
474 469
475 470 fmd_serd_create(hdl, ip->mq_serdnm, GMEM_MQ_SERDN, GMEM_MQ_SERDT);
476 471 (void) fmd_serd_record(hdl, ip->mq_serdnm, ep);
477 472
478 473 tsp = fmd_hdl_zalloc(hdl, sizeof (tstamp_t), FMD_SLEEP);
479 474 tsp->tstamp = now;
480 475 gmem_list_append(&ip->mq_dupce_tstamp, tsp);
481 476 ip->mq_dupce_count++;
482 477 }
483 478
484 479 /*
485 480 * Create a fresh index block for MQSC CE correlation.
486 481 */
487 482 gmem_mq_t *
488 483 mq_create(fmd_hdl_t *hdl, fmd_event_t *ep,
489 484 uint64_t afar, uint16_t upos, uint16_t ckwd, uint64_t now)
490 485 {
491 486 gmem_mq_t *cp;
492 487 tstamp_t *tsp;
493 488
494 489 cp = fmd_hdl_zalloc(hdl, sizeof (gmem_mq_t), FMD_SLEEP);
495 490 cp->mq_tstamp = now;
496 491 cp->mq_ckwd = ckwd;
497 492 cp->mq_phys_addr = afar;
498 493 cp->mq_unit_position = upos;
499 494 cp->mq_ep = ep;
500 495 cp->mq_serdnm =
501 496 gmem_mq_serdnm_create(hdl, "mq", afar, ckwd, upos);
502 497
503 498 tsp = fmd_hdl_zalloc(hdl, sizeof (tstamp_t), FMD_SLEEP);
504 499 tsp->tstamp = now;
505 500 gmem_list_append(&cp->mq_dupce_tstamp, tsp);
506 501 cp->mq_dupce_count = 1;
507 502
508 503 /*
509 504 * Create SERD to keep this event from being removed
510 505 * by fmd which may not know there is an event pointer
511 506 * saved here. This SERD is *never* meant to fire.
512 507 */
513 508 if (fmd_serd_exists(hdl, cp->mq_serdnm))
514 509 fmd_serd_destroy(hdl, cp->mq_serdnm);
515 510
516 511 fmd_serd_create(hdl, cp->mq_serdnm, GMEM_MQ_SERDN, GMEM_MQ_SERDT);
517 512 (void) fmd_serd_record(hdl, cp->mq_serdnm, ep);
518 513
519 514 return (cp);
520 515 }
521 516
522 517 gmem_mq_t *
523 518 mq_destroy(fmd_hdl_t *hdl, gmem_list_t *lp, gmem_mq_t *ip)
524 519 {
525 520 gmem_mq_t *jp = gmem_list_next(ip);
526 521 tstamp_t *tsp, *next;
527 522
528 523
529 524 if (ip->mq_serdnm != NULL) {
530 525 if (fmd_serd_exists(hdl, ip->mq_serdnm))
531 526 fmd_serd_destroy(hdl, ip->mq_serdnm);
532 527 fmd_hdl_strfree(hdl, ip->mq_serdnm);
533 528 ip->mq_serdnm = NULL;
534 529 }
535 530
536 531 for (tsp = gmem_list_next(&ip->mq_dupce_tstamp); tsp != NULL;
537 532 tsp = next) {
538 533 next = gmem_list_next(tsp);
539 534 gmem_list_delete(&ip->mq_dupce_tstamp, &tsp->ts_l);
540 535 fmd_hdl_free(hdl, tsp, sizeof (tstamp_t));
541 536 }
542 537
543 538 gmem_list_delete(lp, &ip->mq_l);
544 539 fmd_hdl_free(hdl, ip, sizeof (gmem_mq_t));
545 540
546 541 return (jp);
547 542 }
548 543
549 544
550 545 /*
551 546 * Add an index block for a new CE, sorted
552 547 * a) by ascending unit position
553 548 * b) order of arrival (~= time order)
554 549 */
555 550 void
556 551 mq_add(fmd_hdl_t *hdl, gmem_dimm_t *dimm, fmd_event_t *ep,
557 552 uint64_t afar, uint16_t unit_position, uint16_t ckwd,
558 553 uint64_t now)
559 554 {
560 555 gmem_mq_t *ip, *jp;
561 556 int cw = (int)ckwd;
562 557
563 558 for (ip = gmem_list_next(&dimm->mq_root[cw]); ip != NULL; ) {
564 559 if (ip->mq_unit_position > unit_position) {
565 560 /* list is in unit position order */
566 561 break;
567 562 } else if (ip->mq_unit_position == unit_position &&
568 563 ip->mq_phys_addr == afar) {
569 564 /*
570 565 * Found a duplicate cw, unit_position, and afar.
571 566 * Delete this node, to be superseded by the new
572 567 * node added below.
573 568 * update the mq_t structure
574 569 */
575 570 mq_update(hdl, ep, ip, now);
576 571 return;
577 572 } else {
578 573 ip = gmem_list_next(ip);
579 574 }
580 575 }
581 576
582 577 jp = mq_create(hdl, ep, afar, unit_position, cw, now);
583 578 if (ip == NULL)
584 579 gmem_list_append(&dimm->mq_root[cw], jp);
585 580 else
586 581 gmem_list_insert_before(&dimm->mq_root[cw], ip, jp);
587 582 }
588 583
589 584 /*
590 585 * Prune the MQSC index lists (one for each checkword), by deleting
591 586 * outdated index blocks from each list.
592 587 */
593 588
594 589 void
595 590 mq_prune(fmd_hdl_t *hdl, gmem_dimm_t *dimm, uint64_t now)
596 591 {
597 592 gmem_mq_t *ip;
598 593 int cw;
599 594
600 595 for (cw = 0; cw < GMEM_MAX_CKWDS; cw++) {
601 596 for (ip = gmem_list_next(&dimm->mq_root[cw]); ip != NULL; ) {
602 597 if (ip->mq_tstamp < now - GMEM_MQ_TIMELIM) {
603 598 /*
604 599 * This event has timed out - delete the
605 600 * mq block as well as serd for the event.
606 601 */
607 602 ip = mq_destroy(hdl, &dimm->mq_root[cw], ip);
608 603 } else {
609 604 mq_prune_dup(hdl, ip, now);
610 605 /* tstamp < now - ce_t */
611 606 ip = gmem_list_next(ip);
612 607 }
613 608 } /* per checkword */
614 609 } /* cw = 0...3 */
615 610 }
616 611
617 612 /*
618 613 * Check the MQSC index lists (one for each checkword) by making a
619 614 * complete pass through each list, checking if the criteria for
620 615 * Rule 4A has been met. Rule 4A checking is done for each checkword.
621 616 *
622 617 * Rule 4A: fault a DIMM "whenever Solaris reports two or more CEs from
623 618 * two or more different physical addresses on each of two or more different
624 619 * bit positions from the same DIMM within 72 hours of each other, and all
625 620 * the addresses are in the same relative checkword (that is, the AFARs
626 621 * are all the same modulo 64). [Note: This means at least 4 CEs; two
627 622 * from one bit position, with unique addresses, and two from another,
628 623 * also with unique addresses, and the lower 6 bits of all the addresses
629 624 * are the same."
630 625 */
631 626
632 627 void
633 628 mq_check(fmd_hdl_t *hdl, gmem_dimm_t *dimm)
634 629 {
635 630 int upos_pairs, curr_upos, cw, i, j;
636 631 nvlist_t *flt, *rsc;
637 632 typedef struct upos_pair {
638 633 int upos;
639 634 gmem_mq_t *mq1;
640 635 gmem_mq_t *mq2;
641 636 } upos_pair_t;
642 637 upos_pair_t upos_array[16]; /* max per cw = 2, * 8 cw's */
643 638 gmem_mq_t *ip;
644 639
645 640 /*
646 641 * Each upos_array[] member represents a pair of CEs for the same
647 642 * unit position (symbol) which is a 4 bit nibble.
648 643 * MQSC rule 4 requires pairs of CEs from the same symbol (same DIMM
649 644 * for rule 4A, and same DRAM for rule 4B) for a violation - this
650 645 * is why CE pairs are tracked.
651 646 */
652 647 upos_pairs = 0;
653 648 upos_array[0].mq1 = NULL;
654 649
655 650 for (cw = 0; cw < GMEM_MAX_CKWDS; cw++) {
656 651 i = upos_pairs;
657 652 curr_upos = -1;
658 653
659 654 /*
660 655 * mq_root[] is an array of cumulative lists of CEs
661 656 * indexed by checkword where the list is in unit position
662 657 * order. Loop through checking for duplicate unit position
663 658 * entries (filled in at mq_create()).
664 659 * The upos_array[] is filled in each time a duplicate
665 660 * unit position is found; the first time through the loop
666 661 * of a unit position sets curr_upos but does not fill in
667 662 * upos_array[] until the second symbol is found.
668 663 */
669 664 for (ip = gmem_list_next(&dimm->mq_root[cw]); ip != NULL;
670 665 ip = gmem_list_next(ip)) {
671 666 if (curr_upos != ip->mq_unit_position) {
672 667 /* Set initial current position */
673 668 curr_upos = ip->mq_unit_position;
674 669 } else if (i > upos_pairs &&
675 670 curr_upos == upos_array[i-1].upos) {
676 671 /*
677 672 * Only keep track of CE pairs; skip
678 673 * triples, quads, etc...
679 674 */
680 675 continue;
681 676 } else if (upos_array[i].mq1 == NULL) {
682 677 /* Have a pair. Add to upos_array[] */
683 678 fmd_hdl_debug(hdl, "pair:upos=%d",
684 679 curr_upos);
685 680 upos_array[i].upos = curr_upos;
686 681 upos_array[i].mq1 = gmem_list_prev(ip);
687 682 upos_array[i].mq2 = ip;
688 683 upos_array[++i].mq1 = NULL;
689 684 }
690 685 }
691 686 if (i - upos_pairs >= 2) {
692 687 /* Rule 4A violation */
693 688 rsc = gmem_find_dimm_rsc(hdl, dimm->dimm_serial);
694 689 flt = fmd_nvl_create_fault(hdl, GMEM_FAULT_DIMM_4A,
695 690 GMEM_FLTMAXCONF, NULL, gmem_dimm_fru(dimm), rsc);
696 691 for (j = upos_pairs; j < i; j++) {
697 692 fmd_case_add_ereport(hdl,
|
↓ open down ↓ |
248 lines elided |
↑ open up ↑ |
698 693 dimm->dimm_case.cc_cp,
699 694 upos_array[j].mq1->mq_ep);
700 695 fmd_case_add_ereport(hdl,
701 696 dimm->dimm_case.cc_cp,
702 697 upos_array[j].mq2->mq_ep);
703 698 }
704 699 dimm->dimm_flags |= GMEM_F_FAULTING;
705 700 gmem_dimm_dirty(hdl, dimm);
706 701 fmd_case_add_suspect(hdl, dimm->dimm_case.cc_cp, flt);
707 702 fmd_case_solve(hdl, dimm->dimm_case.cc_cp);
708 - if (rsc != NULL)
709 - nvlist_free(rsc);
703 + nvlist_free(rsc);
710 704 return;
711 705 }
712 706 upos_pairs = i;
713 707 assert(upos_pairs < 16);
714 708 }
715 709 }
716 710
717 711 /*ARGSUSED*/
718 712 gmem_evdisp_t
719 713 gmem_ce(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class)
720 714 {
721 715 uint16_t symbol_pos, cw;
722 716 uint64_t phyaddr, offset, addr;
723 717 uint32_t filter_ratio = 0;
724 718 gmem_dimm_t *dimm;
725 719 gmem_page_t *page;
726 720 nvlist_t *fru = NULL;
727 721 nvlist_t *topo_rsc = NULL;
728 722 nvlist_t *rsrc, *det;
729 723 const char *uuid;
730 724 ce_dispact_t type;
731 725 boolean_t diagnose;
732 726 char *sn;
733 727 int err, rc;
734 728 uint64_t *now;
735 729 uint_t nelem;
736 730 int skip_error = 0;
737 731
738 732 err = nvlist_lookup_boolean_value(nvl, GMEM_ERPT_PAYLOAD_DIAGNOSE,
739 733 &diagnose);
740 734 if (err != 0 || diagnose == 0)
741 735 return (GMEM_EVD_UNUSED);
742 736
743 737 if ((nvlist_lookup_uint64(nvl, GMEM_ERPT_PAYLOAD_PHYSADDR,
744 738 &phyaddr) != 0) ||
745 739 (nvlist_lookup_uint64(nvl, GMEM_ERPT_PAYLOAD_OFFSET,
746 740 &offset) != 0)) {
747 741 fmd_hdl_debug(hdl, "Can't get page phyaddr or offset");
748 742 return (GMEM_EVD_BAD);
749 743 }
750 744
751 745 fmd_hdl_debug(hdl, "phyaddr %llx offset %llx", phyaddr, offset);
752 746
753 747 if ((page = gmem_page_lookup(phyaddr)) != NULL &&
754 748 page->page_case.cc_cp != NULL &&
755 749 fmd_case_solved(hdl, page->page_case.cc_cp))
756 750 return (GMEM_EVD_REDUND);
757 751
758 752 if (nvlist_lookup_nvlist(nvl, GMEM_ERPT_PAYLOAD_RESOURCE,
759 753 &rsrc) != 0 ||
760 754 nvlist_lookup_string(rsrc, FM_FMRI_HC_SERIAL_ID, &sn) != 0) {
761 755 fmd_hdl_debug(hdl, "Can't get dimm serial\n");
762 756 return (GMEM_EVD_BAD);
763 757 }
764 758
765 759 fmd_hdl_debug(hdl, "serial %s", sn);
766 760
767 761 if (nvlist_lookup_nvlist(nvl, GMEM_ERPT_PAYLOAD_DETECTOR, &det) != 0)
768 762 return (GMEM_EVD_BAD);
769 763
770 764 /*
771 765 * Find dimm fru by serial number.
772 766 */
773 767 fru = gmem_find_dimm_fru(hdl, sn);
774 768
775 769 if (fru == NULL) {
776 770 fmd_hdl_debug(hdl, "Dimm is not present\n");
777 771 return (GMEM_EVD_UNUSED);
778 772 }
779 773
780 774 if ((dimm = gmem_dimm_lookup(hdl, fru)) == NULL &&
781 775 (dimm = gmem_dimm_create(hdl, fru, det)) == NULL) {
782 776 nvlist_free(fru);
783 777 return (GMEM_EVD_UNUSED);
784 778 }
785 779
786 780 if (dimm->dimm_case.cc_cp == NULL) {
787 781 dimm->dimm_case.cc_cp = gmem_case_create(hdl,
788 782 &dimm->dimm_header, GMEM_PTR_DIMM_CASE, &uuid);
789 783 }
790 784
791 785 /*
792 786 * Add to MQSC correlation lists all CEs which pass validity
793 787 * checks above. If there is no symbol_pos & relative ckword
794 788 * in the ereport, skip rule 4A checking.
795 789 */
796 790
797 791 err = nvlist_lookup_uint16(nvl, GMEM_ERPT_PAYLOAD_SYMBOLPOS,
798 792 &symbol_pos);
799 793 err |= nvlist_lookup_uint16(nvl, GMEM_ERPT_PAYLOAD_CKW, &cw);
800 794
801 795 if (err == 0) {
802 796 fmd_hdl_debug(hdl, "symbol_pos=%d cw=%d", symbol_pos, cw);
803 797
804 798 if (nvlist_lookup_uint64_array(nvl,
805 799 "__tod", &now, &nelem) == 0) {
806 800 skip_error = gmem_check_symbol_error(hdl, dimm,
807 801 symbol_pos);
808 802
809 803 if (!skip_error ||
810 804 !(dimm->dimm_flags & GMEM_F_FAULTING))
811 805 mq_add(hdl, dimm, ep, phyaddr, symbol_pos,
812 806 cw, *now);
813 807
814 808 mq_prune(hdl, dimm, *now);
815 809
816 810 if (!skip_error)
817 811 bad_reader_writer_check(hdl, det, dimm);
818 812 if (!(dimm->dimm_flags & GMEM_F_FAULTING)) {
819 813 mq_check(hdl, dimm);
820 814 mq_5b_check(hdl, dimm);
821 815 }
822 816 }
823 817 }
824 818
825 819 type = gmem_mem_name2type(strstr(class, "mem"));
826 820
827 821 switch (type) {
828 822 case CE_DISP_UNKNOWN:
829 823 GMEM_STAT_BUMP(ce_unknown);
830 824 nvlist_free(fru);
831 825 return (GMEM_EVD_UNUSED);
832 826 case CE_DISP_INTERMITTENT:
833 827 GMEM_STAT_BUMP(ce_interm);
834 828 nvlist_free(fru);
835 829 return (GMEM_EVD_UNUSED);
836 830 case CE_DISP_PERS:
837 831 GMEM_STAT_BUMP(ce_clearable_persis);
838 832 break;
839 833 case CE_DISP_STICKY:
840 834 GMEM_STAT_BUMP(ce_sticky);
841 835 break;
842 836 default:
843 837 nvlist_free(fru);
844 838 return (GMEM_EVD_BAD);
845 839 }
846 840
847 841 if (gmem_check_symbol_error(hdl, dimm, symbol_pos)) {
848 842 nvlist_free(fru);
849 843 return (GMEM_EVD_REDUND);
850 844 }
851 845
852 846 if (page == NULL) {
853 847 page = gmem_page_create(hdl, fru, phyaddr, offset);
854 848 if (page == NULL) {
855 849 nvlist_free(fru);
856 850 return (GMEM_EVD_UNUSED);
857 851 }
858 852 }
859 853
860 854 nvlist_free(fru);
861 855
862 856 if (page->page_case.cc_cp == NULL) {
863 857 page->page_case.cc_cp = gmem_case_create(hdl,
864 858 &page->page_header, GMEM_PTR_PAGE_CASE, &uuid);
865 859 }
866 860
867 861 switch (type) {
868 862 case CE_DISP_PERS:
869 863 fmd_hdl_debug(hdl, "adding persistent event to CE serd");
870 864 if (page->page_case.cc_serdnm == NULL)
871 865 gmem_page_serd_create(hdl, page, nvl);
872 866
873 867 filter_ratio = gmem_get_serd_filter_ratio(nvl);
874 868
875 869 fmd_hdl_debug(hdl, "filter_ratio %d\n", filter_ratio);
876 870
877 871 if (gmem_serd_record(hdl, page->page_case.cc_serdnm,
878 872 filter_ratio, ep) == FMD_B_FALSE) {
879 873 return (GMEM_EVD_OK); /* engine hasn't fired */
880 874 }
881 875
882 876 fmd_hdl_debug(hdl, "ce page serd fired\n");
883 877 fmd_case_add_serd(hdl, page->page_case.cc_cp,
884 878 page->page_case.cc_serdnm);
885 879 fmd_serd_reset(hdl, page->page_case.cc_serdnm);
886 880 break; /* to retire */
887 881
888 882 case CE_DISP_STICKY:
889 883 fmd_case_add_ereport(hdl, page->page_case.cc_cp, ep);
890 884 break; /* to retire */
891 885 }
892 886
893 887
894 888 topo_rsc = gmem_find_dimm_rsc(hdl, dimm->dimm_serial);
895 889 rc = gmem_page_fault(hdl, gmem_dimm_fru(dimm), topo_rsc,
896 890 ep, phyaddr, offset);
897 891
898 892 if (rc) {
899 893 gmem_to_hashed_addr(&addr, phyaddr);
900 894
901 895 if (addr > dimm->dimm_phys_addr_hi)
902 896 dimm->dimm_phys_addr_hi = addr;
903 897 if (addr < dimm->dimm_phys_addr_low)
904 898 dimm->dimm_phys_addr_low = addr;
905 899
906 900 dimm->dimm_nretired++;
907 901 dimm->dimm_retstat.fmds_value.ui64++;
908 902 gmem_dimm_dirty(hdl, dimm);
909 903 ce_thresh_check(hdl, dimm);
910 904 }
911 905 return (GMEM_EVD_OK);
912 906 }
913 907
914 908 void
915 909 gmem_dimm_close(fmd_hdl_t *hdl, void *arg)
916 910 {
917 911 gmem_dimm_destroy(hdl, arg);
918 912 }
|
↓ open down ↓ |
199 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX