Print this page
XXXX pass in cpu_pause_func via pause_cpus
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/os/lgrp_topo.c
+++ new/usr/src/uts/common/os/lgrp_topo.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 */
24 24
25 25 /*
26 26 * lgroup topology
27 27 */
28 28
29 29 #include <sys/cpupart.h>
30 30 #include <sys/lgrp.h>
31 31 #include <sys/promif.h>
32 32 #include <sys/types.h>
33 33
34 34
35 35 #define LGRP_TOPO_LEVELS 4 /* default height limit */
36 36 #define LGRP_TOPO_LEVELS_MAX 4 /* max height limit */
37 37
38 38
39 39 /*
40 40 * Only collapse lgroups which have same latency (and resources)
41 41 */
42 42 int lgrp_collapse_equidist = 1;
43 43
44 44 int lgrp_collapse_off = 1; /* disable collapsing of duplicates */
45 45
46 46 /*
47 47 * Height to limit lgroup topology
48 48 */
49 49 unsigned int lgrp_topo_levels = LGRP_TOPO_LEVELS;
50 50
51 51 int lgrp_split_off = 1; /* disable splitting lgroups */
52 52
53 53 #ifdef DEBUG
54 54 /*
55 55 * Debugging output
56 56 * - 0: off
57 57 * - >0: on and bigger means more
58 58 */
59 59 int lgrp_topo_debug = 0;
60 60
61 61
62 62 void
63 63 klgrpset_print(klgrpset_t lgrpset)
64 64 {
65 65 int i;
66 66
67 67
68 68 prom_printf("0x%llx(", (u_longlong_t)lgrpset);
69 69 for (i = 0; i <= lgrp_alloc_max; i++)
70 70 if (klgrpset_ismember(lgrpset, i))
71 71 prom_printf("%d ", i);
72 72 prom_printf(")\n");
73 73 }
74 74
75 75
76 76 void
77 77 lgrp_rsets_print(char *string, klgrpset_t *rsets)
78 78 {
79 79 int i;
80 80
81 81 prom_printf("%s\n", string);
82 82 for (i = 0; i < LGRP_RSRC_COUNT; i++)
83 83 klgrpset_print(rsets[i]);
84 84 }
85 85 #endif /* DEBUG */
86 86
87 87
88 88 /*
89 89 * Add "from" lgroup resources to "to" lgroup resources
90 90 */
91 91 void
92 92 lgrp_rsets_add(klgrpset_t *from, klgrpset_t *to)
93 93 {
94 94 int i;
95 95
96 96 for (i = 0; i < LGRP_RSRC_COUNT; i++)
97 97 klgrpset_or(to[i], from[i]);
98 98 }
99 99
100 100
101 101 /*
102 102 * Copy "from" lgroup resources to "to" lgroup resources
103 103 */
104 104 void
105 105 lgrp_rsets_copy(klgrpset_t *from, klgrpset_t *to)
106 106 {
107 107 int i;
108 108
109 109 for (i = 0; i < LGRP_RSRC_COUNT; i++)
110 110 to[i] = from[i];
111 111 }
112 112
113 113
114 114 /*
115 115 * Delete given lgroup ID from lgroup resource set of specified lgroup
116 116 * and its ancestors if "follow_parent" is set
117 117 */
118 118 void
119 119 lgrp_rsets_delete(lgrp_t *lgrp, lgrp_id_t lgrpid, int follow_parent)
120 120 {
121 121 int i;
122 122
123 123 while (lgrp != NULL) {
124 124 for (i = 0; i < LGRP_RSRC_COUNT; i++)
125 125 klgrpset_del(lgrp->lgrp_set[i], lgrpid);
126 126 if (!follow_parent)
127 127 break;
128 128 lgrp = lgrp->lgrp_parent;
129 129 }
130 130 }
131 131
132 132
133 133 /*
134 134 * Return whether given lgroup resource set empty
135 135 */
136 136 int
137 137 lgrp_rsets_empty(klgrpset_t *rset)
138 138 {
139 139 int i;
140 140
141 141 for (i = 0; i < LGRP_RSRC_COUNT; i++)
142 142 if (!klgrpset_isempty(rset[i]))
143 143 return (0);
144 144
145 145 return (1);
146 146 }
147 147
148 148
149 149 /*
150 150 * Return whether given lgroup resource sets are same
151 151 */
152 152 int
153 153 lgrp_rsets_equal(klgrpset_t *rset1, klgrpset_t *rset2)
154 154 {
155 155 int i;
156 156
157 157 for (i = 0; i < LGRP_RSRC_COUNT; i++)
158 158 if (rset1[i] != rset2[i])
159 159 return (0);
160 160
161 161 return (1);
162 162 }
163 163
164 164
165 165 /*
166 166 * Return whether specified lgroup ID is in given lgroup resource set
167 167 */
168 168 int
169 169 lgrp_rsets_member(klgrpset_t *rset, lgrp_id_t lgrpid)
170 170 {
171 171 int i;
172 172
173 173 for (i = 0; i < LGRP_RSRC_COUNT; i++)
174 174 if (klgrpset_ismember(rset[i], lgrpid))
175 175 return (1);
176 176
177 177 return (0);
178 178 }
179 179
180 180
181 181 /*
182 182 * Return whether specified lgroup ID is in all lgroup resources
183 183 */
184 184 int
185 185 lgrp_rsets_member_all(klgrpset_t *rset, lgrp_id_t lgrpid)
186 186 {
187 187 int i;
188 188
189 189 for (i = 0; i < LGRP_RSRC_COUNT; i++)
190 190 if (!klgrpset_ismember(rset[i], lgrpid))
191 191 return (0);
192 192
193 193 return (1);
194 194 }
195 195
196 196
197 197 /*
198 198 * Replace resources for given lgroup with specified resources at given
199 199 * latency and shift its old resources to its parent and its parent's resources
200 200 * to its parent, etc. until root lgroup reached
201 201 */
202 202 void
203 203 lgrp_rsets_replace(klgrpset_t *rset, int latency, lgrp_t *lgrp, int shift)
204 204 {
205 205 lgrp_t *cur;
206 206 int lat_new;
207 207 int lat_saved;
208 208 klgrpset_t rset_new[LGRP_RSRC_COUNT];
209 209 klgrpset_t rset_saved[LGRP_RSRC_COUNT];
210 210
211 211 cur = lgrp;
212 212 lat_saved = latency;
213 213 lgrp_rsets_copy(rset, rset_saved);
214 214 while (cur && cur != lgrp_root) {
215 215 /*
216 216 * Save current resources and latency to insert in parent and
217 217 * then replace with new resources and latency
218 218 */
219 219 lgrp_rsets_copy(rset_saved, rset_new);
220 220 lgrp_rsets_copy(cur->lgrp_set, rset_saved);
221 221 lgrp_rsets_copy(rset_new, cur->lgrp_set);
222 222
223 223 lat_new = lat_saved;
224 224 lat_saved = cur->lgrp_latency;
225 225 cur->lgrp_latency = lat_new;
226 226 if (!shift)
227 227 break;
228 228 cur = cur->lgrp_parent;
229 229 }
230 230 }
231 231
232 232
233 233 /*
234 234 * Set "to" lgroup resource set with given lgroup ID
235 235 */
236 236 void
237 237 lgrp_rsets_set(klgrpset_t *to, lgrp_id_t lgrpid)
238 238 {
239 239 klgrpset_t from;
240 240 int i;
241 241
242 242 klgrpset_clear(from);
243 243 klgrpset_add(from, lgrpid);
244 244 for (i = 0; i < LGRP_RSRC_COUNT; i++) {
245 245 klgrpset_clear(to[i]);
246 246 klgrpset_or(to[i], from);
247 247 }
248 248 }
249 249
250 250
251 251 /*
252 252 * Delete any ancestors of given child lgroup which don't have any other
253 253 * children
254 254 */
255 255 int
256 256 lgrp_ancestor_delete(lgrp_t *child, klgrpset_t *changed)
257 257 {
258 258 int count;
259 259 lgrp_t *current;
260 260 lgrp_id_t lgrpid;
261 261 lgrp_t *parent;
262 262
263 263 #ifdef DEBUG
264 264 if (lgrp_topo_debug > 1) {
265 265 prom_printf("lgrp_ancestor_delete(0x%p[%d],0x%p)\n",
266 266 (void *)child, child->lgrp_id, (void *)changed);
267 267 }
268 268 #endif /* DEBUG */
269 269
270 270 count = 0;
271 271 if (changed)
272 272 klgrpset_clear(*changed);
273 273
274 274 /*
275 275 * Visit ancestors, decrement child count for each, and remove any
276 276 * that don't have any children left until we reach an ancestor that
277 277 * has multiple children
278 278 */
279 279 current = child;
280 280 parent = child->lgrp_parent;
281 281 lgrpid = current->lgrp_id;
282 282 while (parent != NULL) {
283 283 #ifdef DEBUG
284 284 if (lgrp_topo_debug > 1)
285 285 prom_printf("lgrp_ancestor_delete: parent %d,"
286 286 " current %d\n",
287 287 parent->lgrp_id, lgrpid);
288 288 #endif /* DEBUG */
289 289
290 290 klgrpset_del(parent->lgrp_leaves, lgrpid);
291 291 klgrpset_del(parent->lgrp_children, lgrpid);
292 292 parent->lgrp_childcnt--;
293 293 if (changed)
294 294 klgrpset_add(*changed, parent->lgrp_id);
295 295 count++;
296 296 if (parent->lgrp_childcnt != 0)
297 297 break;
298 298
299 299 current = parent;
300 300 parent = current->lgrp_parent;
301 301 lgrpid = current->lgrp_id;
302 302
303 303 #ifdef DEBUG
304 304 if (lgrp_topo_debug > 0)
305 305 prom_printf("lgrp_ancestor_delete: destroy"
306 306 " lgrp %d at 0x%p\n",
307 307 current->lgrp_id, (void *)current);
308 308 #endif /* DEBUG */
309 309 lgrp_destroy(current);
310 310 }
311 311
312 312 #ifdef DEBUG
313 313 if (lgrp_topo_debug > 1 && changed)
314 314 prom_printf("lgrp_ancestor_delete: changed %d lgrps: 0x%llx\n",
315 315 count, (u_longlong_t)*changed);
316 316 #endif /* DEBUG */
317 317
318 318 return (count);
319 319 }
320 320
321 321
322 322 /*
323 323 * Consolidate lgrp1 into lgrp2
324 324 */
325 325 int
326 326 lgrp_consolidate(lgrp_t *lgrp1, lgrp_t *lgrp2, klgrpset_t *changed)
327 327 {
328 328 klgrpset_t changes;
329 329 lgrp_t *child;
330 330 int count;
331 331 int i;
332 332 lgrp_t *parent;
333 333
334 334 /*
335 335 * Leaf lgroups should never need to be consolidated
336 336 */
337 337 if (lgrp1 == NULL || lgrp2 == NULL || lgrp1->lgrp_childcnt < 1 ||
338 338 lgrp2->lgrp_childcnt < 1)
339 339 return (0);
340 340
341 341 #ifdef DEBUG
342 342 if (lgrp_topo_debug > 0)
343 343 prom_printf("lgrp_consolidate(0x%p[%d],0x%p[%d],0x%p)\n",
344 344 (void *)lgrp1, lgrp1->lgrp_id, (void *)lgrp2,
345 345 lgrp2->lgrp_id, (void *)changed);
346 346 #endif /* DEBUG */
347 347
348 348 count = 0;
349 349 if (changed)
350 350 klgrpset_clear(*changed);
351 351
352 352 /*
353 353 * Lgroup represents resources within certain latency, so need to keep
354 354 * biggest latency value of lgroups being consolidated
355 355 */
356 356 if (lgrp1->lgrp_latency > lgrp2->lgrp_latency)
357 357 lgrp2->lgrp_latency = lgrp1->lgrp_latency;
358 358
359 359 /*
360 360 * Delete ancestors of lgrp1 that don't have any other children
361 361 */
362 362 #ifdef DEBUG
363 363 if (lgrp_topo_debug > 1)
364 364 prom_printf("lgrp_consolidate: delete ancestors\n");
365 365 #endif /* DEBUG */
366 366 count += lgrp_ancestor_delete(lgrp1, &changes);
367 367 if (changed) {
368 368 klgrpset_or(*changed, changes);
369 369 klgrpset_or(*changed, lgrp1->lgrp_id);
370 370 count++;
371 371 }
372 372
373 373 /*
374 374 * Reparent children lgroups of lgrp1 to lgrp2
375 375 */
376 376 for (i = 0; i <= lgrp_alloc_max; i++) {
377 377 if (i == lgrp2->lgrp_id ||
378 378 !klgrpset_ismember(lgrp1->lgrp_children, i))
379 379 continue;
380 380 child = lgrp_table[i];
381 381 if (!LGRP_EXISTS(child))
382 382 continue;
383 383 #ifdef DEBUG
384 384 if (lgrp_topo_debug > 0)
385 385 prom_printf("lgrp_consolidate: reparent "
386 386 "lgrp %d to lgrp %d\n",
387 387 child->lgrp_id, lgrp2->lgrp_id);
388 388 #endif /* DEBUG */
389 389 klgrpset_or(lgrp2->lgrp_leaves, child->lgrp_leaves);
390 390 klgrpset_add(lgrp2->lgrp_children, child->lgrp_id);
391 391 lgrp2->lgrp_childcnt++;
392 392 child->lgrp_parent = lgrp2;
393 393 if (changed) {
394 394 klgrpset_add(*changed, child->lgrp_id);
395 395 klgrpset_add(*changed, lgrp2->lgrp_id);
396 396 }
397 397 count += 2;
398 398 }
399 399
400 400 /*
401 401 * Proprogate leaves from lgrp2 to root
402 402 */
403 403 child = lgrp2;
404 404 parent = child->lgrp_parent;
405 405 while (parent != NULL) {
406 406 klgrpset_or(parent->lgrp_leaves, child->lgrp_leaves);
407 407 if (changed)
408 408 klgrpset_add(*changed, parent->lgrp_id);
409 409 count++;
410 410 child = parent;
411 411 parent = parent->lgrp_parent;
412 412 }
413 413
414 414 #ifdef DEBUG
415 415 if (lgrp_topo_debug > 0)
416 416 prom_printf("lgrp_consolidate: destroy lgrp %d at 0x%p\n",
417 417 lgrp1->lgrp_id, (void *)lgrp1);
418 418 if (lgrp_topo_debug > 1 && changed)
419 419 prom_printf("lgrp_consolidate: changed %d lgrps: 0x%llx\n",
420 420 count, (u_longlong_t)*changed);
421 421 #endif /* DEBUG */
422 422
423 423 lgrp_destroy(lgrp1);
424 424
425 425 return (count);
426 426 }
427 427
428 428 /*
429 429 * Collapse duplicates of target lgroups given
430 430 */
431 431 int
432 432 lgrp_collapse_dups(klgrpset_t target_set, int equidist_only,
433 433 klgrpset_t *changed)
434 434 {
435 435 klgrpset_t changes;
436 436 int count;
437 437 int i;
438 438
439 439 count = 0;
440 440 if (changed)
441 441 klgrpset_clear(*changed);
442 442
443 443 if (lgrp_collapse_off)
444 444 return (0);
445 445
446 446 #ifdef DEBUG
447 447 if (lgrp_topo_debug > 0)
448 448 prom_printf("lgrp_collapse_dups(0x%llx)\n",
449 449 (u_longlong_t)target_set);
450 450 #endif /* DEBUG */
451 451
452 452 /*
453 453 * Look for duplicates of each target lgroup
454 454 */
455 455 for (i = 0; i <= lgrp_alloc_max; i++) {
456 456 int j;
457 457 lgrp_t *keep;
458 458 lgrp_t *target;
459 459
460 460 target = lgrp_table[i];
461 461
462 462 /*
463 463 * Skip to next lgroup if there isn't one here, this is root
464 464 * or leaf lgroup, or this isn't a target lgroup
465 465 */
466 466 if (!LGRP_EXISTS(target) ||
467 467 target == lgrp_root || target->lgrp_childcnt == 0 ||
468 468 !klgrpset_ismember(target_set, target->lgrp_id))
469 469 continue;
470 470
471 471 /*
472 472 * Find all lgroups with same resources and latency
473 473 */
474 474 #ifdef DEBUG
475 475 if (lgrp_topo_debug > 1)
476 476 prom_printf("lgrp_collapse_dups: find "
477 477 "dups of lgrp %d at 0x%p\n",
478 478 target->lgrp_id, (void *)target);
479 479 #endif /* DEBUG */
480 480 keep = NULL;
481 481 for (j = 0; j <= lgrp_alloc_max; j++) {
482 482 lgrp_t *lgrp;
483 483
484 484 lgrp = lgrp_table[j];
485 485
486 486 /*
487 487 * Skip lgroup if there isn't one here, this is root
488 488 * lgroup or leaf (which shouldn't have dups), or this
489 489 * lgroup doesn't have same resources
490 490 */
491 491 if (!LGRP_EXISTS(lgrp) ||
492 492 lgrp->lgrp_childcnt == 0 ||
493 493 !lgrp_rsets_equal(lgrp->lgrp_set,
494 494 target->lgrp_set) ||
495 495 (lgrp->lgrp_latency != target->lgrp_latency &&
496 496 equidist_only))
497 497 continue;
498 498
499 499 /*
500 500 * Keep first matching lgroup (but always keep root)
501 501 * and consolidate other duplicates into it
502 502 */
503 503 if (keep == NULL) {
504 504 keep = lgrp;
505 505 #ifdef DEBUG
506 506 if (lgrp_topo_debug > 1)
507 507 prom_printf("lgrp_collapse_dups: "
508 508 "keep lgrp %d at 0x%p\n",
509 509 keep->lgrp_id, (void *)keep);
510 510 #endif /* DEBUG */
511 511 } else {
512 512 if (lgrp == lgrp_root) {
513 513 lgrp = keep;
514 514 keep = lgrp_root;
515 515 }
516 516 #ifdef DEBUG
517 517 if (lgrp_topo_debug > 0)
518 518 prom_printf("lgrp_collapse_dups:"
519 519 " consolidate lgrp %d at 0x%p"
520 520 " into lgrp %d at 0x%p\n",
521 521 lgrp->lgrp_id, (void *)lgrp,
522 522 keep->lgrp_id, (void *)keep);
523 523 #endif /* DEBUG */
524 524 count += lgrp_consolidate(lgrp, keep,
525 525 &changes);
526 526 if (changed)
527 527 klgrpset_or(*changed, changes);
528 528 }
529 529 }
530 530 }
531 531
532 532 #ifdef DEBUG
533 533 if (lgrp_topo_debug > 1 && changed)
534 534 prom_printf("lgrp_collapse_dups: changed %d lgrps: 0x%llx\n",
535 535 count, (u_longlong_t)*changed);
536 536 #endif /* DEBUG */
537 537
538 538 return (count);
539 539 }
540 540
541 541
542 542 /*
543 543 * Create new parent lgroup with given latency and resources for
544 544 * specified child lgroup, and insert it into hierarchy
545 545 */
546 546 int
547 547 lgrp_new_parent(lgrp_t *child, int latency, klgrpset_t *rset,
548 548 klgrpset_t *changed)
549 549 {
550 550 int count;
551 551 lgrp_t *new;
552 552 lgrp_t *old;
553 553
554 554 count = 0;
555 555 if (changed)
556 556 klgrpset_clear(*changed);
557 557
558 558 /*
559 559 * Create lgroup and set its latency and resources
560 560 */
561 561 new = lgrp_create();
562 562 new->lgrp_latency = latency;
563 563 lgrp_rsets_add(rset, new->lgrp_set);
564 564
565 565 /*
566 566 * Insert new lgroup into hierarchy
567 567 */
568 568 old = child->lgrp_parent;
569 569 new->lgrp_parent = old;
570 570 klgrpset_add(new->lgrp_children, child->lgrp_id);
571 571 new->lgrp_childcnt++;
572 572 klgrpset_add(new->lgrp_children, child->lgrp_id);
573 573 klgrpset_copy(new->lgrp_leaves, child->lgrp_leaves);
574 574
575 575 child->lgrp_parent = new;
576 576 if (old) {
577 577 klgrpset_del(old->lgrp_children, child->lgrp_id);
578 578 klgrpset_add(old->lgrp_children, new->lgrp_id);
579 579 if (changed)
580 580 klgrpset_add(*changed, old->lgrp_id);
581 581 count++;
582 582 }
583 583
584 584 if (changed) {
585 585 klgrpset_add(*changed, child->lgrp_id);
586 586 klgrpset_add(*changed, new->lgrp_id);
587 587 }
588 588 count += 2;
589 589
590 590 #ifdef DEBUG
591 591 if (lgrp_topo_debug > 1 && changed)
592 592 prom_printf("lgrp_new_parent: changed %d lgrps: 0x%llx\n",
593 593 count, (u_longlong_t)*changed);
594 594 #endif /* DEBUG */
595 595
596 596 return (count);
597 597 }
598 598
599 599
600 600 /*
601 601 * Proprogate resources of new leaf into parent lgroup of given child
602 602 */
603 603 int
604 604 lgrp_proprogate(lgrp_t *newleaf, lgrp_t *child, int latency,
605 605 klgrpset_t *changed)
606 606 {
607 607 int count;
608 608 lgrp_t *parent;
609 609
610 610 count = 0;
611 611 if (changed)
612 612 klgrpset_clear(*changed);
613 613
614 614 if (child == NULL || child->lgrp_parent == NULL)
615 615 return (0);
616 616
617 617 parent = child->lgrp_parent;
618 618 klgrpset_or(parent->lgrp_leaves, child->lgrp_leaves);
619 619 if (changed)
620 620 klgrpset_add(*changed, parent->lgrp_id);
621 621 count++;
622 622
623 623 /*
624 624 * Don't proprogate new leaf resources to parent if it already
625 625 * contains these resources
626 626 */
627 627 if (lgrp_rsets_member_all(parent->lgrp_set, newleaf->lgrp_id)) {
628 628 #ifdef DEBUG
629 629 if (lgrp_topo_debug > 1 && changed)
630 630 prom_printf("lgrp_proprogate: changed %d lgrps:"
631 631 " 0x%llx\n",
632 632 count, (u_longlong_t)*changed);
633 633 #endif /* DEBUG */
634 634 return (count);
635 635 }
636 636
637 637 /*
638 638 * Add leaf resources to parent lgroup
639 639 */
640 640 lgrp_rsets_add(newleaf->lgrp_set, parent->lgrp_set);
641 641
642 642 #ifdef DEBUG
643 643 if (lgrp_topo_debug > 1) {
644 644 prom_printf("lgrp_proprogate: newleaf %d(0x%p), "
645 645 "latency %d, child %d(0x%p), parent %d(0x%p)\n",
646 646 newleaf->lgrp_id, (void *)newleaf, latency, child->lgrp_id,
647 647 (void *)child, parent->lgrp_id, (void *)parent);
648 648 prom_printf("lgrp_proprogate: parent's leaves becomes 0x%llx\n",
649 649 (u_longlong_t)parent->lgrp_leaves);
650 650 }
651 651 if (lgrp_topo_debug > 0) {
652 652 prom_printf("lgrp_proprogate: adding to parent %d (0x%p)\n",
653 653 parent->lgrp_id, (void *)parent);
654 654 lgrp_rsets_print("parent resources become:", parent->lgrp_set);
655 655 }
656 656
657 657 if (lgrp_topo_debug > 2 && changed)
658 658 prom_printf("lgrp_proprogate: changed %d lgrps: 0x%llx\n",
659 659 count, (u_longlong_t)*changed);
660 660
661 661 #endif /* DEBUG */
662 662
663 663 return (count);
664 664 }
665 665
666 666
667 667 /*
668 668 * Split parent lgroup of given child if child's leaf decendant (oldleaf) has
669 669 * different latency to new leaf lgroup (newleaf) than leaf lgroups of given
670 670 * child's siblings
671 671 */
672 672 int
673 673 lgrp_split(lgrp_t *oldleaf, lgrp_t *newleaf, lgrp_t *child,
674 674 klgrpset_t *changed)
675 675 {
676 676 klgrpset_t changes;
677 677 int count;
678 678 int i;
679 679 int latency;
680 680 lgrp_t *parent;
681 681
682 682 count = 0;
683 683 if (changed)
684 684 klgrpset_clear(*changed);
685 685
686 686 if (lgrp_split_off || newleaf == NULL || child == NULL)
687 687 return (0);
688 688
689 689 /*
690 690 * Parent must have more than one child to have a child split from it
691 691 * and root lgroup contains all resources and never needs to be split
692 692 */
693 693 parent = child->lgrp_parent;
694 694 if (parent == NULL || parent->lgrp_childcnt < 2 || parent == lgrp_root)
695 695 return (0);
696 696
697 697 #ifdef DEBUG
698 698 if (lgrp_topo_debug > 1)
699 699 prom_printf("lgrp_split(0x%p[%d],0x%p[%d],0x%p[%d],0x%p)\n",
700 700 (void *)oldleaf, oldleaf->lgrp_id,
701 701 (void *)newleaf, newleaf->lgrp_id,
702 702 (void *)child, child->lgrp_id, (void *)changed);
703 703 #endif /* DEBUG */
704 704
705 705 /*
706 706 * Get latency between new leaf and old leaf whose lineage it is
707 707 * being added
708 708 */
709 709 latency = lgrp_plat_latency(oldleaf->lgrp_plathand,
710 710 newleaf->lgrp_plathand);
711 711
712 712 /*
713 713 * Check whether all sibling leaves of given child lgroup have same
714 714 * latency to new leaf
715 715 */
716 716 for (i = 0; i <= lgrp_alloc_max; i++) {
717 717 lgrp_t *grandparent;
718 718 lgrp_t *lgrp;
719 719 int sibling_latency;
720 720
721 721 lgrp = lgrp_table[i];
722 722
723 723 /*
724 724 * Skip non-existent lgroups, old leaf, and any lgroups that
725 725 * don't have parent as common ancestor
726 726 */
727 727 if (!LGRP_EXISTS(lgrp) || lgrp == oldleaf ||
728 728 !klgrpset_ismember(parent->lgrp_leaves, lgrp->lgrp_id))
729 729 continue;
730 730
731 731 /*
732 732 * Same latency, so skip
733 733 */
734 734 sibling_latency = lgrp_plat_latency(lgrp->lgrp_plathand,
735 735 newleaf->lgrp_plathand);
736 736 #ifdef DEBUG
737 737 if (lgrp_topo_debug > 1)
738 738 prom_printf("lgrp_split: latency(%d,%d) %d,"
739 739 " latency(%d,%d) %d\n",
740 740 oldleaf->lgrp_id, newleaf->lgrp_id, latency,
741 741 lgrp->lgrp_id, newleaf->lgrp_id, sibling_latency);
742 742 #endif /* DEBUG */
743 743 if (sibling_latency == latency)
744 744 continue;
745 745
746 746 /*
747 747 * Different latencies, so remove child from its parent and
748 748 * make new parent for old leaf with same latency and same
749 749 * resources
750 750 */
751 751 parent->lgrp_childcnt--;
752 752 klgrpset_del(parent->lgrp_children, child->lgrp_id);
753 753 klgrpset_del(parent->lgrp_leaves, oldleaf->lgrp_id);
754 754 grandparent = parent->lgrp_parent;
755 755 if (grandparent) {
756 756 grandparent->lgrp_childcnt++;
757 757 klgrpset_add(grandparent->lgrp_children,
758 758 child->lgrp_id);
759 759 count++;
760 760 if (changed)
761 761 klgrpset_add(*changed, grandparent->lgrp_id);
762 762 }
763 763 child->lgrp_parent = grandparent;
764 764
765 765 count += lgrp_new_parent(child, parent->lgrp_latency,
766 766 parent->lgrp_set, &changes);
767 767 if (changed) {
768 768 klgrpset_or(*changed, changes);
769 769
770 770 klgrpset_add(*changed, parent->lgrp_id);
771 771 klgrpset_add(*changed, child->lgrp_id);
772 772 count += 2;
773 773 }
774 774
775 775 parent = child->lgrp_parent;
776 776 #ifdef DEBUG
777 777 if (lgrp_topo_debug > 0) {
778 778 prom_printf("lgrp_split: new parent %d (0x%p) for"
779 779 " lgrp %d (0x%p)\n",
780 780 parent->lgrp_id, (void *)parent,
781 781 child->lgrp_id, (void *)child);
782 782 lgrp_rsets_print("new parent resources:",
783 783 parent->lgrp_set);
784 784 }
785 785
786 786 if (lgrp_topo_debug > 1 && changed)
787 787 prom_printf("lgrp_split: changed %d lgrps: 0x%llx\n",
788 788 count, (u_longlong_t)*changed);
789 789 #endif /* DEBUG */
790 790
791 791 return (count);
792 792 }
793 793
794 794 #ifdef DEBUG
795 795 if (lgrp_topo_debug > 1)
796 796 prom_printf("lgrp_split: no changes\n");
797 797 #endif /* DEBUG */
798 798
799 799 return (count);
800 800 }
801 801
802 802
803 803 /*
804 804 * Return height of lgroup topology from given lgroup to root
805 805 */
806 806 int
807 807 lgrp_topo_height(lgrp_t *lgrp)
808 808 {
809 809 int nlevels;
810 810
811 811 if (!LGRP_EXISTS(lgrp))
812 812 return (0);
813 813
814 814 nlevels = 0;
815 815 while (lgrp != NULL) {
816 816 lgrp = lgrp->lgrp_parent;
817 817 nlevels++;
818 818 }
819 819 return (nlevels);
820 820 }
821 821
822 822
823 823 /*
824 824 * Add resources of new leaf to old leaf's lineage
825 825 *
826 826 * Assumes the following:
827 827 * - Lgroup hierarchy consists of at least a root lgroup and its leaves
828 828 * including old and new ones given below
829 829 * - New leaf lgroup has been created and does not need to have its resources
830 830 * added to it
831 831 * - Latencies have been set for root and leaf lgroups
832 832 */
833 833 int
834 834 lgrp_lineage_add(lgrp_t *newleaf, lgrp_t *oldleaf, klgrpset_t *changed)
835 835 {
836 836 klgrpset_t changes;
837 837 lgrp_t *child;
838 838 klgrpset_t collapse;
839 839 int count;
840 840 int latency;
841 841 int nlevels;
842 842 lgrp_t *parent;
843 843 int proprogate;
844 844 int total;
845 845
846 846
847 847 count = total = 0;
848 848 if (changed)
849 849 klgrpset_clear(*changed);
850 850
851 851 if (newleaf == NULL || oldleaf == NULL || newleaf == oldleaf)
852 852 return (0);
853 853
854 854 #ifdef DEBUG
855 855 if (lgrp_topo_debug > 0)
856 856 prom_printf("\nlgrp_lineage_add(0x%p[%d],0x%p[%d],0x%p)\n",
857 857 (void *)newleaf, newleaf->lgrp_id,
858 858 (void *)oldleaf, oldleaf->lgrp_id,
859 859 (void *)changed);
860 860 #endif /* DEBUG */
861 861
862 862 /*
863 863 * Get latency between old and new leaves, so we can determine
864 864 * where the new leaf fits in the old leaf's lineage
865 865 */
866 866 latency = lgrp_plat_latency(oldleaf->lgrp_plathand,
867 867 newleaf->lgrp_plathand);
868 868
869 869 /*
870 870 * Determine height of lgroup topology from old leaf to root lgroup,
871 871 * so height of topology may be limited if necessary
872 872 */
873 873 nlevels = lgrp_topo_height(oldleaf);
874 874
875 875 #ifdef DEBUG
876 876 if (lgrp_topo_debug > 1)
877 877 prom_printf("lgrp_lineage_add: latency(%d,%d) 0x%x, ht %d\n",
878 878 oldleaf->lgrp_id, newleaf->lgrp_id, latency, nlevels);
879 879 #endif /* DEBUG */
880 880
881 881 /*
882 882 * Can't add new leaf to old leaf's lineage if we haven't
883 883 * determined latency between them yet
884 884 */
885 885 if (latency == 0)
886 886 return (0);
887 887
888 888 child = oldleaf;
889 889 parent = child->lgrp_parent;
890 890 proprogate = 0;
891 891 klgrpset_clear(collapse);
892 892
893 893 /*
894 894 * Lineage of old leaf is basically a sorted list of the other leaves
895 895 * from closest to farthest, so find where to add new leaf to the
896 896 * lineage and proprogate its resources from that point up to the root
897 897 * lgroup since parent lgroups contain all the resources of their
898 898 * children
899 899 */
900 900 do {
901 901 klgrpset_t rset[LGRP_RSRC_COUNT];
902 902
903 903 #ifdef DEBUG
904 904 if (lgrp_topo_debug > 1)
905 905 prom_printf("lgrp_lineage_add: child %d (0x%p), parent"
906 906 " %d (0x%p)\n",
907 907 child->lgrp_id, (void *)child,
908 908 parent->lgrp_id, (void *)parent);
909 909 #endif /* DEBUG */
910 910
911 911 /*
912 912 * See whether parent lgroup needs to be split
913 913 *
914 914 * May need to split parent lgroup when it is ancestor to more
915 915 * than one leaf, but all its leaves don't have latency to new
916 916 * leaf within the parent lgroup's latency
917 917 * NOTE: Don't want to collapse this lgroup since we just split
918 918 * it from parent
919 919 */
920 920 count = lgrp_split(oldleaf, newleaf, child, &changes);
921 921 if (count) {
922 922 #ifdef DEBUG
923 923 if (lgrp_topo_debug > 0)
924 924 prom_printf("lgrp_lineage_add: setting parent"
925 925 " for child %d from %d to %d\n",
926 926 child->lgrp_id, parent->lgrp_id,
927 927 child->lgrp_parent->lgrp_id);
928 928 #endif /* DEBUG */
929 929 parent = child->lgrp_parent;
930 930 total += count;
931 931 if (changed)
932 932 klgrpset_or(*changed, changes);
933 933 }
934 934
935 935 /*
936 936 * Already found where resources of new leaf belong in old
937 937 * leaf's lineage, so proprogate resources of new leaf up
938 938 * through rest of ancestors
939 939 */
940 940 if (proprogate) {
941 941 total += lgrp_proprogate(newleaf, child, latency,
942 942 &changes);
943 943 if (changed)
944 944 klgrpset_or(*changed, changes);
945 945
946 946 parent = child->lgrp_parent;
947 947 klgrpset_add(collapse, parent->lgrp_id);
948 948 child = parent;
949 949 parent = parent->lgrp_parent;
950 950 continue;
951 951 }
952 952
953 953 #ifdef DEBUG
954 954 if (lgrp_topo_debug > 1)
955 955 prom_printf("lgrp_lineage_add: latency 0x%x,"
956 956 " parent latency 0x%x\n",
957 957 latency, parent->lgrp_latency);
958 958 #endif /* DEBUG */
959 959 /*
960 960 * As we work our way from the old leaf to the root lgroup,
961 961 * new leaf resources should go in between two lgroups or into
962 962 * one of the parent lgroups somewhere along the line
963 963 */
964 964 if (latency < parent->lgrp_latency) {
965 965 lgrp_t *intermed;
966 966
967 967 /*
968 968 * New leaf resources should go in between current
969 969 * child and parent
970 970 */
971 971 #ifdef DEBUG
972 972 if (lgrp_topo_debug > 0)
973 973 prom_printf("lgrp_lineage_add: "
974 974 "latency < parent latency\n");
975 975 #endif /* DEBUG */
976 976
977 977 /*
978 978 * Create lgroup with desired resources and insert it
979 979 * between child and parent
980 980 */
981 981 lgrp_rsets_copy(child->lgrp_set, rset);
982 982 lgrp_rsets_add(newleaf->lgrp_set, rset);
983 983 if (nlevels >= lgrp_topo_levels) {
984 984
985 985 #ifdef DEBUG
986 986 if (lgrp_topo_debug > 0) {
987 987 prom_printf("lgrp_lineage_add: nlevels "
988 988 "%d > lgrp_topo_levels %d\n",
989 989 nlevels, lgrp_topo_levels);
990 990 lgrp_rsets_print("rset ", rset);
991 991 }
992 992 #endif /* DEBUG */
993 993
994 994 if (parent == lgrp_root) {
995 995 /*
996 996 * Don't proprogate new leaf resources
997 997 * to parent, if it already contains
998 998 * these resources
999 999 */
1000 1000 if (lgrp_rsets_member_all(
1001 1001 parent->lgrp_set, newleaf->lgrp_id))
1002 1002 break;
1003 1003
1004 1004 total += lgrp_proprogate(newleaf, child,
1005 1005 latency, &changes);
1006 1006 break;
1007 1007 }
1008 1008
1009 1009 #ifdef DEBUG
1010 1010 if (lgrp_topo_debug > 0) {
1011 1011 prom_printf("lgrp_lineage_add: "
1012 1012 "replaced parent lgrp %d at 0x%p"
1013 1013 " for lgrp %d\n",
1014 1014 parent->lgrp_id, (void *)parent,
1015 1015 child->lgrp_id);
1016 1016 lgrp_rsets_print("old parent"
1017 1017 " resources:", parent->lgrp_set);
1018 1018 lgrp_rsets_print("new parent "
1019 1019 "resources:", rset);
1020 1020 }
1021 1021 #endif /* DEBUG */
1022 1022 /*
1023 1023 * Replace contents of parent with new
1024 1024 * leaf + child resources since new leaf is
1025 1025 * closer and shift its parent's resources to
1026 1026 * its parent, etc. until root lgroup reached
1027 1027 */
1028 1028 lgrp_rsets_replace(rset, latency, parent, 1);
1029 1029 if (*changed)
1030 1030 klgrpset_or(*changed, parent->lgrp_id);
1031 1031 total++;
1032 1032 proprogate++;
1033 1033 } else {
1034 1034
1035 1035 #ifdef DEBUG
1036 1036 if (lgrp_topo_debug > 0) {
1037 1037 prom_printf("lgrp_lineage_add: "
1038 1038 "lgrp_new_parent(0x%p,%d)\n",
1039 1039 (void *)child, latency);
1040 1040 lgrp_rsets_print("rset ", rset);
1041 1041 }
1042 1042 #endif /* DEBUG */
1043 1043
1044 1044 total += lgrp_new_parent(child, latency, rset,
1045 1045 &changes);
1046 1046 intermed = child->lgrp_parent;
1047 1047 klgrpset_add(collapse, intermed->lgrp_id);
1048 1048 if (changed)
1049 1049 klgrpset_or(*changed, changes);
1050 1050 child = intermed;
1051 1051 proprogate++;
1052 1052 #ifdef DEBUG
1053 1053 if (lgrp_topo_debug > 0) {
1054 1054 prom_printf("lgrp_lineage_add: new "
1055 1055 "parent lgrp %d at 0x%p for "
1056 1056 "lgrp %d\n", intermed->lgrp_id,
1057 1057 (void *)intermed, child->lgrp_id);
1058 1058 lgrp_rsets_print("new parent "
1059 1059 "resources:", rset);
1060 1060 }
1061 1061 #endif /* DEBUG */
1062 1062 continue;
1063 1063 }
1064 1064
1065 1065 } else if (latency == parent->lgrp_latency) {
1066 1066 /*
1067 1067 * New leaf resources should go into parent
1068 1068 */
1069 1069 #ifdef DEBUG
1070 1070 if (lgrp_topo_debug > 0)
1071 1071 prom_printf("lgrp_lineage_add: latency == "
1072 1072 "parent latency\n");
1073 1073 #endif /* DEBUG */
1074 1074
1075 1075 /*
1076 1076 * It's already there, so don't need to do anything.
1077 1077 */
1078 1078 if (lgrp_rsets_member_all(parent->lgrp_set,
1079 1079 newleaf->lgrp_id))
1080 1080 break;
1081 1081
1082 1082 total += lgrp_proprogate(newleaf, child, latency,
1083 1083 &changes);
1084 1084 parent = child->lgrp_parent;
1085 1085 klgrpset_add(collapse, parent->lgrp_id);
1086 1086 if (changed)
1087 1087 klgrpset_or(*changed, changes);
1088 1088
1089 1089 proprogate++;
1090 1090 }
1091 1091
1092 1092 child = parent;
1093 1093 parent = parent->lgrp_parent;
1094 1094 } while (parent != NULL);
1095 1095
1096 1096 /*
1097 1097 * Consolidate any duplicate lgroups of ones just changed
1098 1098 * Assume that there were no duplicates before last round of changes
1099 1099 */
1100 1100 #ifdef DEBUG
1101 1101 if (lgrp_topo_debug > 1)
1102 1102 prom_printf("lgrp_lineage_add: collapsing dups....\n");
1103 1103 #endif /* DEBUG */
1104 1104
1105 1105 total += lgrp_collapse_dups(collapse, lgrp_collapse_equidist,
1106 1106 &changes);
1107 1107 if (changed)
1108 1108 klgrpset_or(*changed, changes);
1109 1109
1110 1110 #ifdef DEBUG
1111 1111 if (lgrp_topo_debug > 1 && changed)
1112 1112 prom_printf("lgrp_lineage_add: changed %d lgrps: 0x%llx\n",
1113 1113 total, (u_longlong_t)*changed);
1114 1114 #endif /* DEBUG */
1115 1115
1116 1116 return (total);
1117 1117 }
1118 1118
1119 1119
1120 1120 /*
1121 1121 * Add leaf lgroup to lgroup topology
1122 1122 */
1123 1123 int
1124 1124 lgrp_leaf_add(lgrp_t *leaf, lgrp_t **lgrps, int lgrp_count,
1125 1125 klgrpset_t *changed)
1126 1126 {
1127 1127 klgrpset_t changes;
1128 1128 int count;
1129 1129 int i;
1130 1130 int latency;
1131 1131
1132 1132 ASSERT(MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0 ||
1133 1133 !lgrp_initialized);
1134 1134
1135 1135 #ifdef DEBUG
1136 1136 if (lgrp_topo_debug > 1)
1137 1137 prom_printf("\nlgrp_leaf_add(0x%p[%d],0x%p,%d,0x%p)\n",
1138 1138 (void *)leaf, leaf->lgrp_id, (void *)lgrps, lgrp_count,
1139 1139 (void *)changed);
1140 1140 #endif /* DEBUG */
1141 1141
1142 1142 count = 0;
1143 1143 if (changed)
1144 1144 klgrpset_clear(*changed);
1145 1145
1146 1146 /*
1147 1147 * Initialize parent of leaf lgroup to root
1148 1148 */
1149 1149 if (leaf->lgrp_parent == NULL) {
1150 1150 leaf->lgrp_parent = lgrp_root;
1151 1151 lgrp_root->lgrp_childcnt++;
1152 1152 klgrpset_add(lgrp_root->lgrp_children, leaf->lgrp_id);
1153 1153
1154 1154 klgrpset_or(lgrp_root->lgrp_leaves, leaf->lgrp_leaves);
1155 1155 lgrp_rsets_add(leaf->lgrp_set, lgrp_root->lgrp_set);
1156 1156
1157 1157 #ifdef DEBUG
1158 1158 if (lgrp_topo_debug > 1)
1159 1159 lgrp_rsets_print("lgrp_leaf_add: root lgrp resources",
1160 1160 lgrp_root->lgrp_set);
1161 1161 #endif /* DEBUG */
1162 1162
1163 1163 if (changed) {
1164 1164 klgrpset_add(*changed, lgrp_root->lgrp_id);
1165 1165 klgrpset_add(*changed, leaf->lgrp_id);
1166 1166 }
1167 1167 count += 2;
1168 1168 }
1169 1169
1170 1170 /*
1171 1171 * Can't add leaf lgroup to rest of topology (and vice versa) unless
1172 1172 * latency for it is available
1173 1173 */
1174 1174 latency = lgrp_plat_latency(leaf->lgrp_plathand, leaf->lgrp_plathand);
1175 1175 if (latency == 0) {
1176 1176 #ifdef DEBUG
1177 1177 if (lgrp_topo_debug > 1 && changed)
1178 1178 prom_printf("lgrp_leaf_add: changed %d lgrps: 0x%llx\n",
1179 1179 count, (u_longlong_t)*changed);
1180 1180 #endif /* DEBUG */
1181 1181 return (count);
1182 1182 }
1183 1183
1184 1184 /*
1185 1185 * Make sure that root and leaf lgroup latencies are set
1186 1186 */
1187 1187 lgrp_root->lgrp_latency = lgrp_plat_latency(lgrp_root->lgrp_plathand,
1188 1188 lgrp_root->lgrp_plathand);
1189 1189 leaf->lgrp_latency = latency;
1190 1190
1191 1191 /*
1192 1192 * Add leaf to lineage of other leaves and vice versa
1193 1193 * since leaves come into existence at different times
1194 1194 */
1195 1195 for (i = 0; i < lgrp_count; i++) {
1196 1196 lgrp_t *lgrp;
1197 1197
1198 1198 lgrp = lgrps[i];
1199 1199
1200 1200 /*
1201 1201 * Skip non-existent lgroups, new leaf lgroup, and
1202 1202 * non-leaf lgroups
1203 1203 */
1204 1204 if (!LGRP_EXISTS(lgrp) || lgrp == leaf ||
1205 1205 lgrp->lgrp_childcnt != 0) {
1206 1206 #ifdef DEBUG
1207 1207 if (lgrp_topo_debug > 1)
1208 1208 prom_printf("lgrp_leaf_add: skip "
1209 1209 "lgrp %d at 0x%p\n",
1210 1210 lgrp->lgrp_id, (void *)lgrp);
1211 1211 #endif /* DEBUG */
1212 1212 continue;
1213 1213 }
1214 1214
1215 1215 #ifdef DEBUG
1216 1216 if (lgrp_topo_debug > 0)
1217 1217 prom_printf("lgrp_leaf_add: lgrp %d (0x%p) =>"
1218 1218 " lgrp %d (0x%p)\n",
1219 1219 leaf->lgrp_id, (void *)leaf, lgrp->lgrp_id,
1220 1220 (void *)lgrp);
1221 1221 #endif /* DEBUG */
1222 1222
1223 1223 count += lgrp_lineage_add(leaf, lgrp, &changes);
1224 1224 if (changed)
1225 1225 klgrpset_or(*changed, changes);
1226 1226
1227 1227 count += lgrp_lineage_add(lgrp, leaf, &changes);
1228 1228 if (changed)
1229 1229 klgrpset_or(*changed, changes);
1230 1230 }
1231 1231
1232 1232 #ifdef DEBUG
1233 1233 if (lgrp_topo_debug > 1 && changed)
1234 1234 prom_printf("lgrp_leaf_add: changed %d lgrps: 0x%llx\n",
1235 1235 count, (u_longlong_t)*changed);
1236 1236 #endif /* DEBUG */
1237 1237
1238 1238 return (count);
1239 1239 }
1240 1240
1241 1241
1242 1242 /*
1243 1243 * Remove resources of leaf from lgroup hierarchy
1244 1244 */
1245 1245 int
1246 1246 lgrp_leaf_delete(lgrp_t *leaf, lgrp_t **lgrps, int lgrp_count,
1247 1247 klgrpset_t *changed)
1248 1248 {
1249 1249 klgrpset_t changes;
1250 1250 klgrpset_t collapse;
1251 1251 int count;
1252 1252 int i;
1253 1253 lgrp_t *lgrp;
1254 1254
1255 1255 ASSERT(MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0 ||
1256 1256 !lgrp_initialized);
1257 1257
1258 1258 count = 0;
1259 1259 klgrpset_clear(collapse);
1260 1260 if (changed)
1261 1261 klgrpset_clear(*changed);
1262 1262
1263 1263 /*
1264 1264 * Nothing to do if no leaf given
1265 1265 */
1266 1266 if (leaf == NULL)
1267 1267 return (0);
1268 1268
1269 1269 #ifdef DEBUG
1270 1270 if (lgrp_topo_debug > 0)
1271 1271 prom_printf("lgrp_leaf_delete(0x%p[%d],0x%p,%d,0x%p)\n",
1272 1272 (void *)leaf, leaf->lgrp_id, (void *)lgrps, lgrp_count,
1273 1273 (void *)changed);
1274 1274 #endif /* DEBUG */
1275 1275
1276 1276 /*
1277 1277 * Remove leaf from any lgroups containing its resources
1278 1278 */
1279 1279 for (i = 0; i < lgrp_count; i++) {
1280 1280 lgrp = lgrps[i];
1281 1281 if (lgrp == NULL || lgrp->lgrp_id == LGRP_NONE ||
1282 1282 !lgrp_rsets_member(lgrp->lgrp_set, leaf->lgrp_id))
1283 1283 continue;
1284 1284
1285 1285 #ifdef DEBUG
1286 1286 if (lgrp_topo_debug > 0)
1287 1287 prom_printf("lgrp_leaf_delete: remove leaf from"
1288 1288 " lgrp %d at %p\n", lgrp->lgrp_id, (void *)lgrp);
1289 1289 #endif /* DEBUG */
1290 1290
1291 1291 lgrp_rsets_delete(lgrp, leaf->lgrp_id, 0);
1292 1292 klgrpset_del(lgrp->lgrp_leaves, leaf->lgrp_id);
1293 1293
1294 1294 klgrpset_add(collapse, lgrp->lgrp_id);
1295 1295 count++;
1296 1296 }
1297 1297
1298 1298 /*
1299 1299 * Remove leaf and its ancestors that don't have any other children
1300 1300 */
1301 1301 #ifdef DEBUG
1302 1302 if (lgrp_topo_debug > 1)
1303 1303 prom_printf("lgrp_leaf_delete: remove leaf and ancestors\n");
1304 1304 #endif /* DEBUG */
1305 1305
1306 1306 count += lgrp_ancestor_delete(leaf, &changes);
1307 1307 klgrpset_or(collapse, changes);
1308 1308 klgrpset_add(collapse, leaf->lgrp_id);
1309 1309 count++;
1310 1310 lgrp_destroy(leaf);
1311 1311
1312 1312 /*
1313 1313 * Consolidate any duplicate lgroups of ones just changed
1314 1314 * Assume that there were no duplicates before last round of changes
1315 1315 */
1316 1316 #ifdef DEBUG
1317 1317 if (lgrp_topo_debug > 1)
1318 1318 prom_printf("lgrp_leaf_delete: collapsing dups\n");
1319 1319 #endif /* DEBUG */
1320 1320 count += lgrp_collapse_dups(collapse, lgrp_collapse_equidist,
1321 1321 &changes);
1322 1322 klgrpset_or(collapse, changes);
1323 1323 if (changed)
1324 1324 klgrpset_copy(*changed, collapse);
1325 1325
1326 1326 #ifdef DEBUG
1327 1327 if (lgrp_topo_debug > 1 && changed)
1328 1328 prom_printf("lgrp_leaf_delete: changed %d lgrps: 0x%llx\n",
1329 1329 count, (u_longlong_t)*changed);
1330 1330 #endif /* DEBUG */
1331 1331
1332 1332 return (count);
1333 1333 }
1334 1334
1335 1335
1336 1336 /*
1337 1337 * Flatten lgroup topology down to height specified
1338 1338 */
1339 1339 int
1340 1340 lgrp_topo_flatten(int levels, lgrp_t **lgrps, int lgrp_count,
1341 1341 klgrpset_t *changed)
1342 1342 {
1343 1343 int count;
1344 1344 int i;
1345 1345 lgrp_t *lgrp;
1346 1346 lgrp_handle_t hdl;
1347 1347
1348 1348 /*
1349 1349 * Only flatten down to 2 level for now
1350 1350 */
1351 1351 if (levels != 2)
1352 1352 return (0);
1353 1353
1354 1354 /*
1355 1355 * Look for non-leaf lgroups to remove and leaf lgroups to reparent
1356 1356 */
1357 1357 count = 0;
1358 1358 for (i = 0; i <= lgrp_count; i++) {
1359 1359 /*
1360 1360 * Skip non-existent lgroups and root
1361 1361 */
1362 1362 lgrp = lgrps[i];
1363 1363 if (!LGRP_EXISTS(lgrp))
1364 1364 continue;
1365 1365
1366 1366 hdl = lgrp->lgrp_plathand;
1367 1367
1368 1368 if (lgrp == lgrp_root) {
1369 1369 lgrp->lgrp_latency = lgrp_plat_latency(hdl, hdl);
1370 1370 continue;
1371 1371 }
1372 1372
1373 1373 if (lgrp->lgrp_childcnt > 0) {
1374 1374 lgrp_t *parent;
1375 1375
1376 1376 /*
1377 1377 * Remove non-leaf lgroup from lgroup topology
1378 1378 */
1379 1379 parent = lgrp->lgrp_parent;
1380 1380 if (changed) {
1381 1381 klgrpset_add(*changed, lgrp->lgrp_id);
1382 1382 klgrpset_add(*changed, parent->lgrp_id);
1383 1383 count += 2;
1384 1384 }
1385 1385 if (parent) {
1386 1386 klgrpset_del(parent->lgrp_children,
1387 1387 lgrp->lgrp_id);
1388 1388 parent->lgrp_childcnt--;
1389 1389 }
1390 1390 lgrp_destroy(lgrp);
1391 1391 } else if (lgrp->lgrp_parent != lgrp_root) {
1392 1392 /*
1393 1393 * Reparent leaf lgroup to root
1394 1394 */
1395 1395 if (changed) {
1396 1396 klgrpset_add(*changed, lgrp_root->lgrp_id);
1397 1397 klgrpset_add(*changed, lgrp->lgrp_id);
1398 1398 count += 2;
1399 1399 }
1400 1400 lgrp->lgrp_parent = lgrp_root;
1401 1401 klgrpset_add(lgrp_root->lgrp_children, lgrp->lgrp_id);
1402 1402 lgrp_root->lgrp_childcnt++;
1403 1403 klgrpset_add(lgrp_root->lgrp_leaves, lgrp->lgrp_id);
1404 1404
1405 1405 lgrp->lgrp_latency = lgrp_plat_latency(hdl, hdl);
1406 1406 }
1407 1407 }
1408 1408
1409 1409 return (count);
1410 1410 }
1411 1411
1412 1412
1413 1413 /*
1414 1414 * Return current height limit for lgroup topology
1415 1415 */
1416 1416 int
1417 1417 lgrp_topo_ht_limit(void)
1418 1418 {
1419 1419 return (lgrp_topo_levels);
1420 1420 }
1421 1421
1422 1422
1423 1423 /*
1424 1424 * Return default height limit for lgroup topology
1425 1425 */
1426 1426 int
1427 1427 lgrp_topo_ht_limit_default(void)
1428 1428 {
1429 1429 return (LGRP_TOPO_LEVELS);
1430 1430 }
1431 1431
1432 1432
1433 1433 /*
1434 1434 * Set height limit for lgroup topology
1435 1435 */
1436 1436 int
1437 1437 lgrp_topo_ht_limit_set(int ht)
1438 1438 {
1439 1439 if (ht > LGRP_TOPO_LEVELS_MAX)
1440 1440 lgrp_topo_levels = LGRP_TOPO_LEVELS_MAX;
1441 1441 else
1442 1442 lgrp_topo_levels = ht;
1443 1443
1444 1444 return (ht);
1445 1445 }
1446 1446
1447 1447
1448 1448 /*
1449 1449 * Update lgroup topology for any leaves that don't have their latency set
1450 1450 *
1451 1451 * This may happen on some machines when the lgroup platform support doesn't
1452 1452 * know the latencies between nodes soon enough to provide it when the
1453 1453 * resources are being added. If the lgroup platform code needs to probe
1454 1454 * memory to determine the latencies between nodes, it must wait until the
1455 1455 * CPUs become active so at least one CPU in each node can probe memory in
1456 1456 * each node.
1457 1457 */
1458 1458 int
1459 1459 lgrp_topo_update(lgrp_t **lgrps, int lgrp_count, klgrpset_t *changed)
1460 1460 {
1461 1461 klgrpset_t changes;
1462 1462 int count;
1463 1463 int i;
1464 1464 lgrp_t *lgrp;
1465 1465
1466 1466 count = 0;
1467 1467 if (changed)
1468 1468 klgrpset_clear(*changed);
1469 1469
1470 1470 /*
1471 1471 * For UMA machines, make sure that root lgroup contains all
1472 1472 * resources. The root lgrp should also name itself as its own leaf
|
↓ open down ↓ |
1472 lines elided |
↑ open up ↑ |
1473 1473 */
1474 1474 if (nlgrps == 1) {
1475 1475 for (i = 0; i < LGRP_RSRC_COUNT; i++)
1476 1476 klgrpset_add(lgrp_root->lgrp_set[i],
1477 1477 lgrp_root->lgrp_id);
1478 1478 klgrpset_add(lgrp_root->lgrp_leaves, lgrp_root->lgrp_id);
1479 1479 return (0);
1480 1480 }
1481 1481
1482 1482 mutex_enter(&cpu_lock);
1483 - pause_cpus(NULL);
1483 + pause_cpus(NULL, NULL);
1484 1484
1485 1485 /*
1486 1486 * Look for any leaf lgroup without its latency set, finish adding it
1487 1487 * to the lgroup topology assuming that it exists and has the root
1488 1488 * lgroup as its parent, and update the memory nodes of all lgroups
1489 1489 * that have it as a memory resource.
1490 1490 */
1491 1491 for (i = 0; i < lgrp_count; i++) {
1492 1492 lgrp = lgrps[i];
1493 1493
1494 1494 /*
1495 1495 * Skip non-existent and non-leaf lgroups and any lgroup
1496 1496 * with its latency set already
1497 1497 */
1498 1498 if (lgrp == NULL || lgrp->lgrp_id == LGRP_NONE ||
1499 1499 lgrp->lgrp_childcnt != 0 || lgrp->lgrp_latency != 0)
1500 1500 continue;
1501 1501
1502 1502 #ifdef DEBUG
1503 1503 if (lgrp_topo_debug > 1) {
1504 1504 prom_printf("\nlgrp_topo_update: updating lineage "
1505 1505 "of lgrp %d at 0x%p\n", lgrp->lgrp_id,
1506 1506 (void *)lgrp);
1507 1507 }
1508 1508 #endif /* DEBUG */
1509 1509
1510 1510 count += lgrp_leaf_add(lgrp, lgrps, lgrp_count, &changes);
1511 1511 if (changed)
1512 1512 klgrpset_or(*changed, changes);
1513 1513
1514 1514 if (!klgrpset_isempty(changes))
1515 1515 (void) lgrp_mnode_update(changes, NULL);
1516 1516
1517 1517 #ifdef DEBUG
1518 1518 if (lgrp_topo_debug > 1 && changed)
1519 1519 prom_printf("lgrp_topo_update: changed %d lgrps: "
1520 1520 "0x%llx\n",
1521 1521 count, (u_longlong_t)*changed);
1522 1522 #endif /* DEBUG */
1523 1523 }
1524 1524
1525 1525 if (lgrp_topo_levels < LGRP_TOPO_LEVELS && lgrp_topo_levels == 2) {
1526 1526 count += lgrp_topo_flatten(2, lgrps, lgrp_count, changed);
1527 1527 (void) lpl_topo_flatten(2);
1528 1528 }
1529 1529
1530 1530 start_cpus();
1531 1531 mutex_exit(&cpu_lock);
1532 1532
1533 1533 return (count);
1534 1534 }
1535 1535
1536 1536 #ifdef DEBUG
1537 1537 void
1538 1538 lgrp_print(lgrp_t *lgrp)
1539 1539 {
1540 1540 lgrp_t *parent;
1541 1541
1542 1542 prom_printf("LGRP %d", lgrp->lgrp_id);
1543 1543 if (lgrp->lgrp_childcnt == 0)
1544 1544 prom_printf(" (plathand %p)\n",
1545 1545 (void *)lgrp->lgrp_plathand);
1546 1546 else
1547 1547 prom_printf("\n");
1548 1548
1549 1549 prom_printf("\tlatency %d\n", lgrp->lgrp_latency);
1550 1550
1551 1551 lgrp_rsets_print("\tresources", lgrp->lgrp_set);
1552 1552
1553 1553 parent = lgrp->lgrp_parent;
1554 1554 prom_printf("\tparent 0x%p", (void *)parent);
1555 1555 if (parent)
1556 1556 prom_printf("[%d]\n", parent->lgrp_id);
1557 1557 else
1558 1558 prom_printf("\n");
1559 1559
1560 1560 prom_printf("\tchild count %d, children ", lgrp->lgrp_childcnt);
1561 1561 klgrpset_print(lgrp->lgrp_children);
1562 1562
1563 1563 prom_printf("\tleaves ");
1564 1564 klgrpset_print(lgrp->lgrp_leaves);
1565 1565 }
1566 1566
1567 1567
1568 1568 void
1569 1569 lgrp_topo_print(lgrp_t **lgrps, int lgrp_max)
1570 1570 {
1571 1571 klgrpset_t siblings;
1572 1572
1573 1573 lgrp_print(lgrp_root);
1574 1574 siblings = lgrp_root->lgrp_children;
1575 1575 while (!klgrpset_isempty(siblings)) {
1576 1576 klgrpset_t children;
1577 1577 int i;
1578 1578
1579 1579 klgrpset_clear(children);
1580 1580 for (i = 0; i <= lgrp_max; i++) {
1581 1581 lgrp_t *lgrp;
1582 1582
1583 1583 lgrp = lgrps[i];
1584 1584 if (lgrp == NULL || !klgrpset_ismember(siblings, i))
1585 1585 continue;
1586 1586 lgrp_print(lgrp);
1587 1587 klgrpset_or(children, lgrp->lgrp_children);
1588 1588 }
1589 1589 klgrpset_copy(siblings, children);
1590 1590 }
1591 1591 }
1592 1592 #endif /* DEBUG */
|
↓ open down ↓ |
99 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX