338 int r;
339 int err;
340 int nchildren;
341
342 ASSERT(MUTEX_HELD(&cpu_lock));
343
344 parent = pg->cmt_parent;
345 if (parent == NULL) {
346 /*
347 * Nothing to do
348 */
349 return;
350 }
351
352 ASSERT(PG_NUM_CPUS((pg_t *)pg) >= PG_NUM_CPUS((pg_t *)parent));
353
354 /*
355 * We're changing around the hierarchy, which is actively traversed
356 * by the dispatcher. Pause CPUS to ensure exclusivity.
357 */
358 pause_cpus(NULL);
359
360 /*
361 * If necessary, update the parent's sibling set, replacing parent
362 * with PG.
363 */
364 if (parent->cmt_siblings) {
365 if (group_remove(parent->cmt_siblings, parent, GRP_NORESIZE)
366 != -1) {
367 r = group_add(parent->cmt_siblings, pg, GRP_NORESIZE);
368 ASSERT(r != -1);
369 }
370 }
371
372 /*
373 * If the parent is at the top of the hierarchy, replace it's entry
374 * in the root lgroup's group of top level PGs.
375 */
376 if (parent->cmt_parent == NULL &&
377 parent->cmt_siblings != &cmt_root->cl_pgs) {
378 if (group_remove(&cmt_root->cl_pgs, parent, GRP_NORESIZE)
1538 group_expand(pg->cmt_siblings,
1539 GROUP_SIZE(pg->cmt_siblings) + cap_needed);
1540
1541 /*
1542 * If this is a top level group, also ensure the
1543 * capacity in the root lgrp level CMT grouping.
1544 */
1545 if (pg->cmt_parent == NULL &&
1546 pg->cmt_siblings != &cmt_root->cl_pgs) {
1547 group_expand(&cmt_root->cl_pgs,
1548 GROUP_SIZE(&cmt_root->cl_pgs) + cap_needed);
1549 cmt_root->cl_npgs += cap_needed;
1550 }
1551 }
1552 }
1553
1554 /*
1555 * We're operating on the PG hierarchy. Pause CPUs to ensure
1556 * exclusivity with respect to the dispatcher.
1557 */
1558 pause_cpus(NULL);
1559
1560 /*
1561 * Prune all PG instances of the hardware sharing relationship
1562 * represented by pg.
1563 */
1564 group_iter_init(&hw_iter);
1565 while ((pg = group_iterate(hwset, &hw_iter)) != NULL) {
1566
1567 /*
1568 * Remove PG from it's group of siblings, if it's there.
1569 */
1570 if (pg->cmt_siblings) {
1571 (void) group_remove(pg->cmt_siblings, pg, GRP_NORESIZE);
1572 }
1573 if (pg->cmt_parent == NULL &&
1574 pg->cmt_siblings != &cmt_root->cl_pgs) {
1575 (void) group_remove(&cmt_root->cl_pgs, pg,
1576 GRP_NORESIZE);
1577 }
1578
1658 * Remove the PG from the CPU's group used for CMT
1659 * scheduling.
1660 */
1661 (void) group_remove(&cpd->cmt_pgs, pg, GRP_NORESIZE);
1662 }
1663 }
1664 start_cpus();
1665 return (0);
1666 }
1667
1668 /*
1669 * Disable CMT scheduling
1670 */
1671 static void
1672 pg_cmt_disable(void)
1673 {
1674 cpu_t *cpu;
1675
1676 ASSERT(MUTEX_HELD(&cpu_lock));
1677
1678 pause_cpus(NULL);
1679 cpu = cpu_list;
1680
1681 do {
1682 if (cpu->cpu_pg)
1683 group_empty(&cpu->cpu_pg->cmt_pgs);
1684 } while ((cpu = cpu->cpu_next) != cpu_list);
1685
1686 cmt_sched_disabled = 1;
1687 start_cpus();
1688 cmn_err(CE_NOTE, "!CMT thread placement optimizations unavailable");
1689 }
1690
1691 /*
1692 * CMT lineage validation
1693 *
1694 * This routine is invoked by pg_cmt_cpu_init() to validate the integrity
1695 * of the PGs in a CPU's lineage. This is necessary because it's possible that
1696 * some groupings (power domain groupings in particular) may be defined by
1697 * sources that are buggy (e.g. BIOS bugs). In such cases, it may not be
1698 * possible to integrate those groupings into the CMT PG hierarchy, if doing
|
338 int r;
339 int err;
340 int nchildren;
341
342 ASSERT(MUTEX_HELD(&cpu_lock));
343
344 parent = pg->cmt_parent;
345 if (parent == NULL) {
346 /*
347 * Nothing to do
348 */
349 return;
350 }
351
352 ASSERT(PG_NUM_CPUS((pg_t *)pg) >= PG_NUM_CPUS((pg_t *)parent));
353
354 /*
355 * We're changing around the hierarchy, which is actively traversed
356 * by the dispatcher. Pause CPUS to ensure exclusivity.
357 */
358 pause_cpus(NULL, NULL);
359
360 /*
361 * If necessary, update the parent's sibling set, replacing parent
362 * with PG.
363 */
364 if (parent->cmt_siblings) {
365 if (group_remove(parent->cmt_siblings, parent, GRP_NORESIZE)
366 != -1) {
367 r = group_add(parent->cmt_siblings, pg, GRP_NORESIZE);
368 ASSERT(r != -1);
369 }
370 }
371
372 /*
373 * If the parent is at the top of the hierarchy, replace it's entry
374 * in the root lgroup's group of top level PGs.
375 */
376 if (parent->cmt_parent == NULL &&
377 parent->cmt_siblings != &cmt_root->cl_pgs) {
378 if (group_remove(&cmt_root->cl_pgs, parent, GRP_NORESIZE)
1538 group_expand(pg->cmt_siblings,
1539 GROUP_SIZE(pg->cmt_siblings) + cap_needed);
1540
1541 /*
1542 * If this is a top level group, also ensure the
1543 * capacity in the root lgrp level CMT grouping.
1544 */
1545 if (pg->cmt_parent == NULL &&
1546 pg->cmt_siblings != &cmt_root->cl_pgs) {
1547 group_expand(&cmt_root->cl_pgs,
1548 GROUP_SIZE(&cmt_root->cl_pgs) + cap_needed);
1549 cmt_root->cl_npgs += cap_needed;
1550 }
1551 }
1552 }
1553
1554 /*
1555 * We're operating on the PG hierarchy. Pause CPUs to ensure
1556 * exclusivity with respect to the dispatcher.
1557 */
1558 pause_cpus(NULL, NULL);
1559
1560 /*
1561 * Prune all PG instances of the hardware sharing relationship
1562 * represented by pg.
1563 */
1564 group_iter_init(&hw_iter);
1565 while ((pg = group_iterate(hwset, &hw_iter)) != NULL) {
1566
1567 /*
1568 * Remove PG from it's group of siblings, if it's there.
1569 */
1570 if (pg->cmt_siblings) {
1571 (void) group_remove(pg->cmt_siblings, pg, GRP_NORESIZE);
1572 }
1573 if (pg->cmt_parent == NULL &&
1574 pg->cmt_siblings != &cmt_root->cl_pgs) {
1575 (void) group_remove(&cmt_root->cl_pgs, pg,
1576 GRP_NORESIZE);
1577 }
1578
1658 * Remove the PG from the CPU's group used for CMT
1659 * scheduling.
1660 */
1661 (void) group_remove(&cpd->cmt_pgs, pg, GRP_NORESIZE);
1662 }
1663 }
1664 start_cpus();
1665 return (0);
1666 }
1667
1668 /*
1669 * Disable CMT scheduling
1670 */
1671 static void
1672 pg_cmt_disable(void)
1673 {
1674 cpu_t *cpu;
1675
1676 ASSERT(MUTEX_HELD(&cpu_lock));
1677
1678 pause_cpus(NULL, NULL);
1679 cpu = cpu_list;
1680
1681 do {
1682 if (cpu->cpu_pg)
1683 group_empty(&cpu->cpu_pg->cmt_pgs);
1684 } while ((cpu = cpu->cpu_next) != cpu_list);
1685
1686 cmt_sched_disabled = 1;
1687 start_cpus();
1688 cmn_err(CE_NOTE, "!CMT thread placement optimizations unavailable");
1689 }
1690
1691 /*
1692 * CMT lineage validation
1693 *
1694 * This routine is invoked by pg_cmt_cpu_init() to validate the integrity
1695 * of the PGs in a CPU's lineage. This is necessary because it's possible that
1696 * some groupings (power domain groupings in particular) may be defined by
1697 * sources that are buggy (e.g. BIOS bugs). In such cases, it may not be
1698 * possible to integrate those groupings into the CMT PG hierarchy, if doing
|