1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright (c) 2012 by Delphix. All rights reserved.
  24  */
  25 
  26 /*
  27  * Architecture-independent CPU control functions.
  28  */
  29 
  30 #include <sys/types.h>
  31 #include <sys/param.h>
  32 #include <sys/var.h>
  33 #include <sys/thread.h>
  34 #include <sys/cpuvar.h>
  35 #include <sys/cpu_event.h>
  36 #include <sys/kstat.h>
  37 #include <sys/uadmin.h>
  38 #include <sys/systm.h>
  39 #include <sys/errno.h>
  40 #include <sys/cmn_err.h>
  41 #include <sys/procset.h>
  42 #include <sys/processor.h>
  43 #include <sys/debug.h>
  44 #include <sys/cpupart.h>
  45 #include <sys/lgrp.h>
  46 #include <sys/pset.h>
  47 #include <sys/pghw.h>
  48 #include <sys/kmem.h>
  49 #include <sys/kmem_impl.h>        /* to set per-cpu kmem_cache offset */
  50 #include <sys/atomic.h>
  51 #include <sys/callb.h>
  52 #include <sys/vtrace.h>
  53 #include <sys/cyclic.h>
  54 #include <sys/bitmap.h>
  55 #include <sys/nvpair.h>
  56 #include <sys/pool_pset.h>
  57 #include <sys/msacct.h>
  58 #include <sys/time.h>
  59 #include <sys/archsystm.h>
  60 #include <sys/sdt.h>
  61 #if defined(__x86) || defined(__amd64)
  62 #include <sys/x86_archext.h>
  63 #endif
  64 #include <sys/callo.h>
  65 
  66 extern int      mp_cpu_start(cpu_t *);
  67 extern int      mp_cpu_stop(cpu_t *);
  68 extern int      mp_cpu_poweron(cpu_t *);
  69 extern int      mp_cpu_poweroff(cpu_t *);
  70 extern int      mp_cpu_configure(int);
  71 extern int      mp_cpu_unconfigure(int);
  72 extern void     mp_cpu_faulted_enter(cpu_t *);
  73 extern void     mp_cpu_faulted_exit(cpu_t *);
  74 
  75 extern int cmp_cpu_to_chip(processorid_t cpuid);
  76 #ifdef __sparcv9
  77 extern char *cpu_fru_fmri(cpu_t *cp);
  78 #endif
  79 
  80 static void cpu_add_active_internal(cpu_t *cp);
  81 static void cpu_remove_active(cpu_t *cp);
  82 static void cpu_info_kstat_create(cpu_t *cp);
  83 static void cpu_info_kstat_destroy(cpu_t *cp);
  84 static void cpu_stats_kstat_create(cpu_t *cp);
  85 static void cpu_stats_kstat_destroy(cpu_t *cp);
  86 
  87 static int cpu_sys_stats_ks_update(kstat_t *ksp, int rw);
  88 static int cpu_vm_stats_ks_update(kstat_t *ksp, int rw);
  89 static int cpu_stat_ks_update(kstat_t *ksp, int rw);
  90 static int cpu_state_change_hooks(int, cpu_setup_t, cpu_setup_t);
  91 
  92 /*
  93  * cpu_lock protects ncpus, ncpus_online, cpu_flag, cpu_list, cpu_active,
  94  * max_cpu_seqid_ever, and dispatch queue reallocations.  The lock ordering with
  95  * respect to related locks is:
  96  *
  97  *      cpu_lock --> thread_free_lock  --->  p_lock  --->  thread_lock()
  98  *
  99  * Warning:  Certain sections of code do not use the cpu_lock when
 100  * traversing the cpu_list (e.g. mutex_vector_enter(), clock()).  Since
 101  * all cpus are paused during modifications to this list, a solution
 102  * to protect the list is too either disable kernel preemption while
 103  * walking the list, *or* recheck the cpu_next pointer at each
 104  * iteration in the loop.  Note that in no cases can any cached
 105  * copies of the cpu pointers be kept as they may become invalid.
 106  */
 107 kmutex_t        cpu_lock;
 108 cpu_t           *cpu_list;              /* list of all CPUs */
 109 cpu_t           *clock_cpu_list;        /* used by clock to walk CPUs */
 110 cpu_t           *cpu_active;            /* list of active CPUs */
 111 static cpuset_t cpu_available;          /* set of available CPUs */
 112 cpuset_t        cpu_seqid_inuse;        /* which cpu_seqids are in use */
 113 
 114 cpu_t           **cpu_seq;              /* ptrs to CPUs, indexed by seq_id */
 115 
 116 /*
 117  * max_ncpus keeps the max cpus the system can have. Initially
 118  * it's NCPU, but since most archs scan the devtree for cpus
 119  * fairly early on during boot, the real max can be known before
 120  * ncpus is set (useful for early NCPU based allocations).
 121  */
 122 int max_ncpus = NCPU;
 123 /*
 124  * platforms that set max_ncpus to maxiumum number of cpus that can be
 125  * dynamically added will set boot_max_ncpus to the number of cpus found
 126  * at device tree scan time during boot.
 127  */
 128 int boot_max_ncpus = -1;
 129 int boot_ncpus = -1;
 130 /*
 131  * Maximum possible CPU id.  This can never be >= NCPU since NCPU is
 132  * used to size arrays that are indexed by CPU id.
 133  */
 134 processorid_t max_cpuid = NCPU - 1;
 135 
 136 /*
 137  * Maximum cpu_seqid was given. This number can only grow and never shrink. It
 138  * can be used to optimize NCPU loops to avoid going through CPUs which were
 139  * never on-line.
 140  */
 141 processorid_t max_cpu_seqid_ever = 0;
 142 
 143 int ncpus = 1;
 144 int ncpus_online = 1;
 145 
 146 /*
 147  * CPU that we're trying to offline.  Protected by cpu_lock.
 148  */
 149 cpu_t *cpu_inmotion;
 150 
 151 /*
 152  * Can be raised to suppress further weakbinding, which are instead
 153  * satisfied by disabling preemption.  Must be raised/lowered under cpu_lock,
 154  * while individual thread weakbinding synchronization is done under thread
 155  * lock.
 156  */
 157 int weakbindingbarrier;
 158 
 159 /*
 160  * Variables used in pause_cpus().
 161  */
 162 static volatile char safe_list[NCPU];
 163 
 164 static struct _cpu_pause_info {
 165         int             cp_spl;         /* spl saved in pause_cpus() */
 166         volatile int    cp_go;          /* Go signal sent after all ready */
 167         int             cp_count;       /* # of CPUs to pause */
 168         ksema_t         cp_sem;         /* synch pause_cpus & cpu_pause */
 169         kthread_id_t    cp_paused;
 170         void            *(*cp_func)(void *);
 171 } cpu_pause_info;
 172 
 173 static kmutex_t pause_free_mutex;
 174 static kcondvar_t pause_free_cv;
 175 
 176 
 177 static struct cpu_sys_stats_ks_data {
 178         kstat_named_t cpu_ticks_idle;
 179         kstat_named_t cpu_ticks_user;
 180         kstat_named_t cpu_ticks_kernel;
 181         kstat_named_t cpu_ticks_wait;
 182         kstat_named_t cpu_nsec_idle;
 183         kstat_named_t cpu_nsec_user;
 184         kstat_named_t cpu_nsec_kernel;
 185         kstat_named_t cpu_nsec_dtrace;
 186         kstat_named_t cpu_nsec_intr;
 187         kstat_named_t cpu_load_intr;
 188         kstat_named_t wait_ticks_io;
 189         kstat_named_t dtrace_probes;
 190         kstat_named_t bread;
 191         kstat_named_t bwrite;
 192         kstat_named_t lread;
 193         kstat_named_t lwrite;
 194         kstat_named_t phread;
 195         kstat_named_t phwrite;
 196         kstat_named_t pswitch;
 197         kstat_named_t trap;
 198         kstat_named_t intr;
 199         kstat_named_t syscall;
 200         kstat_named_t sysread;
 201         kstat_named_t syswrite;
 202         kstat_named_t sysfork;
 203         kstat_named_t sysvfork;
 204         kstat_named_t sysexec;
 205         kstat_named_t readch;
 206         kstat_named_t writech;
 207         kstat_named_t rcvint;
 208         kstat_named_t xmtint;
 209         kstat_named_t mdmint;
 210         kstat_named_t rawch;
 211         kstat_named_t canch;
 212         kstat_named_t outch;
 213         kstat_named_t msg;
 214         kstat_named_t sema;
 215         kstat_named_t namei;
 216         kstat_named_t ufsiget;
 217         kstat_named_t ufsdirblk;
 218         kstat_named_t ufsipage;
 219         kstat_named_t ufsinopage;
 220         kstat_named_t procovf;
 221         kstat_named_t intrthread;
 222         kstat_named_t intrblk;
 223         kstat_named_t intrunpin;
 224         kstat_named_t idlethread;
 225         kstat_named_t inv_swtch;
 226         kstat_named_t nthreads;
 227         kstat_named_t cpumigrate;
 228         kstat_named_t xcalls;
 229         kstat_named_t mutex_adenters;
 230         kstat_named_t rw_rdfails;
 231         kstat_named_t rw_wrfails;
 232         kstat_named_t modload;
 233         kstat_named_t modunload;
 234         kstat_named_t bawrite;
 235         kstat_named_t iowait;
 236 } cpu_sys_stats_ks_data_template = {
 237         { "cpu_ticks_idle",     KSTAT_DATA_UINT64 },
 238         { "cpu_ticks_user",     KSTAT_DATA_UINT64 },
 239         { "cpu_ticks_kernel",   KSTAT_DATA_UINT64 },
 240         { "cpu_ticks_wait",     KSTAT_DATA_UINT64 },
 241         { "cpu_nsec_idle",      KSTAT_DATA_UINT64 },
 242         { "cpu_nsec_user",      KSTAT_DATA_UINT64 },
 243         { "cpu_nsec_kernel",    KSTAT_DATA_UINT64 },
 244         { "cpu_nsec_dtrace",    KSTAT_DATA_UINT64 },
 245         { "cpu_nsec_intr",      KSTAT_DATA_UINT64 },
 246         { "cpu_load_intr",      KSTAT_DATA_UINT64 },
 247         { "wait_ticks_io",      KSTAT_DATA_UINT64 },
 248         { "dtrace_probes",      KSTAT_DATA_UINT64 },
 249         { "bread",              KSTAT_DATA_UINT64 },
 250         { "bwrite",             KSTAT_DATA_UINT64 },
 251         { "lread",              KSTAT_DATA_UINT64 },
 252         { "lwrite",             KSTAT_DATA_UINT64 },
 253         { "phread",             KSTAT_DATA_UINT64 },
 254         { "phwrite",            KSTAT_DATA_UINT64 },
 255         { "pswitch",            KSTAT_DATA_UINT64 },
 256         { "trap",               KSTAT_DATA_UINT64 },
 257         { "intr",               KSTAT_DATA_UINT64 },
 258         { "syscall",            KSTAT_DATA_UINT64 },
 259         { "sysread",            KSTAT_DATA_UINT64 },
 260         { "syswrite",           KSTAT_DATA_UINT64 },
 261         { "sysfork",            KSTAT_DATA_UINT64 },
 262         { "sysvfork",           KSTAT_DATA_UINT64 },
 263         { "sysexec",            KSTAT_DATA_UINT64 },
 264         { "readch",             KSTAT_DATA_UINT64 },
 265         { "writech",            KSTAT_DATA_UINT64 },
 266         { "rcvint",             KSTAT_DATA_UINT64 },
 267         { "xmtint",             KSTAT_DATA_UINT64 },
 268         { "mdmint",             KSTAT_DATA_UINT64 },
 269         { "rawch",              KSTAT_DATA_UINT64 },
 270         { "canch",              KSTAT_DATA_UINT64 },
 271         { "outch",              KSTAT_DATA_UINT64 },
 272         { "msg",                KSTAT_DATA_UINT64 },
 273         { "sema",               KSTAT_DATA_UINT64 },
 274         { "namei",              KSTAT_DATA_UINT64 },
 275         { "ufsiget",            KSTAT_DATA_UINT64 },
 276         { "ufsdirblk",          KSTAT_DATA_UINT64 },
 277         { "ufsipage",           KSTAT_DATA_UINT64 },
 278         { "ufsinopage",         KSTAT_DATA_UINT64 },
 279         { "procovf",            KSTAT_DATA_UINT64 },
 280         { "intrthread",         KSTAT_DATA_UINT64 },
 281         { "intrblk",            KSTAT_DATA_UINT64 },
 282         { "intrunpin",          KSTAT_DATA_UINT64 },
 283         { "idlethread",         KSTAT_DATA_UINT64 },
 284         { "inv_swtch",          KSTAT_DATA_UINT64 },
 285         { "nthreads",           KSTAT_DATA_UINT64 },
 286         { "cpumigrate",         KSTAT_DATA_UINT64 },
 287         { "xcalls",             KSTAT_DATA_UINT64 },
 288         { "mutex_adenters",     KSTAT_DATA_UINT64 },
 289         { "rw_rdfails",         KSTAT_DATA_UINT64 },
 290         { "rw_wrfails",         KSTAT_DATA_UINT64 },
 291         { "modload",            KSTAT_DATA_UINT64 },
 292         { "modunload",          KSTAT_DATA_UINT64 },
 293         { "bawrite",            KSTAT_DATA_UINT64 },
 294         { "iowait",             KSTAT_DATA_UINT64 },
 295 };
 296 
 297 static struct cpu_vm_stats_ks_data {
 298         kstat_named_t pgrec;
 299         kstat_named_t pgfrec;
 300         kstat_named_t pgin;
 301         kstat_named_t pgpgin;
 302         kstat_named_t pgout;
 303         kstat_named_t pgpgout;
 304         kstat_named_t swapin;
 305         kstat_named_t pgswapin;
 306         kstat_named_t swapout;
 307         kstat_named_t pgswapout;
 308         kstat_named_t zfod;
 309         kstat_named_t dfree;
 310         kstat_named_t scan;
 311         kstat_named_t rev;
 312         kstat_named_t hat_fault;
 313         kstat_named_t as_fault;
 314         kstat_named_t maj_fault;
 315         kstat_named_t cow_fault;
 316         kstat_named_t prot_fault;
 317         kstat_named_t softlock;
 318         kstat_named_t kernel_asflt;
 319         kstat_named_t pgrrun;
 320         kstat_named_t execpgin;
 321         kstat_named_t execpgout;
 322         kstat_named_t execfree;
 323         kstat_named_t anonpgin;
 324         kstat_named_t anonpgout;
 325         kstat_named_t anonfree;
 326         kstat_named_t fspgin;
 327         kstat_named_t fspgout;
 328         kstat_named_t fsfree;
 329 } cpu_vm_stats_ks_data_template = {
 330         { "pgrec",              KSTAT_DATA_UINT64 },
 331         { "pgfrec",             KSTAT_DATA_UINT64 },
 332         { "pgin",               KSTAT_DATA_UINT64 },
 333         { "pgpgin",             KSTAT_DATA_UINT64 },
 334         { "pgout",              KSTAT_DATA_UINT64 },
 335         { "pgpgout",            KSTAT_DATA_UINT64 },
 336         { "swapin",             KSTAT_DATA_UINT64 },
 337         { "pgswapin",           KSTAT_DATA_UINT64 },
 338         { "swapout",            KSTAT_DATA_UINT64 },
 339         { "pgswapout",          KSTAT_DATA_UINT64 },
 340         { "zfod",               KSTAT_DATA_UINT64 },
 341         { "dfree",              KSTAT_DATA_UINT64 },
 342         { "scan",               KSTAT_DATA_UINT64 },
 343         { "rev",                KSTAT_DATA_UINT64 },
 344         { "hat_fault",          KSTAT_DATA_UINT64 },
 345         { "as_fault",           KSTAT_DATA_UINT64 },
 346         { "maj_fault",          KSTAT_DATA_UINT64 },
 347         { "cow_fault",          KSTAT_DATA_UINT64 },
 348         { "prot_fault",         KSTAT_DATA_UINT64 },
 349         { "softlock",           KSTAT_DATA_UINT64 },
 350         { "kernel_asflt",       KSTAT_DATA_UINT64 },
 351         { "pgrrun",             KSTAT_DATA_UINT64 },
 352         { "execpgin",           KSTAT_DATA_UINT64 },
 353         { "execpgout",          KSTAT_DATA_UINT64 },
 354         { "execfree",           KSTAT_DATA_UINT64 },
 355         { "anonpgin",           KSTAT_DATA_UINT64 },
 356         { "anonpgout",          KSTAT_DATA_UINT64 },
 357         { "anonfree",           KSTAT_DATA_UINT64 },
 358         { "fspgin",             KSTAT_DATA_UINT64 },
 359         { "fspgout",            KSTAT_DATA_UINT64 },
 360         { "fsfree",             KSTAT_DATA_UINT64 },
 361 };
 362 
 363 /*
 364  * Force the specified thread to migrate to the appropriate processor.
 365  * Called with thread lock held, returns with it dropped.
 366  */
 367 static void
 368 force_thread_migrate(kthread_id_t tp)
 369 {
 370         ASSERT(THREAD_LOCK_HELD(tp));
 371         if (tp == curthread) {
 372                 THREAD_TRANSITION(tp);
 373                 CL_SETRUN(tp);
 374                 thread_unlock_nopreempt(tp);
 375                 swtch();
 376         } else {
 377                 if (tp->t_state == TS_ONPROC) {
 378                         cpu_surrender(tp);
 379                 } else if (tp->t_state == TS_RUN) {
 380                         (void) dispdeq(tp);
 381                         setbackdq(tp);
 382                 }
 383                 thread_unlock(tp);
 384         }
 385 }
 386 
 387 /*
 388  * Set affinity for a specified CPU.
 389  * A reference count is incremented and the affinity is held until the
 390  * reference count is decremented to zero by thread_affinity_clear().
 391  * This is so regions of code requiring affinity can be nested.
 392  * Caller needs to ensure that cpu_id remains valid, which can be
 393  * done by holding cpu_lock across this call, unless the caller
 394  * specifies CPU_CURRENT in which case the cpu_lock will be acquired
 395  * by thread_affinity_set and CPU->cpu_id will be the target CPU.
 396  */
 397 void
 398 thread_affinity_set(kthread_id_t t, int cpu_id)
 399 {
 400         cpu_t           *cp;
 401         int             c;
 402 
 403         ASSERT(!(t == curthread && t->t_weakbound_cpu != NULL));
 404 
 405         if ((c = cpu_id) == CPU_CURRENT) {
 406                 mutex_enter(&cpu_lock);
 407                 cpu_id = CPU->cpu_id;
 408         }
 409         /*
 410          * We should be asserting that cpu_lock is held here, but
 411          * the NCA code doesn't acquire it.  The following assert
 412          * should be uncommented when the NCA code is fixed.
 413          *
 414          * ASSERT(MUTEX_HELD(&cpu_lock));
 415          */
 416         ASSERT((cpu_id >= 0) && (cpu_id < NCPU));
 417         cp = cpu[cpu_id];
 418         ASSERT(cp != NULL);             /* user must provide a good cpu_id */
 419         /*
 420          * If there is already a hard affinity requested, and this affinity
 421          * conflicts with that, panic.
 422          */
 423         thread_lock(t);
 424         if (t->t_affinitycnt > 0 && t->t_bound_cpu != cp) {
 425                 panic("affinity_set: setting %p but already bound to %p",
 426                     (void *)cp, (void *)t->t_bound_cpu);
 427         }
 428         t->t_affinitycnt++;
 429         t->t_bound_cpu = cp;
 430 
 431         /*
 432          * Make sure we're running on the right CPU.
 433          */
 434         if (cp != t->t_cpu || t != curthread) {
 435                 force_thread_migrate(t);        /* drops thread lock */
 436         } else {
 437                 thread_unlock(t);
 438         }
 439 
 440         if (c == CPU_CURRENT)
 441                 mutex_exit(&cpu_lock);
 442 }
 443 
 444 /*
 445  *      Wrapper for backward compatibility.
 446  */
 447 void
 448 affinity_set(int cpu_id)
 449 {
 450         thread_affinity_set(curthread, cpu_id);
 451 }
 452 
 453 /*
 454  * Decrement the affinity reservation count and if it becomes zero,
 455  * clear the CPU affinity for the current thread, or set it to the user's
 456  * software binding request.
 457  */
 458 void
 459 thread_affinity_clear(kthread_id_t t)
 460 {
 461         register processorid_t binding;
 462 
 463         thread_lock(t);
 464         if (--t->t_affinitycnt == 0) {
 465                 if ((binding = t->t_bind_cpu) == PBIND_NONE) {
 466                         /*
 467                          * Adjust disp_max_unbound_pri if necessary.
 468                          */
 469                         disp_adjust_unbound_pri(t);
 470                         t->t_bound_cpu = NULL;
 471                         if (t->t_cpu->cpu_part != t->t_cpupart) {
 472                                 force_thread_migrate(t);
 473                                 return;
 474                         }
 475                 } else {
 476                         t->t_bound_cpu = cpu[binding];
 477                         /*
 478                          * Make sure the thread is running on the bound CPU.
 479                          */
 480                         if (t->t_cpu != t->t_bound_cpu) {
 481                                 force_thread_migrate(t);
 482                                 return;         /* already dropped lock */
 483                         }
 484                 }
 485         }
 486         thread_unlock(t);
 487 }
 488 
 489 /*
 490  * Wrapper for backward compatibility.
 491  */
 492 void
 493 affinity_clear(void)
 494 {
 495         thread_affinity_clear(curthread);
 496 }
 497 
 498 /*
 499  * Weak cpu affinity.  Bind to the "current" cpu for short periods
 500  * of time during which the thread must not block (but may be preempted).
 501  * Use this instead of kpreempt_disable() when it is only "no migration"
 502  * rather than "no preemption" semantics that are required - disabling
 503  * preemption holds higher priority threads off of cpu and if the
 504  * operation that is protected is more than momentary this is not good
 505  * for realtime etc.
 506  *
 507  * Weakly bound threads will not prevent a cpu from being offlined -
 508  * we'll only run them on the cpu to which they are weakly bound but
 509  * (because they do not block) we'll always be able to move them on to
 510  * another cpu at offline time if we give them just a short moment to
 511  * run during which they will unbind.  To give a cpu a chance of offlining,
 512  * however, we require a barrier to weak bindings that may be raised for a
 513  * given cpu (offline/move code may set this and then wait a short time for
 514  * existing weak bindings to drop); the cpu_inmotion pointer is that barrier.
 515  *
 516  * There are few restrictions on the calling context of thread_nomigrate.
 517  * The caller must not hold the thread lock.  Calls may be nested.
 518  *
 519  * After weakbinding a thread must not perform actions that may block.
 520  * In particular it must not call thread_affinity_set; calling that when
 521  * already weakbound is nonsensical anyway.
 522  *
 523  * If curthread is prevented from migrating for other reasons
 524  * (kernel preemption disabled; high pil; strongly bound; interrupt thread)
 525  * then the weak binding will succeed even if this cpu is the target of an
 526  * offline/move request.
 527  */
 528 void
 529 thread_nomigrate(void)
 530 {
 531         cpu_t *cp;
 532         kthread_id_t t = curthread;
 533 
 534 again:
 535         kpreempt_disable();
 536         cp = CPU;
 537 
 538         /*
 539          * A highlevel interrupt must not modify t_nomigrate or
 540          * t_weakbound_cpu of the thread it has interrupted.  A lowlevel
 541          * interrupt thread cannot migrate and we can avoid the
 542          * thread_lock call below by short-circuiting here.  In either
 543          * case we can just return since no migration is possible and
 544          * the condition will persist (ie, when we test for these again
 545          * in thread_allowmigrate they can't have changed).   Migration
 546          * is also impossible if we're at or above DISP_LEVEL pil.
 547          */
 548         if (CPU_ON_INTR(cp) || t->t_flag & T_INTR_THREAD ||
 549             getpil() >= DISP_LEVEL) {
 550                 kpreempt_enable();
 551                 return;
 552         }
 553 
 554         /*
 555          * We must be consistent with existing weak bindings.  Since we
 556          * may be interrupted between the increment of t_nomigrate and
 557          * the store to t_weakbound_cpu below we cannot assume that
 558          * t_weakbound_cpu will be set if t_nomigrate is.  Note that we
 559          * cannot assert t_weakbound_cpu == t_bind_cpu since that is not
 560          * always the case.
 561          */
 562         if (t->t_nomigrate && t->t_weakbound_cpu && t->t_weakbound_cpu != cp) {
 563                 if (!panicstr)
 564                         panic("thread_nomigrate: binding to %p but already "
 565                             "bound to %p", (void *)cp,
 566                             (void *)t->t_weakbound_cpu);
 567         }
 568 
 569         /*
 570          * At this point we have preemption disabled and we don't yet hold
 571          * the thread lock.  So it's possible that somebody else could
 572          * set t_bind_cpu here and not be able to force us across to the
 573          * new cpu (since we have preemption disabled).
 574          */
 575         thread_lock(curthread);
 576 
 577         /*
 578          * If further weak bindings are being (temporarily) suppressed then
 579          * we'll settle for disabling kernel preemption (which assures
 580          * no migration provided the thread does not block which it is
 581          * not allowed to if using thread_nomigrate).  We must remember
 582          * this disposition so we can take appropriate action in
 583          * thread_allowmigrate.  If this is a nested call and the
 584          * thread is already weakbound then fall through as normal.
 585          * We remember the decision to settle for kpreempt_disable through
 586          * negative nesting counting in t_nomigrate.  Once a thread has had one
 587          * weakbinding request satisfied in this way any further (nested)
 588          * requests will continue to be satisfied in the same way,
 589          * even if weak bindings have recommenced.
 590          */
 591         if (t->t_nomigrate < 0 || weakbindingbarrier && t->t_nomigrate == 0) {
 592                 --t->t_nomigrate;
 593                 thread_unlock(curthread);
 594                 return;         /* with kpreempt_disable still active */
 595         }
 596 
 597         /*
 598          * We hold thread_lock so t_bind_cpu cannot change.  We could,
 599          * however, be running on a different cpu to which we are t_bound_cpu
 600          * to (as explained above).  If we grant the weak binding request
 601          * in that case then the dispatcher must favour our weak binding
 602          * over our strong (in which case, just as when preemption is
 603          * disabled, we can continue to run on a cpu other than the one to
 604          * which we are strongbound; the difference in this case is that
 605          * this thread can be preempted and so can appear on the dispatch
 606          * queues of a cpu other than the one it is strongbound to).
 607          *
 608          * If the cpu we are running on does not appear to be a current
 609          * offline target (we check cpu_inmotion to determine this - since
 610          * we don't hold cpu_lock we may not see a recent store to that,
 611          * so it's possible that we at times can grant a weak binding to a
 612          * cpu that is an offline target, but that one request will not
 613          * prevent the offline from succeeding) then we will always grant
 614          * the weak binding request.  This includes the case above where
 615          * we grant a weakbinding not commensurate with our strong binding.
 616          *
 617          * If our cpu does appear to be an offline target then we're inclined
 618          * not to grant the weakbinding request just yet - we'd prefer to
 619          * migrate to another cpu and grant the request there.  The
 620          * exceptions are those cases where going through preemption code
 621          * will not result in us changing cpu:
 622          *
 623          *      . interrupts have already bypassed this case (see above)
 624          *      . we are already weakbound to this cpu (dispatcher code will
 625          *        always return us to the weakbound cpu)
 626          *      . preemption was disabled even before we disabled it above
 627          *      . we are strongbound to this cpu (if we're strongbound to
 628          *      another and not yet running there the trip through the
 629          *      dispatcher will move us to the strongbound cpu and we
 630          *      will grant the weak binding there)
 631          */
 632         if (cp != cpu_inmotion || t->t_nomigrate > 0 || t->t_preempt > 1 ||
 633             t->t_bound_cpu == cp) {
 634                 /*
 635                  * Don't be tempted to store to t_weakbound_cpu only on
 636                  * the first nested bind request - if we're interrupted
 637                  * after the increment of t_nomigrate and before the
 638                  * store to t_weakbound_cpu and the interrupt calls
 639                  * thread_nomigrate then the assertion in thread_allowmigrate
 640                  * would fail.
 641                  */
 642                 t->t_nomigrate++;
 643                 t->t_weakbound_cpu = cp;
 644                 membar_producer();
 645                 thread_unlock(curthread);
 646                 /*
 647                  * Now that we have dropped the thread_lock another thread
 648                  * can set our t_weakbound_cpu, and will try to migrate us
 649                  * to the strongbound cpu (which will not be prevented by
 650                  * preemption being disabled since we're about to enable
 651                  * preemption).  We have granted the weakbinding to the current
 652                  * cpu, so again we are in the position that is is is possible
 653                  * that our weak and strong bindings differ.  Again this
 654                  * is catered for by dispatcher code which will favour our
 655                  * weak binding.
 656                  */
 657                 kpreempt_enable();
 658         } else {
 659                 /*
 660                  * Move to another cpu before granting the request by
 661                  * forcing this thread through preemption code.  When we
 662                  * get to set{front,back}dq called from CL_PREEMPT()
 663                  * cpu_choose() will be used to select a cpu to queue
 664                  * us on - that will see cpu_inmotion and take
 665                  * steps to avoid returning us to this cpu.
 666                  */
 667                 cp->cpu_kprunrun = 1;
 668                 thread_unlock(curthread);
 669                 kpreempt_enable();      /* will call preempt() */
 670                 goto again;
 671         }
 672 }
 673 
 674 void
 675 thread_allowmigrate(void)
 676 {
 677         kthread_id_t t = curthread;
 678 
 679         ASSERT(t->t_weakbound_cpu == CPU ||
 680             (t->t_nomigrate < 0 && t->t_preempt > 0) ||
 681             CPU_ON_INTR(CPU) || t->t_flag & T_INTR_THREAD ||
 682             getpil() >= DISP_LEVEL);
 683 
 684         if (CPU_ON_INTR(CPU) || (t->t_flag & T_INTR_THREAD) ||
 685             getpil() >= DISP_LEVEL)
 686                 return;
 687 
 688         if (t->t_nomigrate < 0) {
 689                 /*
 690                  * This thread was granted "weak binding" in the
 691                  * stronger form of kernel preemption disabling.
 692                  * Undo a level of nesting for both t_nomigrate
 693                  * and t_preempt.
 694                  */
 695                 ++t->t_nomigrate;
 696                 kpreempt_enable();
 697         } else if (--t->t_nomigrate == 0) {
 698                 /*
 699                  * Time to drop the weak binding.  We need to cater
 700                  * for the case where we're weakbound to a different
 701                  * cpu than that to which we're strongbound (a very
 702                  * temporary arrangement that must only persist until
 703                  * weak binding drops).  We don't acquire thread_lock
 704                  * here so even as this code executes t_bound_cpu
 705                  * may be changing.  So we disable preemption and
 706                  * a) in the case that t_bound_cpu changes while we
 707                  * have preemption disabled kprunrun will be set
 708                  * asynchronously, and b) if before disabling
 709                  * preemption we were already on a different cpu to
 710                  * our t_bound_cpu then we set kprunrun ourselves
 711                  * to force a trip through the dispatcher when
 712                  * preemption is enabled.
 713                  */
 714                 kpreempt_disable();
 715                 if (t->t_bound_cpu &&
 716                     t->t_weakbound_cpu != t->t_bound_cpu)
 717                         CPU->cpu_kprunrun = 1;
 718                 t->t_weakbound_cpu = NULL;
 719                 membar_producer();
 720                 kpreempt_enable();
 721         }
 722 }
 723 
 724 /*
 725  * weakbinding_stop can be used to temporarily cause weakbindings made
 726  * with thread_nomigrate to be satisfied through the stronger action of
 727  * kpreempt_disable.  weakbinding_start recommences normal weakbinding.
 728  */
 729 
 730 void
 731 weakbinding_stop(void)
 732 {
 733         ASSERT(MUTEX_HELD(&cpu_lock));
 734         weakbindingbarrier = 1;
 735         membar_producer();      /* make visible before subsequent thread_lock */
 736 }
 737 
 738 void
 739 weakbinding_start(void)
 740 {
 741         ASSERT(MUTEX_HELD(&cpu_lock));
 742         weakbindingbarrier = 0;
 743 }
 744 
 745 void
 746 null_xcall(void)
 747 {
 748 }
 749 
 750 /*
 751  * This routine is called to place the CPUs in a safe place so that
 752  * one of them can be taken off line or placed on line.  What we are
 753  * trying to do here is prevent a thread from traversing the list
 754  * of active CPUs while we are changing it or from getting placed on
 755  * the run queue of a CPU that has just gone off line.  We do this by
 756  * creating a thread with the highest possible prio for each CPU and
 757  * having it call this routine.  The advantage of this method is that
 758  * we can eliminate all checks for CPU_ACTIVE in the disp routines.
 759  * This makes disp faster at the expense of making p_online() slower
 760  * which is a good trade off.
 761  */
 762 static void
 763 cpu_pause(int index)
 764 {
 765         int s;
 766         struct _cpu_pause_info *cpi = &cpu_pause_info;
 767         volatile char *safe = &safe_list[index];
 768         long    lindex = index;
 769 
 770         ASSERT((curthread->t_bound_cpu != NULL) || (*safe == PAUSE_DIE));
 771 
 772         while (*safe != PAUSE_DIE) {
 773                 *safe = PAUSE_READY;
 774                 membar_enter();         /* make sure stores are flushed */
 775                 sema_v(&cpi->cp_sem);    /* signal requesting thread */
 776 
 777                 /*
 778                  * Wait here until all pause threads are running.  That
 779                  * indicates that it's safe to do the spl.  Until
 780                  * cpu_pause_info.cp_go is set, we don't want to spl
 781                  * because that might block clock interrupts needed
 782                  * to preempt threads on other CPUs.
 783                  */
 784                 while (cpi->cp_go == 0)
 785                         ;
 786                 /*
 787                  * Even though we are at the highest disp prio, we need
 788                  * to block out all interrupts below LOCK_LEVEL so that
 789                  * an intr doesn't come in, wake up a thread, and call
 790                  * setbackdq/setfrontdq.
 791                  */
 792                 s = splhigh();
 793                 /*
 794                  * if cp_func has been set then call it using index as the
 795                  * argument, currently only used by cpr_suspend_cpus().
 796                  * This function is used as the code to execute on the
 797                  * "paused" cpu's when a machine comes out of a sleep state
 798                  * and CPU's were powered off.  (could also be used for
 799                  * hotplugging CPU's).
 800                  */
 801                 if (cpi->cp_func != NULL)
 802                         (*cpi->cp_func)((void *)lindex);
 803 
 804                 mach_cpu_pause(safe);
 805 
 806                 splx(s);
 807                 /*
 808                  * Waiting is at an end. Switch out of cpu_pause
 809                  * loop and resume useful work.
 810                  */
 811                 swtch();
 812         }
 813 
 814         mutex_enter(&pause_free_mutex);
 815         *safe = PAUSE_DEAD;
 816         cv_broadcast(&pause_free_cv);
 817         mutex_exit(&pause_free_mutex);
 818 }
 819 
 820 /*
 821  * Allow the cpus to start running again.
 822  */
 823 void
 824 start_cpus()
 825 {
 826         int i;
 827 
 828         ASSERT(MUTEX_HELD(&cpu_lock));
 829         ASSERT(cpu_pause_info.cp_paused);
 830         cpu_pause_info.cp_paused = NULL;
 831         for (i = 0; i < NCPU; i++)
 832                 safe_list[i] = PAUSE_IDLE;
 833         membar_enter();                 /* make sure stores are flushed */
 834         affinity_clear();
 835         splx(cpu_pause_info.cp_spl);
 836         kpreempt_enable();
 837 }
 838 
 839 /*
 840  * Allocate a pause thread for a CPU.
 841  */
 842 static void
 843 cpu_pause_alloc(cpu_t *cp)
 844 {
 845         kthread_id_t    t;
 846         long            cpun = cp->cpu_id;
 847 
 848         /*
 849          * Note, v.v_nglobpris will not change value as long as I hold
 850          * cpu_lock.
 851          */
 852         t = thread_create(NULL, 0, cpu_pause, (void *)cpun,
 853             0, &p0, TS_STOPPED, v.v_nglobpris - 1);
 854         thread_lock(t);
 855         t->t_bound_cpu = cp;
 856         t->t_disp_queue = cp->cpu_disp;
 857         t->t_affinitycnt = 1;
 858         t->t_preempt = 1;
 859         thread_unlock(t);
 860         cp->cpu_pause_thread = t;
 861         /*
 862          * Registering a thread in the callback table is usually done
 863          * in the initialization code of the thread.  In this
 864          * case, we do it right after thread creation because the
 865          * thread itself may never run, and we need to register the
 866          * fact that it is safe for cpr suspend.
 867          */
 868         CALLB_CPR_INIT_SAFE(t, "cpu_pause");
 869 }
 870 
 871 /*
 872  * Free a pause thread for a CPU.
 873  */
 874 static void
 875 cpu_pause_free(cpu_t *cp)
 876 {
 877         kthread_id_t    t;
 878         int             cpun = cp->cpu_id;
 879 
 880         ASSERT(MUTEX_HELD(&cpu_lock));
 881         /*
 882          * We have to get the thread and tell him to die.
 883          */
 884         if ((t = cp->cpu_pause_thread) == NULL) {
 885                 ASSERT(safe_list[cpun] == PAUSE_IDLE);
 886                 return;
 887         }
 888         thread_lock(t);
 889         t->t_cpu = CPU;              /* disp gets upset if last cpu is quiesced. */
 890         t->t_bound_cpu = NULL;       /* Must un-bind; cpu may not be running. */
 891         t->t_pri = v.v_nglobpris - 1;
 892         ASSERT(safe_list[cpun] == PAUSE_IDLE);
 893         safe_list[cpun] = PAUSE_DIE;
 894         THREAD_TRANSITION(t);
 895         setbackdq(t);
 896         thread_unlock_nopreempt(t);
 897 
 898         /*
 899          * If we don't wait for the thread to actually die, it may try to
 900          * run on the wrong cpu as part of an actual call to pause_cpus().
 901          */
 902         mutex_enter(&pause_free_mutex);
 903         while (safe_list[cpun] != PAUSE_DEAD) {
 904                 cv_wait(&pause_free_cv, &pause_free_mutex);
 905         }
 906         mutex_exit(&pause_free_mutex);
 907         safe_list[cpun] = PAUSE_IDLE;
 908 
 909         cp->cpu_pause_thread = NULL;
 910 }
 911 
 912 /*
 913  * Initialize basic structures for pausing CPUs.
 914  */
 915 void
 916 cpu_pause_init()
 917 {
 918         sema_init(&cpu_pause_info.cp_sem, 0, NULL, SEMA_DEFAULT, NULL);
 919         /*
 920          * Create initial CPU pause thread.
 921          */
 922         cpu_pause_alloc(CPU);
 923 }
 924 
 925 /*
 926  * Start the threads used to pause another CPU.
 927  */
 928 static int
 929 cpu_pause_start(processorid_t cpu_id)
 930 {
 931         int     i;
 932         int     cpu_count = 0;
 933 
 934         for (i = 0; i < NCPU; i++) {
 935                 cpu_t           *cp;
 936                 kthread_id_t    t;
 937 
 938                 cp = cpu[i];
 939                 if (!CPU_IN_SET(cpu_available, i) || (i == cpu_id)) {
 940                         safe_list[i] = PAUSE_WAIT;
 941                         continue;
 942                 }
 943 
 944                 /*
 945                  * Skip CPU if it is quiesced or not yet started.
 946                  */
 947                 if ((cp->cpu_flags & (CPU_QUIESCED | CPU_READY)) != CPU_READY) {
 948                         safe_list[i] = PAUSE_WAIT;
 949                         continue;
 950                 }
 951 
 952                 /*
 953                  * Start this CPU's pause thread.
 954                  */
 955                 t = cp->cpu_pause_thread;
 956                 thread_lock(t);
 957                 /*
 958                  * Reset the priority, since nglobpris may have
 959                  * changed since the thread was created, if someone
 960                  * has loaded the RT (or some other) scheduling
 961                  * class.
 962                  */
 963                 t->t_pri = v.v_nglobpris - 1;
 964                 THREAD_TRANSITION(t);
 965                 setbackdq(t);
 966                 thread_unlock_nopreempt(t);
 967                 ++cpu_count;
 968         }
 969         return (cpu_count);
 970 }
 971 
 972 
 973 /*
 974  * Pause all of the CPUs except the one we are on by creating a high
 975  * priority thread bound to those CPUs.
 976  *
 977  * Note that one must be extremely careful regarding code
 978  * executed while CPUs are paused.  Since a CPU may be paused
 979  * while a thread scheduling on that CPU is holding an adaptive
 980  * lock, code executed with CPUs paused must not acquire adaptive
 981  * (or low-level spin) locks.  Also, such code must not block,
 982  * since the thread that is supposed to initiate the wakeup may
 983  * never run.
 984  *
 985  * With a few exceptions, the restrictions on code executed with CPUs
 986  * paused match those for code executed at high-level interrupt
 987  * context.
 988  */
 989 void
 990 pause_cpus(cpu_t *off_cp, void *(*func)(void *))
 991 {
 992         processorid_t   cpu_id;
 993         int             i;
 994         struct _cpu_pause_info  *cpi = &cpu_pause_info;
 995 
 996         ASSERT(MUTEX_HELD(&cpu_lock));
 997         ASSERT(cpi->cp_paused == NULL);
 998         cpi->cp_count = 0;
 999         cpi->cp_go = 0;
1000         for (i = 0; i < NCPU; i++)
1001                 safe_list[i] = PAUSE_IDLE;
1002         kpreempt_disable();
1003 
1004         cpi->cp_func = func;
1005 
1006         /*
1007          * If running on the cpu that is going offline, get off it.
1008          * This is so that it won't be necessary to rechoose a CPU
1009          * when done.
1010          */
1011         if (CPU == off_cp)
1012                 cpu_id = off_cp->cpu_next_part->cpu_id;
1013         else
1014                 cpu_id = CPU->cpu_id;
1015         affinity_set(cpu_id);
1016 
1017         /*
1018          * Start the pause threads and record how many were started
1019          */
1020         cpi->cp_count = cpu_pause_start(cpu_id);
1021 
1022         /*
1023          * Now wait for all CPUs to be running the pause thread.
1024          */
1025         while (cpi->cp_count > 0) {
1026                 /*
1027                  * Spin reading the count without grabbing the disp
1028                  * lock to make sure we don't prevent the pause
1029                  * threads from getting the lock.
1030                  */
1031                 while (sema_held(&cpi->cp_sem))
1032                         ;
1033                 if (sema_tryp(&cpi->cp_sem))
1034                         --cpi->cp_count;
1035         }
1036         cpi->cp_go = 1;                      /* all have reached cpu_pause */
1037 
1038         /*
1039          * Now wait for all CPUs to spl. (Transition from PAUSE_READY
1040          * to PAUSE_WAIT.)
1041          */
1042         for (i = 0; i < NCPU; i++) {
1043                 while (safe_list[i] != PAUSE_WAIT)
1044                         ;
1045         }
1046         cpi->cp_spl = splhigh();     /* block dispatcher on this CPU */
1047         cpi->cp_paused = curthread;
1048 }
1049 
1050 /*
1051  * Check whether the current thread has CPUs paused
1052  */
1053 int
1054 cpus_paused(void)
1055 {
1056         if (cpu_pause_info.cp_paused != NULL) {
1057                 ASSERT(cpu_pause_info.cp_paused == curthread);
1058                 return (1);
1059         }
1060         return (0);
1061 }
1062 
1063 static cpu_t *
1064 cpu_get_all(processorid_t cpun)
1065 {
1066         ASSERT(MUTEX_HELD(&cpu_lock));
1067 
1068         if (cpun >= NCPU || cpun < 0 || !CPU_IN_SET(cpu_available, cpun))
1069                 return (NULL);
1070         return (cpu[cpun]);
1071 }
1072 
1073 /*
1074  * Check whether cpun is a valid processor id and whether it should be
1075  * visible from the current zone. If it is, return a pointer to the
1076  * associated CPU structure.
1077  */
1078 cpu_t *
1079 cpu_get(processorid_t cpun)
1080 {
1081         cpu_t *c;
1082 
1083         ASSERT(MUTEX_HELD(&cpu_lock));
1084         c = cpu_get_all(cpun);
1085         if (c != NULL && !INGLOBALZONE(curproc) && pool_pset_enabled() &&
1086             zone_pset_get(curproc->p_zone) != cpupart_query_cpu(c))
1087                 return (NULL);
1088         return (c);
1089 }
1090 
1091 /*
1092  * The following functions should be used to check CPU states in the kernel.
1093  * They should be invoked with cpu_lock held.  Kernel subsystems interested
1094  * in CPU states should *not* use cpu_get_state() and various P_ONLINE/etc
1095  * states.  Those are for user-land (and system call) use only.
1096  */
1097 
1098 /*
1099  * Determine whether the CPU is online and handling interrupts.
1100  */
1101 int
1102 cpu_is_online(cpu_t *cpu)
1103 {
1104         ASSERT(MUTEX_HELD(&cpu_lock));
1105         return (cpu_flagged_online(cpu->cpu_flags));
1106 }
1107 
1108 /*
1109  * Determine whether the CPU is offline (this includes spare and faulted).
1110  */
1111 int
1112 cpu_is_offline(cpu_t *cpu)
1113 {
1114         ASSERT(MUTEX_HELD(&cpu_lock));
1115         return (cpu_flagged_offline(cpu->cpu_flags));
1116 }
1117 
1118 /*
1119  * Determine whether the CPU is powered off.
1120  */
1121 int
1122 cpu_is_poweredoff(cpu_t *cpu)
1123 {
1124         ASSERT(MUTEX_HELD(&cpu_lock));
1125         return (cpu_flagged_poweredoff(cpu->cpu_flags));
1126 }
1127 
1128 /*
1129  * Determine whether the CPU is handling interrupts.
1130  */
1131 int
1132 cpu_is_nointr(cpu_t *cpu)
1133 {
1134         ASSERT(MUTEX_HELD(&cpu_lock));
1135         return (cpu_flagged_nointr(cpu->cpu_flags));
1136 }
1137 
1138 /*
1139  * Determine whether the CPU is active (scheduling threads).
1140  */
1141 int
1142 cpu_is_active(cpu_t *cpu)
1143 {
1144         ASSERT(MUTEX_HELD(&cpu_lock));
1145         return (cpu_flagged_active(cpu->cpu_flags));
1146 }
1147 
1148 /*
1149  * Same as above, but these require cpu_flags instead of cpu_t pointers.
1150  */
1151 int
1152 cpu_flagged_online(cpu_flag_t cpu_flags)
1153 {
1154         return (cpu_flagged_active(cpu_flags) &&
1155             (cpu_flags & CPU_ENABLE));
1156 }
1157 
1158 int
1159 cpu_flagged_offline(cpu_flag_t cpu_flags)
1160 {
1161         return (((cpu_flags & CPU_POWEROFF) == 0) &&
1162             ((cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY));
1163 }
1164 
1165 int
1166 cpu_flagged_poweredoff(cpu_flag_t cpu_flags)
1167 {
1168         return ((cpu_flags & CPU_POWEROFF) == CPU_POWEROFF);
1169 }
1170 
1171 int
1172 cpu_flagged_nointr(cpu_flag_t cpu_flags)
1173 {
1174         return (cpu_flagged_active(cpu_flags) &&
1175             (cpu_flags & CPU_ENABLE) == 0);
1176 }
1177 
1178 int
1179 cpu_flagged_active(cpu_flag_t cpu_flags)
1180 {
1181         return (((cpu_flags & (CPU_POWEROFF | CPU_FAULTED | CPU_SPARE)) == 0) &&
1182             ((cpu_flags & (CPU_READY | CPU_OFFLINE)) == CPU_READY));
1183 }
1184 
1185 /*
1186  * Bring the indicated CPU online.
1187  */
1188 int
1189 cpu_online(cpu_t *cp)
1190 {
1191         int     error = 0;
1192 
1193         /*
1194          * Handle on-line request.
1195          *      This code must put the new CPU on the active list before
1196          *      starting it because it will not be paused, and will start
1197          *      using the active list immediately.  The real start occurs
1198          *      when the CPU_QUIESCED flag is turned off.
1199          */
1200 
1201         ASSERT(MUTEX_HELD(&cpu_lock));
1202 
1203         /*
1204          * Put all the cpus into a known safe place.
1205          * No mutexes can be entered while CPUs are paused.
1206          */
1207         error = mp_cpu_start(cp);       /* arch-dep hook */
1208         if (error == 0) {
1209                 pg_cpupart_in(cp, cp->cpu_part);
1210                 pause_cpus(NULL, NULL);
1211                 cpu_add_active_internal(cp);
1212                 if (cp->cpu_flags & CPU_FAULTED) {
1213                         cp->cpu_flags &= ~CPU_FAULTED;
1214                         mp_cpu_faulted_exit(cp);
1215                 }
1216                 cp->cpu_flags &= ~(CPU_QUIESCED | CPU_OFFLINE | CPU_FROZEN |
1217                     CPU_SPARE);
1218                 CPU_NEW_GENERATION(cp);
1219                 start_cpus();
1220                 cpu_stats_kstat_create(cp);
1221                 cpu_create_intrstat(cp);
1222                 lgrp_kstat_create(cp);
1223                 cpu_state_change_notify(cp->cpu_id, CPU_ON);
1224                 cpu_intr_enable(cp);    /* arch-dep hook */
1225                 cpu_state_change_notify(cp->cpu_id, CPU_INTR_ON);
1226                 cpu_set_state(cp);
1227                 cyclic_online(cp);
1228                 /*
1229                  * This has to be called only after cyclic_online(). This
1230                  * function uses cyclics.
1231                  */
1232                 callout_cpu_online(cp);
1233                 poke_cpu(cp->cpu_id);
1234         }
1235 
1236         return (error);
1237 }
1238 
1239 /*
1240  * Take the indicated CPU offline.
1241  */
1242 int
1243 cpu_offline(cpu_t *cp, int flags)
1244 {
1245         cpupart_t *pp;
1246         int     error = 0;
1247         cpu_t   *ncp;
1248         int     intr_enable;
1249         int     cyclic_off = 0;
1250         int     callout_off = 0;
1251         int     loop_count;
1252         int     no_quiesce = 0;
1253         int     (*bound_func)(struct cpu *, int);
1254         kthread_t *t;
1255         lpl_t   *cpu_lpl;
1256         proc_t  *p;
1257         int     lgrp_diff_lpl;
1258         boolean_t unbind_all_threads = (flags & CPU_FORCED) != 0;
1259 
1260         ASSERT(MUTEX_HELD(&cpu_lock));
1261 
1262         /*
1263          * If we're going from faulted or spare to offline, just
1264          * clear these flags and update CPU state.
1265          */
1266         if (cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) {
1267                 if (cp->cpu_flags & CPU_FAULTED) {
1268                         cp->cpu_flags &= ~CPU_FAULTED;
1269                         mp_cpu_faulted_exit(cp);
1270                 }
1271                 cp->cpu_flags &= ~CPU_SPARE;
1272                 cpu_set_state(cp);
1273                 return (0);
1274         }
1275 
1276         /*
1277          * Handle off-line request.
1278          */
1279         pp = cp->cpu_part;
1280         /*
1281          * Don't offline last online CPU in partition
1282          */
1283         if (ncpus_online <= 1 || pp->cp_ncpus <= 1 || cpu_intr_count(cp) < 2)
1284                 return (EBUSY);
1285         /*
1286          * Unbind all soft-bound threads bound to our CPU and hard bound threads
1287          * if we were asked to.
1288          */
1289         error = cpu_unbind(cp->cpu_id, unbind_all_threads);
1290         if (error != 0)
1291                 return (error);
1292         /*
1293          * We shouldn't be bound to this CPU ourselves.
1294          */
1295         if (curthread->t_bound_cpu == cp)
1296                 return (EBUSY);
1297 
1298         /*
1299          * Tell interested parties that this CPU is going offline.
1300          */
1301         CPU_NEW_GENERATION(cp);
1302         cpu_state_change_notify(cp->cpu_id, CPU_OFF);
1303 
1304         /*
1305          * Tell the PG subsystem that the CPU is leaving the partition
1306          */
1307         pg_cpupart_out(cp, pp);
1308 
1309         /*
1310          * Take the CPU out of interrupt participation so we won't find
1311          * bound kernel threads.  If the architecture cannot completely
1312          * shut off interrupts on the CPU, don't quiesce it, but don't
1313          * run anything but interrupt thread... this is indicated by
1314          * the CPU_OFFLINE flag being on but the CPU_QUIESCE flag being
1315          * off.
1316          */
1317         intr_enable = cp->cpu_flags & CPU_ENABLE;
1318         if (intr_enable)
1319                 no_quiesce = cpu_intr_disable(cp);
1320 
1321         /*
1322          * Record that we are aiming to offline this cpu.  This acts as
1323          * a barrier to further weak binding requests in thread_nomigrate
1324          * and also causes cpu_choose, disp_lowpri_cpu and setfrontdq to
1325          * lean away from this cpu.  Further strong bindings are already
1326          * avoided since we hold cpu_lock.  Since threads that are set
1327          * runnable around now and others coming off the target cpu are
1328          * directed away from the target, existing strong and weak bindings
1329          * (especially the latter) to the target cpu stand maximum chance of
1330          * being able to unbind during the short delay loop below (if other
1331          * unbound threads compete they may not see cpu in time to unbind
1332          * even if they would do so immediately.
1333          */
1334         cpu_inmotion = cp;
1335         membar_enter();
1336 
1337         /*
1338          * Check for kernel threads (strong or weak) bound to that CPU.
1339          * Strongly bound threads may not unbind, and we'll have to return
1340          * EBUSY.  Weakly bound threads should always disappear - we've
1341          * stopped more weak binding with cpu_inmotion and existing
1342          * bindings will drain imminently (they may not block).  Nonetheless
1343          * we will wait for a fixed period for all bound threads to disappear.
1344          * Inactive interrupt threads are OK (they'll be in TS_FREE
1345          * state).  If test finds some bound threads, wait a few ticks
1346          * to give short-lived threads (such as interrupts) chance to
1347          * complete.  Note that if no_quiesce is set, i.e. this cpu
1348          * is required to service interrupts, then we take the route
1349          * that permits interrupt threads to be active (or bypassed).
1350          */
1351         bound_func = no_quiesce ? disp_bound_threads : disp_bound_anythreads;
1352 
1353 again:  for (loop_count = 0; (*bound_func)(cp, 0); loop_count++) {
1354                 if (loop_count >= 5) {
1355                         error = EBUSY;  /* some threads still bound */
1356                         break;
1357                 }
1358 
1359                 /*
1360                  * If some threads were assigned, give them
1361                  * a chance to complete or move.
1362                  *
1363                  * This assumes that the clock_thread is not bound
1364                  * to any CPU, because the clock_thread is needed to
1365                  * do the delay(hz/100).
1366                  *
1367                  * Note: we still hold the cpu_lock while waiting for
1368                  * the next clock tick.  This is OK since it isn't
1369                  * needed for anything else except processor_bind(2),
1370                  * and system initialization.  If we drop the lock,
1371                  * we would risk another p_online disabling the last
1372                  * processor.
1373                  */
1374                 delay(hz/100);
1375         }
1376 
1377         if (error == 0 && callout_off == 0) {
1378                 callout_cpu_offline(cp);
1379                 callout_off = 1;
1380         }
1381 
1382         if (error == 0 && cyclic_off == 0) {
1383                 if (!cyclic_offline(cp)) {
1384                         /*
1385                          * We must have bound cyclics...
1386                          */
1387                         error = EBUSY;
1388                         goto out;
1389                 }
1390                 cyclic_off = 1;
1391         }
1392 
1393         /*
1394          * Call mp_cpu_stop() to perform any special operations
1395          * needed for this machine architecture to offline a CPU.
1396          */
1397         if (error == 0)
1398                 error = mp_cpu_stop(cp);        /* arch-dep hook */
1399 
1400         /*
1401          * If that all worked, take the CPU offline and decrement
1402          * ncpus_online.
1403          */
1404         if (error == 0) {
1405                 /*
1406                  * Put all the cpus into a known safe place.
1407                  * No mutexes can be entered while CPUs are paused.
1408                  */
1409                 pause_cpus(cp, NULL);
1410                 /*
1411                  * Repeat the operation, if necessary, to make sure that
1412                  * all outstanding low-level interrupts run to completion
1413                  * before we set the CPU_QUIESCED flag.  It's also possible
1414                  * that a thread has weak bound to the cpu despite our raising
1415                  * cpu_inmotion above since it may have loaded that
1416                  * value before the barrier became visible (this would have
1417                  * to be the thread that was on the target cpu at the time
1418                  * we raised the barrier).
1419                  */
1420                 if ((!no_quiesce && cp->cpu_intr_actv != 0) ||
1421                     (*bound_func)(cp, 1)) {
1422                         start_cpus();
1423                         (void) mp_cpu_start(cp);
1424                         goto again;
1425                 }
1426                 ncp = cp->cpu_next_part;
1427                 cpu_lpl = cp->cpu_lpl;
1428                 ASSERT(cpu_lpl != NULL);
1429 
1430                 /*
1431                  * Remove the CPU from the list of active CPUs.
1432                  */
1433                 cpu_remove_active(cp);
1434 
1435                 /*
1436                  * Walk the active process list and look for threads
1437                  * whose home lgroup needs to be updated, or
1438                  * the last CPU they run on is the one being offlined now.
1439                  */
1440 
1441                 ASSERT(curthread->t_cpu != cp);
1442                 for (p = practive; p != NULL; p = p->p_next) {
1443 
1444                         t = p->p_tlist;
1445 
1446                         if (t == NULL)
1447                                 continue;
1448 
1449                         lgrp_diff_lpl = 0;
1450 
1451                         do {
1452                                 ASSERT(t->t_lpl != NULL);
1453                                 /*
1454                                  * Taking last CPU in lpl offline
1455                                  * Rehome thread if it is in this lpl
1456                                  * Otherwise, update the count of how many
1457                                  * threads are in this CPU's lgroup but have
1458                                  * a different lpl.
1459                                  */
1460 
1461                                 if (cpu_lpl->lpl_ncpu == 0) {
1462                                         if (t->t_lpl == cpu_lpl)
1463                                                 lgrp_move_thread(t,
1464                                                     lgrp_choose(t,
1465                                                     t->t_cpupart), 0);
1466                                         else if (t->t_lpl->lpl_lgrpid ==
1467                                             cpu_lpl->lpl_lgrpid)
1468                                                 lgrp_diff_lpl++;
1469                                 }
1470                                 ASSERT(t->t_lpl->lpl_ncpu > 0);
1471 
1472                                 /*
1473                                  * Update CPU last ran on if it was this CPU
1474                                  */
1475                                 if (t->t_cpu == cp && t->t_bound_cpu != cp)
1476                                         t->t_cpu = disp_lowpri_cpu(ncp,
1477                                             t->t_lpl, t->t_pri, NULL);
1478                                 ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1479                                     t->t_weakbound_cpu == cp);
1480 
1481                                 t = t->t_forw;
1482                         } while (t != p->p_tlist);
1483 
1484                         /*
1485                          * Didn't find any threads in the same lgroup as this
1486                          * CPU with a different lpl, so remove the lgroup from
1487                          * the process lgroup bitmask.
1488                          */
1489 
1490                         if (lgrp_diff_lpl == 0)
1491                                 klgrpset_del(p->p_lgrpset, cpu_lpl->lpl_lgrpid);
1492                 }
1493 
1494                 /*
1495                  * Walk thread list looking for threads that need to be
1496                  * rehomed, since there are some threads that are not in
1497                  * their process's p_tlist.
1498                  */
1499 
1500                 t = curthread;
1501                 do {
1502                         ASSERT(t != NULL && t->t_lpl != NULL);
1503 
1504                         /*
1505                          * Rehome threads with same lpl as this CPU when this
1506                          * is the last CPU in the lpl.
1507                          */
1508 
1509                         if ((cpu_lpl->lpl_ncpu == 0) && (t->t_lpl == cpu_lpl))
1510                                 lgrp_move_thread(t,
1511                                     lgrp_choose(t, t->t_cpupart), 1);
1512 
1513                         ASSERT(t->t_lpl->lpl_ncpu > 0);
1514 
1515                         /*
1516                          * Update CPU last ran on if it was this CPU
1517                          */
1518 
1519                         if (t->t_cpu == cp && t->t_bound_cpu != cp) {
1520                                 t->t_cpu = disp_lowpri_cpu(ncp,
1521                                     t->t_lpl, t->t_pri, NULL);
1522                         }
1523                         ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1524                             t->t_weakbound_cpu == cp);
1525                         t = t->t_next;
1526 
1527                 } while (t != curthread);
1528                 ASSERT((cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) == 0);
1529                 cp->cpu_flags |= CPU_OFFLINE;
1530                 disp_cpu_inactive(cp);
1531                 if (!no_quiesce)
1532                         cp->cpu_flags |= CPU_QUIESCED;
1533                 ncpus_online--;
1534                 cpu_set_state(cp);
1535                 cpu_inmotion = NULL;
1536                 start_cpus();
1537                 cpu_stats_kstat_destroy(cp);
1538                 cpu_delete_intrstat(cp);
1539                 lgrp_kstat_destroy(cp);
1540         }
1541 
1542 out:
1543         cpu_inmotion = NULL;
1544 
1545         /*
1546          * If we failed, re-enable interrupts.
1547          * Do this even if cpu_intr_disable returned an error, because
1548          * it may have partially disabled interrupts.
1549          */
1550         if (error && intr_enable)
1551                 cpu_intr_enable(cp);
1552 
1553         /*
1554          * If we failed, but managed to offline the cyclic subsystem on this
1555          * CPU, bring it back online.
1556          */
1557         if (error && cyclic_off)
1558                 cyclic_online(cp);
1559 
1560         /*
1561          * If we failed, but managed to offline callouts on this CPU,
1562          * bring it back online.
1563          */
1564         if (error && callout_off)
1565                 callout_cpu_online(cp);
1566 
1567         /*
1568          * If we failed, tell the PG subsystem that the CPU is back
1569          */
1570         pg_cpupart_in(cp, pp);
1571 
1572         /*
1573          * If we failed, we need to notify everyone that this CPU is back on.
1574          */
1575         if (error != 0) {
1576                 CPU_NEW_GENERATION(cp);
1577                 cpu_state_change_notify(cp->cpu_id, CPU_ON);
1578                 cpu_state_change_notify(cp->cpu_id, CPU_INTR_ON);
1579         }
1580 
1581         return (error);
1582 }
1583 
1584 /*
1585  * Mark the indicated CPU as faulted, taking it offline.
1586  */
1587 int
1588 cpu_faulted(cpu_t *cp, int flags)
1589 {
1590         int     error = 0;
1591 
1592         ASSERT(MUTEX_HELD(&cpu_lock));
1593         ASSERT(!cpu_is_poweredoff(cp));
1594 
1595         if (cpu_is_offline(cp)) {
1596                 cp->cpu_flags &= ~CPU_SPARE;
1597                 cp->cpu_flags |= CPU_FAULTED;
1598                 mp_cpu_faulted_enter(cp);
1599                 cpu_set_state(cp);
1600                 return (0);
1601         }
1602 
1603         if ((error = cpu_offline(cp, flags)) == 0) {
1604                 cp->cpu_flags |= CPU_FAULTED;
1605                 mp_cpu_faulted_enter(cp);
1606                 cpu_set_state(cp);
1607         }
1608 
1609         return (error);
1610 }
1611 
1612 /*
1613  * Mark the indicated CPU as a spare, taking it offline.
1614  */
1615 int
1616 cpu_spare(cpu_t *cp, int flags)
1617 {
1618         int     error = 0;
1619 
1620         ASSERT(MUTEX_HELD(&cpu_lock));
1621         ASSERT(!cpu_is_poweredoff(cp));
1622 
1623         if (cpu_is_offline(cp)) {
1624                 if (cp->cpu_flags & CPU_FAULTED) {
1625                         cp->cpu_flags &= ~CPU_FAULTED;
1626                         mp_cpu_faulted_exit(cp);
1627                 }
1628                 cp->cpu_flags |= CPU_SPARE;
1629                 cpu_set_state(cp);
1630                 return (0);
1631         }
1632 
1633         if ((error = cpu_offline(cp, flags)) == 0) {
1634                 cp->cpu_flags |= CPU_SPARE;
1635                 cpu_set_state(cp);
1636         }
1637 
1638         return (error);
1639 }
1640 
1641 /*
1642  * Take the indicated CPU from poweroff to offline.
1643  */
1644 int
1645 cpu_poweron(cpu_t *cp)
1646 {
1647         int     error = ENOTSUP;
1648 
1649         ASSERT(MUTEX_HELD(&cpu_lock));
1650         ASSERT(cpu_is_poweredoff(cp));
1651 
1652         error = mp_cpu_poweron(cp);     /* arch-dep hook */
1653         if (error == 0)
1654                 cpu_set_state(cp);
1655 
1656         return (error);
1657 }
1658 
1659 /*
1660  * Take the indicated CPU from any inactive state to powered off.
1661  */
1662 int
1663 cpu_poweroff(cpu_t *cp)
1664 {
1665         int     error = ENOTSUP;
1666 
1667         ASSERT(MUTEX_HELD(&cpu_lock));
1668         ASSERT(cpu_is_offline(cp));
1669 
1670         if (!(cp->cpu_flags & CPU_QUIESCED))
1671                 return (EBUSY);         /* not completely idle */
1672 
1673         error = mp_cpu_poweroff(cp);    /* arch-dep hook */
1674         if (error == 0)
1675                 cpu_set_state(cp);
1676 
1677         return (error);
1678 }
1679 
1680 /*
1681  * Initialize the Sequential CPU id lookup table
1682  */
1683 void
1684 cpu_seq_tbl_init()
1685 {
1686         cpu_t   **tbl;
1687 
1688         tbl = kmem_zalloc(sizeof (struct cpu *) * max_ncpus, KM_SLEEP);
1689         tbl[0] = CPU;
1690 
1691         cpu_seq = tbl;
1692 }
1693 
1694 /*
1695  * Initialize the CPU lists for the first CPU.
1696  */
1697 void
1698 cpu_list_init(cpu_t *cp)
1699 {
1700         cp->cpu_next = cp;
1701         cp->cpu_prev = cp;
1702         cpu_list = cp;
1703         clock_cpu_list = cp;
1704 
1705         cp->cpu_next_onln = cp;
1706         cp->cpu_prev_onln = cp;
1707         cpu_active = cp;
1708 
1709         cp->cpu_seqid = 0;
1710         CPUSET_ADD(cpu_seqid_inuse, 0);
1711 
1712         /*
1713          * Bootstrap cpu_seq using cpu_list
1714          * The cpu_seq[] table will be dynamically allocated
1715          * when kmem later becomes available (but before going MP)
1716          */
1717         cpu_seq = &cpu_list;
1718 
1719         cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
1720         cp_default.cp_cpulist = cp;
1721         cp_default.cp_ncpus = 1;
1722         cp->cpu_next_part = cp;
1723         cp->cpu_prev_part = cp;
1724         cp->cpu_part = &cp_default;
1725 
1726         CPUSET_ADD(cpu_available, cp->cpu_id);
1727 }
1728 
1729 /*
1730  * Insert a CPU into the list of available CPUs.
1731  */
1732 void
1733 cpu_add_unit(cpu_t *cp)
1734 {
1735         int seqid;
1736 
1737         ASSERT(MUTEX_HELD(&cpu_lock));
1738         ASSERT(cpu_list != NULL);       /* list started in cpu_list_init */
1739 
1740         lgrp_config(LGRP_CONFIG_CPU_ADD, (uintptr_t)cp, 0);
1741 
1742         /*
1743          * Note: most users of the cpu_list will grab the
1744          * cpu_lock to insure that it isn't modified.  However,
1745          * certain users can't or won't do that.  To allow this
1746          * we pause the other cpus.  Users who walk the list
1747          * without cpu_lock, must disable kernel preemption
1748          * to insure that the list isn't modified underneath
1749          * them.  Also, any cached pointers to cpu structures
1750          * must be revalidated by checking to see if the
1751          * cpu_next pointer points to itself.  This check must
1752          * be done with the cpu_lock held or kernel preemption
1753          * disabled.  This check relies upon the fact that
1754          * old cpu structures are not free'ed or cleared after
1755          * then are removed from the cpu_list.
1756          *
1757          * Note that the clock code walks the cpu list dereferencing
1758          * the cpu_part pointer, so we need to initialize it before
1759          * adding the cpu to the list.
1760          */
1761         cp->cpu_part = &cp_default;
1762         pause_cpus(NULL, NULL);
1763         cp->cpu_next = cpu_list;
1764         cp->cpu_prev = cpu_list->cpu_prev;
1765         cpu_list->cpu_prev->cpu_next = cp;
1766         cpu_list->cpu_prev = cp;
1767         start_cpus();
1768 
1769         for (seqid = 0; CPU_IN_SET(cpu_seqid_inuse, seqid); seqid++)
1770                 continue;
1771         CPUSET_ADD(cpu_seqid_inuse, seqid);
1772         cp->cpu_seqid = seqid;
1773 
1774         if (seqid > max_cpu_seqid_ever)
1775                 max_cpu_seqid_ever = seqid;
1776 
1777         ASSERT(ncpus < max_ncpus);
1778         ncpus++;
1779         cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
1780         cpu[cp->cpu_id] = cp;
1781         CPUSET_ADD(cpu_available, cp->cpu_id);
1782         cpu_seq[cp->cpu_seqid] = cp;
1783 
1784         /*
1785          * allocate a pause thread for this CPU.
1786          */
1787         cpu_pause_alloc(cp);
1788 
1789         /*
1790          * So that new CPUs won't have NULL prev_onln and next_onln pointers,
1791          * link them into a list of just that CPU.
1792          * This is so that disp_lowpri_cpu will work for thread_create in
1793          * pause_cpus() when called from the startup thread in a new CPU.
1794          */
1795         cp->cpu_next_onln = cp;
1796         cp->cpu_prev_onln = cp;
1797         cpu_info_kstat_create(cp);
1798         cp->cpu_next_part = cp;
1799         cp->cpu_prev_part = cp;
1800 
1801         init_cpu_mstate(cp, CMS_SYSTEM);
1802 
1803         pool_pset_mod = gethrtime();
1804 }
1805 
1806 /*
1807  * Do the opposite of cpu_add_unit().
1808  */
1809 void
1810 cpu_del_unit(int cpuid)
1811 {
1812         struct cpu      *cp, *cpnext;
1813 
1814         ASSERT(MUTEX_HELD(&cpu_lock));
1815         cp = cpu[cpuid];
1816         ASSERT(cp != NULL);
1817 
1818         ASSERT(cp->cpu_next_onln == cp);
1819         ASSERT(cp->cpu_prev_onln == cp);
1820         ASSERT(cp->cpu_next_part == cp);
1821         ASSERT(cp->cpu_prev_part == cp);
1822 
1823         /*
1824          * Tear down the CPU's physical ID cache, and update any
1825          * processor groups
1826          */
1827         pg_cpu_fini(cp, NULL);
1828         pghw_physid_destroy(cp);
1829 
1830         /*
1831          * Destroy kstat stuff.
1832          */
1833         cpu_info_kstat_destroy(cp);
1834         term_cpu_mstate(cp);
1835         /*
1836          * Free up pause thread.
1837          */
1838         cpu_pause_free(cp);
1839         CPUSET_DEL(cpu_available, cp->cpu_id);
1840         cpu[cp->cpu_id] = NULL;
1841         cpu_seq[cp->cpu_seqid] = NULL;
1842 
1843         /*
1844          * The clock thread and mutex_vector_enter cannot hold the
1845          * cpu_lock while traversing the cpu list, therefore we pause
1846          * all other threads by pausing the other cpus. These, and any
1847          * other routines holding cpu pointers while possibly sleeping
1848          * must be sure to call kpreempt_disable before processing the
1849          * list and be sure to check that the cpu has not been deleted
1850          * after any sleeps (check cp->cpu_next != NULL). We guarantee
1851          * to keep the deleted cpu structure around.
1852          *
1853          * Note that this MUST be done AFTER cpu_available
1854          * has been updated so that we don't waste time
1855          * trying to pause the cpu we're trying to delete.
1856          */
1857         pause_cpus(NULL, NULL);
1858 
1859         cpnext = cp->cpu_next;
1860         cp->cpu_prev->cpu_next = cp->cpu_next;
1861         cp->cpu_next->cpu_prev = cp->cpu_prev;
1862         if (cp == cpu_list)
1863                 cpu_list = cpnext;
1864 
1865         /*
1866          * Signals that the cpu has been deleted (see above).
1867          */
1868         cp->cpu_next = NULL;
1869         cp->cpu_prev = NULL;
1870 
1871         start_cpus();
1872 
1873         CPUSET_DEL(cpu_seqid_inuse, cp->cpu_seqid);
1874         ncpus--;
1875         lgrp_config(LGRP_CONFIG_CPU_DEL, (uintptr_t)cp, 0);
1876 
1877         pool_pset_mod = gethrtime();
1878 }
1879 
1880 /*
1881  * Add a CPU to the list of active CPUs.
1882  *      This routine must not get any locks, because other CPUs are paused.
1883  */
1884 static void
1885 cpu_add_active_internal(cpu_t *cp)
1886 {
1887         cpupart_t       *pp = cp->cpu_part;
1888 
1889         ASSERT(MUTEX_HELD(&cpu_lock));
1890         ASSERT(cpu_list != NULL);       /* list started in cpu_list_init */
1891 
1892         ncpus_online++;
1893         cpu_set_state(cp);
1894         cp->cpu_next_onln = cpu_active;
1895         cp->cpu_prev_onln = cpu_active->cpu_prev_onln;
1896         cpu_active->cpu_prev_onln->cpu_next_onln = cp;
1897         cpu_active->cpu_prev_onln = cp;
1898 
1899         if (pp->cp_cpulist) {
1900                 cp->cpu_next_part = pp->cp_cpulist;
1901                 cp->cpu_prev_part = pp->cp_cpulist->cpu_prev_part;
1902                 pp->cp_cpulist->cpu_prev_part->cpu_next_part = cp;
1903                 pp->cp_cpulist->cpu_prev_part = cp;
1904         } else {
1905                 ASSERT(pp->cp_ncpus == 0);
1906                 pp->cp_cpulist = cp->cpu_next_part = cp->cpu_prev_part = cp;
1907         }
1908         pp->cp_ncpus++;
1909         if (pp->cp_ncpus == 1) {
1910                 cp_numparts_nonempty++;
1911                 ASSERT(cp_numparts_nonempty != 0);
1912         }
1913 
1914         pg_cpu_active(cp);
1915         lgrp_config(LGRP_CONFIG_CPU_ONLINE, (uintptr_t)cp, 0);
1916 
1917         bzero(&cp->cpu_loadavg, sizeof (cp->cpu_loadavg));
1918 }
1919 
1920 /*
1921  * Add a CPU to the list of active CPUs.
1922  *      This is called from machine-dependent layers when a new CPU is started.
1923  */
1924 void
1925 cpu_add_active(cpu_t *cp)
1926 {
1927         pg_cpupart_in(cp, cp->cpu_part);
1928 
1929         pause_cpus(NULL, NULL);
1930         cpu_add_active_internal(cp);
1931         start_cpus();
1932 
1933         cpu_stats_kstat_create(cp);
1934         cpu_create_intrstat(cp);
1935         lgrp_kstat_create(cp);
1936         cpu_state_change_notify(cp->cpu_id, CPU_INIT);
1937 }
1938 
1939 
1940 /*
1941  * Remove a CPU from the list of active CPUs.
1942  *      This routine must not get any locks, because other CPUs are paused.
1943  */
1944 /* ARGSUSED */
1945 static void
1946 cpu_remove_active(cpu_t *cp)
1947 {
1948         cpupart_t       *pp = cp->cpu_part;
1949 
1950         ASSERT(MUTEX_HELD(&cpu_lock));
1951         ASSERT(cp->cpu_next_onln != cp);     /* not the last one */
1952         ASSERT(cp->cpu_prev_onln != cp);     /* not the last one */
1953 
1954         pg_cpu_inactive(cp);
1955 
1956         lgrp_config(LGRP_CONFIG_CPU_OFFLINE, (uintptr_t)cp, 0);
1957 
1958         if (cp == clock_cpu_list)
1959                 clock_cpu_list = cp->cpu_next_onln;
1960 
1961         cp->cpu_prev_onln->cpu_next_onln = cp->cpu_next_onln;
1962         cp->cpu_next_onln->cpu_prev_onln = cp->cpu_prev_onln;
1963         if (cpu_active == cp) {
1964                 cpu_active = cp->cpu_next_onln;
1965         }
1966         cp->cpu_next_onln = cp;
1967         cp->cpu_prev_onln = cp;
1968 
1969         cp->cpu_prev_part->cpu_next_part = cp->cpu_next_part;
1970         cp->cpu_next_part->cpu_prev_part = cp->cpu_prev_part;
1971         if (pp->cp_cpulist == cp) {
1972                 pp->cp_cpulist = cp->cpu_next_part;
1973                 ASSERT(pp->cp_cpulist != cp);
1974         }
1975         cp->cpu_next_part = cp;
1976         cp->cpu_prev_part = cp;
1977         pp->cp_ncpus--;
1978         if (pp->cp_ncpus == 0) {
1979                 cp_numparts_nonempty--;
1980                 ASSERT(cp_numparts_nonempty != 0);
1981         }
1982 }
1983 
1984 /*
1985  * Routine used to setup a newly inserted CPU in preparation for starting
1986  * it running code.
1987  */
1988 int
1989 cpu_configure(int cpuid)
1990 {
1991         int retval = 0;
1992 
1993         ASSERT(MUTEX_HELD(&cpu_lock));
1994 
1995         /*
1996          * Some structures are statically allocated based upon
1997          * the maximum number of cpus the system supports.  Do not
1998          * try to add anything beyond this limit.
1999          */
2000         if (cpuid < 0 || cpuid >= NCPU) {
2001                 return (EINVAL);
2002         }
2003 
2004         if ((cpu[cpuid] != NULL) && (cpu[cpuid]->cpu_flags != 0)) {
2005                 return (EALREADY);
2006         }
2007 
2008         if ((retval = mp_cpu_configure(cpuid)) != 0) {
2009                 return (retval);
2010         }
2011 
2012         cpu[cpuid]->cpu_flags = CPU_QUIESCED | CPU_OFFLINE | CPU_POWEROFF;
2013         cpu_set_state(cpu[cpuid]);
2014         retval = cpu_state_change_hooks(cpuid, CPU_CONFIG, CPU_UNCONFIG);
2015         if (retval != 0)
2016                 (void) mp_cpu_unconfigure(cpuid);
2017 
2018         return (retval);
2019 }
2020 
2021 /*
2022  * Routine used to cleanup a CPU that has been powered off.  This will
2023  * destroy all per-cpu information related to this cpu.
2024  */
2025 int
2026 cpu_unconfigure(int cpuid)
2027 {
2028         int error;
2029 
2030         ASSERT(MUTEX_HELD(&cpu_lock));
2031 
2032         if (cpu[cpuid] == NULL) {
2033                 return (ENODEV);
2034         }
2035 
2036         if (cpu[cpuid]->cpu_flags == 0) {
2037                 return (EALREADY);
2038         }
2039 
2040         if ((cpu[cpuid]->cpu_flags & CPU_POWEROFF) == 0) {
2041                 return (EBUSY);
2042         }
2043 
2044         if (cpu[cpuid]->cpu_props != NULL) {
2045                 (void) nvlist_free(cpu[cpuid]->cpu_props);
2046                 cpu[cpuid]->cpu_props = NULL;
2047         }
2048 
2049         error = cpu_state_change_hooks(cpuid, CPU_UNCONFIG, CPU_CONFIG);
2050 
2051         if (error != 0)
2052                 return (error);
2053 
2054         return (mp_cpu_unconfigure(cpuid));
2055 }
2056 
2057 /*
2058  * Routines for registering and de-registering cpu_setup callback functions.
2059  *
2060  * Caller's context
2061  *      These routines must not be called from a driver's attach(9E) or
2062  *      detach(9E) entry point.
2063  *
2064  * NOTE: CPU callbacks should not block. They are called with cpu_lock held.
2065  */
2066 
2067 /*
2068  * Ideally, these would be dynamically allocated and put into a linked
2069  * list; however that is not feasible because the registration routine
2070  * has to be available before the kmem allocator is working (in fact,
2071  * it is called by the kmem allocator init code).  In any case, there
2072  * are quite a few extra entries for future users.
2073  */
2074 #define NCPU_SETUPS     20
2075 
2076 struct cpu_setup {
2077         cpu_setup_func_t *func;
2078         void *arg;
2079 } cpu_setups[NCPU_SETUPS];
2080 
2081 void
2082 register_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2083 {
2084         int i;
2085 
2086         ASSERT(MUTEX_HELD(&cpu_lock));
2087 
2088         for (i = 0; i < NCPU_SETUPS; i++)
2089                 if (cpu_setups[i].func == NULL)
2090                         break;
2091         if (i >= NCPU_SETUPS)
2092                 cmn_err(CE_PANIC, "Ran out of cpu_setup callback entries");
2093 
2094         cpu_setups[i].func = func;
2095         cpu_setups[i].arg = arg;
2096 }
2097 
2098 void
2099 unregister_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2100 {
2101         int i;
2102 
2103         ASSERT(MUTEX_HELD(&cpu_lock));
2104 
2105         for (i = 0; i < NCPU_SETUPS; i++)
2106                 if ((cpu_setups[i].func == func) &&
2107                     (cpu_setups[i].arg == arg))
2108                         break;
2109         if (i >= NCPU_SETUPS)
2110                 cmn_err(CE_PANIC, "Could not find cpu_setup callback to "
2111                     "deregister");
2112 
2113         cpu_setups[i].func = NULL;
2114         cpu_setups[i].arg = 0;
2115 }
2116 
2117 /*
2118  * Call any state change hooks for this CPU, ignore any errors.
2119  */
2120 void
2121 cpu_state_change_notify(int id, cpu_setup_t what)
2122 {
2123         int i;
2124 
2125         ASSERT(MUTEX_HELD(&cpu_lock));
2126 
2127         for (i = 0; i < NCPU_SETUPS; i++) {
2128                 if (cpu_setups[i].func != NULL) {
2129                         cpu_setups[i].func(what, id, cpu_setups[i].arg);
2130                 }
2131         }
2132 }
2133 
2134 /*
2135  * Call any state change hooks for this CPU, undo it if error found.
2136  */
2137 static int
2138 cpu_state_change_hooks(int id, cpu_setup_t what, cpu_setup_t undo)
2139 {
2140         int i;
2141         int retval = 0;
2142 
2143         ASSERT(MUTEX_HELD(&cpu_lock));
2144 
2145         for (i = 0; i < NCPU_SETUPS; i++) {
2146                 if (cpu_setups[i].func != NULL) {
2147                         retval = cpu_setups[i].func(what, id,
2148                             cpu_setups[i].arg);
2149                         if (retval) {
2150                                 for (i--; i >= 0; i--) {
2151                                         if (cpu_setups[i].func != NULL)
2152                                                 cpu_setups[i].func(undo,
2153                                                     id, cpu_setups[i].arg);
2154                                 }
2155                                 break;
2156                         }
2157                 }
2158         }
2159         return (retval);
2160 }
2161 
2162 /*
2163  * Export information about this CPU via the kstat mechanism.
2164  */
2165 static struct {
2166         kstat_named_t ci_state;
2167         kstat_named_t ci_state_begin;
2168         kstat_named_t ci_cpu_type;
2169         kstat_named_t ci_fpu_type;
2170         kstat_named_t ci_clock_MHz;
2171         kstat_named_t ci_chip_id;
2172         kstat_named_t ci_implementation;
2173         kstat_named_t ci_brandstr;
2174         kstat_named_t ci_core_id;
2175         kstat_named_t ci_curr_clock_Hz;
2176         kstat_named_t ci_supp_freq_Hz;
2177         kstat_named_t ci_pg_id;
2178 #if defined(__sparcv9)
2179         kstat_named_t ci_device_ID;
2180         kstat_named_t ci_cpu_fru;
2181 #endif
2182 #if defined(__x86)
2183         kstat_named_t ci_vendorstr;
2184         kstat_named_t ci_family;
2185         kstat_named_t ci_model;
2186         kstat_named_t ci_step;
2187         kstat_named_t ci_clogid;
2188         kstat_named_t ci_pkg_core_id;
2189         kstat_named_t ci_ncpuperchip;
2190         kstat_named_t ci_ncoreperchip;
2191         kstat_named_t ci_max_cstates;
2192         kstat_named_t ci_curr_cstate;
2193         kstat_named_t ci_cacheid;
2194         kstat_named_t ci_sktstr;
2195 #endif
2196 } cpu_info_template = {
2197         { "state",                      KSTAT_DATA_CHAR },
2198         { "state_begin",                KSTAT_DATA_LONG },
2199         { "cpu_type",                   KSTAT_DATA_CHAR },
2200         { "fpu_type",                   KSTAT_DATA_CHAR },
2201         { "clock_MHz",                  KSTAT_DATA_LONG },
2202         { "chip_id",                    KSTAT_DATA_LONG },
2203         { "implementation",             KSTAT_DATA_STRING },
2204         { "brand",                      KSTAT_DATA_STRING },
2205         { "core_id",                    KSTAT_DATA_LONG },
2206         { "current_clock_Hz",           KSTAT_DATA_UINT64 },
2207         { "supported_frequencies_Hz",   KSTAT_DATA_STRING },
2208         { "pg_id",                      KSTAT_DATA_LONG },
2209 #if defined(__sparcv9)
2210         { "device_ID",                  KSTAT_DATA_UINT64 },
2211         { "cpu_fru",                    KSTAT_DATA_STRING },
2212 #endif
2213 #if defined(__x86)
2214         { "vendor_id",                  KSTAT_DATA_STRING },
2215         { "family",                     KSTAT_DATA_INT32 },
2216         { "model",                      KSTAT_DATA_INT32 },
2217         { "stepping",                   KSTAT_DATA_INT32 },
2218         { "clog_id",                    KSTAT_DATA_INT32 },
2219         { "pkg_core_id",                KSTAT_DATA_LONG },
2220         { "ncpu_per_chip",              KSTAT_DATA_INT32 },
2221         { "ncore_per_chip",             KSTAT_DATA_INT32 },
2222         { "supported_max_cstates",      KSTAT_DATA_INT32 },
2223         { "current_cstate",             KSTAT_DATA_INT32 },
2224         { "cache_id",                   KSTAT_DATA_INT32 },
2225         { "socket_type",                KSTAT_DATA_STRING },
2226 #endif
2227 };
2228 
2229 static kmutex_t cpu_info_template_lock;
2230 
2231 static int
2232 cpu_info_kstat_update(kstat_t *ksp, int rw)
2233 {
2234         cpu_t   *cp = ksp->ks_private;
2235         const char *pi_state;
2236 
2237         if (rw == KSTAT_WRITE)
2238                 return (EACCES);
2239 
2240 #if defined(__x86)
2241         /* Is the cpu still initialising itself? */
2242         if (cpuid_checkpass(cp, 1) == 0)
2243                 return (ENXIO);
2244 #endif
2245         switch (cp->cpu_type_info.pi_state) {
2246         case P_ONLINE:
2247                 pi_state = PS_ONLINE;
2248                 break;
2249         case P_POWEROFF:
2250                 pi_state = PS_POWEROFF;
2251                 break;
2252         case P_NOINTR:
2253                 pi_state = PS_NOINTR;
2254                 break;
2255         case P_FAULTED:
2256                 pi_state = PS_FAULTED;
2257                 break;
2258         case P_SPARE:
2259                 pi_state = PS_SPARE;
2260                 break;
2261         case P_OFFLINE:
2262                 pi_state = PS_OFFLINE;
2263                 break;
2264         default:
2265                 pi_state = "unknown";
2266         }
2267         (void) strcpy(cpu_info_template.ci_state.value.c, pi_state);
2268         cpu_info_template.ci_state_begin.value.l = cp->cpu_state_begin;
2269         (void) strncpy(cpu_info_template.ci_cpu_type.value.c,
2270             cp->cpu_type_info.pi_processor_type, 15);
2271         (void) strncpy(cpu_info_template.ci_fpu_type.value.c,
2272             cp->cpu_type_info.pi_fputypes, 15);
2273         cpu_info_template.ci_clock_MHz.value.l = cp->cpu_type_info.pi_clock;
2274         cpu_info_template.ci_chip_id.value.l =
2275             pg_plat_hw_instance_id(cp, PGHW_CHIP);
2276         kstat_named_setstr(&cpu_info_template.ci_implementation,
2277             cp->cpu_idstr);
2278         kstat_named_setstr(&cpu_info_template.ci_brandstr, cp->cpu_brandstr);
2279         cpu_info_template.ci_core_id.value.l = pg_plat_get_core_id(cp);
2280         cpu_info_template.ci_curr_clock_Hz.value.ui64 =
2281             cp->cpu_curr_clock;
2282         cpu_info_template.ci_pg_id.value.l =
2283             cp->cpu_pg && cp->cpu_pg->cmt_lineage ?
2284             cp->cpu_pg->cmt_lineage->pg_id : -1;
2285         kstat_named_setstr(&cpu_info_template.ci_supp_freq_Hz,
2286             cp->cpu_supp_freqs);
2287 #if defined(__sparcv9)
2288         cpu_info_template.ci_device_ID.value.ui64 =
2289             cpunodes[cp->cpu_id].device_id;
2290         kstat_named_setstr(&cpu_info_template.ci_cpu_fru, cpu_fru_fmri(cp));
2291 #endif
2292 #if defined(__x86)
2293         kstat_named_setstr(&cpu_info_template.ci_vendorstr,
2294             cpuid_getvendorstr(cp));
2295         cpu_info_template.ci_family.value.l = cpuid_getfamily(cp);
2296         cpu_info_template.ci_model.value.l = cpuid_getmodel(cp);
2297         cpu_info_template.ci_step.value.l = cpuid_getstep(cp);
2298         cpu_info_template.ci_clogid.value.l = cpuid_get_clogid(cp);
2299         cpu_info_template.ci_ncpuperchip.value.l = cpuid_get_ncpu_per_chip(cp);
2300         cpu_info_template.ci_ncoreperchip.value.l =
2301             cpuid_get_ncore_per_chip(cp);
2302         cpu_info_template.ci_pkg_core_id.value.l = cpuid_get_pkgcoreid(cp);
2303         cpu_info_template.ci_max_cstates.value.l = cp->cpu_m.max_cstates;
2304         cpu_info_template.ci_curr_cstate.value.l = cpu_idle_get_cpu_state(cp);
2305         cpu_info_template.ci_cacheid.value.i32 = cpuid_get_cacheid(cp);
2306         kstat_named_setstr(&cpu_info_template.ci_sktstr,
2307             cpuid_getsocketstr(cp));
2308 #endif
2309 
2310         return (0);
2311 }
2312 
2313 static void
2314 cpu_info_kstat_create(cpu_t *cp)
2315 {
2316         zoneid_t zoneid;
2317 
2318         ASSERT(MUTEX_HELD(&cpu_lock));
2319 
2320         if (pool_pset_enabled())
2321                 zoneid = GLOBAL_ZONEID;
2322         else
2323                 zoneid = ALL_ZONES;
2324         if ((cp->cpu_info_kstat = kstat_create_zone("cpu_info", cp->cpu_id,
2325             NULL, "misc", KSTAT_TYPE_NAMED,
2326             sizeof (cpu_info_template) / sizeof (kstat_named_t),
2327             KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_VAR_SIZE, zoneid)) != NULL) {
2328                 cp->cpu_info_kstat->ks_data_size += 2 * CPU_IDSTRLEN;
2329 #if defined(__sparcv9)
2330                 cp->cpu_info_kstat->ks_data_size +=
2331                     strlen(cpu_fru_fmri(cp)) + 1;
2332 #endif
2333 #if defined(__x86)
2334                 cp->cpu_info_kstat->ks_data_size += X86_VENDOR_STRLEN;
2335 #endif
2336                 if (cp->cpu_supp_freqs != NULL)
2337                         cp->cpu_info_kstat->ks_data_size +=
2338                             strlen(cp->cpu_supp_freqs) + 1;
2339                 cp->cpu_info_kstat->ks_lock = &cpu_info_template_lock;
2340                 cp->cpu_info_kstat->ks_data = &cpu_info_template;
2341                 cp->cpu_info_kstat->ks_private = cp;
2342                 cp->cpu_info_kstat->ks_update = cpu_info_kstat_update;
2343                 kstat_install(cp->cpu_info_kstat);
2344         }
2345 }
2346 
2347 static void
2348 cpu_info_kstat_destroy(cpu_t *cp)
2349 {
2350         ASSERT(MUTEX_HELD(&cpu_lock));
2351 
2352         kstat_delete(cp->cpu_info_kstat);
2353         cp->cpu_info_kstat = NULL;
2354 }
2355 
2356 /*
2357  * Create and install kstats for the boot CPU.
2358  */
2359 void
2360 cpu_kstat_init(cpu_t *cp)
2361 {
2362         mutex_enter(&cpu_lock);
2363         cpu_info_kstat_create(cp);
2364         cpu_stats_kstat_create(cp);
2365         cpu_create_intrstat(cp);
2366         cpu_set_state(cp);
2367         mutex_exit(&cpu_lock);
2368 }
2369 
2370 /*
2371  * Make visible to the zone that subset of the cpu information that would be
2372  * initialized when a cpu is configured (but still offline).
2373  */
2374 void
2375 cpu_visibility_configure(cpu_t *cp, zone_t *zone)
2376 {
2377         zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2378 
2379         ASSERT(MUTEX_HELD(&cpu_lock));
2380         ASSERT(pool_pset_enabled());
2381         ASSERT(cp != NULL);
2382 
2383         if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2384                 zone->zone_ncpus++;
2385                 ASSERT(zone->zone_ncpus <= ncpus);
2386         }
2387         if (cp->cpu_info_kstat != NULL)
2388                 kstat_zone_add(cp->cpu_info_kstat, zoneid);
2389 }
2390 
2391 /*
2392  * Make visible to the zone that subset of the cpu information that would be
2393  * initialized when a previously configured cpu is onlined.
2394  */
2395 void
2396 cpu_visibility_online(cpu_t *cp, zone_t *zone)
2397 {
2398         kstat_t *ksp;
2399         char name[sizeof ("cpu_stat") + 10];    /* enough for 32-bit cpuids */
2400         zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2401         processorid_t cpun;
2402 
2403         ASSERT(MUTEX_HELD(&cpu_lock));
2404         ASSERT(pool_pset_enabled());
2405         ASSERT(cp != NULL);
2406         ASSERT(cpu_is_active(cp));
2407 
2408         cpun = cp->cpu_id;
2409         if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2410                 zone->zone_ncpus_online++;
2411                 ASSERT(zone->zone_ncpus_online <= ncpus_online);
2412         }
2413         (void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2414         if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2415             != NULL) {
2416                 kstat_zone_add(ksp, zoneid);
2417                 kstat_rele(ksp);
2418         }
2419         if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2420                 kstat_zone_add(ksp, zoneid);
2421                 kstat_rele(ksp);
2422         }
2423         if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2424                 kstat_zone_add(ksp, zoneid);
2425                 kstat_rele(ksp);
2426         }
2427         if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2428             NULL) {
2429                 kstat_zone_add(ksp, zoneid);
2430                 kstat_rele(ksp);
2431         }
2432 }
2433 
2434 /*
2435  * Update relevant kstats such that cpu is now visible to processes
2436  * executing in specified zone.
2437  */
2438 void
2439 cpu_visibility_add(cpu_t *cp, zone_t *zone)
2440 {
2441         cpu_visibility_configure(cp, zone);
2442         if (cpu_is_active(cp))
2443                 cpu_visibility_online(cp, zone);
2444 }
2445 
2446 /*
2447  * Make invisible to the zone that subset of the cpu information that would be
2448  * torn down when a previously offlined cpu is unconfigured.
2449  */
2450 void
2451 cpu_visibility_unconfigure(cpu_t *cp, zone_t *zone)
2452 {
2453         zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2454 
2455         ASSERT(MUTEX_HELD(&cpu_lock));
2456         ASSERT(pool_pset_enabled());
2457         ASSERT(cp != NULL);
2458 
2459         if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2460                 ASSERT(zone->zone_ncpus != 0);
2461                 zone->zone_ncpus--;
2462         }
2463         if (cp->cpu_info_kstat)
2464                 kstat_zone_remove(cp->cpu_info_kstat, zoneid);
2465 }
2466 
2467 /*
2468  * Make invisible to the zone that subset of the cpu information that would be
2469  * torn down when a cpu is offlined (but still configured).
2470  */
2471 void
2472 cpu_visibility_offline(cpu_t *cp, zone_t *zone)
2473 {
2474         kstat_t *ksp;
2475         char name[sizeof ("cpu_stat") + 10];    /* enough for 32-bit cpuids */
2476         zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2477         processorid_t cpun;
2478 
2479         ASSERT(MUTEX_HELD(&cpu_lock));
2480         ASSERT(pool_pset_enabled());
2481         ASSERT(cp != NULL);
2482         ASSERT(cpu_is_active(cp));
2483 
2484         cpun = cp->cpu_id;
2485         if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2486                 ASSERT(zone->zone_ncpus_online != 0);
2487                 zone->zone_ncpus_online--;
2488         }
2489 
2490         if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2491             NULL) {
2492                 kstat_zone_remove(ksp, zoneid);
2493                 kstat_rele(ksp);
2494         }
2495         if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2496                 kstat_zone_remove(ksp, zoneid);
2497                 kstat_rele(ksp);
2498         }
2499         if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2500                 kstat_zone_remove(ksp, zoneid);
2501                 kstat_rele(ksp);
2502         }
2503         (void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2504         if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2505             != NULL) {
2506                 kstat_zone_remove(ksp, zoneid);
2507                 kstat_rele(ksp);
2508         }
2509 }
2510 
2511 /*
2512  * Update relevant kstats such that cpu is no longer visible to processes
2513  * executing in specified zone.
2514  */
2515 void
2516 cpu_visibility_remove(cpu_t *cp, zone_t *zone)
2517 {
2518         if (cpu_is_active(cp))
2519                 cpu_visibility_offline(cp, zone);
2520         cpu_visibility_unconfigure(cp, zone);
2521 }
2522 
2523 /*
2524  * Bind a thread to a CPU as requested.
2525  */
2526 int
2527 cpu_bind_thread(kthread_id_t tp, processorid_t bind, processorid_t *obind,
2528     int *error)
2529 {
2530         processorid_t   binding;
2531         cpu_t           *cp = NULL;
2532 
2533         ASSERT(MUTEX_HELD(&cpu_lock));
2534         ASSERT(MUTEX_HELD(&ttoproc(tp)->p_lock));
2535 
2536         thread_lock(tp);
2537 
2538         /*
2539          * Record old binding, but change the obind, which was initialized
2540          * to PBIND_NONE, only if this thread has a binding.  This avoids
2541          * reporting PBIND_NONE for a process when some LWPs are bound.
2542          */
2543         binding = tp->t_bind_cpu;
2544         if (binding != PBIND_NONE)
2545                 *obind = binding;       /* record old binding */
2546 
2547         switch (bind) {
2548         case PBIND_QUERY:
2549                 /* Just return the old binding */
2550                 thread_unlock(tp);
2551                 return (0);
2552 
2553         case PBIND_QUERY_TYPE:
2554                 /* Return the binding type */
2555                 *obind = TB_CPU_IS_SOFT(tp) ? PBIND_SOFT : PBIND_HARD;
2556                 thread_unlock(tp);
2557                 return (0);
2558 
2559         case PBIND_SOFT:
2560                 /*
2561                  *  Set soft binding for this thread and return the actual
2562                  *  binding
2563                  */
2564                 TB_CPU_SOFT_SET(tp);
2565                 thread_unlock(tp);
2566                 return (0);
2567 
2568         case PBIND_HARD:
2569                 /*
2570                  *  Set hard binding for this thread and return the actual
2571                  *  binding
2572                  */
2573                 TB_CPU_HARD_SET(tp);
2574                 thread_unlock(tp);
2575                 return (0);
2576 
2577         default:
2578                 break;
2579         }
2580 
2581         /*
2582          * If this thread/LWP cannot be bound because of permission
2583          * problems, just note that and return success so that the
2584          * other threads/LWPs will be bound.  This is the way
2585          * processor_bind() is defined to work.
2586          *
2587          * Binding will get EPERM if the thread is of system class
2588          * or hasprocperm() fails.
2589          */
2590         if (tp->t_cid == 0 || !hasprocperm(tp->t_cred, CRED())) {
2591                 *error = EPERM;
2592                 thread_unlock(tp);
2593                 return (0);
2594         }
2595 
2596         binding = bind;
2597         if (binding != PBIND_NONE) {
2598                 cp = cpu_get((processorid_t)binding);
2599                 /*
2600                  * Make sure binding is valid and is in right partition.
2601                  */
2602                 if (cp == NULL || tp->t_cpupart != cp->cpu_part) {
2603                         *error = EINVAL;
2604                         thread_unlock(tp);
2605                         return (0);
2606                 }
2607         }
2608         tp->t_bind_cpu = binding;    /* set new binding */
2609 
2610         /*
2611          * If there is no system-set reason for affinity, set
2612          * the t_bound_cpu field to reflect the binding.
2613          */
2614         if (tp->t_affinitycnt == 0) {
2615                 if (binding == PBIND_NONE) {
2616                         /*
2617                          * We may need to adjust disp_max_unbound_pri
2618                          * since we're becoming unbound.
2619                          */
2620                         disp_adjust_unbound_pri(tp);
2621 
2622                         tp->t_bound_cpu = NULL;      /* set new binding */
2623 
2624                         /*
2625                          * Move thread to lgroup with strongest affinity
2626                          * after unbinding
2627                          */
2628                         if (tp->t_lgrp_affinity)
2629                                 lgrp_move_thread(tp,
2630                                     lgrp_choose(tp, tp->t_cpupart), 1);
2631 
2632                         if (tp->t_state == TS_ONPROC &&
2633                             tp->t_cpu->cpu_part != tp->t_cpupart)
2634                                 cpu_surrender(tp);
2635                 } else {
2636                         lpl_t   *lpl;
2637 
2638                         tp->t_bound_cpu = cp;
2639                         ASSERT(cp->cpu_lpl != NULL);
2640 
2641                         /*
2642                          * Set home to lgroup with most affinity containing CPU
2643                          * that thread is being bound or minimum bounding
2644                          * lgroup if no affinities set
2645                          */
2646                         if (tp->t_lgrp_affinity)
2647                                 lpl = lgrp_affinity_best(tp, tp->t_cpupart,
2648                                     LGRP_NONE, B_FALSE);
2649                         else
2650                                 lpl = cp->cpu_lpl;
2651 
2652                         if (tp->t_lpl != lpl) {
2653                                 /* can't grab cpu_lock */
2654                                 lgrp_move_thread(tp, lpl, 1);
2655                         }
2656 
2657                         /*
2658                          * Make the thread switch to the bound CPU.
2659                          * If the thread is runnable, we need to
2660                          * requeue it even if t_cpu is already set
2661                          * to the right CPU, since it may be on a
2662                          * kpreempt queue and need to move to a local
2663                          * queue.  We could check t_disp_queue to
2664                          * avoid unnecessary overhead if it's already
2665                          * on the right queue, but since this isn't
2666                          * a performance-critical operation it doesn't
2667                          * seem worth the extra code and complexity.
2668                          *
2669                          * If the thread is weakbound to the cpu then it will
2670                          * resist the new binding request until the weak
2671                          * binding drops.  The cpu_surrender or requeueing
2672                          * below could be skipped in such cases (since it
2673                          * will have no effect), but that would require
2674                          * thread_allowmigrate to acquire thread_lock so
2675                          * we'll take the very occasional hit here instead.
2676                          */
2677                         if (tp->t_state == TS_ONPROC) {
2678                                 cpu_surrender(tp);
2679                         } else if (tp->t_state == TS_RUN) {
2680                                 cpu_t *ocp = tp->t_cpu;
2681 
2682                                 (void) dispdeq(tp);
2683                                 setbackdq(tp);
2684                                 /*
2685                                  * Either on the bound CPU's disp queue now,
2686                                  * or swapped out or on the swap queue.
2687                                  */
2688                                 ASSERT(tp->t_disp_queue == cp->cpu_disp ||
2689                                     tp->t_weakbound_cpu == ocp ||
2690                                     (tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ))
2691                                     != TS_LOAD);
2692                         }
2693                 }
2694         }
2695 
2696         /*
2697          * Our binding has changed; set TP_CHANGEBIND.
2698          */
2699         tp->t_proc_flag |= TP_CHANGEBIND;
2700         aston(tp);
2701 
2702         thread_unlock(tp);
2703 
2704         return (0);
2705 }
2706 
2707 #if CPUSET_WORDS > 1
2708 
2709 /*
2710  * Functions for implementing cpuset operations when a cpuset is more
2711  * than one word.  On platforms where a cpuset is a single word these
2712  * are implemented as macros in cpuvar.h.
2713  */
2714 
2715 void
2716 cpuset_all(cpuset_t *s)
2717 {
2718         int i;
2719 
2720         for (i = 0; i < CPUSET_WORDS; i++)
2721                 s->cpub[i] = ~0UL;
2722 }
2723 
2724 void
2725 cpuset_all_but(cpuset_t *s, uint_t cpu)
2726 {
2727         cpuset_all(s);
2728         CPUSET_DEL(*s, cpu);
2729 }
2730 
2731 void
2732 cpuset_only(cpuset_t *s, uint_t cpu)
2733 {
2734         CPUSET_ZERO(*s);
2735         CPUSET_ADD(*s, cpu);
2736 }
2737 
2738 int
2739 cpuset_isnull(cpuset_t *s)
2740 {
2741         int i;
2742 
2743         for (i = 0; i < CPUSET_WORDS; i++)
2744                 if (s->cpub[i] != 0)
2745                         return (0);
2746         return (1);
2747 }
2748 
2749 int
2750 cpuset_cmp(cpuset_t *s1, cpuset_t *s2)
2751 {
2752         int i;
2753 
2754         for (i = 0; i < CPUSET_WORDS; i++)
2755                 if (s1->cpub[i] != s2->cpub[i])
2756                         return (0);
2757         return (1);
2758 }
2759 
2760 uint_t
2761 cpuset_find(cpuset_t *s)
2762 {
2763 
2764         uint_t  i;
2765         uint_t  cpu = (uint_t)-1;
2766 
2767         /*
2768          * Find a cpu in the cpuset
2769          */
2770         for (i = 0; i < CPUSET_WORDS; i++) {
2771                 cpu = (uint_t)(lowbit(s->cpub[i]) - 1);
2772                 if (cpu != (uint_t)-1) {
2773                         cpu += i * BT_NBIPUL;
2774                         break;
2775                 }
2776         }
2777         return (cpu);
2778 }
2779 
2780 void
2781 cpuset_bounds(cpuset_t *s, uint_t *smallestid, uint_t *largestid)
2782 {
2783         int     i, j;
2784         uint_t  bit;
2785 
2786         /*
2787          * First, find the smallest cpu id in the set.
2788          */
2789         for (i = 0; i < CPUSET_WORDS; i++) {
2790                 if (s->cpub[i] != 0) {
2791                         bit = (uint_t)(lowbit(s->cpub[i]) - 1);
2792                         ASSERT(bit != (uint_t)-1);
2793                         *smallestid = bit + (i * BT_NBIPUL);
2794 
2795                         /*
2796                          * Now find the largest cpu id in
2797                          * the set and return immediately.
2798                          * Done in an inner loop to avoid
2799                          * having to break out of the first
2800                          * loop.
2801                          */
2802                         for (j = CPUSET_WORDS - 1; j >= i; j--) {
2803                                 if (s->cpub[j] != 0) {
2804                                         bit = (uint_t)(highbit(s->cpub[j]) - 1);
2805                                         ASSERT(bit != (uint_t)-1);
2806                                         *largestid = bit + (j * BT_NBIPUL);
2807                                         ASSERT(*largestid >= *smallestid);
2808                                         return;
2809                                 }
2810                         }
2811 
2812                         /*
2813                          * If this code is reached, a
2814                          * smallestid was found, but not a
2815                          * largestid. The cpuset must have
2816                          * been changed during the course
2817                          * of this function call.
2818                          */
2819                         ASSERT(0);
2820                 }
2821         }
2822         *smallestid = *largestid = CPUSET_NOTINSET;
2823 }
2824 
2825 #endif  /* CPUSET_WORDS */
2826 
2827 /*
2828  * Unbind threads bound to specified CPU.
2829  *
2830  * If `unbind_all_threads' is true, unbind all user threads bound to a given
2831  * CPU. Otherwise unbind all soft-bound user threads.
2832  */
2833 int
2834 cpu_unbind(processorid_t cpu, boolean_t unbind_all_threads)
2835 {
2836         processorid_t obind;
2837         kthread_t *tp;
2838         int ret = 0;
2839         proc_t *pp;
2840         int err, berr = 0;
2841 
2842         ASSERT(MUTEX_HELD(&cpu_lock));
2843 
2844         mutex_enter(&pidlock);
2845         for (pp = practive; pp != NULL; pp = pp->p_next) {
2846                 mutex_enter(&pp->p_lock);
2847                 tp = pp->p_tlist;
2848                 /*
2849                  * Skip zombies, kernel processes, and processes in
2850                  * other zones, if called from a non-global zone.
2851                  */
2852                 if (tp == NULL || (pp->p_flag & SSYS) ||
2853                     !HASZONEACCESS(curproc, pp->p_zone->zone_id)) {
2854                         mutex_exit(&pp->p_lock);
2855                         continue;
2856                 }
2857                 do {
2858                         if (tp->t_bind_cpu != cpu)
2859                                 continue;
2860                         /*
2861                          * Skip threads with hard binding when
2862                          * `unbind_all_threads' is not specified.
2863                          */
2864                         if (!unbind_all_threads && TB_CPU_IS_HARD(tp))
2865                                 continue;
2866                         err = cpu_bind_thread(tp, PBIND_NONE, &obind, &berr);
2867                         if (ret == 0)
2868                                 ret = err;
2869                 } while ((tp = tp->t_forw) != pp->p_tlist);
2870                 mutex_exit(&pp->p_lock);
2871         }
2872         mutex_exit(&pidlock);
2873         if (ret == 0)
2874                 ret = berr;
2875         return (ret);
2876 }
2877 
2878 
2879 /*
2880  * Destroy all remaining bound threads on a cpu.
2881  */
2882 void
2883 cpu_destroy_bound_threads(cpu_t *cp)
2884 {
2885         extern id_t syscid;
2886         register kthread_id_t   t, tlist, tnext;
2887 
2888         /*
2889          * Destroy all remaining bound threads on the cpu.  This
2890          * should include both the interrupt threads and the idle thread.
2891          * This requires some care, since we need to traverse the
2892          * thread list with the pidlock mutex locked, but thread_free
2893          * also locks the pidlock mutex.  So, we collect the threads
2894          * we're going to reap in a list headed by "tlist", then we
2895          * unlock the pidlock mutex and traverse the tlist list,
2896          * doing thread_free's on the thread's.  Simple, n'est pas?
2897          * Also, this depends on thread_free not mucking with the
2898          * t_next and t_prev links of the thread.
2899          */
2900 
2901         if ((t = curthread) != NULL) {
2902 
2903                 tlist = NULL;
2904                 mutex_enter(&pidlock);
2905                 do {
2906                         tnext = t->t_next;
2907                         if (t->t_bound_cpu == cp) {
2908 
2909                                 /*
2910                                  * We've found a bound thread, carefully unlink
2911                                  * it out of the thread list, and add it to
2912                                  * our "tlist".  We "know" we don't have to
2913                                  * worry about unlinking curthread (the thread
2914                                  * that is executing this code).
2915                                  */
2916                                 t->t_next->t_prev = t->t_prev;
2917                                 t->t_prev->t_next = t->t_next;
2918                                 t->t_next = tlist;
2919                                 tlist = t;
2920                                 ASSERT(t->t_cid == syscid);
2921                                 /* wake up anyone blocked in thread_join */
2922                                 cv_broadcast(&t->t_joincv);
2923                                 /*
2924                                  * t_lwp set by interrupt threads and not
2925                                  * cleared.
2926                                  */
2927                                 t->t_lwp = NULL;
2928                                 /*
2929                                  * Pause and idle threads always have
2930                                  * t_state set to TS_ONPROC.
2931                                  */
2932                                 t->t_state = TS_FREE;
2933                                 t->t_prev = NULL;    /* Just in case */
2934                         }
2935 
2936                 } while ((t = tnext) != curthread);
2937 
2938                 mutex_exit(&pidlock);
2939 
2940                 mutex_sync();
2941                 for (t = tlist; t != NULL; t = tnext) {
2942                         tnext = t->t_next;
2943                         thread_free(t);
2944                 }
2945         }
2946 }
2947 
2948 /*
2949  * Update the cpu_supp_freqs of this cpu. This information is returned
2950  * as part of cpu_info kstats. If the cpu_info_kstat exists already, then
2951  * maintain the kstat data size.
2952  */
2953 void
2954 cpu_set_supp_freqs(cpu_t *cp, const char *freqs)
2955 {
2956         char clkstr[sizeof ("18446744073709551615") + 1]; /* ui64 MAX */
2957         const char *lfreqs = clkstr;
2958         boolean_t kstat_exists = B_FALSE;
2959         kstat_t *ksp;
2960         size_t len;
2961 
2962         /*
2963          * A NULL pointer means we only support one speed.
2964          */
2965         if (freqs == NULL)
2966                 (void) snprintf(clkstr, sizeof (clkstr), "%"PRIu64,
2967                     cp->cpu_curr_clock);
2968         else
2969                 lfreqs = freqs;
2970 
2971         /*
2972          * Make sure the frequency doesn't change while a snapshot is
2973          * going on. Of course, we only need to worry about this if
2974          * the kstat exists.
2975          */
2976         if ((ksp = cp->cpu_info_kstat) != NULL) {
2977                 mutex_enter(ksp->ks_lock);
2978                 kstat_exists = B_TRUE;
2979         }
2980 
2981         /*
2982          * Free any previously allocated string and if the kstat
2983          * already exists, then update its data size.
2984          */
2985         if (cp->cpu_supp_freqs != NULL) {
2986                 len = strlen(cp->cpu_supp_freqs) + 1;
2987                 kmem_free(cp->cpu_supp_freqs, len);
2988                 if (kstat_exists)
2989                         ksp->ks_data_size -= len;
2990         }
2991 
2992         /*
2993          * Allocate the new string and set the pointer.
2994          */
2995         len = strlen(lfreqs) + 1;
2996         cp->cpu_supp_freqs = kmem_alloc(len, KM_SLEEP);
2997         (void) strcpy(cp->cpu_supp_freqs, lfreqs);
2998 
2999         /*
3000          * If the kstat already exists then update the data size and
3001          * free the lock.
3002          */
3003         if (kstat_exists) {
3004                 ksp->ks_data_size += len;
3005                 mutex_exit(ksp->ks_lock);
3006         }
3007 }
3008 
3009 /*
3010  * Indicate the current CPU's clock freqency (in Hz).
3011  * The calling context must be such that CPU references are safe.
3012  */
3013 void
3014 cpu_set_curr_clock(uint64_t new_clk)
3015 {
3016         uint64_t old_clk;
3017 
3018         old_clk = CPU->cpu_curr_clock;
3019         CPU->cpu_curr_clock = new_clk;
3020 
3021         /*
3022          * The cpu-change-speed DTrace probe exports the frequency in Hz
3023          */
3024         DTRACE_PROBE3(cpu__change__speed, processorid_t, CPU->cpu_id,
3025             uint64_t, old_clk, uint64_t, new_clk);
3026 }
3027 
3028 /*
3029  * processor_info(2) and p_online(2) status support functions
3030  *   The constants returned by the cpu_get_state() and cpu_get_state_str() are
3031  *   for use in communicating processor state information to userland.  Kernel
3032  *   subsystems should only be using the cpu_flags value directly.  Subsystems
3033  *   modifying cpu_flags should record the state change via a call to the
3034  *   cpu_set_state().
3035  */
3036 
3037 /*
3038  * Update the pi_state of this CPU.  This function provides the CPU status for
3039  * the information returned by processor_info(2).
3040  */
3041 void
3042 cpu_set_state(cpu_t *cpu)
3043 {
3044         ASSERT(MUTEX_HELD(&cpu_lock));
3045         cpu->cpu_type_info.pi_state = cpu_get_state(cpu);
3046         cpu->cpu_state_begin = gethrestime_sec();
3047         pool_cpu_mod = gethrtime();
3048 }
3049 
3050 /*
3051  * Return offline/online/other status for the indicated CPU.  Use only for
3052  * communication with user applications; cpu_flags provides the in-kernel
3053  * interface.
3054  */
3055 int
3056 cpu_get_state(cpu_t *cpu)
3057 {
3058         ASSERT(MUTEX_HELD(&cpu_lock));
3059         if (cpu->cpu_flags & CPU_POWEROFF)
3060                 return (P_POWEROFF);
3061         else if (cpu->cpu_flags & CPU_FAULTED)
3062                 return (P_FAULTED);
3063         else if (cpu->cpu_flags & CPU_SPARE)
3064                 return (P_SPARE);
3065         else if ((cpu->cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY)
3066                 return (P_OFFLINE);
3067         else if (cpu->cpu_flags & CPU_ENABLE)
3068                 return (P_ONLINE);
3069         else
3070                 return (P_NOINTR);
3071 }
3072 
3073 /*
3074  * Return processor_info(2) state as a string.
3075  */
3076 const char *
3077 cpu_get_state_str(cpu_t *cpu)
3078 {
3079         const char *string;
3080 
3081         switch (cpu_get_state(cpu)) {
3082         case P_ONLINE:
3083                 string = PS_ONLINE;
3084                 break;
3085         case P_POWEROFF:
3086                 string = PS_POWEROFF;
3087                 break;
3088         case P_NOINTR:
3089                 string = PS_NOINTR;
3090                 break;
3091         case P_SPARE:
3092                 string = PS_SPARE;
3093                 break;
3094         case P_FAULTED:
3095                 string = PS_FAULTED;
3096                 break;
3097         case P_OFFLINE:
3098                 string = PS_OFFLINE;
3099                 break;
3100         default:
3101                 string = "unknown";
3102                 break;
3103         }
3104         return (string);
3105 }
3106 
3107 /*
3108  * Export this CPU's statistics (cpu_stat_t and cpu_stats_t) as raw and named
3109  * kstats, respectively.  This is done when a CPU is initialized or placed
3110  * online via p_online(2).
3111  */
3112 static void
3113 cpu_stats_kstat_create(cpu_t *cp)
3114 {
3115         int     instance = cp->cpu_id;
3116         char    *module = "cpu";
3117         char    *class = "misc";
3118         kstat_t *ksp;
3119         zoneid_t zoneid;
3120 
3121         ASSERT(MUTEX_HELD(&cpu_lock));
3122 
3123         if (pool_pset_enabled())
3124                 zoneid = GLOBAL_ZONEID;
3125         else
3126                 zoneid = ALL_ZONES;
3127         /*
3128          * Create named kstats
3129          */
3130 #define CPU_STATS_KS_CREATE(name, tsize, update_func)                    \
3131         ksp = kstat_create_zone(module, instance, (name), class,         \
3132             KSTAT_TYPE_NAMED, (tsize) / sizeof (kstat_named_t), 0,       \
3133             zoneid);                                                     \
3134         if (ksp != NULL) {                                               \
3135                 ksp->ks_private = cp;                                    \
3136                 ksp->ks_update = (update_func);                          \
3137                 kstat_install(ksp);                                      \
3138         } else                                                           \
3139                 cmn_err(CE_WARN, "cpu: unable to create %s:%d:%s kstat", \
3140                     module, instance, (name));
3141 
3142         CPU_STATS_KS_CREATE("sys", sizeof (cpu_sys_stats_ks_data_template),
3143             cpu_sys_stats_ks_update);
3144         CPU_STATS_KS_CREATE("vm", sizeof (cpu_vm_stats_ks_data_template),
3145             cpu_vm_stats_ks_update);
3146 
3147         /*
3148          * Export the familiar cpu_stat_t KSTAT_TYPE_RAW kstat.
3149          */
3150         ksp = kstat_create_zone("cpu_stat", cp->cpu_id, NULL,
3151             "misc", KSTAT_TYPE_RAW, sizeof (cpu_stat_t), 0, zoneid);
3152         if (ksp != NULL) {
3153                 ksp->ks_update = cpu_stat_ks_update;
3154                 ksp->ks_private = cp;
3155                 kstat_install(ksp);
3156         }
3157 }
3158 
3159 static void
3160 cpu_stats_kstat_destroy(cpu_t *cp)
3161 {
3162         char ks_name[KSTAT_STRLEN];
3163 
3164         (void) sprintf(ks_name, "cpu_stat%d", cp->cpu_id);
3165         kstat_delete_byname("cpu_stat", cp->cpu_id, ks_name);
3166 
3167         kstat_delete_byname("cpu", cp->cpu_id, "sys");
3168         kstat_delete_byname("cpu", cp->cpu_id, "vm");
3169 }
3170 
3171 static int
3172 cpu_sys_stats_ks_update(kstat_t *ksp, int rw)
3173 {
3174         cpu_t *cp = (cpu_t *)ksp->ks_private;
3175         struct cpu_sys_stats_ks_data *csskd;
3176         cpu_sys_stats_t *css;
3177         hrtime_t msnsecs[NCMSTATES];
3178         int     i;
3179 
3180         if (rw == KSTAT_WRITE)
3181                 return (EACCES);
3182 
3183         csskd = ksp->ks_data;
3184         css = &cp->cpu_stats.sys;
3185 
3186         /*
3187          * Read CPU mstate, but compare with the last values we
3188          * received to make sure that the returned kstats never
3189          * decrease.
3190          */
3191 
3192         get_cpu_mstate(cp, msnsecs);
3193         if (csskd->cpu_nsec_idle.value.ui64 > msnsecs[CMS_IDLE])
3194                 msnsecs[CMS_IDLE] = csskd->cpu_nsec_idle.value.ui64;
3195         if (csskd->cpu_nsec_user.value.ui64 > msnsecs[CMS_USER])
3196                 msnsecs[CMS_USER] = csskd->cpu_nsec_user.value.ui64;
3197         if (csskd->cpu_nsec_kernel.value.ui64 > msnsecs[CMS_SYSTEM])
3198                 msnsecs[CMS_SYSTEM] = csskd->cpu_nsec_kernel.value.ui64;
3199 
3200         bcopy(&cpu_sys_stats_ks_data_template, ksp->ks_data,
3201             sizeof (cpu_sys_stats_ks_data_template));
3202 
3203         csskd->cpu_ticks_wait.value.ui64 = 0;
3204         csskd->wait_ticks_io.value.ui64 = 0;
3205 
3206         csskd->cpu_nsec_idle.value.ui64 = msnsecs[CMS_IDLE];
3207         csskd->cpu_nsec_user.value.ui64 = msnsecs[CMS_USER];
3208         csskd->cpu_nsec_kernel.value.ui64 = msnsecs[CMS_SYSTEM];
3209         csskd->cpu_ticks_idle.value.ui64 =
3210             NSEC_TO_TICK(csskd->cpu_nsec_idle.value.ui64);
3211         csskd->cpu_ticks_user.value.ui64 =
3212             NSEC_TO_TICK(csskd->cpu_nsec_user.value.ui64);
3213         csskd->cpu_ticks_kernel.value.ui64 =
3214             NSEC_TO_TICK(csskd->cpu_nsec_kernel.value.ui64);
3215         csskd->cpu_nsec_dtrace.value.ui64 = cp->cpu_dtrace_nsec;
3216         csskd->dtrace_probes.value.ui64 = cp->cpu_dtrace_probes;
3217         csskd->cpu_nsec_intr.value.ui64 = cp->cpu_intrlast;
3218         csskd->cpu_load_intr.value.ui64 = cp->cpu_intrload;
3219         csskd->bread.value.ui64 = css->bread;
3220         csskd->bwrite.value.ui64 = css->bwrite;
3221         csskd->lread.value.ui64 = css->lread;
3222         csskd->lwrite.value.ui64 = css->lwrite;
3223         csskd->phread.value.ui64 = css->phread;
3224         csskd->phwrite.value.ui64 = css->phwrite;
3225         csskd->pswitch.value.ui64 = css->pswitch;
3226         csskd->trap.value.ui64 = css->trap;
3227         csskd->intr.value.ui64 = 0;
3228         for (i = 0; i < PIL_MAX; i++)
3229                 csskd->intr.value.ui64 += css->intr[i];
3230         csskd->syscall.value.ui64 = css->syscall;
3231         csskd->sysread.value.ui64 = css->sysread;
3232         csskd->syswrite.value.ui64 = css->syswrite;
3233         csskd->sysfork.value.ui64 = css->sysfork;
3234         csskd->sysvfork.value.ui64 = css->sysvfork;
3235         csskd->sysexec.value.ui64 = css->sysexec;
3236         csskd->readch.value.ui64 = css->readch;
3237         csskd->writech.value.ui64 = css->writech;
3238         csskd->rcvint.value.ui64 = css->rcvint;
3239         csskd->xmtint.value.ui64 = css->xmtint;
3240         csskd->mdmint.value.ui64 = css->mdmint;
3241         csskd->rawch.value.ui64 = css->rawch;
3242         csskd->canch.value.ui64 = css->canch;
3243         csskd->outch.value.ui64 = css->outch;
3244         csskd->msg.value.ui64 = css->msg;
3245         csskd->sema.value.ui64 = css->sema;
3246         csskd->namei.value.ui64 = css->namei;
3247         csskd->ufsiget.value.ui64 = css->ufsiget;
3248         csskd->ufsdirblk.value.ui64 = css->ufsdirblk;
3249         csskd->ufsipage.value.ui64 = css->ufsipage;
3250         csskd->ufsinopage.value.ui64 = css->ufsinopage;
3251         csskd->procovf.value.ui64 = css->procovf;
3252         csskd->intrthread.value.ui64 = 0;
3253         for (i = 0; i < LOCK_LEVEL - 1; i++)
3254                 csskd->intrthread.value.ui64 += css->intr[i];
3255         csskd->intrblk.value.ui64 = css->intrblk;
3256         csskd->intrunpin.value.ui64 = css->intrunpin;
3257         csskd->idlethread.value.ui64 = css->idlethread;
3258         csskd->inv_swtch.value.ui64 = css->inv_swtch;
3259         csskd->nthreads.value.ui64 = css->nthreads;
3260         csskd->cpumigrate.value.ui64 = css->cpumigrate;
3261         csskd->xcalls.value.ui64 = css->xcalls;
3262         csskd->mutex_adenters.value.ui64 = css->mutex_adenters;
3263         csskd->rw_rdfails.value.ui64 = css->rw_rdfails;
3264         csskd->rw_wrfails.value.ui64 = css->rw_wrfails;
3265         csskd->modload.value.ui64 = css->modload;
3266         csskd->modunload.value.ui64 = css->modunload;
3267         csskd->bawrite.value.ui64 = css->bawrite;
3268         csskd->iowait.value.ui64 = css->iowait;
3269 
3270         return (0);
3271 }
3272 
3273 static int
3274 cpu_vm_stats_ks_update(kstat_t *ksp, int rw)
3275 {
3276         cpu_t *cp = (cpu_t *)ksp->ks_private;
3277         struct cpu_vm_stats_ks_data *cvskd;
3278         cpu_vm_stats_t *cvs;
3279 
3280         if (rw == KSTAT_WRITE)
3281                 return (EACCES);
3282 
3283         cvs = &cp->cpu_stats.vm;
3284         cvskd = ksp->ks_data;
3285 
3286         bcopy(&cpu_vm_stats_ks_data_template, ksp->ks_data,
3287             sizeof (cpu_vm_stats_ks_data_template));
3288         cvskd->pgrec.value.ui64 = cvs->pgrec;
3289         cvskd->pgfrec.value.ui64 = cvs->pgfrec;
3290         cvskd->pgin.value.ui64 = cvs->pgin;
3291         cvskd->pgpgin.value.ui64 = cvs->pgpgin;
3292         cvskd->pgout.value.ui64 = cvs->pgout;
3293         cvskd->pgpgout.value.ui64 = cvs->pgpgout;
3294         cvskd->swapin.value.ui64 = cvs->swapin;
3295         cvskd->pgswapin.value.ui64 = cvs->pgswapin;
3296         cvskd->swapout.value.ui64 = cvs->swapout;
3297         cvskd->pgswapout.value.ui64 = cvs->pgswapout;
3298         cvskd->zfod.value.ui64 = cvs->zfod;
3299         cvskd->dfree.value.ui64 = cvs->dfree;
3300         cvskd->scan.value.ui64 = cvs->scan;
3301         cvskd->rev.value.ui64 = cvs->rev;
3302         cvskd->hat_fault.value.ui64 = cvs->hat_fault;
3303         cvskd->as_fault.value.ui64 = cvs->as_fault;
3304         cvskd->maj_fault.value.ui64 = cvs->maj_fault;
3305         cvskd->cow_fault.value.ui64 = cvs->cow_fault;
3306         cvskd->prot_fault.value.ui64 = cvs->prot_fault;
3307         cvskd->softlock.value.ui64 = cvs->softlock;
3308         cvskd->kernel_asflt.value.ui64 = cvs->kernel_asflt;
3309         cvskd->pgrrun.value.ui64 = cvs->pgrrun;
3310         cvskd->execpgin.value.ui64 = cvs->execpgin;
3311         cvskd->execpgout.value.ui64 = cvs->execpgout;
3312         cvskd->execfree.value.ui64 = cvs->execfree;
3313         cvskd->anonpgin.value.ui64 = cvs->anonpgin;
3314         cvskd->anonpgout.value.ui64 = cvs->anonpgout;
3315         cvskd->anonfree.value.ui64 = cvs->anonfree;
3316         cvskd->fspgin.value.ui64 = cvs->fspgin;
3317         cvskd->fspgout.value.ui64 = cvs->fspgout;
3318         cvskd->fsfree.value.ui64 = cvs->fsfree;
3319 
3320         return (0);
3321 }
3322 
3323 static int
3324 cpu_stat_ks_update(kstat_t *ksp, int rw)
3325 {
3326         cpu_stat_t *cso;
3327         cpu_t *cp;
3328         int i;
3329         hrtime_t msnsecs[NCMSTATES];
3330 
3331         cso = (cpu_stat_t *)ksp->ks_data;
3332         cp = (cpu_t *)ksp->ks_private;
3333 
3334         if (rw == KSTAT_WRITE)
3335                 return (EACCES);
3336 
3337         /*
3338          * Read CPU mstate, but compare with the last values we
3339          * received to make sure that the returned kstats never
3340          * decrease.
3341          */
3342 
3343         get_cpu_mstate(cp, msnsecs);
3344         msnsecs[CMS_IDLE] = NSEC_TO_TICK(msnsecs[CMS_IDLE]);
3345         msnsecs[CMS_USER] = NSEC_TO_TICK(msnsecs[CMS_USER]);
3346         msnsecs[CMS_SYSTEM] = NSEC_TO_TICK(msnsecs[CMS_SYSTEM]);
3347         if (cso->cpu_sysinfo.cpu[CPU_IDLE] < msnsecs[CMS_IDLE])
3348                 cso->cpu_sysinfo.cpu[CPU_IDLE] = msnsecs[CMS_IDLE];
3349         if (cso->cpu_sysinfo.cpu[CPU_USER] < msnsecs[CMS_USER])
3350                 cso->cpu_sysinfo.cpu[CPU_USER] = msnsecs[CMS_USER];
3351         if (cso->cpu_sysinfo.cpu[CPU_KERNEL] < msnsecs[CMS_SYSTEM])
3352                 cso->cpu_sysinfo.cpu[CPU_KERNEL] = msnsecs[CMS_SYSTEM];
3353         cso->cpu_sysinfo.cpu[CPU_WAIT]       = 0;
3354         cso->cpu_sysinfo.wait[W_IO]  = 0;
3355         cso->cpu_sysinfo.wait[W_SWAP]        = 0;
3356         cso->cpu_sysinfo.wait[W_PIO] = 0;
3357         cso->cpu_sysinfo.bread               = CPU_STATS(cp, sys.bread);
3358         cso->cpu_sysinfo.bwrite      = CPU_STATS(cp, sys.bwrite);
3359         cso->cpu_sysinfo.lread               = CPU_STATS(cp, sys.lread);
3360         cso->cpu_sysinfo.lwrite      = CPU_STATS(cp, sys.lwrite);
3361         cso->cpu_sysinfo.phread      = CPU_STATS(cp, sys.phread);
3362         cso->cpu_sysinfo.phwrite     = CPU_STATS(cp, sys.phwrite);
3363         cso->cpu_sysinfo.pswitch     = CPU_STATS(cp, sys.pswitch);
3364         cso->cpu_sysinfo.trap                = CPU_STATS(cp, sys.trap);
3365         cso->cpu_sysinfo.intr                = 0;
3366         for (i = 0; i < PIL_MAX; i++)
3367                 cso->cpu_sysinfo.intr += CPU_STATS(cp, sys.intr[i]);
3368         cso->cpu_sysinfo.syscall     = CPU_STATS(cp, sys.syscall);
3369         cso->cpu_sysinfo.sysread     = CPU_STATS(cp, sys.sysread);
3370         cso->cpu_sysinfo.syswrite    = CPU_STATS(cp, sys.syswrite);
3371         cso->cpu_sysinfo.sysfork     = CPU_STATS(cp, sys.sysfork);
3372         cso->cpu_sysinfo.sysvfork    = CPU_STATS(cp, sys.sysvfork);
3373         cso->cpu_sysinfo.sysexec     = CPU_STATS(cp, sys.sysexec);
3374         cso->cpu_sysinfo.readch              = CPU_STATS(cp, sys.readch);
3375         cso->cpu_sysinfo.writech     = CPU_STATS(cp, sys.writech);
3376         cso->cpu_sysinfo.rcvint              = CPU_STATS(cp, sys.rcvint);
3377         cso->cpu_sysinfo.xmtint              = CPU_STATS(cp, sys.xmtint);
3378         cso->cpu_sysinfo.mdmint              = CPU_STATS(cp, sys.mdmint);
3379         cso->cpu_sysinfo.rawch               = CPU_STATS(cp, sys.rawch);
3380         cso->cpu_sysinfo.canch               = CPU_STATS(cp, sys.canch);
3381         cso->cpu_sysinfo.outch               = CPU_STATS(cp, sys.outch);
3382         cso->cpu_sysinfo.msg         = CPU_STATS(cp, sys.msg);
3383         cso->cpu_sysinfo.sema                = CPU_STATS(cp, sys.sema);
3384         cso->cpu_sysinfo.namei               = CPU_STATS(cp, sys.namei);
3385         cso->cpu_sysinfo.ufsiget     = CPU_STATS(cp, sys.ufsiget);
3386         cso->cpu_sysinfo.ufsdirblk   = CPU_STATS(cp, sys.ufsdirblk);
3387         cso->cpu_sysinfo.ufsipage    = CPU_STATS(cp, sys.ufsipage);
3388         cso->cpu_sysinfo.ufsinopage  = CPU_STATS(cp, sys.ufsinopage);
3389         cso->cpu_sysinfo.inodeovf    = 0;
3390         cso->cpu_sysinfo.fileovf     = 0;
3391         cso->cpu_sysinfo.procovf     = CPU_STATS(cp, sys.procovf);
3392         cso->cpu_sysinfo.intrthread  = 0;
3393         for (i = 0; i < LOCK_LEVEL - 1; i++)
3394                 cso->cpu_sysinfo.intrthread += CPU_STATS(cp, sys.intr[i]);
3395         cso->cpu_sysinfo.intrblk     = CPU_STATS(cp, sys.intrblk);
3396         cso->cpu_sysinfo.idlethread  = CPU_STATS(cp, sys.idlethread);
3397         cso->cpu_sysinfo.inv_swtch   = CPU_STATS(cp, sys.inv_swtch);
3398         cso->cpu_sysinfo.nthreads    = CPU_STATS(cp, sys.nthreads);
3399         cso->cpu_sysinfo.cpumigrate  = CPU_STATS(cp, sys.cpumigrate);
3400         cso->cpu_sysinfo.xcalls              = CPU_STATS(cp, sys.xcalls);
3401         cso->cpu_sysinfo.mutex_adenters      = CPU_STATS(cp, sys.mutex_adenters);
3402         cso->cpu_sysinfo.rw_rdfails  = CPU_STATS(cp, sys.rw_rdfails);
3403         cso->cpu_sysinfo.rw_wrfails  = CPU_STATS(cp, sys.rw_wrfails);
3404         cso->cpu_sysinfo.modload     = CPU_STATS(cp, sys.modload);
3405         cso->cpu_sysinfo.modunload   = CPU_STATS(cp, sys.modunload);
3406         cso->cpu_sysinfo.bawrite     = CPU_STATS(cp, sys.bawrite);
3407         cso->cpu_sysinfo.rw_enters   = 0;
3408         cso->cpu_sysinfo.win_uo_cnt  = 0;
3409         cso->cpu_sysinfo.win_uu_cnt  = 0;
3410         cso->cpu_sysinfo.win_so_cnt  = 0;
3411         cso->cpu_sysinfo.win_su_cnt  = 0;
3412         cso->cpu_sysinfo.win_suo_cnt = 0;
3413 
3414         cso->cpu_syswait.iowait              = CPU_STATS(cp, sys.iowait);
3415         cso->cpu_syswait.swap                = 0;
3416         cso->cpu_syswait.physio              = 0;
3417 
3418         cso->cpu_vminfo.pgrec                = CPU_STATS(cp, vm.pgrec);
3419         cso->cpu_vminfo.pgfrec               = CPU_STATS(cp, vm.pgfrec);
3420         cso->cpu_vminfo.pgin         = CPU_STATS(cp, vm.pgin);
3421         cso->cpu_vminfo.pgpgin               = CPU_STATS(cp, vm.pgpgin);
3422         cso->cpu_vminfo.pgout                = CPU_STATS(cp, vm.pgout);
3423         cso->cpu_vminfo.pgpgout              = CPU_STATS(cp, vm.pgpgout);
3424         cso->cpu_vminfo.swapin               = CPU_STATS(cp, vm.swapin);
3425         cso->cpu_vminfo.pgswapin     = CPU_STATS(cp, vm.pgswapin);
3426         cso->cpu_vminfo.swapout              = CPU_STATS(cp, vm.swapout);
3427         cso->cpu_vminfo.pgswapout    = CPU_STATS(cp, vm.pgswapout);
3428         cso->cpu_vminfo.zfod         = CPU_STATS(cp, vm.zfod);
3429         cso->cpu_vminfo.dfree                = CPU_STATS(cp, vm.dfree);
3430         cso->cpu_vminfo.scan         = CPU_STATS(cp, vm.scan);
3431         cso->cpu_vminfo.rev          = CPU_STATS(cp, vm.rev);
3432         cso->cpu_vminfo.hat_fault    = CPU_STATS(cp, vm.hat_fault);
3433         cso->cpu_vminfo.as_fault     = CPU_STATS(cp, vm.as_fault);
3434         cso->cpu_vminfo.maj_fault    = CPU_STATS(cp, vm.maj_fault);
3435         cso->cpu_vminfo.cow_fault    = CPU_STATS(cp, vm.cow_fault);
3436         cso->cpu_vminfo.prot_fault   = CPU_STATS(cp, vm.prot_fault);
3437         cso->cpu_vminfo.softlock     = CPU_STATS(cp, vm.softlock);
3438         cso->cpu_vminfo.kernel_asflt = CPU_STATS(cp, vm.kernel_asflt);
3439         cso->cpu_vminfo.pgrrun               = CPU_STATS(cp, vm.pgrrun);
3440         cso->cpu_vminfo.execpgin     = CPU_STATS(cp, vm.execpgin);
3441         cso->cpu_vminfo.execpgout    = CPU_STATS(cp, vm.execpgout);
3442         cso->cpu_vminfo.execfree     = CPU_STATS(cp, vm.execfree);
3443         cso->cpu_vminfo.anonpgin     = CPU_STATS(cp, vm.anonpgin);
3444         cso->cpu_vminfo.anonpgout    = CPU_STATS(cp, vm.anonpgout);
3445         cso->cpu_vminfo.anonfree     = CPU_STATS(cp, vm.anonfree);
3446         cso->cpu_vminfo.fspgin               = CPU_STATS(cp, vm.fspgin);
3447         cso->cpu_vminfo.fspgout              = CPU_STATS(cp, vm.fspgout);
3448         cso->cpu_vminfo.fsfree               = CPU_STATS(cp, vm.fsfree);
3449 
3450         return (0);
3451 }