1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright (c) 2012 by Delphix. All rights reserved.
  24  */
  25 
  26 /*
  27  * Architecture-independent CPU control functions.
  28  */
  29 
  30 #include <sys/types.h>
  31 #include <sys/param.h>
  32 #include <sys/var.h>
  33 #include <sys/thread.h>
  34 #include <sys/cpuvar.h>
  35 #include <sys/cpu_event.h>
  36 #include <sys/kstat.h>
  37 #include <sys/uadmin.h>
  38 #include <sys/systm.h>
  39 #include <sys/errno.h>
  40 #include <sys/cmn_err.h>
  41 #include <sys/procset.h>
  42 #include <sys/processor.h>
  43 #include <sys/debug.h>
  44 #include <sys/cpupart.h>
  45 #include <sys/lgrp.h>
  46 #include <sys/pset.h>
  47 #include <sys/pghw.h>
  48 #include <sys/kmem.h>
  49 #include <sys/kmem_impl.h>        /* to set per-cpu kmem_cache offset */
  50 #include <sys/atomic.h>
  51 #include <sys/callb.h>
  52 #include <sys/vtrace.h>
  53 #include <sys/cyclic.h>
  54 #include <sys/bitmap.h>
  55 #include <sys/nvpair.h>
  56 #include <sys/pool_pset.h>
  57 #include <sys/msacct.h>
  58 #include <sys/time.h>
  59 #include <sys/archsystm.h>
  60 #include <sys/sdt.h>
  61 #if defined(__x86) || defined(__amd64)
  62 #include <sys/x86_archext.h>
  63 #endif
  64 #include <sys/callo.h>
  65 
  66 extern int      mp_cpu_start(cpu_t *);
  67 extern int      mp_cpu_stop(cpu_t *);
  68 extern int      mp_cpu_poweron(cpu_t *);
  69 extern int      mp_cpu_poweroff(cpu_t *);
  70 extern int      mp_cpu_configure(int);
  71 extern int      mp_cpu_unconfigure(int);
  72 extern void     mp_cpu_faulted_enter(cpu_t *);
  73 extern void     mp_cpu_faulted_exit(cpu_t *);
  74 
  75 extern int cmp_cpu_to_chip(processorid_t cpuid);
  76 #ifdef __sparcv9
  77 extern char *cpu_fru_fmri(cpu_t *cp);
  78 #endif
  79 
  80 static void cpu_add_active_internal(cpu_t *cp);
  81 static void cpu_remove_active(cpu_t *cp);
  82 static void cpu_info_kstat_create(cpu_t *cp);
  83 static void cpu_info_kstat_destroy(cpu_t *cp);
  84 static void cpu_stats_kstat_create(cpu_t *cp);
  85 static void cpu_stats_kstat_destroy(cpu_t *cp);
  86 
  87 static int cpu_sys_stats_ks_update(kstat_t *ksp, int rw);
  88 static int cpu_vm_stats_ks_update(kstat_t *ksp, int rw);
  89 static int cpu_stat_ks_update(kstat_t *ksp, int rw);
  90 static int cpu_state_change_hooks(int, cpu_setup_t, cpu_setup_t);
  91 
  92 /*
  93  * cpu_lock protects ncpus, ncpus_online, cpu_flag, cpu_list, cpu_active,
  94  * max_cpu_seqid_ever, and dispatch queue reallocations.  The lock ordering with
  95  * respect to related locks is:
  96  *
  97  *      cpu_lock --> thread_free_lock  --->  p_lock  --->  thread_lock()
  98  *
  99  * Warning:  Certain sections of code do not use the cpu_lock when
 100  * traversing the cpu_list (e.g. mutex_vector_enter(), clock()).  Since
 101  * all cpus are paused during modifications to this list, a solution
 102  * to protect the list is too either disable kernel preemption while
 103  * walking the list, *or* recheck the cpu_next pointer at each
 104  * iteration in the loop.  Note that in no cases can any cached
 105  * copies of the cpu pointers be kept as they may become invalid.
 106  */
 107 kmutex_t        cpu_lock;
 108 cpu_t           *cpu_list;              /* list of all CPUs */
 109 cpu_t           *clock_cpu_list;        /* used by clock to walk CPUs */
 110 cpu_t           *cpu_active;            /* list of active CPUs */
 111 static cpuset_t cpu_available;          /* set of available CPUs */
 112 cpuset_t        cpu_seqid_inuse;        /* which cpu_seqids are in use */
 113 
 114 cpu_t           **cpu_seq;              /* ptrs to CPUs, indexed by seq_id */
 115 
 116 /*
 117  * max_ncpus keeps the max cpus the system can have. Initially
 118  * it's NCPU, but since most archs scan the devtree for cpus
 119  * fairly early on during boot, the real max can be known before
 120  * ncpus is set (useful for early NCPU based allocations).
 121  */
 122 int max_ncpus = NCPU;
 123 /*
 124  * platforms that set max_ncpus to maxiumum number of cpus that can be
 125  * dynamically added will set boot_max_ncpus to the number of cpus found
 126  * at device tree scan time during boot.
 127  */
 128 int boot_max_ncpus = -1;
 129 int boot_ncpus = -1;
 130 /*
 131  * Maximum possible CPU id.  This can never be >= NCPU since NCPU is
 132  * used to size arrays that are indexed by CPU id.
 133  */
 134 processorid_t max_cpuid = NCPU - 1;
 135 
 136 /*
 137  * Maximum cpu_seqid was given. This number can only grow and never shrink. It
 138  * can be used to optimize NCPU loops to avoid going through CPUs which were
 139  * never on-line.
 140  */
 141 processorid_t max_cpu_seqid_ever = 0;
 142 
 143 int ncpus = 1;
 144 int ncpus_online = 1;
 145 
 146 /*
 147  * CPU that we're trying to offline.  Protected by cpu_lock.
 148  */
 149 cpu_t *cpu_inmotion;
 150 
 151 /*
 152  * Can be raised to suppress further weakbinding, which are instead
 153  * satisfied by disabling preemption.  Must be raised/lowered under cpu_lock,
 154  * while individual thread weakbinding synchronization is done under thread
 155  * lock.
 156  */
 157 int weakbindingbarrier;
 158 
 159 /*
 160  * Variables used in pause_cpus().
 161  */
 162 static volatile char safe_list[NCPU];
 163 
 164 static struct _cpu_pause_info {
 165         int             cp_spl;         /* spl saved in pause_cpus() */
 166         volatile int    cp_go;          /* Go signal sent after all ready */
 167         int             cp_count;       /* # of CPUs to pause */
 168         ksema_t         cp_sem;         /* synch pause_cpus & cpu_pause */
 169         kthread_id_t    cp_paused;
 170 } cpu_pause_info;
 171 
 172 static kmutex_t pause_free_mutex;
 173 static kcondvar_t pause_free_cv;
 174 
 175 void *(*cpu_pause_func)(void *) = NULL;
 176 
 177 
 178 static struct cpu_sys_stats_ks_data {
 179         kstat_named_t cpu_ticks_idle;
 180         kstat_named_t cpu_ticks_user;
 181         kstat_named_t cpu_ticks_kernel;
 182         kstat_named_t cpu_ticks_wait;
 183         kstat_named_t cpu_nsec_idle;
 184         kstat_named_t cpu_nsec_user;
 185         kstat_named_t cpu_nsec_kernel;
 186         kstat_named_t cpu_nsec_dtrace;
 187         kstat_named_t cpu_nsec_intr;
 188         kstat_named_t cpu_load_intr;
 189         kstat_named_t wait_ticks_io;
 190         kstat_named_t dtrace_probes;
 191         kstat_named_t bread;
 192         kstat_named_t bwrite;
 193         kstat_named_t lread;
 194         kstat_named_t lwrite;
 195         kstat_named_t phread;
 196         kstat_named_t phwrite;
 197         kstat_named_t pswitch;
 198         kstat_named_t trap;
 199         kstat_named_t intr;
 200         kstat_named_t syscall;
 201         kstat_named_t sysread;
 202         kstat_named_t syswrite;
 203         kstat_named_t sysfork;
 204         kstat_named_t sysvfork;
 205         kstat_named_t sysexec;
 206         kstat_named_t readch;
 207         kstat_named_t writech;
 208         kstat_named_t rcvint;
 209         kstat_named_t xmtint;
 210         kstat_named_t mdmint;
 211         kstat_named_t rawch;
 212         kstat_named_t canch;
 213         kstat_named_t outch;
 214         kstat_named_t msg;
 215         kstat_named_t sema;
 216         kstat_named_t namei;
 217         kstat_named_t ufsiget;
 218         kstat_named_t ufsdirblk;
 219         kstat_named_t ufsipage;
 220         kstat_named_t ufsinopage;
 221         kstat_named_t procovf;
 222         kstat_named_t intrthread;
 223         kstat_named_t intrblk;
 224         kstat_named_t intrunpin;
 225         kstat_named_t idlethread;
 226         kstat_named_t inv_swtch;
 227         kstat_named_t nthreads;
 228         kstat_named_t cpumigrate;
 229         kstat_named_t xcalls;
 230         kstat_named_t mutex_adenters;
 231         kstat_named_t rw_rdfails;
 232         kstat_named_t rw_wrfails;
 233         kstat_named_t modload;
 234         kstat_named_t modunload;
 235         kstat_named_t bawrite;
 236         kstat_named_t iowait;
 237 } cpu_sys_stats_ks_data_template = {
 238         { "cpu_ticks_idle",     KSTAT_DATA_UINT64 },
 239         { "cpu_ticks_user",     KSTAT_DATA_UINT64 },
 240         { "cpu_ticks_kernel",   KSTAT_DATA_UINT64 },
 241         { "cpu_ticks_wait",     KSTAT_DATA_UINT64 },
 242         { "cpu_nsec_idle",      KSTAT_DATA_UINT64 },
 243         { "cpu_nsec_user",      KSTAT_DATA_UINT64 },
 244         { "cpu_nsec_kernel",    KSTAT_DATA_UINT64 },
 245         { "cpu_nsec_dtrace",    KSTAT_DATA_UINT64 },
 246         { "cpu_nsec_intr",      KSTAT_DATA_UINT64 },
 247         { "cpu_load_intr",      KSTAT_DATA_UINT64 },
 248         { "wait_ticks_io",      KSTAT_DATA_UINT64 },
 249         { "dtrace_probes",      KSTAT_DATA_UINT64 },
 250         { "bread",              KSTAT_DATA_UINT64 },
 251         { "bwrite",             KSTAT_DATA_UINT64 },
 252         { "lread",              KSTAT_DATA_UINT64 },
 253         { "lwrite",             KSTAT_DATA_UINT64 },
 254         { "phread",             KSTAT_DATA_UINT64 },
 255         { "phwrite",            KSTAT_DATA_UINT64 },
 256         { "pswitch",            KSTAT_DATA_UINT64 },
 257         { "trap",               KSTAT_DATA_UINT64 },
 258         { "intr",               KSTAT_DATA_UINT64 },
 259         { "syscall",            KSTAT_DATA_UINT64 },
 260         { "sysread",            KSTAT_DATA_UINT64 },
 261         { "syswrite",           KSTAT_DATA_UINT64 },
 262         { "sysfork",            KSTAT_DATA_UINT64 },
 263         { "sysvfork",           KSTAT_DATA_UINT64 },
 264         { "sysexec",            KSTAT_DATA_UINT64 },
 265         { "readch",             KSTAT_DATA_UINT64 },
 266         { "writech",            KSTAT_DATA_UINT64 },
 267         { "rcvint",             KSTAT_DATA_UINT64 },
 268         { "xmtint",             KSTAT_DATA_UINT64 },
 269         { "mdmint",             KSTAT_DATA_UINT64 },
 270         { "rawch",              KSTAT_DATA_UINT64 },
 271         { "canch",              KSTAT_DATA_UINT64 },
 272         { "outch",              KSTAT_DATA_UINT64 },
 273         { "msg",                KSTAT_DATA_UINT64 },
 274         { "sema",               KSTAT_DATA_UINT64 },
 275         { "namei",              KSTAT_DATA_UINT64 },
 276         { "ufsiget",            KSTAT_DATA_UINT64 },
 277         { "ufsdirblk",          KSTAT_DATA_UINT64 },
 278         { "ufsipage",           KSTAT_DATA_UINT64 },
 279         { "ufsinopage",         KSTAT_DATA_UINT64 },
 280         { "procovf",            KSTAT_DATA_UINT64 },
 281         { "intrthread",         KSTAT_DATA_UINT64 },
 282         { "intrblk",            KSTAT_DATA_UINT64 },
 283         { "intrunpin",          KSTAT_DATA_UINT64 },
 284         { "idlethread",         KSTAT_DATA_UINT64 },
 285         { "inv_swtch",          KSTAT_DATA_UINT64 },
 286         { "nthreads",           KSTAT_DATA_UINT64 },
 287         { "cpumigrate",         KSTAT_DATA_UINT64 },
 288         { "xcalls",             KSTAT_DATA_UINT64 },
 289         { "mutex_adenters",     KSTAT_DATA_UINT64 },
 290         { "rw_rdfails",         KSTAT_DATA_UINT64 },
 291         { "rw_wrfails",         KSTAT_DATA_UINT64 },
 292         { "modload",            KSTAT_DATA_UINT64 },
 293         { "modunload",          KSTAT_DATA_UINT64 },
 294         { "bawrite",            KSTAT_DATA_UINT64 },
 295         { "iowait",             KSTAT_DATA_UINT64 },
 296 };
 297 
 298 static struct cpu_vm_stats_ks_data {
 299         kstat_named_t pgrec;
 300         kstat_named_t pgfrec;
 301         kstat_named_t pgin;
 302         kstat_named_t pgpgin;
 303         kstat_named_t pgout;
 304         kstat_named_t pgpgout;
 305         kstat_named_t swapin;
 306         kstat_named_t pgswapin;
 307         kstat_named_t swapout;
 308         kstat_named_t pgswapout;
 309         kstat_named_t zfod;
 310         kstat_named_t dfree;
 311         kstat_named_t scan;
 312         kstat_named_t rev;
 313         kstat_named_t hat_fault;
 314         kstat_named_t as_fault;
 315         kstat_named_t maj_fault;
 316         kstat_named_t cow_fault;
 317         kstat_named_t prot_fault;
 318         kstat_named_t softlock;
 319         kstat_named_t kernel_asflt;
 320         kstat_named_t pgrrun;
 321         kstat_named_t execpgin;
 322         kstat_named_t execpgout;
 323         kstat_named_t execfree;
 324         kstat_named_t anonpgin;
 325         kstat_named_t anonpgout;
 326         kstat_named_t anonfree;
 327         kstat_named_t fspgin;
 328         kstat_named_t fspgout;
 329         kstat_named_t fsfree;
 330 } cpu_vm_stats_ks_data_template = {
 331         { "pgrec",              KSTAT_DATA_UINT64 },
 332         { "pgfrec",             KSTAT_DATA_UINT64 },
 333         { "pgin",               KSTAT_DATA_UINT64 },
 334         { "pgpgin",             KSTAT_DATA_UINT64 },
 335         { "pgout",              KSTAT_DATA_UINT64 },
 336         { "pgpgout",            KSTAT_DATA_UINT64 },
 337         { "swapin",             KSTAT_DATA_UINT64 },
 338         { "pgswapin",           KSTAT_DATA_UINT64 },
 339         { "swapout",            KSTAT_DATA_UINT64 },
 340         { "pgswapout",          KSTAT_DATA_UINT64 },
 341         { "zfod",               KSTAT_DATA_UINT64 },
 342         { "dfree",              KSTAT_DATA_UINT64 },
 343         { "scan",               KSTAT_DATA_UINT64 },
 344         { "rev",                KSTAT_DATA_UINT64 },
 345         { "hat_fault",          KSTAT_DATA_UINT64 },
 346         { "as_fault",           KSTAT_DATA_UINT64 },
 347         { "maj_fault",          KSTAT_DATA_UINT64 },
 348         { "cow_fault",          KSTAT_DATA_UINT64 },
 349         { "prot_fault",         KSTAT_DATA_UINT64 },
 350         { "softlock",           KSTAT_DATA_UINT64 },
 351         { "kernel_asflt",       KSTAT_DATA_UINT64 },
 352         { "pgrrun",             KSTAT_DATA_UINT64 },
 353         { "execpgin",           KSTAT_DATA_UINT64 },
 354         { "execpgout",          KSTAT_DATA_UINT64 },
 355         { "execfree",           KSTAT_DATA_UINT64 },
 356         { "anonpgin",           KSTAT_DATA_UINT64 },
 357         { "anonpgout",          KSTAT_DATA_UINT64 },
 358         { "anonfree",           KSTAT_DATA_UINT64 },
 359         { "fspgin",             KSTAT_DATA_UINT64 },
 360         { "fspgout",            KSTAT_DATA_UINT64 },
 361         { "fsfree",             KSTAT_DATA_UINT64 },
 362 };
 363 
 364 /*
 365  * Force the specified thread to migrate to the appropriate processor.
 366  * Called with thread lock held, returns with it dropped.
 367  */
 368 static void
 369 force_thread_migrate(kthread_id_t tp)
 370 {
 371         ASSERT(THREAD_LOCK_HELD(tp));
 372         if (tp == curthread) {
 373                 THREAD_TRANSITION(tp);
 374                 CL_SETRUN(tp);
 375                 thread_unlock_nopreempt(tp);
 376                 swtch();
 377         } else {
 378                 if (tp->t_state == TS_ONPROC) {
 379                         cpu_surrender(tp);
 380                 } else if (tp->t_state == TS_RUN) {
 381                         (void) dispdeq(tp);
 382                         setbackdq(tp);
 383                 }
 384                 thread_unlock(tp);
 385         }
 386 }
 387 
 388 /*
 389  * Set affinity for a specified CPU.
 390  * A reference count is incremented and the affinity is held until the
 391  * reference count is decremented to zero by thread_affinity_clear().
 392  * This is so regions of code requiring affinity can be nested.
 393  * Caller needs to ensure that cpu_id remains valid, which can be
 394  * done by holding cpu_lock across this call, unless the caller
 395  * specifies CPU_CURRENT in which case the cpu_lock will be acquired
 396  * by thread_affinity_set and CPU->cpu_id will be the target CPU.
 397  */
 398 void
 399 thread_affinity_set(kthread_id_t t, int cpu_id)
 400 {
 401         cpu_t           *cp;
 402         int             c;
 403 
 404         ASSERT(!(t == curthread && t->t_weakbound_cpu != NULL));
 405 
 406         if ((c = cpu_id) == CPU_CURRENT) {
 407                 mutex_enter(&cpu_lock);
 408                 cpu_id = CPU->cpu_id;
 409         }
 410         /*
 411          * We should be asserting that cpu_lock is held here, but
 412          * the NCA code doesn't acquire it.  The following assert
 413          * should be uncommented when the NCA code is fixed.
 414          *
 415          * ASSERT(MUTEX_HELD(&cpu_lock));
 416          */
 417         ASSERT((cpu_id >= 0) && (cpu_id < NCPU));
 418         cp = cpu[cpu_id];
 419         ASSERT(cp != NULL);             /* user must provide a good cpu_id */
 420         /*
 421          * If there is already a hard affinity requested, and this affinity
 422          * conflicts with that, panic.
 423          */
 424         thread_lock(t);
 425         if (t->t_affinitycnt > 0 && t->t_bound_cpu != cp) {
 426                 panic("affinity_set: setting %p but already bound to %p",
 427                     (void *)cp, (void *)t->t_bound_cpu);
 428         }
 429         t->t_affinitycnt++;
 430         t->t_bound_cpu = cp;
 431 
 432         /*
 433          * Make sure we're running on the right CPU.
 434          */
 435         if (cp != t->t_cpu || t != curthread) {
 436                 force_thread_migrate(t);        /* drops thread lock */
 437         } else {
 438                 thread_unlock(t);
 439         }
 440 
 441         if (c == CPU_CURRENT)
 442                 mutex_exit(&cpu_lock);
 443 }
 444 
 445 /*
 446  *      Wrapper for backward compatibility.
 447  */
 448 void
 449 affinity_set(int cpu_id)
 450 {
 451         thread_affinity_set(curthread, cpu_id);
 452 }
 453 
 454 /*
 455  * Decrement the affinity reservation count and if it becomes zero,
 456  * clear the CPU affinity for the current thread, or set it to the user's
 457  * software binding request.
 458  */
 459 void
 460 thread_affinity_clear(kthread_id_t t)
 461 {
 462         register processorid_t binding;
 463 
 464         thread_lock(t);
 465         if (--t->t_affinitycnt == 0) {
 466                 if ((binding = t->t_bind_cpu) == PBIND_NONE) {
 467                         /*
 468                          * Adjust disp_max_unbound_pri if necessary.
 469                          */
 470                         disp_adjust_unbound_pri(t);
 471                         t->t_bound_cpu = NULL;
 472                         if (t->t_cpu->cpu_part != t->t_cpupart) {
 473                                 force_thread_migrate(t);
 474                                 return;
 475                         }
 476                 } else {
 477                         t->t_bound_cpu = cpu[binding];
 478                         /*
 479                          * Make sure the thread is running on the bound CPU.
 480                          */
 481                         if (t->t_cpu != t->t_bound_cpu) {
 482                                 force_thread_migrate(t);
 483                                 return;         /* already dropped lock */
 484                         }
 485                 }
 486         }
 487         thread_unlock(t);
 488 }
 489 
 490 /*
 491  * Wrapper for backward compatibility.
 492  */
 493 void
 494 affinity_clear(void)
 495 {
 496         thread_affinity_clear(curthread);
 497 }
 498 
 499 /*
 500  * Weak cpu affinity.  Bind to the "current" cpu for short periods
 501  * of time during which the thread must not block (but may be preempted).
 502  * Use this instead of kpreempt_disable() when it is only "no migration"
 503  * rather than "no preemption" semantics that are required - disabling
 504  * preemption holds higher priority threads off of cpu and if the
 505  * operation that is protected is more than momentary this is not good
 506  * for realtime etc.
 507  *
 508  * Weakly bound threads will not prevent a cpu from being offlined -
 509  * we'll only run them on the cpu to which they are weakly bound but
 510  * (because they do not block) we'll always be able to move them on to
 511  * another cpu at offline time if we give them just a short moment to
 512  * run during which they will unbind.  To give a cpu a chance of offlining,
 513  * however, we require a barrier to weak bindings that may be raised for a
 514  * given cpu (offline/move code may set this and then wait a short time for
 515  * existing weak bindings to drop); the cpu_inmotion pointer is that barrier.
 516  *
 517  * There are few restrictions on the calling context of thread_nomigrate.
 518  * The caller must not hold the thread lock.  Calls may be nested.
 519  *
 520  * After weakbinding a thread must not perform actions that may block.
 521  * In particular it must not call thread_affinity_set; calling that when
 522  * already weakbound is nonsensical anyway.
 523  *
 524  * If curthread is prevented from migrating for other reasons
 525  * (kernel preemption disabled; high pil; strongly bound; interrupt thread)
 526  * then the weak binding will succeed even if this cpu is the target of an
 527  * offline/move request.
 528  */
 529 void
 530 thread_nomigrate(void)
 531 {
 532         cpu_t *cp;
 533         kthread_id_t t = curthread;
 534 
 535 again:
 536         kpreempt_disable();
 537         cp = CPU;
 538 
 539         /*
 540          * A highlevel interrupt must not modify t_nomigrate or
 541          * t_weakbound_cpu of the thread it has interrupted.  A lowlevel
 542          * interrupt thread cannot migrate and we can avoid the
 543          * thread_lock call below by short-circuiting here.  In either
 544          * case we can just return since no migration is possible and
 545          * the condition will persist (ie, when we test for these again
 546          * in thread_allowmigrate they can't have changed).   Migration
 547          * is also impossible if we're at or above DISP_LEVEL pil.
 548          */
 549         if (CPU_ON_INTR(cp) || t->t_flag & T_INTR_THREAD ||
 550             getpil() >= DISP_LEVEL) {
 551                 kpreempt_enable();
 552                 return;
 553         }
 554 
 555         /*
 556          * We must be consistent with existing weak bindings.  Since we
 557          * may be interrupted between the increment of t_nomigrate and
 558          * the store to t_weakbound_cpu below we cannot assume that
 559          * t_weakbound_cpu will be set if t_nomigrate is.  Note that we
 560          * cannot assert t_weakbound_cpu == t_bind_cpu since that is not
 561          * always the case.
 562          */
 563         if (t->t_nomigrate && t->t_weakbound_cpu && t->t_weakbound_cpu != cp) {
 564                 if (!panicstr)
 565                         panic("thread_nomigrate: binding to %p but already "
 566                             "bound to %p", (void *)cp,
 567                             (void *)t->t_weakbound_cpu);
 568         }
 569 
 570         /*
 571          * At this point we have preemption disabled and we don't yet hold
 572          * the thread lock.  So it's possible that somebody else could
 573          * set t_bind_cpu here and not be able to force us across to the
 574          * new cpu (since we have preemption disabled).
 575          */
 576         thread_lock(curthread);
 577 
 578         /*
 579          * If further weak bindings are being (temporarily) suppressed then
 580          * we'll settle for disabling kernel preemption (which assures
 581          * no migration provided the thread does not block which it is
 582          * not allowed to if using thread_nomigrate).  We must remember
 583          * this disposition so we can take appropriate action in
 584          * thread_allowmigrate.  If this is a nested call and the
 585          * thread is already weakbound then fall through as normal.
 586          * We remember the decision to settle for kpreempt_disable through
 587          * negative nesting counting in t_nomigrate.  Once a thread has had one
 588          * weakbinding request satisfied in this way any further (nested)
 589          * requests will continue to be satisfied in the same way,
 590          * even if weak bindings have recommenced.
 591          */
 592         if (t->t_nomigrate < 0 || weakbindingbarrier && t->t_nomigrate == 0) {
 593                 --t->t_nomigrate;
 594                 thread_unlock(curthread);
 595                 return;         /* with kpreempt_disable still active */
 596         }
 597 
 598         /*
 599          * We hold thread_lock so t_bind_cpu cannot change.  We could,
 600          * however, be running on a different cpu to which we are t_bound_cpu
 601          * to (as explained above).  If we grant the weak binding request
 602          * in that case then the dispatcher must favour our weak binding
 603          * over our strong (in which case, just as when preemption is
 604          * disabled, we can continue to run on a cpu other than the one to
 605          * which we are strongbound; the difference in this case is that
 606          * this thread can be preempted and so can appear on the dispatch
 607          * queues of a cpu other than the one it is strongbound to).
 608          *
 609          * If the cpu we are running on does not appear to be a current
 610          * offline target (we check cpu_inmotion to determine this - since
 611          * we don't hold cpu_lock we may not see a recent store to that,
 612          * so it's possible that we at times can grant a weak binding to a
 613          * cpu that is an offline target, but that one request will not
 614          * prevent the offline from succeeding) then we will always grant
 615          * the weak binding request.  This includes the case above where
 616          * we grant a weakbinding not commensurate with our strong binding.
 617          *
 618          * If our cpu does appear to be an offline target then we're inclined
 619          * not to grant the weakbinding request just yet - we'd prefer to
 620          * migrate to another cpu and grant the request there.  The
 621          * exceptions are those cases where going through preemption code
 622          * will not result in us changing cpu:
 623          *
 624          *      . interrupts have already bypassed this case (see above)
 625          *      . we are already weakbound to this cpu (dispatcher code will
 626          *        always return us to the weakbound cpu)
 627          *      . preemption was disabled even before we disabled it above
 628          *      . we are strongbound to this cpu (if we're strongbound to
 629          *      another and not yet running there the trip through the
 630          *      dispatcher will move us to the strongbound cpu and we
 631          *      will grant the weak binding there)
 632          */
 633         if (cp != cpu_inmotion || t->t_nomigrate > 0 || t->t_preempt > 1 ||
 634             t->t_bound_cpu == cp) {
 635                 /*
 636                  * Don't be tempted to store to t_weakbound_cpu only on
 637                  * the first nested bind request - if we're interrupted
 638                  * after the increment of t_nomigrate and before the
 639                  * store to t_weakbound_cpu and the interrupt calls
 640                  * thread_nomigrate then the assertion in thread_allowmigrate
 641                  * would fail.
 642                  */
 643                 t->t_nomigrate++;
 644                 t->t_weakbound_cpu = cp;
 645                 membar_producer();
 646                 thread_unlock(curthread);
 647                 /*
 648                  * Now that we have dropped the thread_lock another thread
 649                  * can set our t_weakbound_cpu, and will try to migrate us
 650                  * to the strongbound cpu (which will not be prevented by
 651                  * preemption being disabled since we're about to enable
 652                  * preemption).  We have granted the weakbinding to the current
 653                  * cpu, so again we are in the position that is is is possible
 654                  * that our weak and strong bindings differ.  Again this
 655                  * is catered for by dispatcher code which will favour our
 656                  * weak binding.
 657                  */
 658                 kpreempt_enable();
 659         } else {
 660                 /*
 661                  * Move to another cpu before granting the request by
 662                  * forcing this thread through preemption code.  When we
 663                  * get to set{front,back}dq called from CL_PREEMPT()
 664                  * cpu_choose() will be used to select a cpu to queue
 665                  * us on - that will see cpu_inmotion and take
 666                  * steps to avoid returning us to this cpu.
 667                  */
 668                 cp->cpu_kprunrun = 1;
 669                 thread_unlock(curthread);
 670                 kpreempt_enable();      /* will call preempt() */
 671                 goto again;
 672         }
 673 }
 674 
 675 void
 676 thread_allowmigrate(void)
 677 {
 678         kthread_id_t t = curthread;
 679 
 680         ASSERT(t->t_weakbound_cpu == CPU ||
 681             (t->t_nomigrate < 0 && t->t_preempt > 0) ||
 682             CPU_ON_INTR(CPU) || t->t_flag & T_INTR_THREAD ||
 683             getpil() >= DISP_LEVEL);
 684 
 685         if (CPU_ON_INTR(CPU) || (t->t_flag & T_INTR_THREAD) ||
 686             getpil() >= DISP_LEVEL)
 687                 return;
 688 
 689         if (t->t_nomigrate < 0) {
 690                 /*
 691                  * This thread was granted "weak binding" in the
 692                  * stronger form of kernel preemption disabling.
 693                  * Undo a level of nesting for both t_nomigrate
 694                  * and t_preempt.
 695                  */
 696                 ++t->t_nomigrate;
 697                 kpreempt_enable();
 698         } else if (--t->t_nomigrate == 0) {
 699                 /*
 700                  * Time to drop the weak binding.  We need to cater
 701                  * for the case where we're weakbound to a different
 702                  * cpu than that to which we're strongbound (a very
 703                  * temporary arrangement that must only persist until
 704                  * weak binding drops).  We don't acquire thread_lock
 705                  * here so even as this code executes t_bound_cpu
 706                  * may be changing.  So we disable preemption and
 707                  * a) in the case that t_bound_cpu changes while we
 708                  * have preemption disabled kprunrun will be set
 709                  * asynchronously, and b) if before disabling
 710                  * preemption we were already on a different cpu to
 711                  * our t_bound_cpu then we set kprunrun ourselves
 712                  * to force a trip through the dispatcher when
 713                  * preemption is enabled.
 714                  */
 715                 kpreempt_disable();
 716                 if (t->t_bound_cpu &&
 717                     t->t_weakbound_cpu != t->t_bound_cpu)
 718                         CPU->cpu_kprunrun = 1;
 719                 t->t_weakbound_cpu = NULL;
 720                 membar_producer();
 721                 kpreempt_enable();
 722         }
 723 }
 724 
 725 /*
 726  * weakbinding_stop can be used to temporarily cause weakbindings made
 727  * with thread_nomigrate to be satisfied through the stronger action of
 728  * kpreempt_disable.  weakbinding_start recommences normal weakbinding.
 729  */
 730 
 731 void
 732 weakbinding_stop(void)
 733 {
 734         ASSERT(MUTEX_HELD(&cpu_lock));
 735         weakbindingbarrier = 1;
 736         membar_producer();      /* make visible before subsequent thread_lock */
 737 }
 738 
 739 void
 740 weakbinding_start(void)
 741 {
 742         ASSERT(MUTEX_HELD(&cpu_lock));
 743         weakbindingbarrier = 0;
 744 }
 745 
 746 void
 747 null_xcall(void)
 748 {
 749 }
 750 
 751 /*
 752  * This routine is called to place the CPUs in a safe place so that
 753  * one of them can be taken off line or placed on line.  What we are
 754  * trying to do here is prevent a thread from traversing the list
 755  * of active CPUs while we are changing it or from getting placed on
 756  * the run queue of a CPU that has just gone off line.  We do this by
 757  * creating a thread with the highest possible prio for each CPU and
 758  * having it call this routine.  The advantage of this method is that
 759  * we can eliminate all checks for CPU_ACTIVE in the disp routines.
 760  * This makes disp faster at the expense of making p_online() slower
 761  * which is a good trade off.
 762  */
 763 static void
 764 cpu_pause(int index)
 765 {
 766         int s;
 767         struct _cpu_pause_info *cpi = &cpu_pause_info;
 768         volatile char *safe = &safe_list[index];
 769         long    lindex = index;
 770 
 771         ASSERT((curthread->t_bound_cpu != NULL) || (*safe == PAUSE_DIE));
 772 
 773         while (*safe != PAUSE_DIE) {
 774                 *safe = PAUSE_READY;
 775                 membar_enter();         /* make sure stores are flushed */
 776                 sema_v(&cpi->cp_sem);    /* signal requesting thread */
 777 
 778                 /*
 779                  * Wait here until all pause threads are running.  That
 780                  * indicates that it's safe to do the spl.  Until
 781                  * cpu_pause_info.cp_go is set, we don't want to spl
 782                  * because that might block clock interrupts needed
 783                  * to preempt threads on other CPUs.
 784                  */
 785                 while (cpi->cp_go == 0)
 786                         ;
 787                 /*
 788                  * Even though we are at the highest disp prio, we need
 789                  * to block out all interrupts below LOCK_LEVEL so that
 790                  * an intr doesn't come in, wake up a thread, and call
 791                  * setbackdq/setfrontdq.
 792                  */
 793                 s = splhigh();
 794                 /*
 795                  * if cpu_pause_func() has been set then call it using
 796                  * index as the argument, currently only used by
 797                  * cpr_suspend_cpus().  This function is used as the
 798                  * code to execute on the "paused" cpu's when a machine
 799                  * comes out of a sleep state and CPU's were powered off.
 800                  * (could also be used for hotplugging CPU's).
 801                  */
 802                 if (cpu_pause_func != NULL)
 803                         (*cpu_pause_func)((void *)lindex);
 804 
 805                 mach_cpu_pause(safe);
 806 
 807                 splx(s);
 808                 /*
 809                  * Waiting is at an end. Switch out of cpu_pause
 810                  * loop and resume useful work.
 811                  */
 812                 swtch();
 813         }
 814 
 815         mutex_enter(&pause_free_mutex);
 816         *safe = PAUSE_DEAD;
 817         cv_broadcast(&pause_free_cv);
 818         mutex_exit(&pause_free_mutex);
 819 }
 820 
 821 /*
 822  * Allow the cpus to start running again.
 823  */
 824 void
 825 start_cpus()
 826 {
 827         int i;
 828 
 829         ASSERT(MUTEX_HELD(&cpu_lock));
 830         ASSERT(cpu_pause_info.cp_paused);
 831         cpu_pause_info.cp_paused = NULL;
 832         for (i = 0; i < NCPU; i++)
 833                 safe_list[i] = PAUSE_IDLE;
 834         membar_enter();                 /* make sure stores are flushed */
 835         affinity_clear();
 836         splx(cpu_pause_info.cp_spl);
 837         kpreempt_enable();
 838 }
 839 
 840 /*
 841  * Allocate a pause thread for a CPU.
 842  */
 843 static void
 844 cpu_pause_alloc(cpu_t *cp)
 845 {
 846         kthread_id_t    t;
 847         long            cpun = cp->cpu_id;
 848 
 849         /*
 850          * Note, v.v_nglobpris will not change value as long as I hold
 851          * cpu_lock.
 852          */
 853         t = thread_create(NULL, 0, cpu_pause, (void *)cpun,
 854             0, &p0, TS_STOPPED, v.v_nglobpris - 1);
 855         thread_lock(t);
 856         t->t_bound_cpu = cp;
 857         t->t_disp_queue = cp->cpu_disp;
 858         t->t_affinitycnt = 1;
 859         t->t_preempt = 1;
 860         thread_unlock(t);
 861         cp->cpu_pause_thread = t;
 862         /*
 863          * Registering a thread in the callback table is usually done
 864          * in the initialization code of the thread.  In this
 865          * case, we do it right after thread creation because the
 866          * thread itself may never run, and we need to register the
 867          * fact that it is safe for cpr suspend.
 868          */
 869         CALLB_CPR_INIT_SAFE(t, "cpu_pause");
 870 }
 871 
 872 /*
 873  * Free a pause thread for a CPU.
 874  */
 875 static void
 876 cpu_pause_free(cpu_t *cp)
 877 {
 878         kthread_id_t    t;
 879         int             cpun = cp->cpu_id;
 880 
 881         ASSERT(MUTEX_HELD(&cpu_lock));
 882         /*
 883          * We have to get the thread and tell him to die.
 884          */
 885         if ((t = cp->cpu_pause_thread) == NULL) {
 886                 ASSERT(safe_list[cpun] == PAUSE_IDLE);
 887                 return;
 888         }
 889         thread_lock(t);
 890         t->t_cpu = CPU;              /* disp gets upset if last cpu is quiesced. */
 891         t->t_bound_cpu = NULL;       /* Must un-bind; cpu may not be running. */
 892         t->t_pri = v.v_nglobpris - 1;
 893         ASSERT(safe_list[cpun] == PAUSE_IDLE);
 894         safe_list[cpun] = PAUSE_DIE;
 895         THREAD_TRANSITION(t);
 896         setbackdq(t);
 897         thread_unlock_nopreempt(t);
 898 
 899         /*
 900          * If we don't wait for the thread to actually die, it may try to
 901          * run on the wrong cpu as part of an actual call to pause_cpus().
 902          */
 903         mutex_enter(&pause_free_mutex);
 904         while (safe_list[cpun] != PAUSE_DEAD) {
 905                 cv_wait(&pause_free_cv, &pause_free_mutex);
 906         }
 907         mutex_exit(&pause_free_mutex);
 908         safe_list[cpun] = PAUSE_IDLE;
 909 
 910         cp->cpu_pause_thread = NULL;
 911 }
 912 
 913 /*
 914  * Initialize basic structures for pausing CPUs.
 915  */
 916 void
 917 cpu_pause_init()
 918 {
 919         sema_init(&cpu_pause_info.cp_sem, 0, NULL, SEMA_DEFAULT, NULL);
 920         /*
 921          * Create initial CPU pause thread.
 922          */
 923         cpu_pause_alloc(CPU);
 924 }
 925 
 926 /*
 927  * Start the threads used to pause another CPU.
 928  */
 929 static int
 930 cpu_pause_start(processorid_t cpu_id)
 931 {
 932         int     i;
 933         int     cpu_count = 0;
 934 
 935         for (i = 0; i < NCPU; i++) {
 936                 cpu_t           *cp;
 937                 kthread_id_t    t;
 938 
 939                 cp = cpu[i];
 940                 if (!CPU_IN_SET(cpu_available, i) || (i == cpu_id)) {
 941                         safe_list[i] = PAUSE_WAIT;
 942                         continue;
 943                 }
 944 
 945                 /*
 946                  * Skip CPU if it is quiesced or not yet started.
 947                  */
 948                 if ((cp->cpu_flags & (CPU_QUIESCED | CPU_READY)) != CPU_READY) {
 949                         safe_list[i] = PAUSE_WAIT;
 950                         continue;
 951                 }
 952 
 953                 /*
 954                  * Start this CPU's pause thread.
 955                  */
 956                 t = cp->cpu_pause_thread;
 957                 thread_lock(t);
 958                 /*
 959                  * Reset the priority, since nglobpris may have
 960                  * changed since the thread was created, if someone
 961                  * has loaded the RT (or some other) scheduling
 962                  * class.
 963                  */
 964                 t->t_pri = v.v_nglobpris - 1;
 965                 THREAD_TRANSITION(t);
 966                 setbackdq(t);
 967                 thread_unlock_nopreempt(t);
 968                 ++cpu_count;
 969         }
 970         return (cpu_count);
 971 }
 972 
 973 
 974 /*
 975  * Pause all of the CPUs except the one we are on by creating a high
 976  * priority thread bound to those CPUs.
 977  *
 978  * Note that one must be extremely careful regarding code
 979  * executed while CPUs are paused.  Since a CPU may be paused
 980  * while a thread scheduling on that CPU is holding an adaptive
 981  * lock, code executed with CPUs paused must not acquire adaptive
 982  * (or low-level spin) locks.  Also, such code must not block,
 983  * since the thread that is supposed to initiate the wakeup may
 984  * never run.
 985  *
 986  * With a few exceptions, the restrictions on code executed with CPUs
 987  * paused match those for code executed at high-level interrupt
 988  * context.
 989  */
 990 void
 991 pause_cpus(cpu_t *off_cp)
 992 {
 993         processorid_t   cpu_id;
 994         int             i;
 995         struct _cpu_pause_info  *cpi = &cpu_pause_info;
 996 
 997         ASSERT(MUTEX_HELD(&cpu_lock));
 998         ASSERT(cpi->cp_paused == NULL);
 999         cpi->cp_count = 0;
1000         cpi->cp_go = 0;
1001         for (i = 0; i < NCPU; i++)
1002                 safe_list[i] = PAUSE_IDLE;
1003         kpreempt_disable();
1004 
1005         /*
1006          * If running on the cpu that is going offline, get off it.
1007          * This is so that it won't be necessary to rechoose a CPU
1008          * when done.
1009          */
1010         if (CPU == off_cp)
1011                 cpu_id = off_cp->cpu_next_part->cpu_id;
1012         else
1013                 cpu_id = CPU->cpu_id;
1014         affinity_set(cpu_id);
1015 
1016         /*
1017          * Start the pause threads and record how many were started
1018          */
1019         cpi->cp_count = cpu_pause_start(cpu_id);
1020 
1021         /*
1022          * Now wait for all CPUs to be running the pause thread.
1023          */
1024         while (cpi->cp_count > 0) {
1025                 /*
1026                  * Spin reading the count without grabbing the disp
1027                  * lock to make sure we don't prevent the pause
1028                  * threads from getting the lock.
1029                  */
1030                 while (sema_held(&cpi->cp_sem))
1031                         ;
1032                 if (sema_tryp(&cpi->cp_sem))
1033                         --cpi->cp_count;
1034         }
1035         cpi->cp_go = 1;                      /* all have reached cpu_pause */
1036 
1037         /*
1038          * Now wait for all CPUs to spl. (Transition from PAUSE_READY
1039          * to PAUSE_WAIT.)
1040          */
1041         for (i = 0; i < NCPU; i++) {
1042                 while (safe_list[i] != PAUSE_WAIT)
1043                         ;
1044         }
1045         cpi->cp_spl = splhigh();     /* block dispatcher on this CPU */
1046         cpi->cp_paused = curthread;
1047 }
1048 
1049 /*
1050  * Check whether the current thread has CPUs paused
1051  */
1052 int
1053 cpus_paused(void)
1054 {
1055         if (cpu_pause_info.cp_paused != NULL) {
1056                 ASSERT(cpu_pause_info.cp_paused == curthread);
1057                 return (1);
1058         }
1059         return (0);
1060 }
1061 
1062 static cpu_t *
1063 cpu_get_all(processorid_t cpun)
1064 {
1065         ASSERT(MUTEX_HELD(&cpu_lock));
1066 
1067         if (cpun >= NCPU || cpun < 0 || !CPU_IN_SET(cpu_available, cpun))
1068                 return (NULL);
1069         return (cpu[cpun]);
1070 }
1071 
1072 /*
1073  * Check whether cpun is a valid processor id and whether it should be
1074  * visible from the current zone. If it is, return a pointer to the
1075  * associated CPU structure.
1076  */
1077 cpu_t *
1078 cpu_get(processorid_t cpun)
1079 {
1080         cpu_t *c;
1081 
1082         ASSERT(MUTEX_HELD(&cpu_lock));
1083         c = cpu_get_all(cpun);
1084         if (c != NULL && !INGLOBALZONE(curproc) && pool_pset_enabled() &&
1085             zone_pset_get(curproc->p_zone) != cpupart_query_cpu(c))
1086                 return (NULL);
1087         return (c);
1088 }
1089 
1090 /*
1091  * The following functions should be used to check CPU states in the kernel.
1092  * They should be invoked with cpu_lock held.  Kernel subsystems interested
1093  * in CPU states should *not* use cpu_get_state() and various P_ONLINE/etc
1094  * states.  Those are for user-land (and system call) use only.
1095  */
1096 
1097 /*
1098  * Determine whether the CPU is online and handling interrupts.
1099  */
1100 int
1101 cpu_is_online(cpu_t *cpu)
1102 {
1103         ASSERT(MUTEX_HELD(&cpu_lock));
1104         return (cpu_flagged_online(cpu->cpu_flags));
1105 }
1106 
1107 /*
1108  * Determine whether the CPU is offline (this includes spare and faulted).
1109  */
1110 int
1111 cpu_is_offline(cpu_t *cpu)
1112 {
1113         ASSERT(MUTEX_HELD(&cpu_lock));
1114         return (cpu_flagged_offline(cpu->cpu_flags));
1115 }
1116 
1117 /*
1118  * Determine whether the CPU is powered off.
1119  */
1120 int
1121 cpu_is_poweredoff(cpu_t *cpu)
1122 {
1123         ASSERT(MUTEX_HELD(&cpu_lock));
1124         return (cpu_flagged_poweredoff(cpu->cpu_flags));
1125 }
1126 
1127 /*
1128  * Determine whether the CPU is handling interrupts.
1129  */
1130 int
1131 cpu_is_nointr(cpu_t *cpu)
1132 {
1133         ASSERT(MUTEX_HELD(&cpu_lock));
1134         return (cpu_flagged_nointr(cpu->cpu_flags));
1135 }
1136 
1137 /*
1138  * Determine whether the CPU is active (scheduling threads).
1139  */
1140 int
1141 cpu_is_active(cpu_t *cpu)
1142 {
1143         ASSERT(MUTEX_HELD(&cpu_lock));
1144         return (cpu_flagged_active(cpu->cpu_flags));
1145 }
1146 
1147 /*
1148  * Same as above, but these require cpu_flags instead of cpu_t pointers.
1149  */
1150 int
1151 cpu_flagged_online(cpu_flag_t cpu_flags)
1152 {
1153         return (cpu_flagged_active(cpu_flags) &&
1154             (cpu_flags & CPU_ENABLE));
1155 }
1156 
1157 int
1158 cpu_flagged_offline(cpu_flag_t cpu_flags)
1159 {
1160         return (((cpu_flags & CPU_POWEROFF) == 0) &&
1161             ((cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY));
1162 }
1163 
1164 int
1165 cpu_flagged_poweredoff(cpu_flag_t cpu_flags)
1166 {
1167         return ((cpu_flags & CPU_POWEROFF) == CPU_POWEROFF);
1168 }
1169 
1170 int
1171 cpu_flagged_nointr(cpu_flag_t cpu_flags)
1172 {
1173         return (cpu_flagged_active(cpu_flags) &&
1174             (cpu_flags & CPU_ENABLE) == 0);
1175 }
1176 
1177 int
1178 cpu_flagged_active(cpu_flag_t cpu_flags)
1179 {
1180         return (((cpu_flags & (CPU_POWEROFF | CPU_FAULTED | CPU_SPARE)) == 0) &&
1181             ((cpu_flags & (CPU_READY | CPU_OFFLINE)) == CPU_READY));
1182 }
1183 
1184 /*
1185  * Bring the indicated CPU online.
1186  */
1187 int
1188 cpu_online(cpu_t *cp)
1189 {
1190         int     error = 0;
1191 
1192         /*
1193          * Handle on-line request.
1194          *      This code must put the new CPU on the active list before
1195          *      starting it because it will not be paused, and will start
1196          *      using the active list immediately.  The real start occurs
1197          *      when the CPU_QUIESCED flag is turned off.
1198          */
1199 
1200         ASSERT(MUTEX_HELD(&cpu_lock));
1201 
1202         /*
1203          * Put all the cpus into a known safe place.
1204          * No mutexes can be entered while CPUs are paused.
1205          */
1206         error = mp_cpu_start(cp);       /* arch-dep hook */
1207         if (error == 0) {
1208                 pg_cpupart_in(cp, cp->cpu_part);
1209                 pause_cpus(NULL);
1210                 cpu_add_active_internal(cp);
1211                 if (cp->cpu_flags & CPU_FAULTED) {
1212                         cp->cpu_flags &= ~CPU_FAULTED;
1213                         mp_cpu_faulted_exit(cp);
1214                 }
1215                 cp->cpu_flags &= ~(CPU_QUIESCED | CPU_OFFLINE | CPU_FROZEN |
1216                     CPU_SPARE);
1217                 CPU_NEW_GENERATION(cp);
1218                 start_cpus();
1219                 cpu_stats_kstat_create(cp);
1220                 cpu_create_intrstat(cp);
1221                 lgrp_kstat_create(cp);
1222                 cpu_state_change_notify(cp->cpu_id, CPU_ON);
1223                 cpu_intr_enable(cp);    /* arch-dep hook */
1224                 cpu_state_change_notify(cp->cpu_id, CPU_INTR_ON);
1225                 cpu_set_state(cp);
1226                 cyclic_online(cp);
1227                 /*
1228                  * This has to be called only after cyclic_online(). This
1229                  * function uses cyclics.
1230                  */
1231                 callout_cpu_online(cp);
1232                 poke_cpu(cp->cpu_id);
1233         }
1234 
1235         return (error);
1236 }
1237 
1238 /*
1239  * Take the indicated CPU offline.
1240  */
1241 int
1242 cpu_offline(cpu_t *cp, int flags)
1243 {
1244         cpupart_t *pp;
1245         int     error = 0;
1246         cpu_t   *ncp;
1247         int     intr_enable;
1248         int     cyclic_off = 0;
1249         int     callout_off = 0;
1250         int     loop_count;
1251         int     no_quiesce = 0;
1252         int     (*bound_func)(struct cpu *, int);
1253         kthread_t *t;
1254         lpl_t   *cpu_lpl;
1255         proc_t  *p;
1256         int     lgrp_diff_lpl;
1257         boolean_t unbind_all_threads = (flags & CPU_FORCED) != 0;
1258 
1259         ASSERT(MUTEX_HELD(&cpu_lock));
1260 
1261         /*
1262          * If we're going from faulted or spare to offline, just
1263          * clear these flags and update CPU state.
1264          */
1265         if (cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) {
1266                 if (cp->cpu_flags & CPU_FAULTED) {
1267                         cp->cpu_flags &= ~CPU_FAULTED;
1268                         mp_cpu_faulted_exit(cp);
1269                 }
1270                 cp->cpu_flags &= ~CPU_SPARE;
1271                 cpu_set_state(cp);
1272                 return (0);
1273         }
1274 
1275         /*
1276          * Handle off-line request.
1277          */
1278         pp = cp->cpu_part;
1279         /*
1280          * Don't offline last online CPU in partition
1281          */
1282         if (ncpus_online <= 1 || pp->cp_ncpus <= 1 || cpu_intr_count(cp) < 2)
1283                 return (EBUSY);
1284         /*
1285          * Unbind all soft-bound threads bound to our CPU and hard bound threads
1286          * if we were asked to.
1287          */
1288         error = cpu_unbind(cp->cpu_id, unbind_all_threads);
1289         if (error != 0)
1290                 return (error);
1291         /*
1292          * We shouldn't be bound to this CPU ourselves.
1293          */
1294         if (curthread->t_bound_cpu == cp)
1295                 return (EBUSY);
1296 
1297         /*
1298          * Tell interested parties that this CPU is going offline.
1299          */
1300         CPU_NEW_GENERATION(cp);
1301         cpu_state_change_notify(cp->cpu_id, CPU_OFF);
1302 
1303         /*
1304          * Tell the PG subsystem that the CPU is leaving the partition
1305          */
1306         pg_cpupart_out(cp, pp);
1307 
1308         /*
1309          * Take the CPU out of interrupt participation so we won't find
1310          * bound kernel threads.  If the architecture cannot completely
1311          * shut off interrupts on the CPU, don't quiesce it, but don't
1312          * run anything but interrupt thread... this is indicated by
1313          * the CPU_OFFLINE flag being on but the CPU_QUIESCE flag being
1314          * off.
1315          */
1316         intr_enable = cp->cpu_flags & CPU_ENABLE;
1317         if (intr_enable)
1318                 no_quiesce = cpu_intr_disable(cp);
1319 
1320         /*
1321          * Record that we are aiming to offline this cpu.  This acts as
1322          * a barrier to further weak binding requests in thread_nomigrate
1323          * and also causes cpu_choose, disp_lowpri_cpu and setfrontdq to
1324          * lean away from this cpu.  Further strong bindings are already
1325          * avoided since we hold cpu_lock.  Since threads that are set
1326          * runnable around now and others coming off the target cpu are
1327          * directed away from the target, existing strong and weak bindings
1328          * (especially the latter) to the target cpu stand maximum chance of
1329          * being able to unbind during the short delay loop below (if other
1330          * unbound threads compete they may not see cpu in time to unbind
1331          * even if they would do so immediately.
1332          */
1333         cpu_inmotion = cp;
1334         membar_enter();
1335 
1336         /*
1337          * Check for kernel threads (strong or weak) bound to that CPU.
1338          * Strongly bound threads may not unbind, and we'll have to return
1339          * EBUSY.  Weakly bound threads should always disappear - we've
1340          * stopped more weak binding with cpu_inmotion and existing
1341          * bindings will drain imminently (they may not block).  Nonetheless
1342          * we will wait for a fixed period for all bound threads to disappear.
1343          * Inactive interrupt threads are OK (they'll be in TS_FREE
1344          * state).  If test finds some bound threads, wait a few ticks
1345          * to give short-lived threads (such as interrupts) chance to
1346          * complete.  Note that if no_quiesce is set, i.e. this cpu
1347          * is required to service interrupts, then we take the route
1348          * that permits interrupt threads to be active (or bypassed).
1349          */
1350         bound_func = no_quiesce ? disp_bound_threads : disp_bound_anythreads;
1351 
1352 again:  for (loop_count = 0; (*bound_func)(cp, 0); loop_count++) {
1353                 if (loop_count >= 5) {
1354                         error = EBUSY;  /* some threads still bound */
1355                         break;
1356                 }
1357 
1358                 /*
1359                  * If some threads were assigned, give them
1360                  * a chance to complete or move.
1361                  *
1362                  * This assumes that the clock_thread is not bound
1363                  * to any CPU, because the clock_thread is needed to
1364                  * do the delay(hz/100).
1365                  *
1366                  * Note: we still hold the cpu_lock while waiting for
1367                  * the next clock tick.  This is OK since it isn't
1368                  * needed for anything else except processor_bind(2),
1369                  * and system initialization.  If we drop the lock,
1370                  * we would risk another p_online disabling the last
1371                  * processor.
1372                  */
1373                 delay(hz/100);
1374         }
1375 
1376         if (error == 0 && callout_off == 0) {
1377                 callout_cpu_offline(cp);
1378                 callout_off = 1;
1379         }
1380 
1381         if (error == 0 && cyclic_off == 0) {
1382                 if (!cyclic_offline(cp)) {
1383                         /*
1384                          * We must have bound cyclics...
1385                          */
1386                         error = EBUSY;
1387                         goto out;
1388                 }
1389                 cyclic_off = 1;
1390         }
1391 
1392         /*
1393          * Call mp_cpu_stop() to perform any special operations
1394          * needed for this machine architecture to offline a CPU.
1395          */
1396         if (error == 0)
1397                 error = mp_cpu_stop(cp);        /* arch-dep hook */
1398 
1399         /*
1400          * If that all worked, take the CPU offline and decrement
1401          * ncpus_online.
1402          */
1403         if (error == 0) {
1404                 /*
1405                  * Put all the cpus into a known safe place.
1406                  * No mutexes can be entered while CPUs are paused.
1407                  */
1408                 pause_cpus(cp);
1409                 /*
1410                  * Repeat the operation, if necessary, to make sure that
1411                  * all outstanding low-level interrupts run to completion
1412                  * before we set the CPU_QUIESCED flag.  It's also possible
1413                  * that a thread has weak bound to the cpu despite our raising
1414                  * cpu_inmotion above since it may have loaded that
1415                  * value before the barrier became visible (this would have
1416                  * to be the thread that was on the target cpu at the time
1417                  * we raised the barrier).
1418                  */
1419                 if ((!no_quiesce && cp->cpu_intr_actv != 0) ||
1420                     (*bound_func)(cp, 1)) {
1421                         start_cpus();
1422                         (void) mp_cpu_start(cp);
1423                         goto again;
1424                 }
1425                 ncp = cp->cpu_next_part;
1426                 cpu_lpl = cp->cpu_lpl;
1427                 ASSERT(cpu_lpl != NULL);
1428 
1429                 /*
1430                  * Remove the CPU from the list of active CPUs.
1431                  */
1432                 cpu_remove_active(cp);
1433 
1434                 /*
1435                  * Walk the active process list and look for threads
1436                  * whose home lgroup needs to be updated, or
1437                  * the last CPU they run on is the one being offlined now.
1438                  */
1439 
1440                 ASSERT(curthread->t_cpu != cp);
1441                 for (p = practive; p != NULL; p = p->p_next) {
1442 
1443                         t = p->p_tlist;
1444 
1445                         if (t == NULL)
1446                                 continue;
1447 
1448                         lgrp_diff_lpl = 0;
1449 
1450                         do {
1451                                 ASSERT(t->t_lpl != NULL);
1452                                 /*
1453                                  * Taking last CPU in lpl offline
1454                                  * Rehome thread if it is in this lpl
1455                                  * Otherwise, update the count of how many
1456                                  * threads are in this CPU's lgroup but have
1457                                  * a different lpl.
1458                                  */
1459 
1460                                 if (cpu_lpl->lpl_ncpu == 0) {
1461                                         if (t->t_lpl == cpu_lpl)
1462                                                 lgrp_move_thread(t,
1463                                                     lgrp_choose(t,
1464                                                     t->t_cpupart), 0);
1465                                         else if (t->t_lpl->lpl_lgrpid ==
1466                                             cpu_lpl->lpl_lgrpid)
1467                                                 lgrp_diff_lpl++;
1468                                 }
1469                                 ASSERT(t->t_lpl->lpl_ncpu > 0);
1470 
1471                                 /*
1472                                  * Update CPU last ran on if it was this CPU
1473                                  */
1474                                 if (t->t_cpu == cp && t->t_bound_cpu != cp)
1475                                         t->t_cpu = disp_lowpri_cpu(ncp,
1476                                             t->t_lpl, t->t_pri, NULL);
1477                                 ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1478                                     t->t_weakbound_cpu == cp);
1479 
1480                                 t = t->t_forw;
1481                         } while (t != p->p_tlist);
1482 
1483                         /*
1484                          * Didn't find any threads in the same lgroup as this
1485                          * CPU with a different lpl, so remove the lgroup from
1486                          * the process lgroup bitmask.
1487                          */
1488 
1489                         if (lgrp_diff_lpl == 0)
1490                                 klgrpset_del(p->p_lgrpset, cpu_lpl->lpl_lgrpid);
1491                 }
1492 
1493                 /*
1494                  * Walk thread list looking for threads that need to be
1495                  * rehomed, since there are some threads that are not in
1496                  * their process's p_tlist.
1497                  */
1498 
1499                 t = curthread;
1500                 do {
1501                         ASSERT(t != NULL && t->t_lpl != NULL);
1502 
1503                         /*
1504                          * Rehome threads with same lpl as this CPU when this
1505                          * is the last CPU in the lpl.
1506                          */
1507 
1508                         if ((cpu_lpl->lpl_ncpu == 0) && (t->t_lpl == cpu_lpl))
1509                                 lgrp_move_thread(t,
1510                                     lgrp_choose(t, t->t_cpupart), 1);
1511 
1512                         ASSERT(t->t_lpl->lpl_ncpu > 0);
1513 
1514                         /*
1515                          * Update CPU last ran on if it was this CPU
1516                          */
1517 
1518                         if (t->t_cpu == cp && t->t_bound_cpu != cp) {
1519                                 t->t_cpu = disp_lowpri_cpu(ncp,
1520                                     t->t_lpl, t->t_pri, NULL);
1521                         }
1522                         ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1523                             t->t_weakbound_cpu == cp);
1524                         t = t->t_next;
1525 
1526                 } while (t != curthread);
1527                 ASSERT((cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) == 0);
1528                 cp->cpu_flags |= CPU_OFFLINE;
1529                 disp_cpu_inactive(cp);
1530                 if (!no_quiesce)
1531                         cp->cpu_flags |= CPU_QUIESCED;
1532                 ncpus_online--;
1533                 cpu_set_state(cp);
1534                 cpu_inmotion = NULL;
1535                 start_cpus();
1536                 cpu_stats_kstat_destroy(cp);
1537                 cpu_delete_intrstat(cp);
1538                 lgrp_kstat_destroy(cp);
1539         }
1540 
1541 out:
1542         cpu_inmotion = NULL;
1543 
1544         /*
1545          * If we failed, re-enable interrupts.
1546          * Do this even if cpu_intr_disable returned an error, because
1547          * it may have partially disabled interrupts.
1548          */
1549         if (error && intr_enable)
1550                 cpu_intr_enable(cp);
1551 
1552         /*
1553          * If we failed, but managed to offline the cyclic subsystem on this
1554          * CPU, bring it back online.
1555          */
1556         if (error && cyclic_off)
1557                 cyclic_online(cp);
1558 
1559         /*
1560          * If we failed, but managed to offline callouts on this CPU,
1561          * bring it back online.
1562          */
1563         if (error && callout_off)
1564                 callout_cpu_online(cp);
1565 
1566         /*
1567          * If we failed, tell the PG subsystem that the CPU is back
1568          */
1569         pg_cpupart_in(cp, pp);
1570 
1571         /*
1572          * If we failed, we need to notify everyone that this CPU is back on.
1573          */
1574         if (error != 0) {
1575                 CPU_NEW_GENERATION(cp);
1576                 cpu_state_change_notify(cp->cpu_id, CPU_ON);
1577                 cpu_state_change_notify(cp->cpu_id, CPU_INTR_ON);
1578         }
1579 
1580         return (error);
1581 }
1582 
1583 /*
1584  * Mark the indicated CPU as faulted, taking it offline.
1585  */
1586 int
1587 cpu_faulted(cpu_t *cp, int flags)
1588 {
1589         int     error = 0;
1590 
1591         ASSERT(MUTEX_HELD(&cpu_lock));
1592         ASSERT(!cpu_is_poweredoff(cp));
1593 
1594         if (cpu_is_offline(cp)) {
1595                 cp->cpu_flags &= ~CPU_SPARE;
1596                 cp->cpu_flags |= CPU_FAULTED;
1597                 mp_cpu_faulted_enter(cp);
1598                 cpu_set_state(cp);
1599                 return (0);
1600         }
1601 
1602         if ((error = cpu_offline(cp, flags)) == 0) {
1603                 cp->cpu_flags |= CPU_FAULTED;
1604                 mp_cpu_faulted_enter(cp);
1605                 cpu_set_state(cp);
1606         }
1607 
1608         return (error);
1609 }
1610 
1611 /*
1612  * Mark the indicated CPU as a spare, taking it offline.
1613  */
1614 int
1615 cpu_spare(cpu_t *cp, int flags)
1616 {
1617         int     error = 0;
1618 
1619         ASSERT(MUTEX_HELD(&cpu_lock));
1620         ASSERT(!cpu_is_poweredoff(cp));
1621 
1622         if (cpu_is_offline(cp)) {
1623                 if (cp->cpu_flags & CPU_FAULTED) {
1624                         cp->cpu_flags &= ~CPU_FAULTED;
1625                         mp_cpu_faulted_exit(cp);
1626                 }
1627                 cp->cpu_flags |= CPU_SPARE;
1628                 cpu_set_state(cp);
1629                 return (0);
1630         }
1631 
1632         if ((error = cpu_offline(cp, flags)) == 0) {
1633                 cp->cpu_flags |= CPU_SPARE;
1634                 cpu_set_state(cp);
1635         }
1636 
1637         return (error);
1638 }
1639 
1640 /*
1641  * Take the indicated CPU from poweroff to offline.
1642  */
1643 int
1644 cpu_poweron(cpu_t *cp)
1645 {
1646         int     error = ENOTSUP;
1647 
1648         ASSERT(MUTEX_HELD(&cpu_lock));
1649         ASSERT(cpu_is_poweredoff(cp));
1650 
1651         error = mp_cpu_poweron(cp);     /* arch-dep hook */
1652         if (error == 0)
1653                 cpu_set_state(cp);
1654 
1655         return (error);
1656 }
1657 
1658 /*
1659  * Take the indicated CPU from any inactive state to powered off.
1660  */
1661 int
1662 cpu_poweroff(cpu_t *cp)
1663 {
1664         int     error = ENOTSUP;
1665 
1666         ASSERT(MUTEX_HELD(&cpu_lock));
1667         ASSERT(cpu_is_offline(cp));
1668 
1669         if (!(cp->cpu_flags & CPU_QUIESCED))
1670                 return (EBUSY);         /* not completely idle */
1671 
1672         error = mp_cpu_poweroff(cp);    /* arch-dep hook */
1673         if (error == 0)
1674                 cpu_set_state(cp);
1675 
1676         return (error);
1677 }
1678 
1679 /*
1680  * Initialize the Sequential CPU id lookup table
1681  */
1682 void
1683 cpu_seq_tbl_init()
1684 {
1685         cpu_t   **tbl;
1686 
1687         tbl = kmem_zalloc(sizeof (struct cpu *) * max_ncpus, KM_SLEEP);
1688         tbl[0] = CPU;
1689 
1690         cpu_seq = tbl;
1691 }
1692 
1693 /*
1694  * Initialize the CPU lists for the first CPU.
1695  */
1696 void
1697 cpu_list_init(cpu_t *cp)
1698 {
1699         cp->cpu_next = cp;
1700         cp->cpu_prev = cp;
1701         cpu_list = cp;
1702         clock_cpu_list = cp;
1703 
1704         cp->cpu_next_onln = cp;
1705         cp->cpu_prev_onln = cp;
1706         cpu_active = cp;
1707 
1708         cp->cpu_seqid = 0;
1709         CPUSET_ADD(cpu_seqid_inuse, 0);
1710 
1711         /*
1712          * Bootstrap cpu_seq using cpu_list
1713          * The cpu_seq[] table will be dynamically allocated
1714          * when kmem later becomes available (but before going MP)
1715          */
1716         cpu_seq = &cpu_list;
1717 
1718         cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
1719         cp_default.cp_cpulist = cp;
1720         cp_default.cp_ncpus = 1;
1721         cp->cpu_next_part = cp;
1722         cp->cpu_prev_part = cp;
1723         cp->cpu_part = &cp_default;
1724 
1725         CPUSET_ADD(cpu_available, cp->cpu_id);
1726 }
1727 
1728 /*
1729  * Insert a CPU into the list of available CPUs.
1730  */
1731 void
1732 cpu_add_unit(cpu_t *cp)
1733 {
1734         int seqid;
1735 
1736         ASSERT(MUTEX_HELD(&cpu_lock));
1737         ASSERT(cpu_list != NULL);       /* list started in cpu_list_init */
1738 
1739         lgrp_config(LGRP_CONFIG_CPU_ADD, (uintptr_t)cp, 0);
1740 
1741         /*
1742          * Note: most users of the cpu_list will grab the
1743          * cpu_lock to insure that it isn't modified.  However,
1744          * certain users can't or won't do that.  To allow this
1745          * we pause the other cpus.  Users who walk the list
1746          * without cpu_lock, must disable kernel preemption
1747          * to insure that the list isn't modified underneath
1748          * them.  Also, any cached pointers to cpu structures
1749          * must be revalidated by checking to see if the
1750          * cpu_next pointer points to itself.  This check must
1751          * be done with the cpu_lock held or kernel preemption
1752          * disabled.  This check relies upon the fact that
1753          * old cpu structures are not free'ed or cleared after
1754          * then are removed from the cpu_list.
1755          *
1756          * Note that the clock code walks the cpu list dereferencing
1757          * the cpu_part pointer, so we need to initialize it before
1758          * adding the cpu to the list.
1759          */
1760         cp->cpu_part = &cp_default;
1761         (void) pause_cpus(NULL);
1762         cp->cpu_next = cpu_list;
1763         cp->cpu_prev = cpu_list->cpu_prev;
1764         cpu_list->cpu_prev->cpu_next = cp;
1765         cpu_list->cpu_prev = cp;
1766         start_cpus();
1767 
1768         for (seqid = 0; CPU_IN_SET(cpu_seqid_inuse, seqid); seqid++)
1769                 continue;
1770         CPUSET_ADD(cpu_seqid_inuse, seqid);
1771         cp->cpu_seqid = seqid;
1772 
1773         if (seqid > max_cpu_seqid_ever)
1774                 max_cpu_seqid_ever = seqid;
1775 
1776         ASSERT(ncpus < max_ncpus);
1777         ncpus++;
1778         cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
1779         cpu[cp->cpu_id] = cp;
1780         CPUSET_ADD(cpu_available, cp->cpu_id);
1781         cpu_seq[cp->cpu_seqid] = cp;
1782 
1783         /*
1784          * allocate a pause thread for this CPU.
1785          */
1786         cpu_pause_alloc(cp);
1787 
1788         /*
1789          * So that new CPUs won't have NULL prev_onln and next_onln pointers,
1790          * link them into a list of just that CPU.
1791          * This is so that disp_lowpri_cpu will work for thread_create in
1792          * pause_cpus() when called from the startup thread in a new CPU.
1793          */
1794         cp->cpu_next_onln = cp;
1795         cp->cpu_prev_onln = cp;
1796         cpu_info_kstat_create(cp);
1797         cp->cpu_next_part = cp;
1798         cp->cpu_prev_part = cp;
1799 
1800         init_cpu_mstate(cp, CMS_SYSTEM);
1801 
1802         pool_pset_mod = gethrtime();
1803 }
1804 
1805 /*
1806  * Do the opposite of cpu_add_unit().
1807  */
1808 void
1809 cpu_del_unit(int cpuid)
1810 {
1811         struct cpu      *cp, *cpnext;
1812 
1813         ASSERT(MUTEX_HELD(&cpu_lock));
1814         cp = cpu[cpuid];
1815         ASSERT(cp != NULL);
1816 
1817         ASSERT(cp->cpu_next_onln == cp);
1818         ASSERT(cp->cpu_prev_onln == cp);
1819         ASSERT(cp->cpu_next_part == cp);
1820         ASSERT(cp->cpu_prev_part == cp);
1821 
1822         /*
1823          * Tear down the CPU's physical ID cache, and update any
1824          * processor groups
1825          */
1826         pg_cpu_fini(cp, NULL);
1827         pghw_physid_destroy(cp);
1828 
1829         /*
1830          * Destroy kstat stuff.
1831          */
1832         cpu_info_kstat_destroy(cp);
1833         term_cpu_mstate(cp);
1834         /*
1835          * Free up pause thread.
1836          */
1837         cpu_pause_free(cp);
1838         CPUSET_DEL(cpu_available, cp->cpu_id);
1839         cpu[cp->cpu_id] = NULL;
1840         cpu_seq[cp->cpu_seqid] = NULL;
1841 
1842         /*
1843          * The clock thread and mutex_vector_enter cannot hold the
1844          * cpu_lock while traversing the cpu list, therefore we pause
1845          * all other threads by pausing the other cpus. These, and any
1846          * other routines holding cpu pointers while possibly sleeping
1847          * must be sure to call kpreempt_disable before processing the
1848          * list and be sure to check that the cpu has not been deleted
1849          * after any sleeps (check cp->cpu_next != NULL). We guarantee
1850          * to keep the deleted cpu structure around.
1851          *
1852          * Note that this MUST be done AFTER cpu_available
1853          * has been updated so that we don't waste time
1854          * trying to pause the cpu we're trying to delete.
1855          */
1856         (void) pause_cpus(NULL);
1857 
1858         cpnext = cp->cpu_next;
1859         cp->cpu_prev->cpu_next = cp->cpu_next;
1860         cp->cpu_next->cpu_prev = cp->cpu_prev;
1861         if (cp == cpu_list)
1862                 cpu_list = cpnext;
1863 
1864         /*
1865          * Signals that the cpu has been deleted (see above).
1866          */
1867         cp->cpu_next = NULL;
1868         cp->cpu_prev = NULL;
1869 
1870         start_cpus();
1871 
1872         CPUSET_DEL(cpu_seqid_inuse, cp->cpu_seqid);
1873         ncpus--;
1874         lgrp_config(LGRP_CONFIG_CPU_DEL, (uintptr_t)cp, 0);
1875 
1876         pool_pset_mod = gethrtime();
1877 }
1878 
1879 /*
1880  * Add a CPU to the list of active CPUs.
1881  *      This routine must not get any locks, because other CPUs are paused.
1882  */
1883 static void
1884 cpu_add_active_internal(cpu_t *cp)
1885 {
1886         cpupart_t       *pp = cp->cpu_part;
1887 
1888         ASSERT(MUTEX_HELD(&cpu_lock));
1889         ASSERT(cpu_list != NULL);       /* list started in cpu_list_init */
1890 
1891         ncpus_online++;
1892         cpu_set_state(cp);
1893         cp->cpu_next_onln = cpu_active;
1894         cp->cpu_prev_onln = cpu_active->cpu_prev_onln;
1895         cpu_active->cpu_prev_onln->cpu_next_onln = cp;
1896         cpu_active->cpu_prev_onln = cp;
1897 
1898         if (pp->cp_cpulist) {
1899                 cp->cpu_next_part = pp->cp_cpulist;
1900                 cp->cpu_prev_part = pp->cp_cpulist->cpu_prev_part;
1901                 pp->cp_cpulist->cpu_prev_part->cpu_next_part = cp;
1902                 pp->cp_cpulist->cpu_prev_part = cp;
1903         } else {
1904                 ASSERT(pp->cp_ncpus == 0);
1905                 pp->cp_cpulist = cp->cpu_next_part = cp->cpu_prev_part = cp;
1906         }
1907         pp->cp_ncpus++;
1908         if (pp->cp_ncpus == 1) {
1909                 cp_numparts_nonempty++;
1910                 ASSERT(cp_numparts_nonempty != 0);
1911         }
1912 
1913         pg_cpu_active(cp);
1914         lgrp_config(LGRP_CONFIG_CPU_ONLINE, (uintptr_t)cp, 0);
1915 
1916         bzero(&cp->cpu_loadavg, sizeof (cp->cpu_loadavg));
1917 }
1918 
1919 /*
1920  * Add a CPU to the list of active CPUs.
1921  *      This is called from machine-dependent layers when a new CPU is started.
1922  */
1923 void
1924 cpu_add_active(cpu_t *cp)
1925 {
1926         pg_cpupart_in(cp, cp->cpu_part);
1927 
1928         pause_cpus(NULL);
1929         cpu_add_active_internal(cp);
1930         start_cpus();
1931 
1932         cpu_stats_kstat_create(cp);
1933         cpu_create_intrstat(cp);
1934         lgrp_kstat_create(cp);
1935         cpu_state_change_notify(cp->cpu_id, CPU_INIT);
1936 }
1937 
1938 
1939 /*
1940  * Remove a CPU from the list of active CPUs.
1941  *      This routine must not get any locks, because other CPUs are paused.
1942  */
1943 /* ARGSUSED */
1944 static void
1945 cpu_remove_active(cpu_t *cp)
1946 {
1947         cpupart_t       *pp = cp->cpu_part;
1948 
1949         ASSERT(MUTEX_HELD(&cpu_lock));
1950         ASSERT(cp->cpu_next_onln != cp);     /* not the last one */
1951         ASSERT(cp->cpu_prev_onln != cp);     /* not the last one */
1952 
1953         pg_cpu_inactive(cp);
1954 
1955         lgrp_config(LGRP_CONFIG_CPU_OFFLINE, (uintptr_t)cp, 0);
1956 
1957         if (cp == clock_cpu_list)
1958                 clock_cpu_list = cp->cpu_next_onln;
1959 
1960         cp->cpu_prev_onln->cpu_next_onln = cp->cpu_next_onln;
1961         cp->cpu_next_onln->cpu_prev_onln = cp->cpu_prev_onln;
1962         if (cpu_active == cp) {
1963                 cpu_active = cp->cpu_next_onln;
1964         }
1965         cp->cpu_next_onln = cp;
1966         cp->cpu_prev_onln = cp;
1967 
1968         cp->cpu_prev_part->cpu_next_part = cp->cpu_next_part;
1969         cp->cpu_next_part->cpu_prev_part = cp->cpu_prev_part;
1970         if (pp->cp_cpulist == cp) {
1971                 pp->cp_cpulist = cp->cpu_next_part;
1972                 ASSERT(pp->cp_cpulist != cp);
1973         }
1974         cp->cpu_next_part = cp;
1975         cp->cpu_prev_part = cp;
1976         pp->cp_ncpus--;
1977         if (pp->cp_ncpus == 0) {
1978                 cp_numparts_nonempty--;
1979                 ASSERT(cp_numparts_nonempty != 0);
1980         }
1981 }
1982 
1983 /*
1984  * Routine used to setup a newly inserted CPU in preparation for starting
1985  * it running code.
1986  */
1987 int
1988 cpu_configure(int cpuid)
1989 {
1990         int retval = 0;
1991 
1992         ASSERT(MUTEX_HELD(&cpu_lock));
1993 
1994         /*
1995          * Some structures are statically allocated based upon
1996          * the maximum number of cpus the system supports.  Do not
1997          * try to add anything beyond this limit.
1998          */
1999         if (cpuid < 0 || cpuid >= NCPU) {
2000                 return (EINVAL);
2001         }
2002 
2003         if ((cpu[cpuid] != NULL) && (cpu[cpuid]->cpu_flags != 0)) {
2004                 return (EALREADY);
2005         }
2006 
2007         if ((retval = mp_cpu_configure(cpuid)) != 0) {
2008                 return (retval);
2009         }
2010 
2011         cpu[cpuid]->cpu_flags = CPU_QUIESCED | CPU_OFFLINE | CPU_POWEROFF;
2012         cpu_set_state(cpu[cpuid]);
2013         retval = cpu_state_change_hooks(cpuid, CPU_CONFIG, CPU_UNCONFIG);
2014         if (retval != 0)
2015                 (void) mp_cpu_unconfigure(cpuid);
2016 
2017         return (retval);
2018 }
2019 
2020 /*
2021  * Routine used to cleanup a CPU that has been powered off.  This will
2022  * destroy all per-cpu information related to this cpu.
2023  */
2024 int
2025 cpu_unconfigure(int cpuid)
2026 {
2027         int error;
2028 
2029         ASSERT(MUTEX_HELD(&cpu_lock));
2030 
2031         if (cpu[cpuid] == NULL) {
2032                 return (ENODEV);
2033         }
2034 
2035         if (cpu[cpuid]->cpu_flags == 0) {
2036                 return (EALREADY);
2037         }
2038 
2039         if ((cpu[cpuid]->cpu_flags & CPU_POWEROFF) == 0) {
2040                 return (EBUSY);
2041         }
2042 
2043         if (cpu[cpuid]->cpu_props != NULL) {
2044                 (void) nvlist_free(cpu[cpuid]->cpu_props);
2045                 cpu[cpuid]->cpu_props = NULL;
2046         }
2047 
2048         error = cpu_state_change_hooks(cpuid, CPU_UNCONFIG, CPU_CONFIG);
2049 
2050         if (error != 0)
2051                 return (error);
2052 
2053         return (mp_cpu_unconfigure(cpuid));
2054 }
2055 
2056 /*
2057  * Routines for registering and de-registering cpu_setup callback functions.
2058  *
2059  * Caller's context
2060  *      These routines must not be called from a driver's attach(9E) or
2061  *      detach(9E) entry point.
2062  *
2063  * NOTE: CPU callbacks should not block. They are called with cpu_lock held.
2064  */
2065 
2066 /*
2067  * Ideally, these would be dynamically allocated and put into a linked
2068  * list; however that is not feasible because the registration routine
2069  * has to be available before the kmem allocator is working (in fact,
2070  * it is called by the kmem allocator init code).  In any case, there
2071  * are quite a few extra entries for future users.
2072  */
2073 #define NCPU_SETUPS     20
2074 
2075 struct cpu_setup {
2076         cpu_setup_func_t *func;
2077         void *arg;
2078 } cpu_setups[NCPU_SETUPS];
2079 
2080 void
2081 register_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2082 {
2083         int i;
2084 
2085         ASSERT(MUTEX_HELD(&cpu_lock));
2086 
2087         for (i = 0; i < NCPU_SETUPS; i++)
2088                 if (cpu_setups[i].func == NULL)
2089                         break;
2090         if (i >= NCPU_SETUPS)
2091                 cmn_err(CE_PANIC, "Ran out of cpu_setup callback entries");
2092 
2093         cpu_setups[i].func = func;
2094         cpu_setups[i].arg = arg;
2095 }
2096 
2097 void
2098 unregister_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2099 {
2100         int i;
2101 
2102         ASSERT(MUTEX_HELD(&cpu_lock));
2103 
2104         for (i = 0; i < NCPU_SETUPS; i++)
2105                 if ((cpu_setups[i].func == func) &&
2106                     (cpu_setups[i].arg == arg))
2107                         break;
2108         if (i >= NCPU_SETUPS)
2109                 cmn_err(CE_PANIC, "Could not find cpu_setup callback to "
2110                     "deregister");
2111 
2112         cpu_setups[i].func = NULL;
2113         cpu_setups[i].arg = 0;
2114 }
2115 
2116 /*
2117  * Call any state change hooks for this CPU, ignore any errors.
2118  */
2119 void
2120 cpu_state_change_notify(int id, cpu_setup_t what)
2121 {
2122         int i;
2123 
2124         ASSERT(MUTEX_HELD(&cpu_lock));
2125 
2126         for (i = 0; i < NCPU_SETUPS; i++) {
2127                 if (cpu_setups[i].func != NULL) {
2128                         cpu_setups[i].func(what, id, cpu_setups[i].arg);
2129                 }
2130         }
2131 }
2132 
2133 /*
2134  * Call any state change hooks for this CPU, undo it if error found.
2135  */
2136 static int
2137 cpu_state_change_hooks(int id, cpu_setup_t what, cpu_setup_t undo)
2138 {
2139         int i;
2140         int retval = 0;
2141 
2142         ASSERT(MUTEX_HELD(&cpu_lock));
2143 
2144         for (i = 0; i < NCPU_SETUPS; i++) {
2145                 if (cpu_setups[i].func != NULL) {
2146                         retval = cpu_setups[i].func(what, id,
2147                             cpu_setups[i].arg);
2148                         if (retval) {
2149                                 for (i--; i >= 0; i--) {
2150                                         if (cpu_setups[i].func != NULL)
2151                                                 cpu_setups[i].func(undo,
2152                                                     id, cpu_setups[i].arg);
2153                                 }
2154                                 break;
2155                         }
2156                 }
2157         }
2158         return (retval);
2159 }
2160 
2161 /*
2162  * Export information about this CPU via the kstat mechanism.
2163  */
2164 static struct {
2165         kstat_named_t ci_state;
2166         kstat_named_t ci_state_begin;
2167         kstat_named_t ci_cpu_type;
2168         kstat_named_t ci_fpu_type;
2169         kstat_named_t ci_clock_MHz;
2170         kstat_named_t ci_chip_id;
2171         kstat_named_t ci_implementation;
2172         kstat_named_t ci_brandstr;
2173         kstat_named_t ci_core_id;
2174         kstat_named_t ci_curr_clock_Hz;
2175         kstat_named_t ci_supp_freq_Hz;
2176         kstat_named_t ci_pg_id;
2177 #if defined(__sparcv9)
2178         kstat_named_t ci_device_ID;
2179         kstat_named_t ci_cpu_fru;
2180 #endif
2181 #if defined(__x86)
2182         kstat_named_t ci_vendorstr;
2183         kstat_named_t ci_family;
2184         kstat_named_t ci_model;
2185         kstat_named_t ci_step;
2186         kstat_named_t ci_clogid;
2187         kstat_named_t ci_pkg_core_id;
2188         kstat_named_t ci_ncpuperchip;
2189         kstat_named_t ci_ncoreperchip;
2190         kstat_named_t ci_max_cstates;
2191         kstat_named_t ci_curr_cstate;
2192         kstat_named_t ci_cacheid;
2193         kstat_named_t ci_sktstr;
2194 #endif
2195 } cpu_info_template = {
2196         { "state",                      KSTAT_DATA_CHAR },
2197         { "state_begin",                KSTAT_DATA_LONG },
2198         { "cpu_type",                   KSTAT_DATA_CHAR },
2199         { "fpu_type",                   KSTAT_DATA_CHAR },
2200         { "clock_MHz",                  KSTAT_DATA_LONG },
2201         { "chip_id",                    KSTAT_DATA_LONG },
2202         { "implementation",             KSTAT_DATA_STRING },
2203         { "brand",                      KSTAT_DATA_STRING },
2204         { "core_id",                    KSTAT_DATA_LONG },
2205         { "current_clock_Hz",           KSTAT_DATA_UINT64 },
2206         { "supported_frequencies_Hz",   KSTAT_DATA_STRING },
2207         { "pg_id",                      KSTAT_DATA_LONG },
2208 #if defined(__sparcv9)
2209         { "device_ID",                  KSTAT_DATA_UINT64 },
2210         { "cpu_fru",                    KSTAT_DATA_STRING },
2211 #endif
2212 #if defined(__x86)
2213         { "vendor_id",                  KSTAT_DATA_STRING },
2214         { "family",                     KSTAT_DATA_INT32 },
2215         { "model",                      KSTAT_DATA_INT32 },
2216         { "stepping",                   KSTAT_DATA_INT32 },
2217         { "clog_id",                    KSTAT_DATA_INT32 },
2218         { "pkg_core_id",                KSTAT_DATA_LONG },
2219         { "ncpu_per_chip",              KSTAT_DATA_INT32 },
2220         { "ncore_per_chip",             KSTAT_DATA_INT32 },
2221         { "supported_max_cstates",      KSTAT_DATA_INT32 },
2222         { "current_cstate",             KSTAT_DATA_INT32 },
2223         { "cache_id",                   KSTAT_DATA_INT32 },
2224         { "socket_type",                KSTAT_DATA_STRING },
2225 #endif
2226 };
2227 
2228 static kmutex_t cpu_info_template_lock;
2229 
2230 static int
2231 cpu_info_kstat_update(kstat_t *ksp, int rw)
2232 {
2233         cpu_t   *cp = ksp->ks_private;
2234         const char *pi_state;
2235 
2236         if (rw == KSTAT_WRITE)
2237                 return (EACCES);
2238 
2239 #if defined(__x86)
2240         /* Is the cpu still initialising itself? */
2241         if (cpuid_checkpass(cp, 1) == 0)
2242                 return (ENXIO);
2243 #endif
2244         switch (cp->cpu_type_info.pi_state) {
2245         case P_ONLINE:
2246                 pi_state = PS_ONLINE;
2247                 break;
2248         case P_POWEROFF:
2249                 pi_state = PS_POWEROFF;
2250                 break;
2251         case P_NOINTR:
2252                 pi_state = PS_NOINTR;
2253                 break;
2254         case P_FAULTED:
2255                 pi_state = PS_FAULTED;
2256                 break;
2257         case P_SPARE:
2258                 pi_state = PS_SPARE;
2259                 break;
2260         case P_OFFLINE:
2261                 pi_state = PS_OFFLINE;
2262                 break;
2263         default:
2264                 pi_state = "unknown";
2265         }
2266         (void) strcpy(cpu_info_template.ci_state.value.c, pi_state);
2267         cpu_info_template.ci_state_begin.value.l = cp->cpu_state_begin;
2268         (void) strncpy(cpu_info_template.ci_cpu_type.value.c,
2269             cp->cpu_type_info.pi_processor_type, 15);
2270         (void) strncpy(cpu_info_template.ci_fpu_type.value.c,
2271             cp->cpu_type_info.pi_fputypes, 15);
2272         cpu_info_template.ci_clock_MHz.value.l = cp->cpu_type_info.pi_clock;
2273         cpu_info_template.ci_chip_id.value.l =
2274             pg_plat_hw_instance_id(cp, PGHW_CHIP);
2275         kstat_named_setstr(&cpu_info_template.ci_implementation,
2276             cp->cpu_idstr);
2277         kstat_named_setstr(&cpu_info_template.ci_brandstr, cp->cpu_brandstr);
2278         cpu_info_template.ci_core_id.value.l = pg_plat_get_core_id(cp);
2279         cpu_info_template.ci_curr_clock_Hz.value.ui64 =
2280             cp->cpu_curr_clock;
2281         cpu_info_template.ci_pg_id.value.l =
2282             cp->cpu_pg && cp->cpu_pg->cmt_lineage ?
2283             cp->cpu_pg->cmt_lineage->pg_id : -1;
2284         kstat_named_setstr(&cpu_info_template.ci_supp_freq_Hz,
2285             cp->cpu_supp_freqs);
2286 #if defined(__sparcv9)
2287         cpu_info_template.ci_device_ID.value.ui64 =
2288             cpunodes[cp->cpu_id].device_id;
2289         kstat_named_setstr(&cpu_info_template.ci_cpu_fru, cpu_fru_fmri(cp));
2290 #endif
2291 #if defined(__x86)
2292         kstat_named_setstr(&cpu_info_template.ci_vendorstr,
2293             cpuid_getvendorstr(cp));
2294         cpu_info_template.ci_family.value.l = cpuid_getfamily(cp);
2295         cpu_info_template.ci_model.value.l = cpuid_getmodel(cp);
2296         cpu_info_template.ci_step.value.l = cpuid_getstep(cp);
2297         cpu_info_template.ci_clogid.value.l = cpuid_get_clogid(cp);
2298         cpu_info_template.ci_ncpuperchip.value.l = cpuid_get_ncpu_per_chip(cp);
2299         cpu_info_template.ci_ncoreperchip.value.l =
2300             cpuid_get_ncore_per_chip(cp);
2301         cpu_info_template.ci_pkg_core_id.value.l = cpuid_get_pkgcoreid(cp);
2302         cpu_info_template.ci_max_cstates.value.l = cp->cpu_m.max_cstates;
2303         cpu_info_template.ci_curr_cstate.value.l = cpu_idle_get_cpu_state(cp);
2304         cpu_info_template.ci_cacheid.value.i32 = cpuid_get_cacheid(cp);
2305         kstat_named_setstr(&cpu_info_template.ci_sktstr,
2306             cpuid_getsocketstr(cp));
2307 #endif
2308 
2309         return (0);
2310 }
2311 
2312 static void
2313 cpu_info_kstat_create(cpu_t *cp)
2314 {
2315         zoneid_t zoneid;
2316 
2317         ASSERT(MUTEX_HELD(&cpu_lock));
2318 
2319         if (pool_pset_enabled())
2320                 zoneid = GLOBAL_ZONEID;
2321         else
2322                 zoneid = ALL_ZONES;
2323         if ((cp->cpu_info_kstat = kstat_create_zone("cpu_info", cp->cpu_id,
2324             NULL, "misc", KSTAT_TYPE_NAMED,
2325             sizeof (cpu_info_template) / sizeof (kstat_named_t),
2326             KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_VAR_SIZE, zoneid)) != NULL) {
2327                 cp->cpu_info_kstat->ks_data_size += 2 * CPU_IDSTRLEN;
2328 #if defined(__sparcv9)
2329                 cp->cpu_info_kstat->ks_data_size +=
2330                     strlen(cpu_fru_fmri(cp)) + 1;
2331 #endif
2332 #if defined(__x86)
2333                 cp->cpu_info_kstat->ks_data_size += X86_VENDOR_STRLEN;
2334 #endif
2335                 if (cp->cpu_supp_freqs != NULL)
2336                         cp->cpu_info_kstat->ks_data_size +=
2337                             strlen(cp->cpu_supp_freqs) + 1;
2338                 cp->cpu_info_kstat->ks_lock = &cpu_info_template_lock;
2339                 cp->cpu_info_kstat->ks_data = &cpu_info_template;
2340                 cp->cpu_info_kstat->ks_private = cp;
2341                 cp->cpu_info_kstat->ks_update = cpu_info_kstat_update;
2342                 kstat_install(cp->cpu_info_kstat);
2343         }
2344 }
2345 
2346 static void
2347 cpu_info_kstat_destroy(cpu_t *cp)
2348 {
2349         ASSERT(MUTEX_HELD(&cpu_lock));
2350 
2351         kstat_delete(cp->cpu_info_kstat);
2352         cp->cpu_info_kstat = NULL;
2353 }
2354 
2355 /*
2356  * Create and install kstats for the boot CPU.
2357  */
2358 void
2359 cpu_kstat_init(cpu_t *cp)
2360 {
2361         mutex_enter(&cpu_lock);
2362         cpu_info_kstat_create(cp);
2363         cpu_stats_kstat_create(cp);
2364         cpu_create_intrstat(cp);
2365         cpu_set_state(cp);
2366         mutex_exit(&cpu_lock);
2367 }
2368 
2369 /*
2370  * Make visible to the zone that subset of the cpu information that would be
2371  * initialized when a cpu is configured (but still offline).
2372  */
2373 void
2374 cpu_visibility_configure(cpu_t *cp, zone_t *zone)
2375 {
2376         zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2377 
2378         ASSERT(MUTEX_HELD(&cpu_lock));
2379         ASSERT(pool_pset_enabled());
2380         ASSERT(cp != NULL);
2381 
2382         if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2383                 zone->zone_ncpus++;
2384                 ASSERT(zone->zone_ncpus <= ncpus);
2385         }
2386         if (cp->cpu_info_kstat != NULL)
2387                 kstat_zone_add(cp->cpu_info_kstat, zoneid);
2388 }
2389 
2390 /*
2391  * Make visible to the zone that subset of the cpu information that would be
2392  * initialized when a previously configured cpu is onlined.
2393  */
2394 void
2395 cpu_visibility_online(cpu_t *cp, zone_t *zone)
2396 {
2397         kstat_t *ksp;
2398         char name[sizeof ("cpu_stat") + 10];    /* enough for 32-bit cpuids */
2399         zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2400         processorid_t cpun;
2401 
2402         ASSERT(MUTEX_HELD(&cpu_lock));
2403         ASSERT(pool_pset_enabled());
2404         ASSERT(cp != NULL);
2405         ASSERT(cpu_is_active(cp));
2406 
2407         cpun = cp->cpu_id;
2408         if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2409                 zone->zone_ncpus_online++;
2410                 ASSERT(zone->zone_ncpus_online <= ncpus_online);
2411         }
2412         (void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2413         if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2414             != NULL) {
2415                 kstat_zone_add(ksp, zoneid);
2416                 kstat_rele(ksp);
2417         }
2418         if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2419                 kstat_zone_add(ksp, zoneid);
2420                 kstat_rele(ksp);
2421         }
2422         if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2423                 kstat_zone_add(ksp, zoneid);
2424                 kstat_rele(ksp);
2425         }
2426         if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2427             NULL) {
2428                 kstat_zone_add(ksp, zoneid);
2429                 kstat_rele(ksp);
2430         }
2431 }
2432 
2433 /*
2434  * Update relevant kstats such that cpu is now visible to processes
2435  * executing in specified zone.
2436  */
2437 void
2438 cpu_visibility_add(cpu_t *cp, zone_t *zone)
2439 {
2440         cpu_visibility_configure(cp, zone);
2441         if (cpu_is_active(cp))
2442                 cpu_visibility_online(cp, zone);
2443 }
2444 
2445 /*
2446  * Make invisible to the zone that subset of the cpu information that would be
2447  * torn down when a previously offlined cpu is unconfigured.
2448  */
2449 void
2450 cpu_visibility_unconfigure(cpu_t *cp, zone_t *zone)
2451 {
2452         zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2453 
2454         ASSERT(MUTEX_HELD(&cpu_lock));
2455         ASSERT(pool_pset_enabled());
2456         ASSERT(cp != NULL);
2457 
2458         if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2459                 ASSERT(zone->zone_ncpus != 0);
2460                 zone->zone_ncpus--;
2461         }
2462         if (cp->cpu_info_kstat)
2463                 kstat_zone_remove(cp->cpu_info_kstat, zoneid);
2464 }
2465 
2466 /*
2467  * Make invisible to the zone that subset of the cpu information that would be
2468  * torn down when a cpu is offlined (but still configured).
2469  */
2470 void
2471 cpu_visibility_offline(cpu_t *cp, zone_t *zone)
2472 {
2473         kstat_t *ksp;
2474         char name[sizeof ("cpu_stat") + 10];    /* enough for 32-bit cpuids */
2475         zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2476         processorid_t cpun;
2477 
2478         ASSERT(MUTEX_HELD(&cpu_lock));
2479         ASSERT(pool_pset_enabled());
2480         ASSERT(cp != NULL);
2481         ASSERT(cpu_is_active(cp));
2482 
2483         cpun = cp->cpu_id;
2484         if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2485                 ASSERT(zone->zone_ncpus_online != 0);
2486                 zone->zone_ncpus_online--;
2487         }
2488 
2489         if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2490             NULL) {
2491                 kstat_zone_remove(ksp, zoneid);
2492                 kstat_rele(ksp);
2493         }
2494         if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2495                 kstat_zone_remove(ksp, zoneid);
2496                 kstat_rele(ksp);
2497         }
2498         if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2499                 kstat_zone_remove(ksp, zoneid);
2500                 kstat_rele(ksp);
2501         }
2502         (void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2503         if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2504             != NULL) {
2505                 kstat_zone_remove(ksp, zoneid);
2506                 kstat_rele(ksp);
2507         }
2508 }
2509 
2510 /*
2511  * Update relevant kstats such that cpu is no longer visible to processes
2512  * executing in specified zone.
2513  */
2514 void
2515 cpu_visibility_remove(cpu_t *cp, zone_t *zone)
2516 {
2517         if (cpu_is_active(cp))
2518                 cpu_visibility_offline(cp, zone);
2519         cpu_visibility_unconfigure(cp, zone);
2520 }
2521 
2522 /*
2523  * Bind a thread to a CPU as requested.
2524  */
2525 int
2526 cpu_bind_thread(kthread_id_t tp, processorid_t bind, processorid_t *obind,
2527     int *error)
2528 {
2529         processorid_t   binding;
2530         cpu_t           *cp = NULL;
2531 
2532         ASSERT(MUTEX_HELD(&cpu_lock));
2533         ASSERT(MUTEX_HELD(&ttoproc(tp)->p_lock));
2534 
2535         thread_lock(tp);
2536 
2537         /*
2538          * Record old binding, but change the obind, which was initialized
2539          * to PBIND_NONE, only if this thread has a binding.  This avoids
2540          * reporting PBIND_NONE for a process when some LWPs are bound.
2541          */
2542         binding = tp->t_bind_cpu;
2543         if (binding != PBIND_NONE)
2544                 *obind = binding;       /* record old binding */
2545 
2546         switch (bind) {
2547         case PBIND_QUERY:
2548                 /* Just return the old binding */
2549                 thread_unlock(tp);
2550                 return (0);
2551 
2552         case PBIND_QUERY_TYPE:
2553                 /* Return the binding type */
2554                 *obind = TB_CPU_IS_SOFT(tp) ? PBIND_SOFT : PBIND_HARD;
2555                 thread_unlock(tp);
2556                 return (0);
2557 
2558         case PBIND_SOFT:
2559                 /*
2560                  *  Set soft binding for this thread and return the actual
2561                  *  binding
2562                  */
2563                 TB_CPU_SOFT_SET(tp);
2564                 thread_unlock(tp);
2565                 return (0);
2566 
2567         case PBIND_HARD:
2568                 /*
2569                  *  Set hard binding for this thread and return the actual
2570                  *  binding
2571                  */
2572                 TB_CPU_HARD_SET(tp);
2573                 thread_unlock(tp);
2574                 return (0);
2575 
2576         default:
2577                 break;
2578         }
2579 
2580         /*
2581          * If this thread/LWP cannot be bound because of permission
2582          * problems, just note that and return success so that the
2583          * other threads/LWPs will be bound.  This is the way
2584          * processor_bind() is defined to work.
2585          *
2586          * Binding will get EPERM if the thread is of system class
2587          * or hasprocperm() fails.
2588          */
2589         if (tp->t_cid == 0 || !hasprocperm(tp->t_cred, CRED())) {
2590                 *error = EPERM;
2591                 thread_unlock(tp);
2592                 return (0);
2593         }
2594 
2595         binding = bind;
2596         if (binding != PBIND_NONE) {
2597                 cp = cpu_get((processorid_t)binding);
2598                 /*
2599                  * Make sure binding is valid and is in right partition.
2600                  */
2601                 if (cp == NULL || tp->t_cpupart != cp->cpu_part) {
2602                         *error = EINVAL;
2603                         thread_unlock(tp);
2604                         return (0);
2605                 }
2606         }
2607         tp->t_bind_cpu = binding;    /* set new binding */
2608 
2609         /*
2610          * If there is no system-set reason for affinity, set
2611          * the t_bound_cpu field to reflect the binding.
2612          */
2613         if (tp->t_affinitycnt == 0) {
2614                 if (binding == PBIND_NONE) {
2615                         /*
2616                          * We may need to adjust disp_max_unbound_pri
2617                          * since we're becoming unbound.
2618                          */
2619                         disp_adjust_unbound_pri(tp);
2620 
2621                         tp->t_bound_cpu = NULL;      /* set new binding */
2622 
2623                         /*
2624                          * Move thread to lgroup with strongest affinity
2625                          * after unbinding
2626                          */
2627                         if (tp->t_lgrp_affinity)
2628                                 lgrp_move_thread(tp,
2629                                     lgrp_choose(tp, tp->t_cpupart), 1);
2630 
2631                         if (tp->t_state == TS_ONPROC &&
2632                             tp->t_cpu->cpu_part != tp->t_cpupart)
2633                                 cpu_surrender(tp);
2634                 } else {
2635                         lpl_t   *lpl;
2636 
2637                         tp->t_bound_cpu = cp;
2638                         ASSERT(cp->cpu_lpl != NULL);
2639 
2640                         /*
2641                          * Set home to lgroup with most affinity containing CPU
2642                          * that thread is being bound or minimum bounding
2643                          * lgroup if no affinities set
2644                          */
2645                         if (tp->t_lgrp_affinity)
2646                                 lpl = lgrp_affinity_best(tp, tp->t_cpupart,
2647                                     LGRP_NONE, B_FALSE);
2648                         else
2649                                 lpl = cp->cpu_lpl;
2650 
2651                         if (tp->t_lpl != lpl) {
2652                                 /* can't grab cpu_lock */
2653                                 lgrp_move_thread(tp, lpl, 1);
2654                         }
2655 
2656                         /*
2657                          * Make the thread switch to the bound CPU.
2658                          * If the thread is runnable, we need to
2659                          * requeue it even if t_cpu is already set
2660                          * to the right CPU, since it may be on a
2661                          * kpreempt queue and need to move to a local
2662                          * queue.  We could check t_disp_queue to
2663                          * avoid unnecessary overhead if it's already
2664                          * on the right queue, but since this isn't
2665                          * a performance-critical operation it doesn't
2666                          * seem worth the extra code and complexity.
2667                          *
2668                          * If the thread is weakbound to the cpu then it will
2669                          * resist the new binding request until the weak
2670                          * binding drops.  The cpu_surrender or requeueing
2671                          * below could be skipped in such cases (since it
2672                          * will have no effect), but that would require
2673                          * thread_allowmigrate to acquire thread_lock so
2674                          * we'll take the very occasional hit here instead.
2675                          */
2676                         if (tp->t_state == TS_ONPROC) {
2677                                 cpu_surrender(tp);
2678                         } else if (tp->t_state == TS_RUN) {
2679                                 cpu_t *ocp = tp->t_cpu;
2680 
2681                                 (void) dispdeq(tp);
2682                                 setbackdq(tp);
2683                                 /*
2684                                  * Either on the bound CPU's disp queue now,
2685                                  * or swapped out or on the swap queue.
2686                                  */
2687                                 ASSERT(tp->t_disp_queue == cp->cpu_disp ||
2688                                     tp->t_weakbound_cpu == ocp ||
2689                                     (tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ))
2690                                     != TS_LOAD);
2691                         }
2692                 }
2693         }
2694 
2695         /*
2696          * Our binding has changed; set TP_CHANGEBIND.
2697          */
2698         tp->t_proc_flag |= TP_CHANGEBIND;
2699         aston(tp);
2700 
2701         thread_unlock(tp);
2702 
2703         return (0);
2704 }
2705 
2706 #if CPUSET_WORDS > 1
2707 
2708 /*
2709  * Functions for implementing cpuset operations when a cpuset is more
2710  * than one word.  On platforms where a cpuset is a single word these
2711  * are implemented as macros in cpuvar.h.
2712  */
2713 
2714 void
2715 cpuset_all(cpuset_t *s)
2716 {
2717         int i;
2718 
2719         for (i = 0; i < CPUSET_WORDS; i++)
2720                 s->cpub[i] = ~0UL;
2721 }
2722 
2723 void
2724 cpuset_all_but(cpuset_t *s, uint_t cpu)
2725 {
2726         cpuset_all(s);
2727         CPUSET_DEL(*s, cpu);
2728 }
2729 
2730 void
2731 cpuset_only(cpuset_t *s, uint_t cpu)
2732 {
2733         CPUSET_ZERO(*s);
2734         CPUSET_ADD(*s, cpu);
2735 }
2736 
2737 int
2738 cpuset_isnull(cpuset_t *s)
2739 {
2740         int i;
2741 
2742         for (i = 0; i < CPUSET_WORDS; i++)
2743                 if (s->cpub[i] != 0)
2744                         return (0);
2745         return (1);
2746 }
2747 
2748 int
2749 cpuset_cmp(cpuset_t *s1, cpuset_t *s2)
2750 {
2751         int i;
2752 
2753         for (i = 0; i < CPUSET_WORDS; i++)
2754                 if (s1->cpub[i] != s2->cpub[i])
2755                         return (0);
2756         return (1);
2757 }
2758 
2759 uint_t
2760 cpuset_find(cpuset_t *s)
2761 {
2762 
2763         uint_t  i;
2764         uint_t  cpu = (uint_t)-1;
2765 
2766         /*
2767          * Find a cpu in the cpuset
2768          */
2769         for (i = 0; i < CPUSET_WORDS; i++) {
2770                 cpu = (uint_t)(lowbit(s->cpub[i]) - 1);
2771                 if (cpu != (uint_t)-1) {
2772                         cpu += i * BT_NBIPUL;
2773                         break;
2774                 }
2775         }
2776         return (cpu);
2777 }
2778 
2779 void
2780 cpuset_bounds(cpuset_t *s, uint_t *smallestid, uint_t *largestid)
2781 {
2782         int     i, j;
2783         uint_t  bit;
2784 
2785         /*
2786          * First, find the smallest cpu id in the set.
2787          */
2788         for (i = 0; i < CPUSET_WORDS; i++) {
2789                 if (s->cpub[i] != 0) {
2790                         bit = (uint_t)(lowbit(s->cpub[i]) - 1);
2791                         ASSERT(bit != (uint_t)-1);
2792                         *smallestid = bit + (i * BT_NBIPUL);
2793 
2794                         /*
2795                          * Now find the largest cpu id in
2796                          * the set and return immediately.
2797                          * Done in an inner loop to avoid
2798                          * having to break out of the first
2799                          * loop.
2800                          */
2801                         for (j = CPUSET_WORDS - 1; j >= i; j--) {
2802                                 if (s->cpub[j] != 0) {
2803                                         bit = (uint_t)(highbit(s->cpub[j]) - 1);
2804                                         ASSERT(bit != (uint_t)-1);
2805                                         *largestid = bit + (j * BT_NBIPUL);
2806                                         ASSERT(*largestid >= *smallestid);
2807                                         return;
2808                                 }
2809                         }
2810 
2811                         /*
2812                          * If this code is reached, a
2813                          * smallestid was found, but not a
2814                          * largestid. The cpuset must have
2815                          * been changed during the course
2816                          * of this function call.
2817                          */
2818                         ASSERT(0);
2819                 }
2820         }
2821         *smallestid = *largestid = CPUSET_NOTINSET;
2822 }
2823 
2824 #endif  /* CPUSET_WORDS */
2825 
2826 /*
2827  * Unbind threads bound to specified CPU.
2828  *
2829  * If `unbind_all_threads' is true, unbind all user threads bound to a given
2830  * CPU. Otherwise unbind all soft-bound user threads.
2831  */
2832 int
2833 cpu_unbind(processorid_t cpu, boolean_t unbind_all_threads)
2834 {
2835         processorid_t obind;
2836         kthread_t *tp;
2837         int ret = 0;
2838         proc_t *pp;
2839         int err, berr = 0;
2840 
2841         ASSERT(MUTEX_HELD(&cpu_lock));
2842 
2843         mutex_enter(&pidlock);
2844         for (pp = practive; pp != NULL; pp = pp->p_next) {
2845                 mutex_enter(&pp->p_lock);
2846                 tp = pp->p_tlist;
2847                 /*
2848                  * Skip zombies, kernel processes, and processes in
2849                  * other zones, if called from a non-global zone.
2850                  */
2851                 if (tp == NULL || (pp->p_flag & SSYS) ||
2852                     !HASZONEACCESS(curproc, pp->p_zone->zone_id)) {
2853                         mutex_exit(&pp->p_lock);
2854                         continue;
2855                 }
2856                 do {
2857                         if (tp->t_bind_cpu != cpu)
2858                                 continue;
2859                         /*
2860                          * Skip threads with hard binding when
2861                          * `unbind_all_threads' is not specified.
2862                          */
2863                         if (!unbind_all_threads && TB_CPU_IS_HARD(tp))
2864                                 continue;
2865                         err = cpu_bind_thread(tp, PBIND_NONE, &obind, &berr);
2866                         if (ret == 0)
2867                                 ret = err;
2868                 } while ((tp = tp->t_forw) != pp->p_tlist);
2869                 mutex_exit(&pp->p_lock);
2870         }
2871         mutex_exit(&pidlock);
2872         if (ret == 0)
2873                 ret = berr;
2874         return (ret);
2875 }
2876 
2877 
2878 /*
2879  * Destroy all remaining bound threads on a cpu.
2880  */
2881 void
2882 cpu_destroy_bound_threads(cpu_t *cp)
2883 {
2884         extern id_t syscid;
2885         register kthread_id_t   t, tlist, tnext;
2886 
2887         /*
2888          * Destroy all remaining bound threads on the cpu.  This
2889          * should include both the interrupt threads and the idle thread.
2890          * This requires some care, since we need to traverse the
2891          * thread list with the pidlock mutex locked, but thread_free
2892          * also locks the pidlock mutex.  So, we collect the threads
2893          * we're going to reap in a list headed by "tlist", then we
2894          * unlock the pidlock mutex and traverse the tlist list,
2895          * doing thread_free's on the thread's.  Simple, n'est pas?
2896          * Also, this depends on thread_free not mucking with the
2897          * t_next and t_prev links of the thread.
2898          */
2899 
2900         if ((t = curthread) != NULL) {
2901 
2902                 tlist = NULL;
2903                 mutex_enter(&pidlock);
2904                 do {
2905                         tnext = t->t_next;
2906                         if (t->t_bound_cpu == cp) {
2907 
2908                                 /*
2909                                  * We've found a bound thread, carefully unlink
2910                                  * it out of the thread list, and add it to
2911                                  * our "tlist".  We "know" we don't have to
2912                                  * worry about unlinking curthread (the thread
2913                                  * that is executing this code).
2914                                  */
2915                                 t->t_next->t_prev = t->t_prev;
2916                                 t->t_prev->t_next = t->t_next;
2917                                 t->t_next = tlist;
2918                                 tlist = t;
2919                                 ASSERT(t->t_cid == syscid);
2920                                 /* wake up anyone blocked in thread_join */
2921                                 cv_broadcast(&t->t_joincv);
2922                                 /*
2923                                  * t_lwp set by interrupt threads and not
2924                                  * cleared.
2925                                  */
2926                                 t->t_lwp = NULL;
2927                                 /*
2928                                  * Pause and idle threads always have
2929                                  * t_state set to TS_ONPROC.
2930                                  */
2931                                 t->t_state = TS_FREE;
2932                                 t->t_prev = NULL;    /* Just in case */
2933                         }
2934 
2935                 } while ((t = tnext) != curthread);
2936 
2937                 mutex_exit(&pidlock);
2938 
2939                 mutex_sync();
2940                 for (t = tlist; t != NULL; t = tnext) {
2941                         tnext = t->t_next;
2942                         thread_free(t);
2943                 }
2944         }
2945 }
2946 
2947 /*
2948  * Update the cpu_supp_freqs of this cpu. This information is returned
2949  * as part of cpu_info kstats. If the cpu_info_kstat exists already, then
2950  * maintain the kstat data size.
2951  */
2952 void
2953 cpu_set_supp_freqs(cpu_t *cp, const char *freqs)
2954 {
2955         char clkstr[sizeof ("18446744073709551615") + 1]; /* ui64 MAX */
2956         const char *lfreqs = clkstr;
2957         boolean_t kstat_exists = B_FALSE;
2958         kstat_t *ksp;
2959         size_t len;
2960 
2961         /*
2962          * A NULL pointer means we only support one speed.
2963          */
2964         if (freqs == NULL)
2965                 (void) snprintf(clkstr, sizeof (clkstr), "%"PRIu64,
2966                     cp->cpu_curr_clock);
2967         else
2968                 lfreqs = freqs;
2969 
2970         /*
2971          * Make sure the frequency doesn't change while a snapshot is
2972          * going on. Of course, we only need to worry about this if
2973          * the kstat exists.
2974          */
2975         if ((ksp = cp->cpu_info_kstat) != NULL) {
2976                 mutex_enter(ksp->ks_lock);
2977                 kstat_exists = B_TRUE;
2978         }
2979 
2980         /*
2981          * Free any previously allocated string and if the kstat
2982          * already exists, then update its data size.
2983          */
2984         if (cp->cpu_supp_freqs != NULL) {
2985                 len = strlen(cp->cpu_supp_freqs) + 1;
2986                 kmem_free(cp->cpu_supp_freqs, len);
2987                 if (kstat_exists)
2988                         ksp->ks_data_size -= len;
2989         }
2990 
2991         /*
2992          * Allocate the new string and set the pointer.
2993          */
2994         len = strlen(lfreqs) + 1;
2995         cp->cpu_supp_freqs = kmem_alloc(len, KM_SLEEP);
2996         (void) strcpy(cp->cpu_supp_freqs, lfreqs);
2997 
2998         /*
2999          * If the kstat already exists then update the data size and
3000          * free the lock.
3001          */
3002         if (kstat_exists) {
3003                 ksp->ks_data_size += len;
3004                 mutex_exit(ksp->ks_lock);
3005         }
3006 }
3007 
3008 /*
3009  * Indicate the current CPU's clock freqency (in Hz).
3010  * The calling context must be such that CPU references are safe.
3011  */
3012 void
3013 cpu_set_curr_clock(uint64_t new_clk)
3014 {
3015         uint64_t old_clk;
3016 
3017         old_clk = CPU->cpu_curr_clock;
3018         CPU->cpu_curr_clock = new_clk;
3019 
3020         /*
3021          * The cpu-change-speed DTrace probe exports the frequency in Hz
3022          */
3023         DTRACE_PROBE3(cpu__change__speed, processorid_t, CPU->cpu_id,
3024             uint64_t, old_clk, uint64_t, new_clk);
3025 }
3026 
3027 /*
3028  * processor_info(2) and p_online(2) status support functions
3029  *   The constants returned by the cpu_get_state() and cpu_get_state_str() are
3030  *   for use in communicating processor state information to userland.  Kernel
3031  *   subsystems should only be using the cpu_flags value directly.  Subsystems
3032  *   modifying cpu_flags should record the state change via a call to the
3033  *   cpu_set_state().
3034  */
3035 
3036 /*
3037  * Update the pi_state of this CPU.  This function provides the CPU status for
3038  * the information returned by processor_info(2).
3039  */
3040 void
3041 cpu_set_state(cpu_t *cpu)
3042 {
3043         ASSERT(MUTEX_HELD(&cpu_lock));
3044         cpu->cpu_type_info.pi_state = cpu_get_state(cpu);
3045         cpu->cpu_state_begin = gethrestime_sec();
3046         pool_cpu_mod = gethrtime();
3047 }
3048 
3049 /*
3050  * Return offline/online/other status for the indicated CPU.  Use only for
3051  * communication with user applications; cpu_flags provides the in-kernel
3052  * interface.
3053  */
3054 int
3055 cpu_get_state(cpu_t *cpu)
3056 {
3057         ASSERT(MUTEX_HELD(&cpu_lock));
3058         if (cpu->cpu_flags & CPU_POWEROFF)
3059                 return (P_POWEROFF);
3060         else if (cpu->cpu_flags & CPU_FAULTED)
3061                 return (P_FAULTED);
3062         else if (cpu->cpu_flags & CPU_SPARE)
3063                 return (P_SPARE);
3064         else if ((cpu->cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY)
3065                 return (P_OFFLINE);
3066         else if (cpu->cpu_flags & CPU_ENABLE)
3067                 return (P_ONLINE);
3068         else
3069                 return (P_NOINTR);
3070 }
3071 
3072 /*
3073  * Return processor_info(2) state as a string.
3074  */
3075 const char *
3076 cpu_get_state_str(cpu_t *cpu)
3077 {
3078         const char *string;
3079 
3080         switch (cpu_get_state(cpu)) {
3081         case P_ONLINE:
3082                 string = PS_ONLINE;
3083                 break;
3084         case P_POWEROFF:
3085                 string = PS_POWEROFF;
3086                 break;
3087         case P_NOINTR:
3088                 string = PS_NOINTR;
3089                 break;
3090         case P_SPARE:
3091                 string = PS_SPARE;
3092                 break;
3093         case P_FAULTED:
3094                 string = PS_FAULTED;
3095                 break;
3096         case P_OFFLINE:
3097                 string = PS_OFFLINE;
3098                 break;
3099         default:
3100                 string = "unknown";
3101                 break;
3102         }
3103         return (string);
3104 }
3105 
3106 /*
3107  * Export this CPU's statistics (cpu_stat_t and cpu_stats_t) as raw and named
3108  * kstats, respectively.  This is done when a CPU is initialized or placed
3109  * online via p_online(2).
3110  */
3111 static void
3112 cpu_stats_kstat_create(cpu_t *cp)
3113 {
3114         int     instance = cp->cpu_id;
3115         char    *module = "cpu";
3116         char    *class = "misc";
3117         kstat_t *ksp;
3118         zoneid_t zoneid;
3119 
3120         ASSERT(MUTEX_HELD(&cpu_lock));
3121 
3122         if (pool_pset_enabled())
3123                 zoneid = GLOBAL_ZONEID;
3124         else
3125                 zoneid = ALL_ZONES;
3126         /*
3127          * Create named kstats
3128          */
3129 #define CPU_STATS_KS_CREATE(name, tsize, update_func)                    \
3130         ksp = kstat_create_zone(module, instance, (name), class,         \
3131             KSTAT_TYPE_NAMED, (tsize) / sizeof (kstat_named_t), 0,       \
3132             zoneid);                                                     \
3133         if (ksp != NULL) {                                               \
3134                 ksp->ks_private = cp;                                    \
3135                 ksp->ks_update = (update_func);                          \
3136                 kstat_install(ksp);                                      \
3137         } else                                                           \
3138                 cmn_err(CE_WARN, "cpu: unable to create %s:%d:%s kstat", \
3139                     module, instance, (name));
3140 
3141         CPU_STATS_KS_CREATE("sys", sizeof (cpu_sys_stats_ks_data_template),
3142             cpu_sys_stats_ks_update);
3143         CPU_STATS_KS_CREATE("vm", sizeof (cpu_vm_stats_ks_data_template),
3144             cpu_vm_stats_ks_update);
3145 
3146         /*
3147          * Export the familiar cpu_stat_t KSTAT_TYPE_RAW kstat.
3148          */
3149         ksp = kstat_create_zone("cpu_stat", cp->cpu_id, NULL,
3150             "misc", KSTAT_TYPE_RAW, sizeof (cpu_stat_t), 0, zoneid);
3151         if (ksp != NULL) {
3152                 ksp->ks_update = cpu_stat_ks_update;
3153                 ksp->ks_private = cp;
3154                 kstat_install(ksp);
3155         }
3156 }
3157 
3158 static void
3159 cpu_stats_kstat_destroy(cpu_t *cp)
3160 {
3161         char ks_name[KSTAT_STRLEN];
3162 
3163         (void) sprintf(ks_name, "cpu_stat%d", cp->cpu_id);
3164         kstat_delete_byname("cpu_stat", cp->cpu_id, ks_name);
3165 
3166         kstat_delete_byname("cpu", cp->cpu_id, "sys");
3167         kstat_delete_byname("cpu", cp->cpu_id, "vm");
3168 }
3169 
3170 static int
3171 cpu_sys_stats_ks_update(kstat_t *ksp, int rw)
3172 {
3173         cpu_t *cp = (cpu_t *)ksp->ks_private;
3174         struct cpu_sys_stats_ks_data *csskd;
3175         cpu_sys_stats_t *css;
3176         hrtime_t msnsecs[NCMSTATES];
3177         int     i;
3178 
3179         if (rw == KSTAT_WRITE)
3180                 return (EACCES);
3181 
3182         csskd = ksp->ks_data;
3183         css = &cp->cpu_stats.sys;
3184 
3185         /*
3186          * Read CPU mstate, but compare with the last values we
3187          * received to make sure that the returned kstats never
3188          * decrease.
3189          */
3190 
3191         get_cpu_mstate(cp, msnsecs);
3192         if (csskd->cpu_nsec_idle.value.ui64 > msnsecs[CMS_IDLE])
3193                 msnsecs[CMS_IDLE] = csskd->cpu_nsec_idle.value.ui64;
3194         if (csskd->cpu_nsec_user.value.ui64 > msnsecs[CMS_USER])
3195                 msnsecs[CMS_USER] = csskd->cpu_nsec_user.value.ui64;
3196         if (csskd->cpu_nsec_kernel.value.ui64 > msnsecs[CMS_SYSTEM])
3197                 msnsecs[CMS_SYSTEM] = csskd->cpu_nsec_kernel.value.ui64;
3198 
3199         bcopy(&cpu_sys_stats_ks_data_template, ksp->ks_data,
3200             sizeof (cpu_sys_stats_ks_data_template));
3201 
3202         csskd->cpu_ticks_wait.value.ui64 = 0;
3203         csskd->wait_ticks_io.value.ui64 = 0;
3204 
3205         csskd->cpu_nsec_idle.value.ui64 = msnsecs[CMS_IDLE];
3206         csskd->cpu_nsec_user.value.ui64 = msnsecs[CMS_USER];
3207         csskd->cpu_nsec_kernel.value.ui64 = msnsecs[CMS_SYSTEM];
3208         csskd->cpu_ticks_idle.value.ui64 =
3209             NSEC_TO_TICK(csskd->cpu_nsec_idle.value.ui64);
3210         csskd->cpu_ticks_user.value.ui64 =
3211             NSEC_TO_TICK(csskd->cpu_nsec_user.value.ui64);
3212         csskd->cpu_ticks_kernel.value.ui64 =
3213             NSEC_TO_TICK(csskd->cpu_nsec_kernel.value.ui64);
3214         csskd->cpu_nsec_dtrace.value.ui64 = cp->cpu_dtrace_nsec;
3215         csskd->dtrace_probes.value.ui64 = cp->cpu_dtrace_probes;
3216         csskd->cpu_nsec_intr.value.ui64 = cp->cpu_intrlast;
3217         csskd->cpu_load_intr.value.ui64 = cp->cpu_intrload;
3218         csskd->bread.value.ui64 = css->bread;
3219         csskd->bwrite.value.ui64 = css->bwrite;
3220         csskd->lread.value.ui64 = css->lread;
3221         csskd->lwrite.value.ui64 = css->lwrite;
3222         csskd->phread.value.ui64 = css->phread;
3223         csskd->phwrite.value.ui64 = css->phwrite;
3224         csskd->pswitch.value.ui64 = css->pswitch;
3225         csskd->trap.value.ui64 = css->trap;
3226         csskd->intr.value.ui64 = 0;
3227         for (i = 0; i < PIL_MAX; i++)
3228                 csskd->intr.value.ui64 += css->intr[i];
3229         csskd->syscall.value.ui64 = css->syscall;
3230         csskd->sysread.value.ui64 = css->sysread;
3231         csskd->syswrite.value.ui64 = css->syswrite;
3232         csskd->sysfork.value.ui64 = css->sysfork;
3233         csskd->sysvfork.value.ui64 = css->sysvfork;
3234         csskd->sysexec.value.ui64 = css->sysexec;
3235         csskd->readch.value.ui64 = css->readch;
3236         csskd->writech.value.ui64 = css->writech;
3237         csskd->rcvint.value.ui64 = css->rcvint;
3238         csskd->xmtint.value.ui64 = css->xmtint;
3239         csskd->mdmint.value.ui64 = css->mdmint;
3240         csskd->rawch.value.ui64 = css->rawch;
3241         csskd->canch.value.ui64 = css->canch;
3242         csskd->outch.value.ui64 = css->outch;
3243         csskd->msg.value.ui64 = css->msg;
3244         csskd->sema.value.ui64 = css->sema;
3245         csskd->namei.value.ui64 = css->namei;
3246         csskd->ufsiget.value.ui64 = css->ufsiget;
3247         csskd->ufsdirblk.value.ui64 = css->ufsdirblk;
3248         csskd->ufsipage.value.ui64 = css->ufsipage;
3249         csskd->ufsinopage.value.ui64 = css->ufsinopage;
3250         csskd->procovf.value.ui64 = css->procovf;
3251         csskd->intrthread.value.ui64 = 0;
3252         for (i = 0; i < LOCK_LEVEL - 1; i++)
3253                 csskd->intrthread.value.ui64 += css->intr[i];
3254         csskd->intrblk.value.ui64 = css->intrblk;
3255         csskd->intrunpin.value.ui64 = css->intrunpin;
3256         csskd->idlethread.value.ui64 = css->idlethread;
3257         csskd->inv_swtch.value.ui64 = css->inv_swtch;
3258         csskd->nthreads.value.ui64 = css->nthreads;
3259         csskd->cpumigrate.value.ui64 = css->cpumigrate;
3260         csskd->xcalls.value.ui64 = css->xcalls;
3261         csskd->mutex_adenters.value.ui64 = css->mutex_adenters;
3262         csskd->rw_rdfails.value.ui64 = css->rw_rdfails;
3263         csskd->rw_wrfails.value.ui64 = css->rw_wrfails;
3264         csskd->modload.value.ui64 = css->modload;
3265         csskd->modunload.value.ui64 = css->modunload;
3266         csskd->bawrite.value.ui64 = css->bawrite;
3267         csskd->iowait.value.ui64 = css->iowait;
3268 
3269         return (0);
3270 }
3271 
3272 static int
3273 cpu_vm_stats_ks_update(kstat_t *ksp, int rw)
3274 {
3275         cpu_t *cp = (cpu_t *)ksp->ks_private;
3276         struct cpu_vm_stats_ks_data *cvskd;
3277         cpu_vm_stats_t *cvs;
3278 
3279         if (rw == KSTAT_WRITE)
3280                 return (EACCES);
3281 
3282         cvs = &cp->cpu_stats.vm;
3283         cvskd = ksp->ks_data;
3284 
3285         bcopy(&cpu_vm_stats_ks_data_template, ksp->ks_data,
3286             sizeof (cpu_vm_stats_ks_data_template));
3287         cvskd->pgrec.value.ui64 = cvs->pgrec;
3288         cvskd->pgfrec.value.ui64 = cvs->pgfrec;
3289         cvskd->pgin.value.ui64 = cvs->pgin;
3290         cvskd->pgpgin.value.ui64 = cvs->pgpgin;
3291         cvskd->pgout.value.ui64 = cvs->pgout;
3292         cvskd->pgpgout.value.ui64 = cvs->pgpgout;
3293         cvskd->swapin.value.ui64 = cvs->swapin;
3294         cvskd->pgswapin.value.ui64 = cvs->pgswapin;
3295         cvskd->swapout.value.ui64 = cvs->swapout;
3296         cvskd->pgswapout.value.ui64 = cvs->pgswapout;
3297         cvskd->zfod.value.ui64 = cvs->zfod;
3298         cvskd->dfree.value.ui64 = cvs->dfree;
3299         cvskd->scan.value.ui64 = cvs->scan;
3300         cvskd->rev.value.ui64 = cvs->rev;
3301         cvskd->hat_fault.value.ui64 = cvs->hat_fault;
3302         cvskd->as_fault.value.ui64 = cvs->as_fault;
3303         cvskd->maj_fault.value.ui64 = cvs->maj_fault;
3304         cvskd->cow_fault.value.ui64 = cvs->cow_fault;
3305         cvskd->prot_fault.value.ui64 = cvs->prot_fault;
3306         cvskd->softlock.value.ui64 = cvs->softlock;
3307         cvskd->kernel_asflt.value.ui64 = cvs->kernel_asflt;
3308         cvskd->pgrrun.value.ui64 = cvs->pgrrun;
3309         cvskd->execpgin.value.ui64 = cvs->execpgin;
3310         cvskd->execpgout.value.ui64 = cvs->execpgout;
3311         cvskd->execfree.value.ui64 = cvs->execfree;
3312         cvskd->anonpgin.value.ui64 = cvs->anonpgin;
3313         cvskd->anonpgout.value.ui64 = cvs->anonpgout;
3314         cvskd->anonfree.value.ui64 = cvs->anonfree;
3315         cvskd->fspgin.value.ui64 = cvs->fspgin;
3316         cvskd->fspgout.value.ui64 = cvs->fspgout;
3317         cvskd->fsfree.value.ui64 = cvs->fsfree;
3318 
3319         return (0);
3320 }
3321 
3322 static int
3323 cpu_stat_ks_update(kstat_t *ksp, int rw)
3324 {
3325         cpu_stat_t *cso;
3326         cpu_t *cp;
3327         int i;
3328         hrtime_t msnsecs[NCMSTATES];
3329 
3330         cso = (cpu_stat_t *)ksp->ks_data;
3331         cp = (cpu_t *)ksp->ks_private;
3332 
3333         if (rw == KSTAT_WRITE)
3334                 return (EACCES);
3335 
3336         /*
3337          * Read CPU mstate, but compare with the last values we
3338          * received to make sure that the returned kstats never
3339          * decrease.
3340          */
3341 
3342         get_cpu_mstate(cp, msnsecs);
3343         msnsecs[CMS_IDLE] = NSEC_TO_TICK(msnsecs[CMS_IDLE]);
3344         msnsecs[CMS_USER] = NSEC_TO_TICK(msnsecs[CMS_USER]);
3345         msnsecs[CMS_SYSTEM] = NSEC_TO_TICK(msnsecs[CMS_SYSTEM]);
3346         if (cso->cpu_sysinfo.cpu[CPU_IDLE] < msnsecs[CMS_IDLE])
3347                 cso->cpu_sysinfo.cpu[CPU_IDLE] = msnsecs[CMS_IDLE];
3348         if (cso->cpu_sysinfo.cpu[CPU_USER] < msnsecs[CMS_USER])
3349                 cso->cpu_sysinfo.cpu[CPU_USER] = msnsecs[CMS_USER];
3350         if (cso->cpu_sysinfo.cpu[CPU_KERNEL] < msnsecs[CMS_SYSTEM])
3351                 cso->cpu_sysinfo.cpu[CPU_KERNEL] = msnsecs[CMS_SYSTEM];
3352         cso->cpu_sysinfo.cpu[CPU_WAIT]       = 0;
3353         cso->cpu_sysinfo.wait[W_IO]  = 0;
3354         cso->cpu_sysinfo.wait[W_SWAP]        = 0;
3355         cso->cpu_sysinfo.wait[W_PIO] = 0;
3356         cso->cpu_sysinfo.bread               = CPU_STATS(cp, sys.bread);
3357         cso->cpu_sysinfo.bwrite      = CPU_STATS(cp, sys.bwrite);
3358         cso->cpu_sysinfo.lread               = CPU_STATS(cp, sys.lread);
3359         cso->cpu_sysinfo.lwrite      = CPU_STATS(cp, sys.lwrite);
3360         cso->cpu_sysinfo.phread      = CPU_STATS(cp, sys.phread);
3361         cso->cpu_sysinfo.phwrite     = CPU_STATS(cp, sys.phwrite);
3362         cso->cpu_sysinfo.pswitch     = CPU_STATS(cp, sys.pswitch);
3363         cso->cpu_sysinfo.trap                = CPU_STATS(cp, sys.trap);
3364         cso->cpu_sysinfo.intr                = 0;
3365         for (i = 0; i < PIL_MAX; i++)
3366                 cso->cpu_sysinfo.intr += CPU_STATS(cp, sys.intr[i]);
3367         cso->cpu_sysinfo.syscall     = CPU_STATS(cp, sys.syscall);
3368         cso->cpu_sysinfo.sysread     = CPU_STATS(cp, sys.sysread);
3369         cso->cpu_sysinfo.syswrite    = CPU_STATS(cp, sys.syswrite);
3370         cso->cpu_sysinfo.sysfork     = CPU_STATS(cp, sys.sysfork);
3371         cso->cpu_sysinfo.sysvfork    = CPU_STATS(cp, sys.sysvfork);
3372         cso->cpu_sysinfo.sysexec     = CPU_STATS(cp, sys.sysexec);
3373         cso->cpu_sysinfo.readch              = CPU_STATS(cp, sys.readch);
3374         cso->cpu_sysinfo.writech     = CPU_STATS(cp, sys.writech);
3375         cso->cpu_sysinfo.rcvint              = CPU_STATS(cp, sys.rcvint);
3376         cso->cpu_sysinfo.xmtint              = CPU_STATS(cp, sys.xmtint);
3377         cso->cpu_sysinfo.mdmint              = CPU_STATS(cp, sys.mdmint);
3378         cso->cpu_sysinfo.rawch               = CPU_STATS(cp, sys.rawch);
3379         cso->cpu_sysinfo.canch               = CPU_STATS(cp, sys.canch);
3380         cso->cpu_sysinfo.outch               = CPU_STATS(cp, sys.outch);
3381         cso->cpu_sysinfo.msg         = CPU_STATS(cp, sys.msg);
3382         cso->cpu_sysinfo.sema                = CPU_STATS(cp, sys.sema);
3383         cso->cpu_sysinfo.namei               = CPU_STATS(cp, sys.namei);
3384         cso->cpu_sysinfo.ufsiget     = CPU_STATS(cp, sys.ufsiget);
3385         cso->cpu_sysinfo.ufsdirblk   = CPU_STATS(cp, sys.ufsdirblk);
3386         cso->cpu_sysinfo.ufsipage    = CPU_STATS(cp, sys.ufsipage);
3387         cso->cpu_sysinfo.ufsinopage  = CPU_STATS(cp, sys.ufsinopage);
3388         cso->cpu_sysinfo.inodeovf    = 0;
3389         cso->cpu_sysinfo.fileovf     = 0;
3390         cso->cpu_sysinfo.procovf     = CPU_STATS(cp, sys.procovf);
3391         cso->cpu_sysinfo.intrthread  = 0;
3392         for (i = 0; i < LOCK_LEVEL - 1; i++)
3393                 cso->cpu_sysinfo.intrthread += CPU_STATS(cp, sys.intr[i]);
3394         cso->cpu_sysinfo.intrblk     = CPU_STATS(cp, sys.intrblk);
3395         cso->cpu_sysinfo.idlethread  = CPU_STATS(cp, sys.idlethread);
3396         cso->cpu_sysinfo.inv_swtch   = CPU_STATS(cp, sys.inv_swtch);
3397         cso->cpu_sysinfo.nthreads    = CPU_STATS(cp, sys.nthreads);
3398         cso->cpu_sysinfo.cpumigrate  = CPU_STATS(cp, sys.cpumigrate);
3399         cso->cpu_sysinfo.xcalls              = CPU_STATS(cp, sys.xcalls);
3400         cso->cpu_sysinfo.mutex_adenters      = CPU_STATS(cp, sys.mutex_adenters);
3401         cso->cpu_sysinfo.rw_rdfails  = CPU_STATS(cp, sys.rw_rdfails);
3402         cso->cpu_sysinfo.rw_wrfails  = CPU_STATS(cp, sys.rw_wrfails);
3403         cso->cpu_sysinfo.modload     = CPU_STATS(cp, sys.modload);
3404         cso->cpu_sysinfo.modunload   = CPU_STATS(cp, sys.modunload);
3405         cso->cpu_sysinfo.bawrite     = CPU_STATS(cp, sys.bawrite);
3406         cso->cpu_sysinfo.rw_enters   = 0;
3407         cso->cpu_sysinfo.win_uo_cnt  = 0;
3408         cso->cpu_sysinfo.win_uu_cnt  = 0;
3409         cso->cpu_sysinfo.win_so_cnt  = 0;
3410         cso->cpu_sysinfo.win_su_cnt  = 0;
3411         cso->cpu_sysinfo.win_suo_cnt = 0;
3412 
3413         cso->cpu_syswait.iowait              = CPU_STATS(cp, sys.iowait);
3414         cso->cpu_syswait.swap                = 0;
3415         cso->cpu_syswait.physio              = 0;
3416 
3417         cso->cpu_vminfo.pgrec                = CPU_STATS(cp, vm.pgrec);
3418         cso->cpu_vminfo.pgfrec               = CPU_STATS(cp, vm.pgfrec);
3419         cso->cpu_vminfo.pgin         = CPU_STATS(cp, vm.pgin);
3420         cso->cpu_vminfo.pgpgin               = CPU_STATS(cp, vm.pgpgin);
3421         cso->cpu_vminfo.pgout                = CPU_STATS(cp, vm.pgout);
3422         cso->cpu_vminfo.pgpgout              = CPU_STATS(cp, vm.pgpgout);
3423         cso->cpu_vminfo.swapin               = CPU_STATS(cp, vm.swapin);
3424         cso->cpu_vminfo.pgswapin     = CPU_STATS(cp, vm.pgswapin);
3425         cso->cpu_vminfo.swapout              = CPU_STATS(cp, vm.swapout);
3426         cso->cpu_vminfo.pgswapout    = CPU_STATS(cp, vm.pgswapout);
3427         cso->cpu_vminfo.zfod         = CPU_STATS(cp, vm.zfod);
3428         cso->cpu_vminfo.dfree                = CPU_STATS(cp, vm.dfree);
3429         cso->cpu_vminfo.scan         = CPU_STATS(cp, vm.scan);
3430         cso->cpu_vminfo.rev          = CPU_STATS(cp, vm.rev);
3431         cso->cpu_vminfo.hat_fault    = CPU_STATS(cp, vm.hat_fault);
3432         cso->cpu_vminfo.as_fault     = CPU_STATS(cp, vm.as_fault);
3433         cso->cpu_vminfo.maj_fault    = CPU_STATS(cp, vm.maj_fault);
3434         cso->cpu_vminfo.cow_fault    = CPU_STATS(cp, vm.cow_fault);
3435         cso->cpu_vminfo.prot_fault   = CPU_STATS(cp, vm.prot_fault);
3436         cso->cpu_vminfo.softlock     = CPU_STATS(cp, vm.softlock);
3437         cso->cpu_vminfo.kernel_asflt = CPU_STATS(cp, vm.kernel_asflt);
3438         cso->cpu_vminfo.pgrrun               = CPU_STATS(cp, vm.pgrrun);
3439         cso->cpu_vminfo.execpgin     = CPU_STATS(cp, vm.execpgin);
3440         cso->cpu_vminfo.execpgout    = CPU_STATS(cp, vm.execpgout);
3441         cso->cpu_vminfo.execfree     = CPU_STATS(cp, vm.execfree);
3442         cso->cpu_vminfo.anonpgin     = CPU_STATS(cp, vm.anonpgin);
3443         cso->cpu_vminfo.anonpgout    = CPU_STATS(cp, vm.anonpgout);
3444         cso->cpu_vminfo.anonfree     = CPU_STATS(cp, vm.anonfree);
3445         cso->cpu_vminfo.fspgin               = CPU_STATS(cp, vm.fspgin);
3446         cso->cpu_vminfo.fspgout              = CPU_STATS(cp, vm.fspgout);
3447         cso->cpu_vminfo.fsfree               = CPU_STATS(cp, vm.fsfree);
3448 
3449         return (0);
3450 }