Print this page
patch delete-swapped_lock
patch sched-cleanup
patch remove-useless-var2
patch remove-dead-sched-code
patch remove-as_swapout
patch remove-load-flag
patch remove-on-swapq-flag
patch remove-swapenq-flag
patch remove-dont-swap-flag
patch remove-swapinout-class-ops
patch remove-useless-var


  37 #include <sys/tuneable.h>
  38 #include <sys/cmn_err.h>
  39 #include <sys/buf.h>
  40 #include <sys/disp.h>
  41 #include <sys/vmsystm.h>
  42 #include <sys/vmparam.h>
  43 #include <sys/class.h>
  44 #include <sys/vtrace.h>
  45 #include <sys/modctl.h>
  46 #include <sys/debug.h>
  47 #include <sys/tnf_probe.h>
  48 #include <sys/procfs.h>
  49 
  50 #include <vm/seg.h>
  51 #include <vm/seg_kp.h>
  52 #include <vm/as.h>
  53 #include <vm/rm.h>
  54 #include <vm/seg_kmem.h>
  55 #include <sys/callb.h>
  56 
  57 /*
  58  * The swapper sleeps on runout when there is no one to swap in.
  59  * It sleeps on runin when it could not find space to swap someone
  60  * in or after swapping someone in.
  61  */
  62 char    runout;
  63 char    runin;
  64 char    wake_sched;     /* flag tells clock to wake swapper on next tick */
  65 char    wake_sched_sec; /* flag tells clock to wake swapper after a second */
  66 
  67 /*
  68  * The swapper swaps processes to reduce memory demand and runs
  69  * when avefree < desfree.  The swapper resorts to SOFTSWAP when
  70  * avefree < desfree which results in swapping out all processes
  71  * sleeping for more than maxslp seconds.  HARDSWAP occurs when the
  72  * system is on the verge of thrashing and this results in swapping
  73  * out runnable threads or threads sleeping for less than maxslp secs.
  74  *
  75  * The swapper runs through all the active processes in the system
  76  * and invokes the scheduling class specific swapin/swapout routine
  77  * for every thread in the process to obtain an effective priority
  78  * for the process.  A priority of -1 implies that the thread isn't
  79  * swappable.  This effective priority is used to find the most
  80  * eligible process to swapout or swapin.
  81  *
  82  * NOTE:  Threads which have been swapped are not linked on any
  83  *        queue and their dispatcher lock points at the "swapped_lock".
  84  *
  85  * Processes containing threads with the TS_DONT_SWAP flag set cannot be
  86  * swapped out immediately by the swapper.  This is due to the fact that
  87  * such threads may be holding locks which may be needed by the swapper
  88  * to push its pages out.  The TS_SWAPENQ flag is set on such threads
  89  * to prevent them running in user mode.  When such threads reach a
  90  * safe point (i.e., are not holding any locks - CL_TRAPRET), they
  91  * queue themseleves onto the swap queue which is processed by the
  92  * swapper.  This results in reducing memory demand when the system
  93  * is desparate for memory as the thread can't run in user mode.
  94  *
  95  * The swap queue consists of threads, linked via t_link, which are
  96  * haven't been swapped, are runnable but not on the run queue.  The
  97  * swap queue is protected by the "swapped_lock".  The dispatcher
  98  * lock (t_lockp) of all threads on the swap queue points at the
  99  * "swapped_lock".  Thus, the entire queue and/or threads on the
 100  * queue can be locked by acquiring "swapped_lock".
 101  */
 102 static kthread_t *tswap_queue;
 103 extern disp_lock_t swapped_lock; /* protects swap queue and threads on it */
 104 
 105 int     maxslp = 0;
 106 pgcnt_t avefree;        /* 5 sec moving average of free memory */
 107 pgcnt_t avefree30;      /* 30 sec moving average of free memory */
 108 
 109 /*
 110  * Minimum size used to decide if sufficient memory is available
 111  * before a process is swapped in.  This is necessary since in most
 112  * cases the actual size of a process (p_swrss) being swapped in
 113  * is usually 2 pages (kernel stack pages).  This is due to the fact
 114  * almost all user pages of a process are stolen by pageout before
 115  * the swapper decides to swapout it out.
 116  */
 117 int     min_procsize = 12;
 118 
 119 static int      swapin(proc_t *);
 120 static int      swapout(proc_t *, uint_t *, int);
 121 static void     process_swap_queue();
 122 
 123 #ifdef __sparc
 124 extern void lwp_swapin(kthread_t *);
 125 #endif /* __sparc */
 126 
 127 /*
 128  * Counters to keep track of the number of swapins or swapouts.
 129  */
 130 uint_t tot_swapped_in, tot_swapped_out;
 131 uint_t softswap, hardswap, swapqswap;
 132 
 133 /*
 134  * Macro to determine if a process is eligble to be swapped.
 135  */
 136 #define not_swappable(p)                                        \
 137         (((p)->p_flag & SSYS) || (p)->p_stat == SIDL ||               \
 138             (p)->p_stat == SZOMB || (p)->p_as == NULL ||  \
 139             (p)->p_as == &kas)
 140 
 141 /*
 142  * Memory scheduler.
 143  */
 144 void
 145 sched()
 146 {
 147         kthread_id_t    t;
 148         pri_t           proc_pri;
 149         pri_t           thread_pri;
 150         pri_t           swapin_pri;
 151         int             desperate;
 152         pgcnt_t         needs;
 153         int             divisor;
 154         proc_t          *prp;
 155         proc_t          *swapout_prp;
 156         proc_t          *swapin_prp;
 157         spgcnt_t        avail;
 158         int             chosen_pri;
 159         time_t          swapout_time;
 160         time_t          swapin_proc_time;
 161         callb_cpr_t     cprinfo;
 162         kmutex_t        swap_cpr_lock;
 163 
 164         mutex_init(&swap_cpr_lock, NULL, MUTEX_DEFAULT, NULL);
 165         CALLB_CPR_INIT(&cprinfo, &swap_cpr_lock, callb_generic_cpr, "sched");
 166         if (maxslp == 0)
 167                 maxslp = MAXSLP;
 168 loop:
 169         needs = 0;
 170         desperate = 0;
 171 
 172         swapin_pri = v.v_nglobpris;
 173         swapin_prp = NULL;
 174         chosen_pri = -1;
 175 
 176         process_swap_queue();
 177 
 178         /*
 179          * Set desperate if
 180          *      1.  At least 2 runnable processes (on average).
 181          *      2.  Short (5 sec) and longer (30 sec) average is less
 182          *          than minfree and desfree respectively.
 183          *      3.  Pagein + pageout rate is excessive.
 184          */
 185         if (avenrun[0] >= 2 * FSCALE &&
 186             (MAX(avefree, avefree30) < desfree) &&
 187             (pginrate + pgoutrate > maxpgio || avefree < minfree)) {
 188                 TRACE_4(TR_FAC_SCHED, TR_DESPERATE,
 189                     "desp:avefree: %d, avefree30: %d, freemem: %d"
 190                     " pginrate: %d\n", avefree, avefree30, freemem, pginrate);
 191                 desperate = 1;
 192                 goto unload;
 193         }
 194 
 195         /*
 196          * Search list of processes to swapin and swapout deadwood.
 197          */
 198         swapin_proc_time = 0;
 199 top:
 200         mutex_enter(&pidlock);
 201         for (prp = practive; prp != NULL; prp = prp->p_next) {
 202                 if (not_swappable(prp))
 203                         continue;
 204 
 205                 /*
 206                  * Look at processes with at least one swapped lwp.
 207                  */
 208                 if (prp->p_swapcnt) {
 209                         time_t proc_time;
 210 
 211                         /*
 212                          * Higher priority processes are good candidates
 213                          * to swapin.
 214                          */
 215                         mutex_enter(&prp->p_lock);
 216                         proc_pri = -1;
 217                         t = prp->p_tlist;
 218                         proc_time = 0;
 219                         do {
 220                                 if (t->t_schedflag & TS_LOAD)
 221                                         continue;
 222 
 223                                 thread_lock(t);
 224                                 thread_pri = CL_SWAPIN(t, 0);
 225                                 thread_unlock(t);
 226 
 227                                 if (t->t_stime - proc_time > 0)
 228                                         proc_time = t->t_stime;
 229                                 if (thread_pri > proc_pri)
 230                                         proc_pri = thread_pri;
 231                         } while ((t = t->t_forw) != prp->p_tlist);
 232                         mutex_exit(&prp->p_lock);
 233 
 234                         if (proc_pri == -1)
 235                                 continue;
 236 
 237                         TRACE_3(TR_FAC_SCHED, TR_CHOOSE_SWAPIN,
 238                             "prp %p epri %d proc_time %d",
 239                             prp, proc_pri, proc_time);
 240 
 241                         /*
 242                          * Swapin processes with a high effective priority.
 243                          */
 244                         if (swapin_prp == NULL || proc_pri > chosen_pri) {
 245                                 swapin_prp = prp;
 246                                 chosen_pri = proc_pri;
 247                                 swapin_pri = proc_pri;
 248                                 swapin_proc_time = proc_time;
 249                         }
 250                 } else {
 251                         /*
 252                          * No need to soft swap if we have sufficient
 253                          * memory.
 254                          */
 255                         if (avefree > desfree ||
 256                             avefree < desfree && freemem > desfree)
 257                                 continue;
 258 
 259                         /*
 260                          * Skip processes that are exiting
 261                          * or whose address spaces are locked.
 262                          */
 263                         mutex_enter(&prp->p_lock);
 264                         if ((prp->p_flag & SEXITING) ||
 265                             (prp->p_as != NULL && AS_ISPGLCK(prp->p_as))) {
 266                                 mutex_exit(&prp->p_lock);
 267                                 continue;
 268                         }
 269 
 270                         /*
 271                          * Softswapping to kick out deadwood.
 272                          */
 273                         proc_pri = -1;
 274                         t = prp->p_tlist;
 275                         do {
 276                                 if ((t->t_schedflag & (TS_SWAPENQ |
 277                                     TS_ON_SWAPQ | TS_LOAD)) != TS_LOAD)
 278                                         continue;
 279 
 280                                 thread_lock(t);
 281                                 thread_pri = CL_SWAPOUT(t, SOFTSWAP);
 282                                 thread_unlock(t);
 283                                 if (thread_pri > proc_pri)
 284                                         proc_pri = thread_pri;
 285                         } while ((t = t->t_forw) != prp->p_tlist);
 286 
 287                         if (proc_pri != -1) {
 288                                 uint_t swrss;
 289 
 290                                 mutex_exit(&pidlock);
 291 
 292                                 TRACE_1(TR_FAC_SCHED, TR_SOFTSWAP,
 293                                     "softswap:prp %p", prp);
 294 
 295                                 (void) swapout(prp, &swrss, SOFTSWAP);
 296                                 softswap++;
 297                                 prp->p_swrss += swrss;
 298                                 mutex_exit(&prp->p_lock);
 299                                 goto top;
 300                         }
 301                         mutex_exit(&prp->p_lock);
 302                 }
 303         }
 304         if (swapin_prp != NULL)
 305                 mutex_enter(&swapin_prp->p_lock);
 306         mutex_exit(&pidlock);
 307 
 308         if (swapin_prp == NULL) {
 309                 TRACE_3(TR_FAC_SCHED, TR_RUNOUT,
 310                 "schedrunout:runout nswapped: %d, avefree: %ld freemem: %ld",
 311                     nswapped, avefree, freemem);
 312 
 313                 t = curthread;
 314                 thread_lock(t);
 315                 runout++;
 316                 t->t_schedflag |= (TS_ALLSTART & ~TS_CSTART);
 317                 t->t_whystop = PR_SUSPENDED;
 318                 t->t_whatstop = SUSPEND_NORMAL;
 319                 (void) new_mstate(t, LMS_SLEEP);
 320                 mutex_enter(&swap_cpr_lock);
 321                 CALLB_CPR_SAFE_BEGIN(&cprinfo);
 322                 mutex_exit(&swap_cpr_lock);
 323                 thread_stop(t);         /* change state and drop lock */
 324                 swtch();
 325                 mutex_enter(&swap_cpr_lock);
 326                 CALLB_CPR_SAFE_END(&cprinfo, &swap_cpr_lock);
 327                 mutex_exit(&swap_cpr_lock);
 328                 goto loop;
 329         }
 330 
 331         /*
 332          * Decide how deserving this process is to be brought in.
 333          * Needs is an estimate of how much core the process will
 334          * need.  If the process has been out for a while, then we
 335          * will bring it in with 1/2 the core needed, otherwise
 336          * we are conservative.
 337          */
 338         divisor = 1;
 339         swapout_time = (ddi_get_lbolt() - swapin_proc_time) / hz;
 340         if (swapout_time > maxslp / 2)
 341                 divisor = 2;
 342 
 343         needs = MIN(swapin_prp->p_swrss, lotsfree);
 344         needs = MAX(needs, min_procsize);
 345         needs = needs / divisor;
 346 
 347         /*
 348          * Use freemem, since we want processes to be swapped
 349          * in quickly.
 350          */
 351         avail = freemem - deficit;
 352         if (avail > (spgcnt_t)needs) {
 353                 deficit += needs;
 354 
 355                 TRACE_2(TR_FAC_SCHED, TR_SWAPIN_VALUES,
 356                     "swapin_values: prp %p needs %lu", swapin_prp, needs);
 357 
 358                 if (swapin(swapin_prp)) {
 359                         mutex_exit(&swapin_prp->p_lock);
 360                         goto loop;
 361                 }
 362                 deficit -= MIN(needs, deficit);
 363                 mutex_exit(&swapin_prp->p_lock);
 364         } else {
 365                 mutex_exit(&swapin_prp->p_lock);
 366                 /*
 367                  * If deficit is high, too many processes have been
 368                  * swapped in so wait a sec before attempting to
 369                  * swapin more.
 370                  */
 371                 if (freemem > needs) {
 372                         TRACE_2(TR_FAC_SCHED, TR_HIGH_DEFICIT,
 373                             "deficit: prp %p needs %lu", swapin_prp, needs);
 374                         goto block;
 375                 }
 376         }
 377 
 378         TRACE_2(TR_FAC_SCHED, TR_UNLOAD,
 379             "unload: prp %p needs %lu", swapin_prp, needs);
 380 
 381 unload:
 382         /*
 383          * Unload all unloadable modules, free all other memory
 384          * resources we can find, then look for a thread to hardswap.

 385          */
 386         modreap();
 387         segkp_cache_free();
 388 
 389         swapout_prp = NULL;
 390         mutex_enter(&pidlock);
 391         for (prp = practive; prp != NULL; prp = prp->p_next) {
 392 
 393                 /*
 394                  * No need to soft swap if we have sufficient
 395                  * memory.
 396                  */
 397                 if (not_swappable(prp))
 398                         continue;
 399 
 400                 if (avefree > minfree ||
 401                     avefree < minfree && freemem > desfree) {
 402                         swapout_prp = NULL;
 403                         break;
 404                 }
 405 
 406                 /*
 407                  * Skip processes that are exiting
 408                  * or whose address spaces are locked.
 409                  */
 410                 mutex_enter(&prp->p_lock);
 411                 if ((prp->p_flag & SEXITING) ||
 412                     (prp->p_as != NULL && AS_ISPGLCK(prp->p_as))) {
 413                         mutex_exit(&prp->p_lock);
 414                         continue;
 415                 }
 416 
 417                 proc_pri = -1;
 418                 t = prp->p_tlist;
 419                 do {
 420                         if ((t->t_schedflag & (TS_SWAPENQ |
 421                             TS_ON_SWAPQ | TS_LOAD)) != TS_LOAD)
 422                                 continue;
 423 
 424                         thread_lock(t);
 425                         thread_pri = CL_SWAPOUT(t, HARDSWAP);
 426                         thread_unlock(t);
 427                         if (thread_pri > proc_pri)
 428                                 proc_pri = thread_pri;
 429                 } while ((t = t->t_forw) != prp->p_tlist);
 430 
 431                 mutex_exit(&prp->p_lock);
 432                 if (proc_pri == -1)
 433                         continue;
 434 
 435                 /*
 436                  * Swapout processes sleeping with a lower priority
 437                  * than the one currently being swapped in, if any.
 438                  */
 439                 if (swapin_prp == NULL || swapin_pri > proc_pri) {
 440                         TRACE_2(TR_FAC_SCHED, TR_CHOOSE_SWAPOUT,
 441                             "hardswap: prp %p needs %lu", prp, needs);
 442 
 443                         if (swapout_prp == NULL || proc_pri < chosen_pri) {
 444                                 swapout_prp = prp;
 445                                 chosen_pri = proc_pri;
 446                         }
 447                 }
 448         }
 449 
 450         /*
 451          * Acquire the "p_lock" before dropping "pidlock"
 452          * to prevent the proc structure from being freed
 453          * if the process exits before swapout completes.
 454          */
 455         if (swapout_prp != NULL)
 456                 mutex_enter(&swapout_prp->p_lock);
 457         mutex_exit(&pidlock);
 458 
 459         if ((prp = swapout_prp) != NULL) {
 460                 uint_t swrss = 0;
 461                 int swapped;
 462 
 463                 swapped = swapout(prp, &swrss, HARDSWAP);
 464                 if (swapped) {
 465                         /*
 466                          * If desperate, we want to give the space obtained
 467                          * by swapping this process out to processes in core,
 468                          * so we give them a chance by increasing deficit.
 469                          */
 470                         prp->p_swrss += swrss;
 471                         if (desperate)
 472                                 deficit += MIN(prp->p_swrss, lotsfree);
 473                         hardswap++;
 474                 }
 475                 mutex_exit(&swapout_prp->p_lock);
 476 
 477                 if (swapped)
 478                         goto loop;
 479         }
 480 
 481         /*
 482          * Delay for 1 second and look again later.
 483          */
 484         TRACE_3(TR_FAC_SCHED, TR_RUNIN,
 485             "schedrunin:runin nswapped: %d, avefree: %ld freemem: %ld",
 486             nswapped, avefree, freemem);
 487 
 488 block:
 489         t = curthread;
 490         thread_lock(t);
 491         runin++;
 492         t->t_schedflag |= (TS_ALLSTART & ~TS_CSTART);
 493         t->t_whystop = PR_SUSPENDED;
 494         t->t_whatstop = SUSPEND_NORMAL;
 495         (void) new_mstate(t, LMS_SLEEP);
 496         mutex_enter(&swap_cpr_lock);
 497         CALLB_CPR_SAFE_BEGIN(&cprinfo);
 498         mutex_exit(&swap_cpr_lock);
 499         thread_stop(t);         /* change to stop state and drop lock */
 500         swtch();
 501         mutex_enter(&swap_cpr_lock);
 502         CALLB_CPR_SAFE_END(&cprinfo, &swap_cpr_lock);
 503         mutex_exit(&swap_cpr_lock);
 504         goto loop;
 505 }
 506 
 507 /*
 508  * Remove the specified thread from the swap queue.
 509  */
 510 static void
 511 swapdeq(kthread_id_t tp)
 512 {
 513         kthread_id_t *tpp;
 514 
 515         ASSERT(THREAD_LOCK_HELD(tp));
 516         ASSERT(tp->t_schedflag & TS_ON_SWAPQ);
 517 
 518         tpp = &tswap_queue;
 519         for (;;) {
 520                 ASSERT(*tpp != NULL);
 521                 if (*tpp == tp)
 522                         break;
 523                 tpp = &(*tpp)->t_link;
 524         }
 525         *tpp = tp->t_link;
 526         tp->t_schedflag &= ~TS_ON_SWAPQ;
 527 }
 528 
 529 /*
 530  * Swap in lwps.  Returns nonzero on success (i.e., if at least one lwp is
 531  * swapped in) and 0 on failure.
 532  */
 533 static int
 534 swapin(proc_t *pp)
 535 {
 536         kthread_id_t tp;
 537         int err;
 538         int num_swapped_in = 0;
 539         struct cpu *cpup = CPU;
 540         pri_t thread_pri;
 541 
 542         ASSERT(MUTEX_HELD(&pp->p_lock));
 543         ASSERT(pp->p_swapcnt);
 544 
 545 top:
 546         tp = pp->p_tlist;
 547         do {
 548                 /*
 549                  * Only swapin eligible lwps (specified by the scheduling
 550                  * class) which are unloaded and ready to run.
 551                  */
 552                 thread_lock(tp);
 553                 thread_pri = CL_SWAPIN(tp, 0);
 554                 if (thread_pri != -1 && tp->t_state == TS_RUN &&
 555                     (tp->t_schedflag & TS_LOAD) == 0) {
 556                         size_t stack_size;
 557                         pgcnt_t stack_pages;
 558 
 559                         ASSERT((tp->t_schedflag & TS_ON_SWAPQ) == 0);
 560 
 561                         thread_unlock(tp);
 562                         /*
 563                          * Now drop the p_lock since the stack needs
 564                          * to brought in.
 565                          */
 566                         mutex_exit(&pp->p_lock);
 567 
 568                         stack_size = swapsize(tp->t_swap);
 569                         stack_pages = btopr(stack_size);
 570                         /* Kernel probe */
 571                         TNF_PROBE_4(swapin_lwp, "vm swap swapin", /* CSTYLED */,
 572                             tnf_pid,            pid,            pp->p_pid,
 573                             tnf_lwpid,          lwpid,          tp->t_tid,
 574                             tnf_kthread_id,     tid,            tp,
 575                             tnf_ulong,          page_count,     stack_pages);
 576 
 577                         rw_enter(&kas.a_lock, RW_READER);
 578                         err = segkp_fault(segkp->s_as->a_hat, segkp,
 579                             tp->t_swap, stack_size, F_SOFTLOCK, S_OTHER);
 580                         rw_exit(&kas.a_lock);
 581 
 582                         /*
 583                          * Re-acquire the p_lock.
 584                          */
 585                         mutex_enter(&pp->p_lock);
 586                         if (err) {
 587                                 num_swapped_in = 0;
 588                                 break;
 589                         } else {
 590 #ifdef __sparc
 591                                 lwp_swapin(tp);
 592 #endif /* __sparc */
 593                                 CPU_STATS_ADDQ(cpup, vm, swapin, 1);
 594                                 CPU_STATS_ADDQ(cpup, vm, pgswapin,
 595                                     stack_pages);
 596 
 597                                 pp->p_swapcnt--;
 598                                 pp->p_swrss -= stack_pages;
 599 
 600                                 thread_lock(tp);
 601                                 tp->t_schedflag |= TS_LOAD;
 602                                 dq_sruninc(tp);
 603 
 604                                 /* set swapin time */
 605                                 tp->t_stime = ddi_get_lbolt();
 606                                 thread_unlock(tp);
 607 
 608                                 nswapped--;
 609                                 tot_swapped_in++;
 610                                 num_swapped_in++;
 611 
 612                                 TRACE_2(TR_FAC_SCHED, TR_SWAPIN,
 613                                     "swapin: pp %p stack_pages %lu",
 614                                     pp, stack_pages);
 615                                 goto top;
 616                         }
 617                 }
 618                 thread_unlock(tp);
 619         } while ((tp = tp->t_forw) != pp->p_tlist);
 620         return (num_swapped_in);
 621 }
 622 
 623 /*
 624  * Swap out lwps.  Returns nonzero on success (i.e., if at least one lwp is
 625  * swapped out) and 0 on failure.
 626  */
 627 static int
 628 swapout(proc_t *pp, uint_t *swrss, int swapflags)
 629 {
 630         kthread_id_t tp;
 631         pgcnt_t ws_pages = 0;
 632         int err;
 633         int swapped_lwps = 0;
 634         struct as *as = pp->p_as;
 635         struct cpu *cpup = CPU;
 636         pri_t thread_pri;
 637 
 638         ASSERT(MUTEX_HELD(&pp->p_lock));
 639 
 640         if (pp->p_flag & SEXITING)
 641                 return (0);
 642 
 643 top:
 644         tp = pp->p_tlist;
 645         do {
 646                 klwp_t *lwp = ttolwp(tp);
 647 
 648                 /*
 649                  * Swapout eligible lwps (specified by the scheduling
 650                  * class) which don't have TS_DONT_SWAP set.  Set the
 651                  * "intent to swap" flag (TS_SWAPENQ) on threads
 652                  * which have TS_DONT_SWAP set so that they can be
 653                  * swapped if and when they reach a safe point.
 654                  */
 655                 thread_lock(tp);
 656                 thread_pri = CL_SWAPOUT(tp, swapflags);
 657                 if (thread_pri != -1) {
 658                         if (tp->t_schedflag & TS_DONT_SWAP) {
 659                                 tp->t_schedflag |= TS_SWAPENQ;
 660                                 tp->t_trapret = 1;
 661                                 aston(tp);
 662                         } else {
 663                                 pgcnt_t stack_pages;
 664                                 size_t stack_size;
 665 
 666                                 ASSERT((tp->t_schedflag &
 667                                     (TS_DONT_SWAP | TS_LOAD)) == TS_LOAD);
 668 
 669                                 if (lock_try(&tp->t_lock)) {
 670                                         /*
 671                                          * Remove thread from the swap_queue.
 672                                          */
 673                                         if (tp->t_schedflag & TS_ON_SWAPQ) {
 674                                                 ASSERT(!(tp->t_schedflag &
 675                                                     TS_SWAPENQ));
 676                                                 swapdeq(tp);
 677                                         } else if (tp->t_state == TS_RUN)
 678                                                 dq_srundec(tp);
 679 
 680                                         tp->t_schedflag &=
 681                                             ~(TS_LOAD | TS_SWAPENQ);
 682                                         lock_clear(&tp->t_lock);
 683 
 684                                         /*
 685                                          * Set swapout time if the thread isn't
 686                                          * sleeping.
 687                                          */
 688                                         if (tp->t_state != TS_SLEEP)
 689                                                 tp->t_stime = ddi_get_lbolt();
 690                                         thread_unlock(tp);
 691 
 692                                         nswapped++;
 693                                         tot_swapped_out++;
 694 
 695                                         lwp->lwp_ru.nswap++;
 696 
 697                                         /*
 698                                          * Now drop the p_lock since the
 699                                          * stack needs to pushed out.
 700                                          */
 701                                         mutex_exit(&pp->p_lock);
 702 
 703                                         stack_size = swapsize(tp->t_swap);
 704                                         stack_pages = btopr(stack_size);
 705                                         ws_pages += stack_pages;
 706                                         /* Kernel probe */
 707                                         TNF_PROBE_4(swapout_lwp,
 708                                             "vm swap swapout",
 709                                             /* CSTYLED */,
 710                                             tnf_pid, pid, pp->p_pid,
 711                                             tnf_lwpid, lwpid, tp->t_tid,
 712                                             tnf_kthread_id, tid, tp,
 713                                             tnf_ulong, page_count,
 714                                             stack_pages);
 715 
 716                                         rw_enter(&kas.a_lock, RW_READER);
 717                                         err = segkp_fault(segkp->s_as->a_hat,
 718                                             segkp, tp->t_swap, stack_size,
 719                                             F_SOFTUNLOCK, S_WRITE);
 720                                         rw_exit(&kas.a_lock);
 721 
 722                                         if (err) {
 723                                                 cmn_err(CE_PANIC,
 724                                                     "swapout: segkp_fault "
 725                                                     "failed err: %d", err);
 726                                         }
 727                                         CPU_STATS_ADDQ(cpup,
 728                                             vm, pgswapout, stack_pages);
 729 
 730                                         mutex_enter(&pp->p_lock);
 731                                         pp->p_swapcnt++;
 732                                         swapped_lwps++;
 733                                         goto top;
 734                                 }
 735                         }
 736                 }
 737                 thread_unlock(tp);
 738         } while ((tp = tp->t_forw) != pp->p_tlist);
 739 
 740         /*
 741          * Unload address space when all lwps are swapped out.
 742          */
 743         if (pp->p_swapcnt == pp->p_lwpcnt) {
 744                 size_t as_size = 0;
 745 
 746                 /*
 747                  * Avoid invoking as_swapout() if the process has
 748                  * no MMU resources since pageout will eventually
 749                  * steal pages belonging to this address space.  This
 750                  * saves CPU cycles as the number of pages that are
 751                  * potentially freed or pushed out by the segment
 752                  * swapout operation is very small.
 753                  */
 754                 if (rm_asrss(pp->p_as) != 0)
 755                         as_size = as_swapout(as);
 756 
 757                 CPU_STATS_ADDQ(cpup, vm, pgswapout, btop(as_size));
 758                 CPU_STATS_ADDQ(cpup, vm, swapout, 1);
 759                 ws_pages += btop(as_size);
 760 
 761                 TRACE_2(TR_FAC_SCHED, TR_SWAPOUT,
 762                     "swapout: pp %p pages_pushed %lu", pp, ws_pages);
 763                 /* Kernel probe */
 764                 TNF_PROBE_2(swapout_process, "vm swap swapout", /* CSTYLED */,
 765                     tnf_pid,    pid,            pp->p_pid,
 766                     tnf_ulong,  page_count,     ws_pages);
 767         }
 768         *swrss = ws_pages;
 769         return (swapped_lwps);
 770 }
 771 
 772 void
 773 swapout_lwp(klwp_t *lwp)
 774 {
 775         kthread_id_t tp = curthread;
 776 
 777         ASSERT(curthread == lwptot(lwp));
 778 
 779         /*
 780          * Don't insert the thread onto the swap queue if
 781          * sufficient memory is available.
 782          */
 783         if (avefree > desfree || avefree < desfree && freemem > desfree) {
 784                 thread_lock(tp);
 785                 tp->t_schedflag &= ~TS_SWAPENQ;
 786                 thread_unlock(tp);
 787                 return;
 788         }
 789 
 790         /*
 791          * Lock the thread, then move it to the swapped queue from the
 792          * onproc queue and set its state to be TS_RUN.
 793          */
 794         thread_lock(tp);
 795         ASSERT(tp->t_state == TS_ONPROC);
 796         if (tp->t_schedflag & TS_SWAPENQ) {
 797                 tp->t_schedflag &= ~TS_SWAPENQ;
 798 
 799                 /*
 800                  * Set the state of this thread to be runnable
 801                  * and move it from the onproc queue to the swap queue.
 802                  */
 803                 disp_swapped_enq(tp);
 804 
 805                 /*
 806                  * Insert the thread onto the swap queue.
 807                  */
 808                 tp->t_link = tswap_queue;
 809                 tswap_queue = tp;
 810                 tp->t_schedflag |= TS_ON_SWAPQ;
 811 
 812                 thread_unlock_nopreempt(tp);
 813 
 814                 TRACE_1(TR_FAC_SCHED, TR_SWAPOUT_LWP, "swapout_lwp:%x", lwp);
 815 
 816                 swtch();
 817         } else {
 818                 thread_unlock(tp);
 819         }
 820 }
 821 
 822 /*
 823  * Swap all threads on the swap queue.
 824  */
 825 static void
 826 process_swap_queue(void)
 827 {
 828         kthread_id_t tp;
 829         uint_t ws_pages;
 830         proc_t *pp;
 831         struct cpu *cpup = CPU;
 832         klwp_t *lwp;
 833         int err;
 834 
 835         if (tswap_queue == NULL)
 836                 return;
 837 
 838         /*
 839          * Acquire the "swapped_lock" which locks the swap queue,
 840          * and unload the stacks of all threads on it.
 841          */
 842         disp_lock_enter(&swapped_lock);
 843         while ((tp = tswap_queue) != NULL) {
 844                 pgcnt_t stack_pages;
 845                 size_t stack_size;
 846 
 847                 tswap_queue = tp->t_link;
 848                 tp->t_link = NULL;
 849 
 850                 /*
 851                  * Drop the "dispatcher lock" before acquiring "t_lock"
 852                  * to avoid spinning on it since the thread at the front
 853                  * of the swap queue could be pinned before giving up
 854                  * its "t_lock" in resume.
 855                  */
 856                 disp_lock_exit(&swapped_lock);
 857                 lock_set(&tp->t_lock);
 858 
 859                 /*
 860                  * Now, re-acquire the "swapped_lock".  Acquiring this lock
 861                  * results in locking the thread since its dispatcher lock
 862                  * (t_lockp) is the "swapped_lock".
 863                  */
 864                 disp_lock_enter(&swapped_lock);
 865                 ASSERT(tp->t_state == TS_RUN);
 866                 ASSERT(tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ));
 867 
 868                 tp->t_schedflag &= ~(TS_LOAD | TS_ON_SWAPQ);
 869                 tp->t_stime = ddi_get_lbolt();               /* swapout time */
 870                 disp_lock_exit(&swapped_lock);
 871                 lock_clear(&tp->t_lock);
 872 
 873                 lwp = ttolwp(tp);
 874                 lwp->lwp_ru.nswap++;
 875 
 876                 pp = ttoproc(tp);
 877                 stack_size = swapsize(tp->t_swap);
 878                 stack_pages = btopr(stack_size);
 879 
 880                 /* Kernel probe */
 881                 TNF_PROBE_4(swapout_lwp, "vm swap swapout", /* CSTYLED */,
 882                     tnf_pid,            pid,            pp->p_pid,
 883                     tnf_lwpid,          lwpid,          tp->t_tid,
 884                     tnf_kthread_id,     tid,            tp,
 885                     tnf_ulong,          page_count,     stack_pages);
 886 
 887                 rw_enter(&kas.a_lock, RW_READER);
 888                 err = segkp_fault(segkp->s_as->a_hat, segkp, tp->t_swap,
 889                     stack_size, F_SOFTUNLOCK, S_WRITE);
 890                 rw_exit(&kas.a_lock);
 891 
 892                 if (err) {
 893                         cmn_err(CE_PANIC,
 894                         "process_swap_list: segkp_fault failed err: %d", err);
 895                 }
 896                 CPU_STATS_ADDQ(cpup, vm, pgswapout, stack_pages);
 897 
 898                 nswapped++;
 899                 tot_swapped_out++;
 900                 swapqswap++;
 901 
 902                 /*
 903                  * Don't need p_lock since the swapper is the only
 904                  * thread which increments/decrements p_swapcnt and p_swrss.
 905                  */
 906                 ws_pages = stack_pages;
 907                 pp->p_swapcnt++;
 908 
 909                 TRACE_1(TR_FAC_SCHED, TR_SWAPQ_LWP, "swaplist: pp %p", pp);
 910 
 911                 /*
 912                  * Unload address space when all lwps are swapped out.
 913                  */
 914                 if (pp->p_swapcnt == pp->p_lwpcnt) {
 915                         size_t as_size = 0;
 916 
 917                         if (rm_asrss(pp->p_as) != 0)
 918                                 as_size = as_swapout(pp->p_as);
 919 
 920                         CPU_STATS_ADDQ(cpup, vm, pgswapout,
 921                             btop(as_size));
 922                         CPU_STATS_ADDQ(cpup, vm, swapout, 1);
 923 
 924                         ws_pages += btop(as_size);
 925 
 926                         TRACE_2(TR_FAC_SCHED, TR_SWAPQ_PROC,
 927                             "swaplist_proc: pp %p pages_pushed: %lu",
 928                             pp, ws_pages);
 929                         /* Kernel probe */
 930                         TNF_PROBE_2(swapout_process, "vm swap swapout",
 931                             /* CSTYLED */,
 932                             tnf_pid,    pid,            pp->p_pid,
 933                             tnf_ulong,  page_count,     ws_pages);
 934                 }
 935                 pp->p_swrss += ws_pages;
 936                 disp_lock_enter(&swapped_lock);
 937         }
 938         disp_lock_exit(&swapped_lock);
 939 }


  37 #include <sys/tuneable.h>
  38 #include <sys/cmn_err.h>
  39 #include <sys/buf.h>
  40 #include <sys/disp.h>
  41 #include <sys/vmsystm.h>
  42 #include <sys/vmparam.h>
  43 #include <sys/class.h>
  44 #include <sys/vtrace.h>
  45 #include <sys/modctl.h>
  46 #include <sys/debug.h>
  47 #include <sys/tnf_probe.h>
  48 #include <sys/procfs.h>
  49 
  50 #include <vm/seg.h>
  51 #include <vm/seg_kp.h>
  52 #include <vm/as.h>
  53 #include <vm/rm.h>
  54 #include <vm/seg_kmem.h>
  55 #include <sys/callb.h>
  56 

















































  57 pgcnt_t avefree;        /* 5 sec moving average of free memory */
  58 pgcnt_t avefree30;      /* 30 sec moving average of free memory */
  59 
  60 /*
































  61  * Memory scheduler.
  62  */
  63 void
  64 sched()
  65 {
  66         kthread_id_t    t;













  67         callb_cpr_t     cprinfo;
  68         kmutex_t        swap_cpr_lock;
  69 
  70         mutex_init(&swap_cpr_lock, NULL, MUTEX_DEFAULT, NULL);
  71         CALLB_CPR_INIT(&cprinfo, &swap_cpr_lock, callb_generic_cpr, "sched");





  72 
  73         for (;;) {












  74                 if (avenrun[0] >= 2 * FSCALE &&
  75                     (MAX(avefree, avefree30) < desfree) &&
  76                     (pginrate + pgoutrate > maxpgio || avefree < minfree)) {


































































































































































































  77                         /*
  78                          * Unload all unloadable modules, free all other memory
  79                          * resources we can find, then look for a thread to
  80                          * hardswap.
  81                          */
  82                         modreap();
  83                         segkp_cache_free();



























































































  84                 }
  85 








  86                 t = curthread;
  87                 thread_lock(t);

  88                 t->t_schedflag |= (TS_ALLSTART & ~TS_CSTART);
  89                 t->t_whystop = PR_SUSPENDED;
  90                 t->t_whatstop = SUSPEND_NORMAL;
  91                 (void) new_mstate(t, LMS_SLEEP);
  92                 mutex_enter(&swap_cpr_lock);
  93                 CALLB_CPR_SAFE_BEGIN(&cprinfo);
  94                 mutex_exit(&swap_cpr_lock);
  95                 thread_stop(t);         /* change to stop state and drop lock */
  96                 swtch();
  97                 mutex_enter(&swap_cpr_lock);
  98                 CALLB_CPR_SAFE_END(&cprinfo, &swap_cpr_lock);
  99                 mutex_exit(&swap_cpr_lock);

















































































































































































































































































































































































































































 100         }

 101 }