1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 #include <sys/types.h>
  27 #include <sys/cmn_err.h>
  28 #include <sys/vmem.h>
  29 #include <sys/kmem.h>
  30 #include <sys/systm.h>
  31 #include <sys/machsystm.h>        /* for page_freelist_coalesce() */
  32 #include <sys/errno.h>
  33 #include <sys/memnode.h>
  34 #include <sys/memlist.h>
  35 #include <sys/memlist_impl.h>
  36 #include <sys/tuneable.h>
  37 #include <sys/proc.h>
  38 #include <sys/disp.h>
  39 #include <sys/debug.h>
  40 #include <sys/vm.h>
  41 #include <sys/callb.h>
  42 #include <sys/memlist_plat.h>     /* for installed_top_size() */
  43 #include <sys/condvar_impl.h>     /* for CV_HAS_WAITERS() */
  44 #include <sys/dumphdr.h>  /* for dump_resize() */
  45 #include <sys/atomic.h>           /* for use in stats collection */
  46 #include <sys/rwlock.h>
  47 #include <sys/cpuvar.h>
  48 #include <vm/seg_kmem.h>
  49 #include <vm/seg_kpm.h>
  50 #include <vm/page.h>
  51 #include <vm/vm_dep.h>
  52 #define SUNDDI_IMPL             /* so sunddi.h will not redefine splx() et al */
  53 #include <sys/sunddi.h>
  54 #include <sys/mem_config.h>
  55 #include <sys/mem_cage.h>
  56 #include <sys/lgrp.h>
  57 #include <sys/ddi.h>
  58 #include <sys/modctl.h>
  59 
  60 extern struct memlist *phys_avail;
  61 
  62 extern uint_t page_ctrs_adjust(int);
  63 void page_ctrs_cleanup(void);
  64 static void kphysm_setup_post_add(pgcnt_t);
  65 static int kphysm_setup_pre_del(pgcnt_t);
  66 static void kphysm_setup_post_del(pgcnt_t, int);
  67 
  68 static int kphysm_split_memseg(pfn_t base, pgcnt_t npgs);
  69 
  70 static int delspan_reserve(pfn_t, pgcnt_t);
  71 static void delspan_unreserve(pfn_t, pgcnt_t);
  72 
  73 kmutex_t memseg_lists_lock;
  74 struct memseg *memseg_va_avail;
  75 struct memseg *memseg_alloc(void);
  76 static struct memseg *memseg_delete_junk;
  77 static struct memseg *memseg_edit_junk;
  78 void memseg_remap_init(void);
  79 static void memseg_remap_to_dummy(struct memseg *);
  80 static void kphysm_addmem_error_undospan(pfn_t, pgcnt_t);
  81 static struct memseg *memseg_reuse(pgcnt_t);
  82 
  83 static struct kmem_cache *memseg_cache;
  84 
  85 /*
  86  * Interfaces to manage externally allocated
  87  * page_t memory (metadata) for a memseg.
  88  */
  89 #pragma weak    memseg_alloc_meta
  90 #pragma weak    memseg_free_meta
  91 #pragma weak    memseg_get_metapfn
  92 #pragma weak    memseg_remap_meta
  93 
  94 extern int ppvm_enable;
  95 extern page_t *ppvm_base;
  96 extern int memseg_alloc_meta(pfn_t, pgcnt_t, void **, pgcnt_t *);
  97 extern void memseg_free_meta(void *, pgcnt_t);
  98 extern pfn_t memseg_get_metapfn(void *, pgcnt_t);
  99 extern void memseg_remap_meta(struct memseg *);
 100 static int memseg_is_dynamic(struct memseg *);
 101 static int memseg_includes_meta(struct memseg *);
 102 pfn_t memseg_get_start(struct memseg *);
 103 static void memseg_cpu_vm_flush(void);
 104 
 105 int meta_alloc_enable;
 106 
 107 #ifdef  DEBUG
 108 static int memseg_debug;
 109 #define MEMSEG_DEBUG(args...) if (memseg_debug) printf(args)
 110 #else
 111 #define MEMSEG_DEBUG(...)
 112 #endif
 113 
 114 /*
 115  * Add a chunk of memory to the system.
 116  * base: starting PAGESIZE page of new memory.
 117  * npgs: length in PAGESIZE pages.
 118  *
 119  * Adding mem this way doesn't increase the size of the hash tables;
 120  * growing them would be too hard.  This should be OK, but adding memory
 121  * dynamically most likely means more hash misses, since the tables will
 122  * be smaller than they otherwise would be.
 123  */
 124 int
 125 kphysm_add_memory_dynamic(pfn_t base, pgcnt_t npgs)
 126 {
 127         page_t *pp;
 128         page_t          *opp, *oepp, *segpp;
 129         struct memseg   *seg;
 130         uint64_t        avmem;
 131         pfn_t           pfn;
 132         pfn_t           pt_base = base;
 133         pgcnt_t         tpgs = npgs;
 134         pgcnt_t         metapgs = 0;
 135         int             exhausted;
 136         pfn_t           pnum;
 137         int             mnode;
 138         caddr_t         vaddr;
 139         int             reuse;
 140         int             mlret;
 141         int             rv;
 142         int             flags;
 143         int             meta_alloc = 0;
 144         void            *mapva;
 145         void            *metabase = (void *)base;
 146         pgcnt_t         nkpmpgs = 0;
 147         offset_t        kpm_pages_off;
 148 
 149         cmn_err(CE_CONT,
 150             "?kphysm_add_memory_dynamic: adding %ldK at 0x%" PRIx64 "\n",
 151             npgs << (PAGESHIFT - 10), (uint64_t)base << PAGESHIFT);
 152 
 153         /*
 154          * Add this span in the delete list to prevent interactions.
 155          */
 156         if (!delspan_reserve(base, npgs)) {
 157                 return (KPHYSM_ESPAN);
 158         }
 159         /*
 160          * Check to see if any of the memory span has been added
 161          * by trying an add to the installed memory list. This
 162          * forms the interlocking process for add.
 163          */
 164 
 165         memlist_write_lock();
 166 
 167         mlret = memlist_add_span((uint64_t)(pt_base) << PAGESHIFT,
 168             (uint64_t)(tpgs) << PAGESHIFT, &phys_install);
 169 
 170         if (mlret == MEML_SPANOP_OK)
 171                 installed_top_size(phys_install, &physmax, &physinstalled);
 172 
 173         memlist_write_unlock();
 174 
 175         if (mlret != MEML_SPANOP_OK) {
 176                 if (mlret == MEML_SPANOP_EALLOC) {
 177                         delspan_unreserve(pt_base, tpgs);
 178                         return (KPHYSM_ERESOURCE);
 179                 } else if (mlret == MEML_SPANOP_ESPAN) {
 180                         delspan_unreserve(pt_base, tpgs);
 181                         return (KPHYSM_ESPAN);
 182                 } else {
 183                         delspan_unreserve(pt_base, tpgs);
 184                         return (KPHYSM_ERESOURCE);
 185                 }
 186         }
 187 
 188         if (meta_alloc_enable) {
 189                 /*
 190                  * Allocate the page_t's from existing memory;
 191                  * if that fails, allocate from the incoming memory.
 192                  */
 193                 rv = memseg_alloc_meta(base, npgs, &metabase, &metapgs);
 194                 if (rv == KPHYSM_OK) {
 195                         ASSERT(metapgs);
 196                         ASSERT(btopr(npgs * sizeof (page_t)) <= metapgs);
 197                         meta_alloc = 1;
 198                         goto mapalloc;
 199                 }
 200         }
 201 
 202         /*
 203          * We store the page_t's for this new memory in the first
 204          * few pages of the chunk. Here, we go and get'em ...
 205          */
 206 
 207         /*
 208          * The expression after the '-' gives the number of pages
 209          * that will fit in the new memory based on a requirement
 210          * of (PAGESIZE + sizeof (page_t)) bytes per page.
 211          */
 212         metapgs = npgs - (((uint64_t)(npgs) << PAGESHIFT) /
 213             (PAGESIZE + sizeof (page_t)));
 214 
 215         npgs -= metapgs;
 216         base += metapgs;
 217 
 218         ASSERT(btopr(npgs * sizeof (page_t)) <= metapgs);
 219 
 220         exhausted = (metapgs == 0 || npgs == 0);
 221 
 222         if (kpm_enable && !exhausted) {
 223                 pgcnt_t start, end, nkpmpgs_prelim;
 224                 size_t  ptsz;
 225 
 226                 /*
 227                  * A viable kpm large page mapping must not overlap two
 228                  * dynamic memsegs. Therefore the total size is checked
 229                  * to be at least kpm_pgsz and also whether start and end
 230                  * points are at least kpm_pgsz aligned.
 231                  */
 232                 if (ptokpmp(tpgs) < 1 || pmodkpmp(pt_base) ||
 233                     pmodkpmp(base + npgs)) {
 234 
 235                         kphysm_addmem_error_undospan(pt_base, tpgs);
 236 
 237                         /*
 238                          * There is no specific error code for violating
 239                          * kpm granularity constraints.
 240                          */
 241                         return (KPHYSM_ENOTVIABLE);
 242                 }
 243 
 244                 start = kpmptop(ptokpmp(base));
 245                 end = kpmptop(ptokpmp(base + npgs));
 246                 nkpmpgs_prelim = ptokpmp(end - start);
 247                 ptsz = npgs * sizeof (page_t);
 248                 metapgs = btopr(ptsz + nkpmpgs_prelim * KPMPAGE_T_SZ);
 249                 exhausted = (tpgs <= metapgs);
 250                 if (!exhausted) {
 251                         npgs = tpgs - metapgs;
 252                         base = pt_base + metapgs;
 253 
 254                         /* final nkpmpgs */
 255                         start = kpmptop(ptokpmp(base));
 256                         nkpmpgs = ptokpmp(end - start);
 257                         kpm_pages_off = ptsz +
 258                             (nkpmpgs_prelim - nkpmpgs) * KPMPAGE_T_SZ;
 259                 }
 260         }
 261 
 262         /*
 263          * Is memory area supplied too small?
 264          */
 265         if (exhausted) {
 266                 kphysm_addmem_error_undospan(pt_base, tpgs);
 267                 /*
 268                  * There is no specific error code for 'too small'.
 269                  */
 270                 return (KPHYSM_ERESOURCE);
 271         }
 272 
 273 mapalloc:
 274         /*
 275          * We may re-use a previously allocated VA space for the page_ts
 276          * eventually, but we need to initialize and lock the pages first.
 277          */
 278 
 279         /*
 280          * Get an address in the kernel address map, map
 281          * the page_t pages and see if we can touch them.
 282          */
 283 
 284         mapva = vmem_alloc(heap_arena, ptob(metapgs), VM_NOSLEEP);
 285         if (mapva == NULL) {
 286                 cmn_err(CE_WARN, "kphysm_add_memory_dynamic:"
 287                     " Can't allocate VA for page_ts");
 288 
 289                 if (meta_alloc)
 290                         memseg_free_meta(metabase, metapgs);
 291                 kphysm_addmem_error_undospan(pt_base, tpgs);
 292 
 293                 return (KPHYSM_ERESOURCE);
 294         }
 295         pp = mapva;
 296 
 297         if (physmax < (pt_base + tpgs))
 298                 physmax = (pt_base + tpgs);
 299 
 300         /*
 301          * In the remapping code we map one page at a time so we must do
 302          * the same here to match mapping sizes.
 303          */
 304         pfn = pt_base;
 305         vaddr = (caddr_t)pp;
 306         for (pnum = 0; pnum < metapgs; pnum++) {
 307                 if (meta_alloc)
 308                         pfn = memseg_get_metapfn(metabase, (pgcnt_t)pnum);
 309                 hat_devload(kas.a_hat, vaddr, ptob(1), pfn,
 310                     PROT_READ | PROT_WRITE,
 311                     HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
 312                 pfn++;
 313                 vaddr += ptob(1);
 314         }
 315 
 316         if (ddi_peek32((dev_info_t *)NULL,
 317             (int32_t *)pp, (int32_t *)0) == DDI_FAILURE) {
 318 
 319                 cmn_err(CE_WARN, "kphysm_add_memory_dynamic:"
 320                     " Can't access pp array at 0x%p [phys 0x%lx]",
 321                     (void *)pp, pt_base);
 322 
 323                 hat_unload(kas.a_hat, (caddr_t)pp, ptob(metapgs),
 324                     HAT_UNLOAD_UNMAP|HAT_UNLOAD_UNLOCK);
 325 
 326                 vmem_free(heap_arena, mapva, ptob(metapgs));
 327                 if (meta_alloc)
 328                         memseg_free_meta(metabase, metapgs);
 329                 kphysm_addmem_error_undospan(pt_base, tpgs);
 330 
 331                 return (KPHYSM_EFAULT);
 332         }
 333 
 334         /*
 335          * Add this memory slice to its memory node translation.
 336          *
 337          * Note that right now, each node may have only one slice;
 338          * this may change with COD or in larger SSM systems with
 339          * nested latency groups, so we must not assume that the
 340          * node does not yet exist.
 341          *
 342          * Note that there may be multiple memory nodes associated with
 343          * a single lgrp node on x86 systems.
 344          */
 345         pnum = pt_base + tpgs - 1;
 346         mem_node_add_range(pt_base, pnum);
 347 
 348         /*
 349          * Allocate or resize page counters as necessary to accommodate
 350          * the increase in memory pages.
 351          */
 352         mnode = PFN_2_MEM_NODE(pnum);
 353         PAGE_CTRS_ADJUST(base, npgs, rv);
 354         if (rv) {
 355 
 356                 mem_node_del_range(pt_base, pnum);
 357 
 358                 /* cleanup the  page counters */
 359                 page_ctrs_cleanup();
 360 
 361                 hat_unload(kas.a_hat, (caddr_t)pp, ptob(metapgs),
 362                     HAT_UNLOAD_UNMAP|HAT_UNLOAD_UNLOCK);
 363 
 364                 vmem_free(heap_arena, mapva, ptob(metapgs));
 365                 if (meta_alloc)
 366                         memseg_free_meta(metabase, metapgs);
 367                 kphysm_addmem_error_undospan(pt_base, tpgs);
 368 
 369                 return (KPHYSM_ERESOURCE);
 370         }
 371 
 372         /*
 373          * Update the phys_avail memory list.
 374          * The phys_install list was done at the start.
 375          */
 376 
 377         memlist_write_lock();
 378 
 379         mlret = memlist_add_span((uint64_t)(base) << PAGESHIFT,
 380             (uint64_t)(npgs) << PAGESHIFT, &phys_avail);
 381         ASSERT(mlret == MEML_SPANOP_OK);
 382 
 383         memlist_write_unlock();
 384 
 385         /* See if we can find a memseg to re-use. */
 386         if (meta_alloc) {
 387                 seg = memseg_reuse(0);
 388                 reuse = 1;      /* force unmapping of temp mapva */
 389                 flags = MEMSEG_DYNAMIC | MEMSEG_META_ALLOC;
 390                 /*
 391                  * There is a 1:1 fixed relationship between a pfn
 392                  * and a page_t VA.  The pfn is used as an index into
 393                  * the ppvm_base page_t table in order to calculate
 394                  * the page_t base address for a given pfn range.
 395                  */
 396                 segpp = ppvm_base + base;
 397         } else {
 398                 seg = memseg_reuse(metapgs);
 399                 reuse = (seg != NULL);
 400                 flags = MEMSEG_DYNAMIC | MEMSEG_META_INCL;
 401                 segpp = pp;
 402         }
 403 
 404         /*
 405          * Initialize the memseg structure representing this memory
 406          * and add it to the existing list of memsegs. Do some basic
 407          * initialization and add the memory to the system.
 408          * In order to prevent lock deadlocks, the add_physmem()
 409          * code is repeated here, but split into several stages.
 410          *
 411          * If a memseg is reused, invalidate memseg pointers in
 412          * all cpu vm caches.  We need to do this this since the check
 413          *      pp >= seg->pages && pp < seg->epages
 414          * used in various places is not atomic and so the first compare
 415          * can happen before reuse and the second compare after reuse.
 416          * The invalidation ensures that a memseg is not deferenced while
 417          * it's page/pfn pointers are changing.
 418          */
 419         if (seg == NULL) {
 420                 seg = memseg_alloc();
 421                 ASSERT(seg != NULL);
 422                 seg->msegflags = flags;
 423                 MEMSEG_DEBUG("memseg_get: alloc seg=0x%p, pages=0x%p",
 424                     (void *)seg, (void *)(seg->pages));
 425                 seg->pages = segpp;
 426         } else {
 427                 ASSERT(seg->msegflags == flags);
 428                 ASSERT(seg->pages_base == seg->pages_end);
 429                 MEMSEG_DEBUG("memseg_get: reuse seg=0x%p, pages=0x%p",
 430                     (void *)seg, (void *)(seg->pages));
 431                 if (meta_alloc) {
 432                         memseg_cpu_vm_flush();
 433                         seg->pages = segpp;
 434                 }
 435         }
 436 
 437         seg->epages = seg->pages + npgs;
 438         seg->pages_base = base;
 439         seg->pages_end = base + npgs;
 440 
 441         /*
 442          * Initialize metadata. The page_ts are set to locked state
 443          * ready to be freed.
 444          */
 445         bzero((caddr_t)pp, ptob(metapgs));
 446 
 447         pfn = seg->pages_base;
 448         /* Save the original pp base in case we reuse a memseg. */
 449         opp = pp;
 450         oepp = opp + npgs;
 451         for (pp = opp; pp < oepp; pp++) {
 452                 pp->p_pagenum = pfn;
 453                 pfn++;
 454                 page_iolock_init(pp);
 455                 while (!page_lock(pp, SE_EXCL, (kmutex_t *)NULL, P_RECLAIM))
 456                         continue;
 457                 pp->p_offset = (u_offset_t)-1;
 458         }
 459 
 460         if (reuse) {
 461                 /* Remap our page_ts to the re-used memseg VA space. */
 462                 pfn = pt_base;
 463                 vaddr = (caddr_t)seg->pages;
 464                 for (pnum = 0; pnum < metapgs; pnum++) {
 465                         if (meta_alloc)
 466                                 pfn = memseg_get_metapfn(metabase,
 467                                     (pgcnt_t)pnum);
 468                         hat_devload(kas.a_hat, vaddr, ptob(1), pfn,
 469                             PROT_READ | PROT_WRITE,
 470                             HAT_LOAD_REMAP | HAT_LOAD | HAT_LOAD_NOCONSIST);
 471                         pfn++;
 472                         vaddr += ptob(1);
 473                 }
 474 
 475                 hat_unload(kas.a_hat, (caddr_t)opp, ptob(metapgs),
 476                     HAT_UNLOAD_UNMAP|HAT_UNLOAD_UNLOCK);
 477 
 478                 vmem_free(heap_arena, mapva, ptob(metapgs));
 479         }
 480 
 481         hat_kpm_addmem_mseg_update(seg, nkpmpgs, kpm_pages_off);
 482 
 483         memsegs_lock(1);
 484 
 485         /*
 486          * The new memseg is inserted at the beginning of the list.
 487          * Not only does this save searching for the tail, but in the
 488          * case of a re-used memseg, it solves the problem of what
 489          * happens if some process has still got a pointer to the
 490          * memseg and follows the next pointer to continue traversing
 491          * the memsegs list.
 492          */
 493 
 494         hat_kpm_addmem_mseg_insert(seg);
 495 
 496         seg->next = memsegs;
 497         membar_producer();
 498 
 499         hat_kpm_addmem_memsegs_update(seg);
 500 
 501         memsegs = seg;
 502 
 503         build_pfn_hash();
 504 
 505         total_pages += npgs;
 506 
 507         /*
 508          * Recalculate the paging parameters now total_pages has changed.
 509          * This will also cause the clock hands to be reset before next use.
 510          */
 511         setupclock(1);
 512 
 513         memsegs_unlock(1);
 514 
 515         PLCNT_MODIFY_MAX(seg->pages_base, (long)npgs);
 516 
 517         /*
 518          * Free the pages outside the lock to avoid locking loops.
 519          */
 520         for (pp = seg->pages; pp < seg->epages; pp++) {
 521                 page_free(pp, 1);
 522         }
 523 
 524         /*
 525          * Now that we've updated the appropriate memory lists we
 526          * need to reset a number of globals, since we've increased memory.
 527          * Several have already been updated for us as noted above. The
 528          * globals we're interested in at this point are:
 529          *   physmax - highest page frame number.
 530          *   physinstalled - number of pages currently installed (done earlier)
 531          *   maxmem - max free pages in the system
 532          *   physmem - physical memory pages available
 533          *   availrmem - real memory available
 534          */
 535 
 536         mutex_enter(&freemem_lock);
 537         maxmem += npgs;
 538         physmem += npgs;
 539         availrmem += npgs;
 540         availrmem_initial += npgs;
 541 
 542         mutex_exit(&freemem_lock);
 543 
 544         dump_resize();
 545 
 546         page_freelist_coalesce_all(mnode);
 547 
 548         kphysm_setup_post_add(npgs);
 549 
 550         cmn_err(CE_CONT, "?kphysm_add_memory_dynamic: mem = %ldK "
 551             "(0x%" PRIx64 ")\n",
 552             physinstalled << (PAGESHIFT - 10),
 553             (uint64_t)physinstalled << PAGESHIFT);
 554 
 555         avmem = (uint64_t)freemem << PAGESHIFT;
 556         cmn_err(CE_CONT, "?kphysm_add_memory_dynamic: "
 557             "avail mem = %" PRId64 "\n", avmem);
 558 
 559         /*
 560          * Update lgroup generation number on single lgroup systems
 561          */
 562         if (nlgrps == 1)
 563                 lgrp_config(LGRP_CONFIG_GEN_UPDATE, 0, 0);
 564 
 565         /*
 566          * Inform DDI of update
 567          */
 568         ddi_mem_update((uint64_t)(pt_base) << PAGESHIFT,
 569             (uint64_t)(tpgs) << PAGESHIFT);
 570 
 571         delspan_unreserve(pt_base, tpgs);
 572 
 573         return (KPHYSM_OK);             /* Successfully added system memory */
 574 }
 575 
 576 /*
 577  * There are various error conditions in kphysm_add_memory_dynamic()
 578  * which require a rollback of already changed global state.
 579  */
 580 static void
 581 kphysm_addmem_error_undospan(pfn_t pt_base, pgcnt_t tpgs)
 582 {
 583         int mlret;
 584 
 585         /* Unreserve memory span. */
 586         memlist_write_lock();
 587 
 588         mlret = memlist_delete_span(
 589             (uint64_t)(pt_base) << PAGESHIFT,
 590             (uint64_t)(tpgs) << PAGESHIFT, &phys_install);
 591 
 592         ASSERT(mlret == MEML_SPANOP_OK);
 593         phys_install_has_changed();
 594         installed_top_size(phys_install, &physmax, &physinstalled);
 595 
 596         memlist_write_unlock();
 597         delspan_unreserve(pt_base, tpgs);
 598 }
 599 
 600 /*
 601  * Only return an available memseg of exactly the right size
 602  * if size is required.
 603  * When the meta data area has it's own virtual address space
 604  * we will need to manage this more carefully and do best fit
 605  * allocations, possibly splitting an available area.
 606  */
 607 struct memseg *
 608 memseg_reuse(pgcnt_t metapgs)
 609 {
 610         int type;
 611         struct memseg **segpp, *seg;
 612 
 613         mutex_enter(&memseg_lists_lock);
 614 
 615         segpp = &memseg_va_avail;
 616         for (; (seg = *segpp) != NULL; segpp = &seg->lnext) {
 617                 caddr_t end;
 618 
 619                 /*
 620                  * Make sure we are reusing the right segment type.
 621                  */
 622                 type = metapgs ? MEMSEG_META_INCL : MEMSEG_META_ALLOC;
 623 
 624                 if ((seg->msegflags & (MEMSEG_META_INCL | MEMSEG_META_ALLOC))
 625                     != type)
 626                         continue;
 627 
 628                 if (kpm_enable)
 629                         end = hat_kpm_mseg_reuse(seg);
 630                 else
 631                         end = (caddr_t)seg->epages;
 632 
 633                 /*
 634                  * Check for the right size if it is provided.
 635                  */
 636                 if (!metapgs || btopr(end - (caddr_t)seg->pages) == metapgs) {
 637                         *segpp = seg->lnext;
 638                         seg->lnext = NULL;
 639                         break;
 640                 }
 641         }
 642         mutex_exit(&memseg_lists_lock);
 643 
 644         return (seg);
 645 }
 646 
 647 static uint_t handle_gen;
 648 
 649 struct memdelspan {
 650         struct memdelspan *mds_next;
 651         pfn_t           mds_base;
 652         pgcnt_t         mds_npgs;
 653         uint_t          *mds_bitmap;
 654         uint_t          *mds_bitmap_retired;
 655 };
 656 
 657 #define NBPBMW          (sizeof (uint_t) * NBBY)
 658 #define MDS_BITMAPBYTES(MDSP) \
 659         ((((MDSP)->mds_npgs + NBPBMW - 1) / NBPBMW) * sizeof (uint_t))
 660 
 661 struct transit_list {
 662         struct transit_list     *trl_next;
 663         struct memdelspan       *trl_spans;
 664         int                     trl_collect;
 665 };
 666 
 667 struct transit_list_head {
 668         kmutex_t                trh_lock;
 669         struct transit_list     *trh_head;
 670 };
 671 
 672 static struct transit_list_head transit_list_head;
 673 
 674 struct mem_handle;
 675 static void transit_list_collect(struct mem_handle *, int);
 676 static void transit_list_insert(struct transit_list *);
 677 static void transit_list_remove(struct transit_list *);
 678 
 679 #ifdef DEBUG
 680 #define MEM_DEL_STATS
 681 #endif /* DEBUG */
 682 
 683 #ifdef MEM_DEL_STATS
 684 static int mem_del_stat_print = 0;
 685 struct mem_del_stat {
 686         uint_t  nloop;
 687         uint_t  need_free;
 688         uint_t  free_loop;
 689         uint_t  free_low;
 690         uint_t  free_failed;
 691         uint_t  ncheck;
 692         uint_t  nopaget;
 693         uint_t  lockfail;
 694         uint_t  nfree;
 695         uint_t  nreloc;
 696         uint_t  nrelocfail;
 697         uint_t  already_done;
 698         uint_t  first_notfree;
 699         uint_t  npplocked;
 700         uint_t  nlockreloc;
 701         uint_t  nnorepl;
 702         uint_t  nmodreloc;
 703         uint_t  ndestroy;
 704         uint_t  nputpage;
 705         uint_t  nnoreclaim;
 706         uint_t  ndelay;
 707         uint_t  demotefail;
 708         uint64_t nticks_total;
 709         uint64_t nticks_pgrp;
 710         uint_t  retired;
 711         uint_t  toxic;
 712         uint_t  failing;
 713         uint_t  modtoxic;
 714         uint_t  npplkdtoxic;
 715         uint_t  gptlmodfail;
 716         uint_t  gptllckfail;
 717 };
 718 /*
 719  * The stat values are only incremented in the delete thread
 720  * so no locking or atomic required.
 721  */
 722 #define MDSTAT_INCR(MHP, FLD)   (MHP)->mh_delstat.FLD++
 723 #define MDSTAT_TOTAL(MHP, ntck) ((MHP)->mh_delstat.nticks_total += (ntck))
 724 #define MDSTAT_PGRP(MHP, ntck)  ((MHP)->mh_delstat.nticks_pgrp += (ntck))
 725 static void mem_del_stat_print_func(struct mem_handle *);
 726 #define MDSTAT_PRINT(MHP)       mem_del_stat_print_func((MHP))
 727 #else /* MEM_DEL_STATS */
 728 #define MDSTAT_INCR(MHP, FLD)
 729 #define MDSTAT_TOTAL(MHP, ntck)
 730 #define MDSTAT_PGRP(MHP, ntck)
 731 #define MDSTAT_PRINT(MHP)
 732 #endif /* MEM_DEL_STATS */
 733 
 734 typedef enum mhnd_state {MHND_FREE = 0, MHND_INIT, MHND_STARTING,
 735         MHND_RUNNING, MHND_DONE, MHND_RELEASE} mhnd_state_t;
 736 
 737 /*
 738  * mh_mutex must be taken to examine or change mh_exthandle and mh_state.
 739  * The mutex may not be required for other fields, dependent on mh_state.
 740  */
 741 struct mem_handle {
 742         kmutex_t        mh_mutex;
 743         struct mem_handle *mh_next;
 744         memhandle_t     mh_exthandle;
 745         mhnd_state_t    mh_state;
 746         struct transit_list mh_transit;
 747         pgcnt_t         mh_phys_pages;
 748         pgcnt_t         mh_vm_pages;
 749         pgcnt_t         mh_hold_todo;
 750         void            (*mh_delete_complete)(void *, int error);
 751         void            *mh_delete_complete_arg;
 752         volatile uint_t mh_cancel;
 753         volatile uint_t mh_dr_aio_cleanup_cancel;
 754         volatile uint_t mh_aio_cleanup_done;
 755         kcondvar_t      mh_cv;
 756         kthread_id_t    mh_thread_id;
 757         page_t          *mh_deleted;    /* link through p_next */
 758 #ifdef MEM_DEL_STATS
 759         struct mem_del_stat mh_delstat;
 760 #endif /* MEM_DEL_STATS */
 761 };
 762 
 763 static struct mem_handle *mem_handle_head;
 764 static kmutex_t mem_handle_list_mutex;
 765 
 766 static struct mem_handle *
 767 kphysm_allocate_mem_handle()
 768 {
 769         struct mem_handle *mhp;
 770 
 771         mhp = kmem_zalloc(sizeof (struct mem_handle), KM_SLEEP);
 772         mutex_init(&mhp->mh_mutex, NULL, MUTEX_DEFAULT, NULL);
 773         mutex_enter(&mem_handle_list_mutex);
 774         mutex_enter(&mhp->mh_mutex);
 775         /* handle_gen is protected by list mutex. */
 776         mhp->mh_exthandle = (memhandle_t)(uintptr_t)(++handle_gen);
 777         mhp->mh_next = mem_handle_head;
 778         mem_handle_head = mhp;
 779         mutex_exit(&mem_handle_list_mutex);
 780 
 781         return (mhp);
 782 }
 783 
 784 static void
 785 kphysm_free_mem_handle(struct mem_handle *mhp)
 786 {
 787         struct mem_handle **mhpp;
 788 
 789         ASSERT(mutex_owned(&mhp->mh_mutex));
 790         ASSERT(mhp->mh_state == MHND_FREE);
 791         /*
 792          * Exit the mutex to preserve locking order. This is OK
 793          * here as once in the FREE state, the handle cannot
 794          * be found by a lookup.
 795          */
 796         mutex_exit(&mhp->mh_mutex);
 797 
 798         mutex_enter(&mem_handle_list_mutex);
 799         mhpp = &mem_handle_head;
 800         while (*mhpp != NULL && *mhpp != mhp)
 801                 mhpp = &(*mhpp)->mh_next;
 802         ASSERT(*mhpp == mhp);
 803         /*
 804          * No need to lock the handle (mh_mutex) as only
 805          * mh_next changing and this is the only thread that
 806          * can be referncing mhp.
 807          */
 808         *mhpp = mhp->mh_next;
 809         mutex_exit(&mem_handle_list_mutex);
 810 
 811         mutex_destroy(&mhp->mh_mutex);
 812         kmem_free(mhp, sizeof (struct mem_handle));
 813 }
 814 
 815 /*
 816  * This function finds the internal mem_handle corresponding to an
 817  * external handle and returns it with the mh_mutex held.
 818  */
 819 static struct mem_handle *
 820 kphysm_lookup_mem_handle(memhandle_t handle)
 821 {
 822         struct mem_handle *mhp;
 823 
 824         mutex_enter(&mem_handle_list_mutex);
 825         for (mhp = mem_handle_head; mhp != NULL; mhp = mhp->mh_next) {
 826                 if (mhp->mh_exthandle == handle) {
 827                         mutex_enter(&mhp->mh_mutex);
 828                         /*
 829                          * The state of the handle could have been changed
 830                          * by kphysm_del_release() while waiting for mh_mutex.
 831                          */
 832                         if (mhp->mh_state == MHND_FREE) {
 833                                 mutex_exit(&mhp->mh_mutex);
 834                                 continue;
 835                         }
 836                         break;
 837                 }
 838         }
 839         mutex_exit(&mem_handle_list_mutex);
 840         return (mhp);
 841 }
 842 
 843 int
 844 kphysm_del_gethandle(memhandle_t *xmhp)
 845 {
 846         struct mem_handle *mhp;
 847 
 848         mhp = kphysm_allocate_mem_handle();
 849         /*
 850          * The handle is allocated using KM_SLEEP, so cannot fail.
 851          * If the implementation is changed, the correct error to return
 852          * here would be KPHYSM_ENOHANDLES.
 853          */
 854         ASSERT(mhp->mh_state == MHND_FREE);
 855         mhp->mh_state = MHND_INIT;
 856         *xmhp = mhp->mh_exthandle;
 857         mutex_exit(&mhp->mh_mutex);
 858         return (KPHYSM_OK);
 859 }
 860 
 861 static int
 862 overlapping(pfn_t b1, pgcnt_t l1, pfn_t b2, pgcnt_t l2)
 863 {
 864         pfn_t e1, e2;
 865 
 866         e1 = b1 + l1;
 867         e2 = b2 + l2;
 868 
 869         return (!(b2 >= e1 || b1 >= e2));
 870 }
 871 
 872 static int can_remove_pgs(pgcnt_t);
 873 
 874 static struct memdelspan *
 875 span_to_install(pfn_t base, pgcnt_t npgs)
 876 {
 877         struct memdelspan *mdsp;
 878         struct memdelspan *mdsp_new;
 879         uint64_t address, size, thislen;
 880         struct memlist *mlp;
 881 
 882         mdsp_new = NULL;
 883 
 884         address = (uint64_t)base << PAGESHIFT;
 885         size = (uint64_t)npgs << PAGESHIFT;
 886         while (size != 0) {
 887                 memlist_read_lock();
 888                 for (mlp = phys_install; mlp != NULL; mlp = mlp->ml_next) {
 889                         if (address >= (mlp->ml_address + mlp->ml_size))
 890                                 continue;
 891                         if ((address + size) > mlp->ml_address)
 892                                 break;
 893                 }
 894                 if (mlp == NULL) {
 895                         address += size;
 896                         size = 0;
 897                         thislen = 0;
 898                 } else {
 899                         if (address < mlp->ml_address) {
 900                                 size -= (mlp->ml_address - address);
 901                                 address = mlp->ml_address;
 902                         }
 903                         ASSERT(address >= mlp->ml_address);
 904                         if ((address + size) >
 905                             (mlp->ml_address + mlp->ml_size)) {
 906                                 thislen =
 907                                     mlp->ml_size - (address - mlp->ml_address);
 908                         } else {
 909                                 thislen = size;
 910                         }
 911                 }
 912                 memlist_read_unlock();
 913                 /* TODO: phys_install could change now */
 914                 if (thislen == 0)
 915                         continue;
 916                 mdsp = kmem_zalloc(sizeof (struct memdelspan), KM_SLEEP);
 917                 mdsp->mds_base = btop(address);
 918                 mdsp->mds_npgs = btop(thislen);
 919                 mdsp->mds_next = mdsp_new;
 920                 mdsp_new = mdsp;
 921                 address += thislen;
 922                 size -= thislen;
 923         }
 924         return (mdsp_new);
 925 }
 926 
 927 static void
 928 free_delspans(struct memdelspan *mdsp)
 929 {
 930         struct memdelspan *amdsp;
 931 
 932         while ((amdsp = mdsp) != NULL) {
 933                 mdsp = amdsp->mds_next;
 934                 kmem_free(amdsp, sizeof (struct memdelspan));
 935         }
 936 }
 937 
 938 /*
 939  * Concatenate lists. No list ordering is required.
 940  */
 941 
 942 static void
 943 delspan_concat(struct memdelspan **mdspp, struct memdelspan *mdsp)
 944 {
 945         while (*mdspp != NULL)
 946                 mdspp = &(*mdspp)->mds_next;
 947 
 948         *mdspp = mdsp;
 949 }
 950 
 951 /*
 952  * Given a new list of delspans, check there is no overlap with
 953  * all existing span activity (add or delete) and then concatenate
 954  * the new spans to the given list.
 955  * Return 1 for OK, 0 if overlapping.
 956  */
 957 static int
 958 delspan_insert(
 959         struct transit_list *my_tlp,
 960         struct memdelspan *mdsp_new)
 961 {
 962         struct transit_list_head *trh;
 963         struct transit_list *tlp;
 964         int ret;
 965 
 966         trh = &transit_list_head;
 967 
 968         ASSERT(my_tlp != NULL);
 969         ASSERT(mdsp_new != NULL);
 970 
 971         ret = 1;
 972         mutex_enter(&trh->trh_lock);
 973         /* ASSERT(my_tlp->trl_spans == NULL || tlp_in_list(trh, my_tlp)); */
 974         for (tlp = trh->trh_head; tlp != NULL; tlp = tlp->trl_next) {
 975                 struct memdelspan *mdsp;
 976 
 977                 for (mdsp = tlp->trl_spans; mdsp != NULL;
 978                     mdsp = mdsp->mds_next) {
 979                         struct memdelspan *nmdsp;
 980 
 981                         for (nmdsp = mdsp_new; nmdsp != NULL;
 982                             nmdsp = nmdsp->mds_next) {
 983                                 if (overlapping(mdsp->mds_base, mdsp->mds_npgs,
 984                                     nmdsp->mds_base, nmdsp->mds_npgs)) {
 985                                         ret = 0;
 986                                         goto done;
 987                                 }
 988                         }
 989                 }
 990         }
 991 done:
 992         if (ret != 0) {
 993                 if (my_tlp->trl_spans == NULL)
 994                         transit_list_insert(my_tlp);
 995                 delspan_concat(&my_tlp->trl_spans, mdsp_new);
 996         }
 997         mutex_exit(&trh->trh_lock);
 998         return (ret);
 999 }
1000 
1001 static void
1002 delspan_remove(
1003         struct transit_list *my_tlp,
1004         pfn_t base,
1005         pgcnt_t npgs)
1006 {
1007         struct transit_list_head *trh;
1008         struct memdelspan *mdsp;
1009 
1010         trh = &transit_list_head;
1011 
1012         ASSERT(my_tlp != NULL);
1013 
1014         mutex_enter(&trh->trh_lock);
1015         if ((mdsp = my_tlp->trl_spans) != NULL) {
1016                 if (npgs == 0) {
1017                         my_tlp->trl_spans = NULL;
1018                         free_delspans(mdsp);
1019                         transit_list_remove(my_tlp);
1020                 } else {
1021                         struct memdelspan **prv;
1022 
1023                         prv = &my_tlp->trl_spans;
1024                         while (mdsp != NULL) {
1025                                 pfn_t p_end;
1026 
1027                                 p_end = mdsp->mds_base + mdsp->mds_npgs;
1028                                 if (mdsp->mds_base >= base &&
1029                                     p_end <= (base + npgs)) {
1030                                         *prv = mdsp->mds_next;
1031                                         mdsp->mds_next = NULL;
1032                                         free_delspans(mdsp);
1033                                 } else {
1034                                         prv = &mdsp->mds_next;
1035                                 }
1036                                 mdsp = *prv;
1037                         }
1038                         if (my_tlp->trl_spans == NULL)
1039                                 transit_list_remove(my_tlp);
1040                 }
1041         }
1042         mutex_exit(&trh->trh_lock);
1043 }
1044 
1045 /*
1046  * Reserve interface for add to stop delete before add finished.
1047  * This list is only accessed through the delspan_insert/remove
1048  * functions and so is fully protected by the mutex in struct transit_list.
1049  */
1050 
1051 static struct transit_list reserve_transit;
1052 
1053 static int
1054 delspan_reserve(pfn_t base, pgcnt_t npgs)
1055 {
1056         struct memdelspan *mdsp;
1057         int ret;
1058 
1059         mdsp = kmem_zalloc(sizeof (struct memdelspan), KM_SLEEP);
1060         mdsp->mds_base = base;
1061         mdsp->mds_npgs = npgs;
1062         if ((ret = delspan_insert(&reserve_transit, mdsp)) == 0) {
1063                 free_delspans(mdsp);
1064         }
1065         return (ret);
1066 }
1067 
1068 static void
1069 delspan_unreserve(pfn_t base, pgcnt_t npgs)
1070 {
1071         delspan_remove(&reserve_transit, base, npgs);
1072 }
1073 
1074 /*
1075  * Return whether memseg was created by kphysm_add_memory_dynamic().
1076  */
1077 static int
1078 memseg_is_dynamic(struct memseg *seg)
1079 {
1080         return (seg->msegflags & MEMSEG_DYNAMIC);
1081 }
1082 
1083 int
1084 kphysm_del_span(
1085         memhandle_t handle,
1086         pfn_t base,
1087         pgcnt_t npgs)
1088 {
1089         struct mem_handle *mhp;
1090         struct memseg *seg;
1091         struct memdelspan *mdsp;
1092         struct memdelspan *mdsp_new;
1093         pgcnt_t phys_pages, vm_pages;
1094         pfn_t p_end;
1095         page_t *pp;
1096         int ret;
1097 
1098         mhp = kphysm_lookup_mem_handle(handle);
1099         if (mhp == NULL) {
1100                 return (KPHYSM_EHANDLE);
1101         }
1102         if (mhp->mh_state != MHND_INIT) {
1103                 mutex_exit(&mhp->mh_mutex);
1104                 return (KPHYSM_ESEQUENCE);
1105         }
1106 
1107         /*
1108          * Intersect the span with the installed memory list (phys_install).
1109          */
1110         mdsp_new = span_to_install(base, npgs);
1111         if (mdsp_new == NULL) {
1112                 /*
1113                  * No physical memory in this range. Is this an
1114                  * error? If an attempt to start the delete is made
1115                  * for OK returns from del_span such as this, start will
1116                  * return an error.
1117                  * Could return KPHYSM_ENOWORK.
1118                  */
1119                 /*
1120                  * It is assumed that there are no error returns
1121                  * from span_to_install() due to kmem_alloc failure.
1122                  */
1123                 mutex_exit(&mhp->mh_mutex);
1124                 return (KPHYSM_OK);
1125         }
1126         /*
1127          * Does this span overlap an existing span?
1128          */
1129         if (delspan_insert(&mhp->mh_transit, mdsp_new) == 0) {
1130                 /*
1131                  * Differentiate between already on list for this handle
1132                  * (KPHYSM_EDUP) and busy elsewhere (KPHYSM_EBUSY).
1133                  */
1134                 ret = KPHYSM_EBUSY;
1135                 for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
1136                     mdsp = mdsp->mds_next) {
1137                         if (overlapping(mdsp->mds_base, mdsp->mds_npgs,
1138                             base, npgs)) {
1139                                 ret = KPHYSM_EDUP;
1140                                 break;
1141                         }
1142                 }
1143                 mutex_exit(&mhp->mh_mutex);
1144                 free_delspans(mdsp_new);
1145                 return (ret);
1146         }
1147         /*
1148          * At this point the spans in mdsp_new have been inserted into the
1149          * list of spans for this handle and thereby to the global list of
1150          * spans being processed. Each of these spans must now be checked
1151          * for relocatability. As a side-effect segments in the memseg list
1152          * may be split.
1153          *
1154          * Note that mdsp_new can no longer be used as it is now part of
1155          * a larger list. Select elements of this larger list based
1156          * on base and npgs.
1157          */
1158 restart:
1159         phys_pages = 0;
1160         vm_pages = 0;
1161         ret = KPHYSM_OK;
1162         for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
1163             mdsp = mdsp->mds_next) {
1164                 pgcnt_t pages_checked;
1165 
1166                 if (!overlapping(mdsp->mds_base, mdsp->mds_npgs, base, npgs)) {
1167                         continue;
1168                 }
1169                 p_end = mdsp->mds_base + mdsp->mds_npgs;
1170                 /*
1171                  * The pages_checked count is a hack. All pages should be
1172                  * checked for relocatability. Those not covered by memsegs
1173                  * should be tested with arch_kphysm_del_span_ok().
1174                  */
1175                 pages_checked = 0;
1176                 for (seg = memsegs; seg; seg = seg->next) {
1177                         pfn_t mseg_start;
1178 
1179                         if (seg->pages_base >= p_end ||
1180                             seg->pages_end <= mdsp->mds_base) {
1181                                 /* Span and memseg don't overlap. */
1182                                 continue;
1183                         }
1184                         mseg_start = memseg_get_start(seg);
1185                         /* Check that segment is suitable for delete. */
1186                         if (memseg_includes_meta(seg)) {
1187                                 /*
1188                                  * Check that this segment is completely
1189                                  * within the span.
1190                                  */
1191                                 if (mseg_start < mdsp->mds_base ||
1192                                     seg->pages_end > p_end) {
1193                                         ret = KPHYSM_EBUSY;
1194                                         break;
1195                                 }
1196                                 pages_checked += seg->pages_end - mseg_start;
1197                         } else {
1198                                 /*
1199                                  * If this segment is larger than the span,
1200                                  * try to split it. After the split, it
1201                                  * is necessary to restart.
1202                                  */
1203                                 if (seg->pages_base < mdsp->mds_base ||
1204                                     seg->pages_end > p_end) {
1205                                         pfn_t abase;
1206                                         pgcnt_t anpgs;
1207                                         int s_ret;
1208 
1209                                         /* Split required.  */
1210                                         if (mdsp->mds_base < seg->pages_base)
1211                                                 abase = seg->pages_base;
1212                                         else
1213                                                 abase = mdsp->mds_base;
1214                                         if (p_end > seg->pages_end)
1215                                                 anpgs = seg->pages_end - abase;
1216                                         else
1217                                                 anpgs = p_end - abase;
1218                                         s_ret = kphysm_split_memseg(abase,
1219                                             anpgs);
1220                                         if (s_ret == 0) {
1221                                                 /* Split failed. */
1222                                                 ret = KPHYSM_ERESOURCE;
1223                                                 break;
1224                                         }
1225                                         goto restart;
1226                                 }
1227                                 pages_checked +=
1228                                     seg->pages_end - seg->pages_base;
1229                         }
1230                         /*
1231                          * The memseg is wholly within the delete span.
1232                          * The individual pages can now be checked.
1233                          */
1234                         /* Cage test. */
1235                         for (pp = seg->pages; pp < seg->epages; pp++) {
1236                                 if (PP_ISNORELOC(pp)) {
1237                                         ret = KPHYSM_ENONRELOC;
1238                                         break;
1239                                 }
1240                         }
1241                         if (ret != KPHYSM_OK) {
1242                                 break;
1243                         }
1244                         phys_pages += (seg->pages_end - mseg_start);
1245                         vm_pages += MSEG_NPAGES(seg);
1246                 }
1247                 if (ret != KPHYSM_OK)
1248                         break;
1249                 if (pages_checked != mdsp->mds_npgs) {
1250                         ret = KPHYSM_ENONRELOC;
1251                         break;
1252                 }
1253         }
1254 
1255         if (ret == KPHYSM_OK) {
1256                 mhp->mh_phys_pages += phys_pages;
1257                 mhp->mh_vm_pages += vm_pages;
1258         } else {
1259                 /*
1260                  * Keep holding the mh_mutex to prevent it going away.
1261                  */
1262                 delspan_remove(&mhp->mh_transit, base, npgs);
1263         }
1264         mutex_exit(&mhp->mh_mutex);
1265         return (ret);
1266 }
1267 
1268 int
1269 kphysm_del_span_query(
1270         pfn_t base,
1271         pgcnt_t npgs,
1272         memquery_t *mqp)
1273 {
1274         struct memdelspan *mdsp;
1275         struct memdelspan *mdsp_new;
1276         int done_first_nonreloc;
1277 
1278         mqp->phys_pages = 0;
1279         mqp->managed = 0;
1280         mqp->nonrelocatable = 0;
1281         mqp->first_nonrelocatable = 0;
1282         mqp->last_nonrelocatable = 0;
1283 
1284         mdsp_new = span_to_install(base, npgs);
1285         /*
1286          * It is OK to proceed here if mdsp_new == NULL.
1287          */
1288         done_first_nonreloc = 0;
1289         for (mdsp = mdsp_new; mdsp != NULL; mdsp = mdsp->mds_next) {
1290                 pfn_t sbase;
1291                 pgcnt_t snpgs;
1292 
1293                 mqp->phys_pages += mdsp->mds_npgs;
1294                 sbase = mdsp->mds_base;
1295                 snpgs = mdsp->mds_npgs;
1296                 while (snpgs != 0) {
1297                         struct memseg *lseg, *seg;
1298                         pfn_t p_end;
1299                         page_t *pp;
1300                         pfn_t mseg_start;
1301 
1302                         p_end = sbase + snpgs;
1303                         /*
1304                          * Find the lowest addressed memseg that starts
1305                          * after sbase and account for it.
1306                          * This is to catch dynamic memsegs whose start
1307                          * is hidden.
1308                          */
1309                         seg = NULL;
1310                         for (lseg = memsegs; lseg != NULL; lseg = lseg->next) {
1311                                 if ((lseg->pages_base >= sbase) ||
1312                                     (lseg->pages_base < p_end &&
1313                                     lseg->pages_end > sbase)) {
1314                                         if (seg == NULL ||
1315                                             seg->pages_base > lseg->pages_base)
1316                                                 seg = lseg;
1317                                 }
1318                         }
1319                         if (seg != NULL) {
1320                                 mseg_start = memseg_get_start(seg);
1321                                 /*
1322                                  * Now have the full extent of the memseg so
1323                                  * do the range check.
1324                                  */
1325                                 if (mseg_start >= p_end ||
1326                                     seg->pages_end <= sbase) {
1327                                         /* Span does not overlap memseg. */
1328                                         seg = NULL;
1329                                 }
1330                         }
1331                         /*
1332                          * Account for gap either before the segment if
1333                          * there is one or to the end of the span.
1334                          */
1335                         if (seg == NULL || mseg_start > sbase) {
1336                                 pfn_t a_end;
1337 
1338                                 a_end = (seg == NULL) ? p_end : mseg_start;
1339                                 /*
1340                                  * Check with arch layer for relocatability.
1341                                  */
1342                                 if (arch_kphysm_del_span_ok(sbase,
1343                                     (a_end - sbase))) {
1344                                         /*
1345                                          * No non-relocatble pages in this
1346                                          * area, avoid the fine-grained
1347                                          * test.
1348                                          */
1349                                         snpgs -= (a_end - sbase);
1350                                         sbase = a_end;
1351                                 }
1352                                 while (sbase < a_end) {
1353                                         if (!arch_kphysm_del_span_ok(sbase,
1354                                             1)) {
1355                                                 mqp->nonrelocatable++;
1356                                                 if (!done_first_nonreloc) {
1357                                                         mqp->
1358                                                             first_nonrelocatable
1359                                                             = sbase;
1360                                                         done_first_nonreloc = 1;
1361                                                 }
1362                                                 mqp->last_nonrelocatable =
1363                                                     sbase;
1364                                         }
1365                                         sbase++;
1366                                         snpgs--;
1367                                 }
1368                         }
1369                         if (seg != NULL) {
1370                                 ASSERT(mseg_start <= sbase);
1371                                 if (seg->pages_base != mseg_start &&
1372                                     seg->pages_base > sbase) {
1373                                         pgcnt_t skip_pgs;
1374 
1375                                         /*
1376                                          * Skip the page_t area of a
1377                                          * dynamic memseg.
1378                                          */
1379                                         skip_pgs = seg->pages_base - sbase;
1380                                         if (snpgs <= skip_pgs) {
1381                                                 sbase += snpgs;
1382                                                 snpgs = 0;
1383                                                 continue;
1384                                         }
1385                                         snpgs -= skip_pgs;
1386                                         sbase += skip_pgs;
1387                                 }
1388                                 ASSERT(snpgs != 0);
1389                                 ASSERT(seg->pages_base <= sbase);
1390                                 /*
1391                                  * The individual pages can now be checked.
1392                                  */
1393                                 for (pp = seg->pages +
1394                                     (sbase - seg->pages_base);
1395                                     snpgs != 0 && pp < seg->epages; pp++) {
1396                                         mqp->managed++;
1397                                         if (PP_ISNORELOC(pp)) {
1398                                                 mqp->nonrelocatable++;
1399                                                 if (!done_first_nonreloc) {
1400                                                         mqp->
1401                                                             first_nonrelocatable
1402                                                             = sbase;
1403                                                         done_first_nonreloc = 1;
1404                                                 }
1405                                                 mqp->last_nonrelocatable =
1406                                                     sbase;
1407                                         }
1408                                         sbase++;
1409                                         snpgs--;
1410                                 }
1411                         }
1412                 }
1413         }
1414 
1415         free_delspans(mdsp_new);
1416 
1417         return (KPHYSM_OK);
1418 }
1419 
1420 /*
1421  * This release function can be called at any stage as follows:
1422  *      _gethandle only called
1423  *      _span(s) only called
1424  *      _start called but failed
1425  *      delete thread exited
1426  */
1427 int
1428 kphysm_del_release(memhandle_t handle)
1429 {
1430         struct mem_handle *mhp;
1431 
1432         mhp = kphysm_lookup_mem_handle(handle);
1433         if (mhp == NULL) {
1434                 return (KPHYSM_EHANDLE);
1435         }
1436         switch (mhp->mh_state) {
1437         case MHND_STARTING:
1438         case MHND_RUNNING:
1439                 mutex_exit(&mhp->mh_mutex);
1440                 return (KPHYSM_ENOTFINISHED);
1441         case MHND_FREE:
1442                 ASSERT(mhp->mh_state != MHND_FREE);
1443                 mutex_exit(&mhp->mh_mutex);
1444                 return (KPHYSM_EHANDLE);
1445         case MHND_INIT:
1446                 break;
1447         case MHND_DONE:
1448                 break;
1449         case MHND_RELEASE:
1450                 mutex_exit(&mhp->mh_mutex);
1451                 return (KPHYSM_ESEQUENCE);
1452         default:
1453 #ifdef DEBUG
1454                 cmn_err(CE_WARN, "kphysm_del_release(0x%p) state corrupt %d",
1455                     (void *)mhp, mhp->mh_state);
1456 #endif /* DEBUG */
1457                 mutex_exit(&mhp->mh_mutex);
1458                 return (KPHYSM_EHANDLE);
1459         }
1460         /*
1461          * Set state so that we can wait if necessary.
1462          * Also this means that we have read/write access to all
1463          * fields except mh_exthandle and mh_state.
1464          */
1465         mhp->mh_state = MHND_RELEASE;
1466         /*
1467          * The mem_handle cannot be de-allocated by any other operation
1468          * now, so no need to hold mh_mutex.
1469          */
1470         mutex_exit(&mhp->mh_mutex);
1471 
1472         delspan_remove(&mhp->mh_transit, 0, 0);
1473         mhp->mh_phys_pages = 0;
1474         mhp->mh_vm_pages = 0;
1475         mhp->mh_hold_todo = 0;
1476         mhp->mh_delete_complete = NULL;
1477         mhp->mh_delete_complete_arg = NULL;
1478         mhp->mh_cancel = 0;
1479 
1480         mutex_enter(&mhp->mh_mutex);
1481         ASSERT(mhp->mh_state == MHND_RELEASE);
1482         mhp->mh_state = MHND_FREE;
1483 
1484         kphysm_free_mem_handle(mhp);
1485 
1486         return (KPHYSM_OK);
1487 }
1488 
1489 /*
1490  * This cancel function can only be called with the thread running.
1491  */
1492 int
1493 kphysm_del_cancel(memhandle_t handle)
1494 {
1495         struct mem_handle *mhp;
1496 
1497         mhp = kphysm_lookup_mem_handle(handle);
1498         if (mhp == NULL) {
1499                 return (KPHYSM_EHANDLE);
1500         }
1501         if (mhp->mh_state != MHND_STARTING && mhp->mh_state != MHND_RUNNING) {
1502                 mutex_exit(&mhp->mh_mutex);
1503                 return (KPHYSM_ENOTRUNNING);
1504         }
1505         /*
1506          * Set the cancel flag and wake the delete thread up.
1507          * The thread may be waiting on I/O, so the effect of the cancel
1508          * may be delayed.
1509          */
1510         if (mhp->mh_cancel == 0) {
1511                 mhp->mh_cancel = KPHYSM_ECANCELLED;
1512                 cv_signal(&mhp->mh_cv);
1513         }
1514         mutex_exit(&mhp->mh_mutex);
1515         return (KPHYSM_OK);
1516 }
1517 
1518 int
1519 kphysm_del_status(
1520         memhandle_t handle,
1521         memdelstat_t *mdstp)
1522 {
1523         struct mem_handle *mhp;
1524 
1525         mhp = kphysm_lookup_mem_handle(handle);
1526         if (mhp == NULL) {
1527                 return (KPHYSM_EHANDLE);
1528         }
1529         /*
1530          * Calling kphysm_del_status() is allowed before the delete
1531          * is started to allow for status display.
1532          */
1533         if (mhp->mh_state != MHND_INIT && mhp->mh_state != MHND_STARTING &&
1534             mhp->mh_state != MHND_RUNNING) {
1535                 mutex_exit(&mhp->mh_mutex);
1536                 return (KPHYSM_ENOTRUNNING);
1537         }
1538         mdstp->phys_pages = mhp->mh_phys_pages;
1539         mdstp->managed = mhp->mh_vm_pages;
1540         mdstp->collected = mhp->mh_vm_pages - mhp->mh_hold_todo;
1541         mutex_exit(&mhp->mh_mutex);
1542         return (KPHYSM_OK);
1543 }
1544 
1545 static int mem_delete_additional_pages = 100;
1546 
1547 static int
1548 can_remove_pgs(pgcnt_t npgs)
1549 {
1550         /*
1551          * If all pageable pages were paged out, freemem would
1552          * equal availrmem.  There is a minimum requirement for
1553          * availrmem.
1554          */
1555         if ((availrmem - (tune.t_minarmem + mem_delete_additional_pages))
1556             < npgs)
1557                 return (0);
1558         /* TODO: check swap space, etc. */
1559         return (1);
1560 }
1561 
1562 static int
1563 get_availrmem(pgcnt_t npgs)
1564 {
1565         int ret;
1566 
1567         mutex_enter(&freemem_lock);
1568         ret = can_remove_pgs(npgs);
1569         if (ret != 0)
1570                 availrmem -= npgs;
1571         mutex_exit(&freemem_lock);
1572         return (ret);
1573 }
1574 
1575 static void
1576 put_availrmem(pgcnt_t npgs)
1577 {
1578         mutex_enter(&freemem_lock);
1579         availrmem += npgs;
1580         mutex_exit(&freemem_lock);
1581 }
1582 
1583 #define FREEMEM_INCR    100
1584 static pgcnt_t freemem_incr = FREEMEM_INCR;
1585 #define DEL_FREE_WAIT_FRAC      4
1586 #define DEL_FREE_WAIT_TICKS     ((hz+DEL_FREE_WAIT_FRAC-1)/DEL_FREE_WAIT_FRAC)
1587 
1588 #define DEL_BUSY_WAIT_FRAC      20
1589 #define DEL_BUSY_WAIT_TICKS     ((hz+DEL_BUSY_WAIT_FRAC-1)/DEL_BUSY_WAIT_FRAC)
1590 
1591 static void kphysm_del_cleanup(struct mem_handle *);
1592 
1593 static void page_delete_collect(page_t *, struct mem_handle *);
1594 
1595 static pgcnt_t
1596 delthr_get_freemem(struct mem_handle *mhp)
1597 {
1598         pgcnt_t free_get;
1599         int ret;
1600 
1601         ASSERT(MUTEX_HELD(&mhp->mh_mutex));
1602 
1603         MDSTAT_INCR(mhp, need_free);
1604         /*
1605          * Get up to freemem_incr pages.
1606          */
1607         free_get = freemem_incr;
1608         if (free_get > mhp->mh_hold_todo)
1609                 free_get = mhp->mh_hold_todo;
1610         /*
1611          * Take free_get pages away from freemem,
1612          * waiting if necessary.
1613          */
1614 
1615         while (!mhp->mh_cancel) {
1616                 mutex_exit(&mhp->mh_mutex);
1617                 MDSTAT_INCR(mhp, free_loop);
1618                 /*
1619                  * Duplicate test from page_create_throttle()
1620                  * but don't override with !PG_WAIT.
1621                  */
1622                 if (freemem < (free_get + throttlefree)) {
1623                         MDSTAT_INCR(mhp, free_low);
1624                         ret = 0;
1625                 } else {
1626                         ret = page_create_wait(free_get, 0);
1627                         if (ret == 0) {
1628                                 /* EMPTY */
1629                                 MDSTAT_INCR(mhp, free_failed);
1630                         }
1631                 }
1632                 if (ret != 0) {
1633                         mutex_enter(&mhp->mh_mutex);
1634                         return (free_get);
1635                 }
1636 
1637                 /*
1638                  * Put pressure on pageout.
1639                  */
1640                 page_needfree(free_get);
1641                 cv_signal(&proc_pageout->p_cv);
1642 
1643                 mutex_enter(&mhp->mh_mutex);
1644                 (void) cv_reltimedwait(&mhp->mh_cv, &mhp->mh_mutex,
1645                     DEL_FREE_WAIT_TICKS, TR_CLOCK_TICK);
1646                 mutex_exit(&mhp->mh_mutex);
1647                 page_needfree(-(spgcnt_t)free_get);
1648 
1649                 mutex_enter(&mhp->mh_mutex);
1650         }
1651         return (0);
1652 }
1653 
1654 #define DR_AIO_CLEANUP_DELAY    25000   /* 0.025secs, in usec */
1655 #define DR_AIO_CLEANUP_MAXLOOPS_NODELAY 100
1656 /*
1657  * This function is run as a helper thread for delete_memory_thread.
1658  * It is needed in order to force kaio cleanup, so that pages used in kaio
1659  * will be unlocked and subsequently relocated by delete_memory_thread.
1660  * The address of the delete_memory_threads's mem_handle is passed in to
1661  * this thread function, and is used to set the mh_aio_cleanup_done member
1662  * prior to calling thread_exit().
1663  */
1664 static void
1665 dr_aio_cleanup_thread(caddr_t amhp)
1666 {
1667         proc_t *procp;
1668         int (*aio_cleanup_dr_delete_memory)(proc_t *);
1669         int cleaned;
1670         int n = 0;
1671         struct mem_handle *mhp;
1672         volatile uint_t *pcancel;
1673 
1674         mhp = (struct mem_handle *)amhp;
1675         ASSERT(mhp != NULL);
1676         pcancel = &mhp->mh_dr_aio_cleanup_cancel;
1677         if (modload("sys", "kaio") == -1) {
1678                 mhp->mh_aio_cleanup_done = 1;
1679                 cmn_err(CE_WARN, "dr_aio_cleanup_thread: cannot load kaio");
1680                 thread_exit();
1681         }
1682         aio_cleanup_dr_delete_memory = (int (*)(proc_t *))
1683             modgetsymvalue("aio_cleanup_dr_delete_memory", 0);
1684         if (aio_cleanup_dr_delete_memory == NULL) {
1685                 mhp->mh_aio_cleanup_done = 1;
1686                 cmn_err(CE_WARN,
1687             "aio_cleanup_dr_delete_memory not found in kaio");
1688                 thread_exit();
1689         }
1690         do {
1691                 cleaned = 0;
1692                 mutex_enter(&pidlock);
1693                 for (procp = practive; (*pcancel == 0) && (procp != NULL);
1694                     procp = procp->p_next) {
1695                         mutex_enter(&procp->p_lock);
1696                         if (procp->p_aio != NULL) {
1697                                 /* cleanup proc's outstanding kaio */
1698                                 cleaned +=
1699                                     (*aio_cleanup_dr_delete_memory)(procp);
1700                         }
1701                         mutex_exit(&procp->p_lock);
1702                 }
1703                 mutex_exit(&pidlock);
1704                 if ((*pcancel == 0) &&
1705                     (!cleaned || (++n == DR_AIO_CLEANUP_MAXLOOPS_NODELAY))) {
1706                         /* delay a bit before retrying all procs again */
1707                         delay(drv_usectohz(DR_AIO_CLEANUP_DELAY));
1708                         n = 0;
1709                 }
1710         } while (*pcancel == 0);
1711         mhp->mh_aio_cleanup_done = 1;
1712         thread_exit();
1713 }
1714 
1715 static void
1716 delete_memory_thread(caddr_t amhp)
1717 {
1718         struct mem_handle *mhp;
1719         struct memdelspan *mdsp;
1720         callb_cpr_t cprinfo;
1721         page_t *pp_targ;
1722         spgcnt_t freemem_left;
1723         void (*del_complete_funcp)(void *, int error);
1724         void *del_complete_arg;
1725         int comp_code;
1726         int ret;
1727         int first_scan;
1728         uint_t szc;
1729 #ifdef MEM_DEL_STATS
1730         uint64_t start_total, ntick_total;
1731         uint64_t start_pgrp, ntick_pgrp;
1732 #endif /* MEM_DEL_STATS */
1733 
1734         mhp = (struct mem_handle *)amhp;
1735 
1736 #ifdef MEM_DEL_STATS
1737         start_total = ddi_get_lbolt();
1738 #endif /* MEM_DEL_STATS */
1739 
1740         CALLB_CPR_INIT(&cprinfo, &mhp->mh_mutex,
1741             callb_generic_cpr, "memdel");
1742 
1743         mutex_enter(&mhp->mh_mutex);
1744         ASSERT(mhp->mh_state == MHND_STARTING);
1745 
1746         mhp->mh_state = MHND_RUNNING;
1747         mhp->mh_thread_id = curthread;
1748 
1749         mhp->mh_hold_todo = mhp->mh_vm_pages;
1750         mutex_exit(&mhp->mh_mutex);
1751 
1752         /* Allocate the remap pages now, if necessary. */
1753         memseg_remap_init();
1754 
1755         /*
1756          * Subtract from availrmem now if possible as availrmem
1757          * may not be available by the end of the delete.
1758          */
1759         if (!get_availrmem(mhp->mh_vm_pages)) {
1760                 comp_code = KPHYSM_ENOTVIABLE;
1761                 mutex_enter(&mhp->mh_mutex);
1762                 goto early_exit;
1763         }
1764 
1765         ret = kphysm_setup_pre_del(mhp->mh_vm_pages);
1766 
1767         mutex_enter(&mhp->mh_mutex);
1768 
1769         if (ret != 0) {
1770                 mhp->mh_cancel = KPHYSM_EREFUSED;
1771                 goto refused;
1772         }
1773 
1774         transit_list_collect(mhp, 1);
1775 
1776         for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
1777             mdsp = mdsp->mds_next) {
1778                 ASSERT(mdsp->mds_bitmap == NULL);
1779                 mdsp->mds_bitmap = kmem_zalloc(MDS_BITMAPBYTES(mdsp), KM_SLEEP);
1780                 mdsp->mds_bitmap_retired = kmem_zalloc(MDS_BITMAPBYTES(mdsp),
1781                     KM_SLEEP);
1782         }
1783 
1784         first_scan = 1;
1785         freemem_left = 0;
1786         /*
1787          * Start dr_aio_cleanup_thread, which periodically iterates
1788          * through the process list and invokes aio cleanup.  This
1789          * is needed in order to avoid a deadly embrace between the
1790          * delete_memory_thread (waiting on writer lock for page, with the
1791          * exclusive-wanted bit set), kaio read request threads (waiting for a
1792          * reader lock on the same page that is wanted by the
1793          * delete_memory_thread), and threads waiting for kaio completion
1794          * (blocked on spt_amp->lock).
1795          */
1796         mhp->mh_dr_aio_cleanup_cancel = 0;
1797         mhp->mh_aio_cleanup_done = 0;
1798         (void) thread_create(NULL, 0, dr_aio_cleanup_thread,
1799             (caddr_t)mhp, 0, &p0, TS_RUN, maxclsyspri - 1);
1800         while ((mhp->mh_hold_todo != 0) && (mhp->mh_cancel == 0)) {
1801                 pgcnt_t collected;
1802 
1803                 MDSTAT_INCR(mhp, nloop);
1804                 collected = 0;
1805                 for (mdsp = mhp->mh_transit.trl_spans; (mdsp != NULL) &&
1806                     (mhp->mh_cancel == 0); mdsp = mdsp->mds_next) {
1807                         pfn_t pfn, p_end;
1808 
1809                         p_end = mdsp->mds_base + mdsp->mds_npgs;
1810                         for (pfn = mdsp->mds_base; (pfn < p_end) &&
1811                             (mhp->mh_cancel == 0); pfn++) {
1812                                 page_t *pp, *tpp, *tpp_targ;
1813                                 pgcnt_t bit;
1814                                 struct vnode *vp;
1815                                 u_offset_t offset;
1816                                 int mod, result;
1817                                 spgcnt_t pgcnt;
1818 
1819                                 bit = pfn - mdsp->mds_base;
1820                                 if ((mdsp->mds_bitmap[bit / NBPBMW] &
1821                                     (1 << (bit % NBPBMW))) != 0) {
1822                                         MDSTAT_INCR(mhp, already_done);
1823                                         continue;
1824                                 }
1825                                 if (freemem_left == 0) {
1826                                         freemem_left += delthr_get_freemem(mhp);
1827                                         if (freemem_left == 0)
1828                                                 break;
1829                                 }
1830 
1831                                 /*
1832                                  * Release mh_mutex - some of this
1833                                  * stuff takes some time (eg PUTPAGE).
1834                                  */
1835 
1836                                 mutex_exit(&mhp->mh_mutex);
1837                                 MDSTAT_INCR(mhp, ncheck);
1838 
1839                                 pp = page_numtopp_nolock(pfn);
1840                                 if (pp == NULL) {
1841                                         /*
1842                                          * Not covered by a page_t - will
1843                                          * be dealt with elsewhere.
1844                                          */
1845                                         MDSTAT_INCR(mhp, nopaget);
1846                                         mutex_enter(&mhp->mh_mutex);
1847                                         mdsp->mds_bitmap[bit / NBPBMW] |=
1848                                             (1 << (bit % NBPBMW));
1849                                         continue;
1850                                 }
1851 
1852                                 if (!page_try_reclaim_lock(pp, SE_EXCL,
1853                                     SE_EXCL_WANTED | SE_RETIRED)) {
1854                                         /*
1855                                          * Page in use elsewhere.  Skip it.
1856                                          */
1857                                         MDSTAT_INCR(mhp, lockfail);
1858                                         mutex_enter(&mhp->mh_mutex);
1859                                         continue;
1860                                 }
1861                                 /*
1862                                  * See if the cage expanded into the delete.
1863                                  * This can happen as we have to allow the
1864                                  * cage to expand.
1865                                  */
1866                                 if (PP_ISNORELOC(pp)) {
1867                                         page_unlock(pp);
1868                                         mutex_enter(&mhp->mh_mutex);
1869                                         mhp->mh_cancel = KPHYSM_ENONRELOC;
1870                                         break;
1871                                 }
1872                                 if (PP_RETIRED(pp)) {
1873                                         /*
1874                                          * Page has been retired and is
1875                                          * not part of the cage so we
1876                                          * can now do the accounting for
1877                                          * it.
1878                                          */
1879                                         MDSTAT_INCR(mhp, retired);
1880                                         mutex_enter(&mhp->mh_mutex);
1881                                         mdsp->mds_bitmap[bit / NBPBMW]
1882                                             |= (1 << (bit % NBPBMW));
1883                                         mdsp->mds_bitmap_retired[bit /
1884                                             NBPBMW] |=
1885                                             (1 << (bit % NBPBMW));
1886                                         mhp->mh_hold_todo--;
1887                                         continue;
1888                                 }
1889                                 ASSERT(freemem_left != 0);
1890                                 if (PP_ISFREE(pp)) {
1891                                         /*
1892                                          * Like page_reclaim() only 'freemem'
1893                                          * processing is already done.
1894                                          */
1895                                         MDSTAT_INCR(mhp, nfree);
1896                                 free_page_collect:
1897                                         if (PP_ISAGED(pp)) {
1898                                                 page_list_sub(pp,
1899                                                     PG_FREE_LIST);
1900                                         } else {
1901                                                 page_list_sub(pp,
1902                                                     PG_CACHE_LIST);
1903                                         }
1904                                         PP_CLRFREE(pp);
1905                                         PP_CLRAGED(pp);
1906                                         collected++;
1907                                         mutex_enter(&mhp->mh_mutex);
1908                                         page_delete_collect(pp, mhp);
1909                                         mdsp->mds_bitmap[bit / NBPBMW] |=
1910                                             (1 << (bit % NBPBMW));
1911                                         freemem_left--;
1912                                         continue;
1913                                 }
1914                                 ASSERT(pp->p_vnode != NULL);
1915                                 if (first_scan) {
1916                                         MDSTAT_INCR(mhp, first_notfree);
1917                                         page_unlock(pp);
1918                                         mutex_enter(&mhp->mh_mutex);
1919                                         continue;
1920                                 }
1921                                 /*
1922                                  * Keep stats on pages encountered that
1923                                  * are marked for retirement.
1924                                  */
1925                                 if (PP_TOXIC(pp)) {
1926                                         MDSTAT_INCR(mhp, toxic);
1927                                 } else if (PP_PR_REQ(pp)) {
1928                                         MDSTAT_INCR(mhp, failing);
1929                                 }
1930                                 /*
1931                                  * In certain cases below, special exceptions
1932                                  * are made for pages that are toxic.  This
1933                                  * is because the current meaning of toxic
1934                                  * is that an uncorrectable error has been
1935                                  * previously associated with the page.
1936                                  */
1937                                 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) {
1938                                         if (!PP_TOXIC(pp)) {
1939                                                 /*
1940                                                  * Must relocate locked in
1941                                                  * memory pages.
1942                                                  */
1943 #ifdef MEM_DEL_STATS
1944                                                 start_pgrp = ddi_get_lbolt();
1945 #endif /* MEM_DEL_STATS */
1946                                                 /*
1947                                                  * Lock all constituent pages
1948                                                  * of a large page to ensure
1949                                                  * that p_szc won't change.
1950                                                  */
1951                                                 if (!group_page_trylock(pp,
1952                                                     SE_EXCL)) {
1953                                                         MDSTAT_INCR(mhp,
1954                                                             gptllckfail);
1955                                                         page_unlock(pp);
1956                                                         mutex_enter(
1957                                                             &mhp->mh_mutex);
1958                                                         continue;
1959                                                 }
1960                                                 MDSTAT_INCR(mhp, npplocked);
1961                                                 pp_targ =
1962                                                     page_get_replacement_page(
1963                                                     pp, NULL, 0);
1964                                                 if (pp_targ != NULL) {
1965 #ifdef MEM_DEL_STATS
1966                                                         ntick_pgrp =
1967                                                             (uint64_t)
1968                                                             ddi_get_lbolt() -
1969                                                             start_pgrp;
1970 #endif /* MEM_DEL_STATS */
1971                                                         MDSTAT_PGRP(mhp,
1972                                                             ntick_pgrp);
1973                                                         MDSTAT_INCR(mhp,
1974                                                             nlockreloc);
1975                                                         goto reloc;
1976                                                 }
1977                                                 group_page_unlock(pp);
1978                                                 page_unlock(pp);
1979 #ifdef MEM_DEL_STATS
1980                                                 ntick_pgrp =
1981                                                     (uint64_t)ddi_get_lbolt() -
1982                                                     start_pgrp;
1983 #endif /* MEM_DEL_STATS */
1984                                                 MDSTAT_PGRP(mhp, ntick_pgrp);
1985                                                 MDSTAT_INCR(mhp, nnorepl);
1986                                                 mutex_enter(&mhp->mh_mutex);
1987                                                 continue;
1988                                         } else {
1989                                                 /*
1990                                                  * Cannot do anything about
1991                                                  * this page because it is
1992                                                  * toxic.
1993                                                  */
1994                                                 MDSTAT_INCR(mhp, npplkdtoxic);
1995                                                 page_unlock(pp);
1996                                                 mutex_enter(&mhp->mh_mutex);
1997                                                 continue;
1998                                         }
1999                                 }
2000                                 /*
2001                                  * Unload the mappings and check if mod bit
2002                                  * is set.
2003                                  */
2004                                 ASSERT(!PP_ISKAS(pp));
2005                                 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
2006                                 mod = hat_ismod(pp);
2007 
2008 #ifdef MEM_DEL_STATS
2009                                 start_pgrp = ddi_get_lbolt();
2010 #endif /* MEM_DEL_STATS */
2011                                 if (mod && !PP_TOXIC(pp)) {
2012                                         /*
2013                                          * Lock all constituent pages
2014                                          * of a large page to ensure
2015                                          * that p_szc won't change.
2016                                          */
2017                                         if (!group_page_trylock(pp, SE_EXCL)) {
2018                                                 MDSTAT_INCR(mhp, gptlmodfail);
2019                                                 page_unlock(pp);
2020                                                 mutex_enter(&mhp->mh_mutex);
2021                                                 continue;
2022                                         }
2023                                         pp_targ = page_get_replacement_page(pp,
2024                                             NULL, 0);
2025                                         if (pp_targ != NULL) {
2026                                                 MDSTAT_INCR(mhp, nmodreloc);
2027 #ifdef MEM_DEL_STATS
2028                                                 ntick_pgrp =
2029                                                     (uint64_t)ddi_get_lbolt() -
2030                                                     start_pgrp;
2031 #endif /* MEM_DEL_STATS */
2032                                                 MDSTAT_PGRP(mhp, ntick_pgrp);
2033                                                 goto reloc;
2034                                         }
2035                                         group_page_unlock(pp);
2036                                 }
2037 
2038                                 if (!page_try_demote_pages(pp)) {
2039                                         MDSTAT_INCR(mhp, demotefail);
2040                                         page_unlock(pp);
2041 #ifdef MEM_DEL_STATS
2042                                         ntick_pgrp = (uint64_t)ddi_get_lbolt() -
2043                                             start_pgrp;
2044 #endif /* MEM_DEL_STATS */
2045                                         MDSTAT_PGRP(mhp, ntick_pgrp);
2046                                         mutex_enter(&mhp->mh_mutex);
2047                                         continue;
2048                                 }
2049 
2050                                 /*
2051                                  * Regular 'page-out'.
2052                                  */
2053                                 if (!mod) {
2054                                         MDSTAT_INCR(mhp, ndestroy);
2055                                         page_destroy(pp, 1);
2056                                         /*
2057                                          * page_destroy was called with
2058                                          * dontfree. As long as p_lckcnt
2059                                          * and p_cowcnt are both zero, the
2060                                          * only additional action of
2061                                          * page_destroy with !dontfree is to
2062                                          * call page_free, so we can collect
2063                                          * the page here.
2064                                          */
2065                                         collected++;
2066 #ifdef MEM_DEL_STATS
2067                                         ntick_pgrp = (uint64_t)ddi_get_lbolt() -
2068                                             start_pgrp;
2069 #endif /* MEM_DEL_STATS */
2070                                         MDSTAT_PGRP(mhp, ntick_pgrp);
2071                                         mutex_enter(&mhp->mh_mutex);
2072                                         page_delete_collect(pp, mhp);
2073                                         mdsp->mds_bitmap[bit / NBPBMW] |=
2074                                             (1 << (bit % NBPBMW));
2075                                         continue;
2076                                 }
2077                                 /*
2078                                  * The page is toxic and the mod bit is
2079                                  * set, we cannot do anything here to deal
2080                                  * with it.
2081                                  */
2082                                 if (PP_TOXIC(pp)) {
2083                                         page_unlock(pp);
2084 #ifdef MEM_DEL_STATS
2085                                         ntick_pgrp = (uint64_t)ddi_get_lbolt() -
2086                                             start_pgrp;
2087 #endif /* MEM_DEL_STATS */
2088                                         MDSTAT_PGRP(mhp, ntick_pgrp);
2089                                         MDSTAT_INCR(mhp, modtoxic);
2090                                         mutex_enter(&mhp->mh_mutex);
2091                                         continue;
2092                                 }
2093                                 MDSTAT_INCR(mhp, nputpage);
2094                                 vp = pp->p_vnode;
2095                                 offset = pp->p_offset;
2096                                 VN_HOLD(vp);
2097                                 page_unlock(pp);
2098                                 (void) VOP_PUTPAGE(vp, offset, PAGESIZE,
2099                                     B_INVAL|B_FORCE, kcred, NULL);
2100                                 VN_RELE(vp);
2101 #ifdef MEM_DEL_STATS
2102                                 ntick_pgrp = (uint64_t)ddi_get_lbolt() -
2103                                     start_pgrp;
2104 #endif /* MEM_DEL_STATS */
2105                                 MDSTAT_PGRP(mhp, ntick_pgrp);
2106                                 /*
2107                                  * Try to get the page back immediately
2108                                  * so that it can be collected.
2109                                  */
2110                                 pp = page_numtopp_nolock(pfn);
2111                                 if (pp == NULL) {
2112                                         MDSTAT_INCR(mhp, nnoreclaim);
2113                                         /*
2114                                          * This should not happen as this
2115                                          * thread is deleting the page.
2116                                          * If this code is generalized, this
2117                                          * becomes a reality.
2118                                          */
2119 #ifdef DEBUG
2120                                         cmn_err(CE_WARN,
2121                                             "delete_memory_thread(0x%p) "
2122                                             "pfn 0x%lx has no page_t",
2123                                             (void *)mhp, pfn);
2124 #endif /* DEBUG */
2125                                         mutex_enter(&mhp->mh_mutex);
2126                                         continue;
2127                                 }
2128                                 if (page_try_reclaim_lock(pp, SE_EXCL,
2129                                     SE_EXCL_WANTED | SE_RETIRED)) {
2130                                         if (PP_ISFREE(pp)) {
2131                                                 goto free_page_collect;
2132                                         }
2133                                         page_unlock(pp);
2134                                 }
2135                                 MDSTAT_INCR(mhp, nnoreclaim);
2136                                 mutex_enter(&mhp->mh_mutex);
2137                                 continue;
2138 
2139                         reloc:
2140                                 /*
2141                                  * Got some freemem and a target
2142                                  * page, so move the data to avoid
2143                                  * I/O and lock problems.
2144                                  */
2145                                 ASSERT(!page_iolock_assert(pp));
2146                                 MDSTAT_INCR(mhp, nreloc);
2147                                 /*
2148                                  * page_relocate() will return pgcnt: the
2149                                  * number of consecutive pages relocated.
2150                                  * If it is successful, pp will be a
2151                                  * linked list of the page structs that
2152                                  * were relocated. If page_relocate() is
2153                                  * unsuccessful, pp will be unmodified.
2154                                  */
2155 #ifdef MEM_DEL_STATS
2156                                 start_pgrp = ddi_get_lbolt();
2157 #endif /* MEM_DEL_STATS */
2158                                 result = page_relocate(&pp, &pp_targ, 0, 0,
2159                                     &pgcnt, NULL);
2160 #ifdef MEM_DEL_STATS
2161                                 ntick_pgrp = (uint64_t)ddi_get_lbolt() -
2162                                     start_pgrp;
2163 #endif /* MEM_DEL_STATS */
2164                                 MDSTAT_PGRP(mhp, ntick_pgrp);
2165                                 if (result != 0) {
2166                                         MDSTAT_INCR(mhp, nrelocfail);
2167                                         /*
2168                                          * We did not succeed. We need
2169                                          * to give the pp_targ pages back.
2170                                          * page_free(pp_targ, 1) without
2171                                          * the freemem accounting.
2172                                          */
2173                                         group_page_unlock(pp);
2174                                         page_free_replacement_page(pp_targ);
2175                                         page_unlock(pp);
2176                                         mutex_enter(&mhp->mh_mutex);
2177                                         continue;
2178                                 }
2179 
2180                                 /*
2181                                  * We will then collect pgcnt pages.
2182                                  */
2183                                 ASSERT(pgcnt > 0);
2184                                 mutex_enter(&mhp->mh_mutex);
2185                                 /*
2186                                  * We need to make sure freemem_left is
2187                                  * large enough.
2188                                  */
2189                                 while ((freemem_left < pgcnt) &&
2190                                     (!mhp->mh_cancel)) {
2191                                         freemem_left +=
2192                                             delthr_get_freemem(mhp);
2193                                 }
2194 
2195                                 /*
2196                                  * Do not proceed if mh_cancel is set.
2197                                  */
2198                                 if (mhp->mh_cancel) {
2199                                         while (pp_targ != NULL) {
2200                                                 /*
2201                                                  * Unlink and unlock each page.
2202                                                  */
2203                                                 tpp_targ = pp_targ;
2204                                                 page_sub(&pp_targ, tpp_targ);
2205                                                 page_unlock(tpp_targ);
2206                                         }
2207                                         /*
2208                                          * We need to give the pp pages back.
2209                                          * page_free(pp, 1) without the
2210                                          * freemem accounting.
2211                                          */
2212                                         page_free_replacement_page(pp);
2213                                         break;
2214                                 }
2215 
2216                                 /* Now remove pgcnt from freemem_left */
2217                                 freemem_left -= pgcnt;
2218                                 ASSERT(freemem_left >= 0);
2219                                 szc = pp->p_szc;
2220                                 while (pp != NULL) {
2221                                         /*
2222                                          * pp and pp_targ were passed back as
2223                                          * a linked list of pages.
2224                                          * Unlink and unlock each page.
2225                                          */
2226                                         tpp_targ = pp_targ;
2227                                         page_sub(&pp_targ, tpp_targ);
2228                                         page_unlock(tpp_targ);
2229                                         /*
2230                                          * The original page is now free
2231                                          * so remove it from the linked
2232                                          * list and collect it.
2233                                          */
2234                                         tpp = pp;
2235                                         page_sub(&pp, tpp);
2236                                         pfn = page_pptonum(tpp);
2237                                         collected++;
2238                                         ASSERT(PAGE_EXCL(tpp));
2239                                         ASSERT(tpp->p_vnode == NULL);
2240                                         ASSERT(!hat_page_is_mapped(tpp));
2241                                         ASSERT(tpp->p_szc == szc);
2242                                         tpp->p_szc = 0;
2243                                         page_delete_collect(tpp, mhp);
2244                                         bit = pfn - mdsp->mds_base;
2245                                         mdsp->mds_bitmap[bit / NBPBMW] |=
2246                                             (1 << (bit % NBPBMW));
2247                                 }
2248                                 ASSERT(pp_targ == NULL);
2249                         }
2250                 }
2251                 first_scan = 0;
2252                 if ((mhp->mh_cancel == 0) && (mhp->mh_hold_todo != 0) &&
2253                     (collected == 0)) {
2254                         /*
2255                          * This code is needed as we cannot wait
2256                          * for a page to be locked OR the delete to
2257                          * be cancelled.  Also, we must delay so
2258                          * that other threads get a chance to run
2259                          * on our cpu, otherwise page locks may be
2260                          * held indefinitely by those threads.
2261                          */
2262                         MDSTAT_INCR(mhp, ndelay);
2263                         CALLB_CPR_SAFE_BEGIN(&cprinfo);
2264                         (void) cv_reltimedwait(&mhp->mh_cv, &mhp->mh_mutex,
2265                             DEL_BUSY_WAIT_TICKS, TR_CLOCK_TICK);
2266                         CALLB_CPR_SAFE_END(&cprinfo, &mhp->mh_mutex);
2267                 }
2268         }
2269         /* stop the dr aio cleanup thread */
2270         mhp->mh_dr_aio_cleanup_cancel = 1;
2271         transit_list_collect(mhp, 0);
2272         if (freemem_left != 0) {
2273                 /* Return any surplus. */
2274                 page_create_putback(freemem_left);
2275                 freemem_left = 0;
2276         }
2277 #ifdef MEM_DEL_STATS
2278         ntick_total = (uint64_t)ddi_get_lbolt() - start_total;
2279 #endif /* MEM_DEL_STATS */
2280         MDSTAT_TOTAL(mhp, ntick_total);
2281         MDSTAT_PRINT(mhp);
2282 
2283         /*
2284          * If the memory delete was cancelled, exclusive-wanted bits must
2285          * be cleared. If there are retired pages being deleted, they need
2286          * to be unretired.
2287          */
2288         for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
2289             mdsp = mdsp->mds_next) {
2290                 pfn_t pfn, p_end;
2291 
2292                 p_end = mdsp->mds_base + mdsp->mds_npgs;
2293                 for (pfn = mdsp->mds_base; pfn < p_end; pfn++) {
2294                         page_t *pp;
2295                         pgcnt_t bit;
2296 
2297                         bit = pfn - mdsp->mds_base;
2298                         if (mhp->mh_cancel) {
2299                                 pp = page_numtopp_nolock(pfn);
2300                                 if (pp != NULL) {
2301                                         if ((mdsp->mds_bitmap[bit / NBPBMW] &
2302                                             (1 << (bit % NBPBMW))) == 0) {
2303                                                 page_lock_clr_exclwanted(pp);
2304                                         }
2305                                 }
2306                         } else {
2307                                 pp = NULL;
2308                         }
2309                         if ((mdsp->mds_bitmap_retired[bit / NBPBMW] &
2310                             (1 << (bit % NBPBMW))) != 0) {
2311                                 /* do we already have pp? */
2312                                 if (pp == NULL) {
2313                                         pp = page_numtopp_nolock(pfn);
2314                                 }
2315                                 ASSERT(pp != NULL);
2316                                 ASSERT(PP_RETIRED(pp));
2317                                 if (mhp->mh_cancel != 0) {
2318                                         page_unlock(pp);
2319                                         /*
2320                                          * To satisfy ASSERT below in
2321                                          * cancel code.
2322                                          */
2323                                         mhp->mh_hold_todo++;
2324                                 } else {
2325                                         (void) page_unretire_pp(pp,
2326                                             PR_UNR_CLEAN);
2327                                 }
2328                         }
2329                 }
2330         }
2331         /*
2332          * Free retired page bitmap and collected page bitmap
2333          */
2334         for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
2335             mdsp = mdsp->mds_next) {
2336                 ASSERT(mdsp->mds_bitmap_retired != NULL);
2337                 kmem_free(mdsp->mds_bitmap_retired, MDS_BITMAPBYTES(mdsp));
2338                 mdsp->mds_bitmap_retired = NULL;     /* Paranoia. */
2339                 ASSERT(mdsp->mds_bitmap != NULL);
2340                 kmem_free(mdsp->mds_bitmap, MDS_BITMAPBYTES(mdsp));
2341                 mdsp->mds_bitmap = NULL;     /* Paranoia. */
2342         }
2343 
2344         /* wait for our dr aio cancel thread to exit */
2345         while (!(mhp->mh_aio_cleanup_done)) {
2346                 CALLB_CPR_SAFE_BEGIN(&cprinfo);
2347                 delay(drv_usectohz(DR_AIO_CLEANUP_DELAY));
2348                 CALLB_CPR_SAFE_END(&cprinfo, &mhp->mh_mutex);
2349         }
2350 refused:
2351         if (mhp->mh_cancel != 0) {
2352                 page_t *pp;
2353 
2354                 comp_code = mhp->mh_cancel;
2355                 /*
2356                  * Go through list of deleted pages (mh_deleted) freeing
2357                  * them.
2358                  */
2359                 while ((pp = mhp->mh_deleted) != NULL) {
2360                         mhp->mh_deleted = pp->p_next;
2361                         mhp->mh_hold_todo++;
2362                         mutex_exit(&mhp->mh_mutex);
2363                         /* Restore p_next. */
2364                         pp->p_next = pp->p_prev;
2365                         if (PP_ISFREE(pp)) {
2366                                 cmn_err(CE_PANIC,
2367                                     "page %p is free",
2368                                     (void *)pp);
2369                         }
2370                         page_free(pp, 1);
2371                         mutex_enter(&mhp->mh_mutex);
2372                 }
2373                 ASSERT(mhp->mh_hold_todo == mhp->mh_vm_pages);
2374 
2375                 mutex_exit(&mhp->mh_mutex);
2376                 put_availrmem(mhp->mh_vm_pages);
2377                 mutex_enter(&mhp->mh_mutex);
2378 
2379                 goto t_exit;
2380         }
2381 
2382         /*
2383          * All the pages are no longer in use and are exclusively locked.
2384          */
2385 
2386         mhp->mh_deleted = NULL;
2387 
2388         kphysm_del_cleanup(mhp);
2389 
2390         /*
2391          * mem_node_del_range needs to be after kphysm_del_cleanup so
2392          * that the mem_node_config[] will remain intact for the cleanup.
2393          */
2394         for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
2395             mdsp = mdsp->mds_next) {
2396                 mem_node_del_range(mdsp->mds_base,
2397                     mdsp->mds_base + mdsp->mds_npgs - 1);
2398         }
2399         /* cleanup the page counters */
2400         page_ctrs_cleanup();
2401 
2402         comp_code = KPHYSM_OK;
2403 
2404 t_exit:
2405         mutex_exit(&mhp->mh_mutex);
2406         kphysm_setup_post_del(mhp->mh_vm_pages,
2407             (comp_code == KPHYSM_OK) ? 0 : 1);
2408         mutex_enter(&mhp->mh_mutex);
2409 
2410 early_exit:
2411         /* mhp->mh_mutex exited by CALLB_CPR_EXIT() */
2412         mhp->mh_state = MHND_DONE;
2413         del_complete_funcp = mhp->mh_delete_complete;
2414         del_complete_arg = mhp->mh_delete_complete_arg;
2415         CALLB_CPR_EXIT(&cprinfo);
2416         (*del_complete_funcp)(del_complete_arg, comp_code);
2417         thread_exit();
2418         /*NOTREACHED*/
2419 }
2420 
2421 /*
2422  * Start the delete of the memory from the system.
2423  */
2424 int
2425 kphysm_del_start(
2426         memhandle_t handle,
2427         void (*complete)(void *, int),
2428         void *complete_arg)
2429 {
2430         struct mem_handle *mhp;
2431 
2432         mhp = kphysm_lookup_mem_handle(handle);
2433         if (mhp == NULL) {
2434                 return (KPHYSM_EHANDLE);
2435         }
2436         switch (mhp->mh_state) {
2437         case MHND_FREE:
2438                 ASSERT(mhp->mh_state != MHND_FREE);
2439                 mutex_exit(&mhp->mh_mutex);
2440                 return (KPHYSM_EHANDLE);
2441         case MHND_INIT:
2442                 break;
2443         case MHND_STARTING:
2444         case MHND_RUNNING:
2445                 mutex_exit(&mhp->mh_mutex);
2446                 return (KPHYSM_ESEQUENCE);
2447         case MHND_DONE:
2448                 mutex_exit(&mhp->mh_mutex);
2449                 return (KPHYSM_ESEQUENCE);
2450         case MHND_RELEASE:
2451                 mutex_exit(&mhp->mh_mutex);
2452                 return (KPHYSM_ESEQUENCE);
2453         default:
2454 #ifdef DEBUG
2455                 cmn_err(CE_WARN, "kphysm_del_start(0x%p) state corrupt %d",
2456                     (void *)mhp, mhp->mh_state);
2457 #endif /* DEBUG */
2458                 mutex_exit(&mhp->mh_mutex);
2459                 return (KPHYSM_EHANDLE);
2460         }
2461 
2462         if (mhp->mh_transit.trl_spans == NULL) {
2463                 mutex_exit(&mhp->mh_mutex);
2464                 return (KPHYSM_ENOWORK);
2465         }
2466 
2467         ASSERT(complete != NULL);
2468         mhp->mh_delete_complete = complete;
2469         mhp->mh_delete_complete_arg = complete_arg;
2470         mhp->mh_state = MHND_STARTING;
2471         /*
2472          * Release the mutex in case thread_create sleeps.
2473          */
2474         mutex_exit(&mhp->mh_mutex);
2475 
2476         /*
2477          * The "obvious" process for this thread is pageout (proc_pageout)
2478          * but this gives the thread too much power over freemem
2479          * which results in freemem starvation.
2480          */
2481         (void) thread_create(NULL, 0, delete_memory_thread, mhp, 0, &p0,
2482             TS_RUN, maxclsyspri - 1);
2483 
2484         return (KPHYSM_OK);
2485 }
2486 
2487 static kmutex_t pp_dummy_lock;          /* Protects init. of pp_dummy. */
2488 static caddr_t pp_dummy;
2489 static pgcnt_t pp_dummy_npages;
2490 static pfn_t *pp_dummy_pfn;     /* Array of dummy pfns. */
2491 
2492 static void
2493 memseg_remap_init_pages(page_t *pages, page_t *epages)
2494 {
2495         page_t *pp;
2496 
2497         for (pp = pages; pp < epages; pp++) {
2498                 pp->p_pagenum = PFN_INVALID; /* XXXX */
2499                 pp->p_offset = (u_offset_t)-1;
2500                 page_iolock_init(pp);
2501                 while (!page_lock(pp, SE_EXCL, (kmutex_t *)NULL, P_RECLAIM))
2502                         continue;
2503                 page_lock_delete(pp);
2504         }
2505 }
2506 
2507 void
2508 memseg_remap_init()
2509 {
2510         mutex_enter(&pp_dummy_lock);
2511         if (pp_dummy == NULL) {
2512                 uint_t dpages;
2513                 int i;
2514 
2515                 /*
2516                  * dpages starts off as the size of the structure and
2517                  * ends up as the minimum number of pages that will
2518                  * hold a whole number of page_t structures.
2519                  */
2520                 dpages = sizeof (page_t);
2521                 ASSERT(dpages != 0);
2522                 ASSERT(dpages <= MMU_PAGESIZE);
2523 
2524                 while ((dpages & 1) == 0)
2525                         dpages >>= 1;
2526 
2527                 pp_dummy_npages = dpages;
2528                 /*
2529                  * Allocate pp_dummy pages directly from static_arena,
2530                  * since these are whole page allocations and are
2531                  * referenced by physical address.  This also has the
2532                  * nice fringe benefit of hiding the memory from
2533                  * ::findleaks since it doesn't deal well with allocated
2534                  * kernel heap memory that doesn't have any mappings.
2535                  */
2536                 pp_dummy = vmem_xalloc(static_arena, ptob(pp_dummy_npages),
2537                     PAGESIZE, 0, 0, NULL, NULL, VM_SLEEP);
2538                 bzero(pp_dummy, ptob(pp_dummy_npages));
2539                 ASSERT(((uintptr_t)pp_dummy & MMU_PAGEOFFSET) == 0);
2540                 pp_dummy_pfn = kmem_alloc(sizeof (*pp_dummy_pfn) *
2541                     pp_dummy_npages, KM_SLEEP);
2542                 for (i = 0; i < pp_dummy_npages; i++) {
2543                         pp_dummy_pfn[i] = hat_getpfnum(kas.a_hat,
2544                             &pp_dummy[MMU_PAGESIZE * i]);
2545                         ASSERT(pp_dummy_pfn[i] != PFN_INVALID);
2546                 }
2547                 /*
2548                  * Initialize the page_t's to a known 'deleted' state
2549                  * that matches the state of deleted pages.
2550                  */
2551                 memseg_remap_init_pages((page_t *)pp_dummy,
2552                     (page_t *)(pp_dummy + ptob(pp_dummy_npages)));
2553                 /* Remove kmem mappings for the pages for safety. */
2554                 hat_unload(kas.a_hat, pp_dummy, ptob(pp_dummy_npages),
2555                     HAT_UNLOAD_UNLOCK);
2556                 /* Leave pp_dummy pointer set as flag that init is done. */
2557         }
2558         mutex_exit(&pp_dummy_lock);
2559 }
2560 
2561 /*
2562  * Remap a page-aglined range of page_t's to dummy pages.
2563  */
2564 void
2565 remap_to_dummy(caddr_t va, pgcnt_t metapgs)
2566 {
2567         int phase;
2568 
2569         ASSERT(IS_P2ALIGNED((uint64_t)(uintptr_t)va, PAGESIZE));
2570 
2571         /*
2572          * We may start remapping at a non-zero page offset
2573          * within the dummy pages since the low/high ends
2574          * of the outgoing pp's could be shared by other
2575          * memsegs (see memseg_remap_meta).
2576          */
2577         phase = btop((uint64_t)(uintptr_t)va) % pp_dummy_npages;
2578         /*CONSTCOND*/
2579         ASSERT(PAGESIZE % sizeof (page_t) || phase == 0);
2580 
2581         while (metapgs != 0) {
2582                 pgcnt_t n;
2583                 int i, j;
2584 
2585                 n = pp_dummy_npages;
2586                 if (n > metapgs)
2587                         n = metapgs;
2588                 for (i = 0; i < n; i++) {
2589                         j = (i + phase) % pp_dummy_npages;
2590                         hat_devload(kas.a_hat, va, ptob(1), pp_dummy_pfn[j],
2591                             PROT_READ,
2592                             HAT_LOAD | HAT_LOAD_NOCONSIST |
2593                             HAT_LOAD_REMAP);
2594                         va += ptob(1);
2595                 }
2596                 metapgs -= n;
2597         }
2598 }
2599 
2600 static void
2601 memseg_remap_to_dummy(struct memseg *seg)
2602 {
2603         caddr_t pp;
2604         pgcnt_t metapgs;
2605 
2606         ASSERT(memseg_is_dynamic(seg));
2607         ASSERT(pp_dummy != NULL);
2608 
2609 
2610         if (!memseg_includes_meta(seg)) {
2611                 memseg_remap_meta(seg);
2612                 return;
2613         }
2614 
2615         pp = (caddr_t)seg->pages;
2616         metapgs = seg->pages_base - memseg_get_start(seg);
2617         ASSERT(metapgs != 0);
2618 
2619         seg->pages_end = seg->pages_base;
2620 
2621         remap_to_dummy(pp, metapgs);
2622 }
2623 
2624 /*
2625  * Transition all the deleted pages to the deleted state so that
2626  * page_lock will not wait. The page_lock_delete call will
2627  * also wake up any waiters.
2628  */
2629 static void
2630 memseg_lock_delete_all(struct memseg *seg)
2631 {
2632         page_t *pp;
2633 
2634         for (pp = seg->pages; pp < seg->epages; pp++) {
2635                 pp->p_pagenum = PFN_INVALID; /* XXXX */
2636                 page_lock_delete(pp);
2637         }
2638 }
2639 
2640 static void
2641 kphysm_del_cleanup(struct mem_handle *mhp)
2642 {
2643         struct memdelspan       *mdsp;
2644         struct memseg           *seg;
2645         struct memseg           **segpp;
2646         struct memseg           *seglist;
2647         pfn_t                   p_end;
2648         uint64_t                avmem;
2649         pgcnt_t                 avpgs;
2650         pgcnt_t                 npgs;
2651 
2652         avpgs = mhp->mh_vm_pages;
2653 
2654         memsegs_lock(1);
2655 
2656         /*
2657          * remove from main segment list.
2658          */
2659         npgs = 0;
2660         seglist = NULL;
2661         for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
2662             mdsp = mdsp->mds_next) {
2663                 p_end = mdsp->mds_base + mdsp->mds_npgs;
2664                 for (segpp = &memsegs; (seg = *segpp) != NULL; ) {
2665                         if (seg->pages_base >= p_end ||
2666                             seg->pages_end <= mdsp->mds_base) {
2667                                 /* Span and memseg don't overlap. */
2668                                 segpp = &((*segpp)->next);
2669                                 continue;
2670                         }
2671                         ASSERT(seg->pages_base >= mdsp->mds_base);
2672                         ASSERT(seg->pages_end <= p_end);
2673 
2674                         PLCNT_MODIFY_MAX(seg->pages_base,
2675                             seg->pages_base - seg->pages_end);
2676 
2677                         /* Hide the memseg from future scans. */
2678                         hat_kpm_delmem_mseg_update(seg, segpp);
2679                         *segpp = seg->next;
2680                         membar_producer();      /* TODO: Needed? */
2681                         npgs += MSEG_NPAGES(seg);
2682 
2683                         /*
2684                          * Leave the deleted segment's next pointer intact
2685                          * in case a memsegs scanning loop is walking this
2686                          * segment concurrently.
2687                          */
2688                         seg->lnext = seglist;
2689                         seglist = seg;
2690                 }
2691         }
2692 
2693         build_pfn_hash();
2694 
2695         ASSERT(npgs < total_pages);
2696         total_pages -= npgs;
2697 
2698         /*
2699          * Recalculate the paging parameters now total_pages has changed.
2700          * This will also cause the clock hands to be reset before next use.
2701          */
2702         setupclock(1);
2703 
2704         memsegs_unlock(1);
2705 
2706         mutex_exit(&mhp->mh_mutex);
2707 
2708         while ((seg = seglist) != NULL) {
2709                 pfn_t mseg_start;
2710                 pfn_t mseg_base, mseg_end;
2711                 pgcnt_t mseg_npgs;
2712                 int mlret;
2713 
2714                 seglist = seg->lnext;
2715 
2716                 /*
2717                  * Put the page_t's into the deleted state to stop
2718                  * cv_wait()s on the pages. When we remap, the dummy
2719                  * page_t's will be in the same state.
2720                  */
2721                 memseg_lock_delete_all(seg);
2722                 /*
2723                  * Collect up information based on pages_base and pages_end
2724                  * early so that we can flag early that the memseg has been
2725                  * deleted by setting pages_end == pages_base.
2726                  */
2727                 mseg_base = seg->pages_base;
2728                 mseg_end = seg->pages_end;
2729                 mseg_npgs = MSEG_NPAGES(seg);
2730                 mseg_start = memseg_get_start(seg);
2731 
2732                 if (memseg_is_dynamic(seg)) {
2733                         /* Remap the meta data to our special dummy area. */
2734                         memseg_remap_to_dummy(seg);
2735 
2736                         mutex_enter(&memseg_lists_lock);
2737                         seg->lnext = memseg_va_avail;
2738                         memseg_va_avail = seg;
2739                         mutex_exit(&memseg_lists_lock);
2740                 } else {
2741                         /*
2742                          * For memory whose page_ts were allocated
2743                          * at boot, we need to find a new use for
2744                          * the page_t memory.
2745                          * For the moment, just leak it.
2746                          * (It is held in the memseg_delete_junk list.)
2747                          */
2748                         seg->pages_end = seg->pages_base;
2749 
2750                         mutex_enter(&memseg_lists_lock);
2751                         seg->lnext = memseg_delete_junk;
2752                         memseg_delete_junk = seg;
2753                         mutex_exit(&memseg_lists_lock);
2754                 }
2755 
2756                 /* Must not use seg now as it could be re-used. */
2757 
2758                 memlist_write_lock();
2759 
2760                 mlret = memlist_delete_span(
2761                     (uint64_t)(mseg_base) << PAGESHIFT,
2762                     (uint64_t)(mseg_npgs) << PAGESHIFT,
2763                     &phys_avail);
2764                 ASSERT(mlret == MEML_SPANOP_OK);
2765 
2766                 mlret = memlist_delete_span(
2767                     (uint64_t)(mseg_start) << PAGESHIFT,
2768                     (uint64_t)(mseg_end - mseg_start) <<
2769                     PAGESHIFT,
2770                     &phys_install);
2771                 ASSERT(mlret == MEML_SPANOP_OK);
2772                 phys_install_has_changed();
2773 
2774                 memlist_write_unlock();
2775         }
2776 
2777         memlist_read_lock();
2778         installed_top_size(phys_install, &physmax, &physinstalled);
2779         memlist_read_unlock();
2780 
2781         mutex_enter(&freemem_lock);
2782         maxmem -= avpgs;
2783         physmem -= avpgs;
2784         /* availrmem is adjusted during the delete. */
2785         availrmem_initial -= avpgs;
2786 
2787         mutex_exit(&freemem_lock);
2788 
2789         dump_resize();
2790 
2791         cmn_err(CE_CONT, "?kphysm_delete: mem = %ldK "
2792             "(0x%" PRIx64 ")\n",
2793             physinstalled << (PAGESHIFT - 10),
2794             (uint64_t)physinstalled << PAGESHIFT);
2795 
2796         avmem = (uint64_t)freemem << PAGESHIFT;
2797         cmn_err(CE_CONT, "?kphysm_delete: "
2798             "avail mem = %" PRId64 "\n", avmem);
2799 
2800         /*
2801          * Update lgroup generation number on single lgroup systems
2802          */
2803         if (nlgrps == 1)
2804                 lgrp_config(LGRP_CONFIG_GEN_UPDATE, 0, 0);
2805 
2806         /* Successfully deleted system memory */
2807         mutex_enter(&mhp->mh_mutex);
2808 }
2809 
2810 static uint_t mdel_nullvp_waiter;
2811 
2812 static void
2813 page_delete_collect(
2814         page_t *pp,
2815         struct mem_handle *mhp)
2816 {
2817         if (pp->p_vnode) {
2818                 page_hashout(pp, (kmutex_t *)NULL);
2819                 /* do not do PP_SETAGED(pp); */
2820         } else {
2821                 kmutex_t *sep;
2822 
2823                 sep = page_se_mutex(pp);
2824                 mutex_enter(sep);
2825                 if (CV_HAS_WAITERS(&pp->p_cv)) {
2826                         mdel_nullvp_waiter++;
2827                         cv_broadcast(&pp->p_cv);
2828                 }
2829                 mutex_exit(sep);
2830         }
2831         ASSERT(pp->p_next == pp->p_prev);
2832         ASSERT(pp->p_next == NULL || pp->p_next == pp);
2833         pp->p_next = mhp->mh_deleted;
2834         mhp->mh_deleted = pp;
2835         ASSERT(mhp->mh_hold_todo != 0);
2836         mhp->mh_hold_todo--;
2837 }
2838 
2839 static void
2840 transit_list_collect(struct mem_handle *mhp, int v)
2841 {
2842         struct transit_list_head *trh;
2843 
2844         trh = &transit_list_head;
2845         mutex_enter(&trh->trh_lock);
2846         mhp->mh_transit.trl_collect = v;
2847         mutex_exit(&trh->trh_lock);
2848 }
2849 
2850 static void
2851 transit_list_insert(struct transit_list *tlp)
2852 {
2853         struct transit_list_head *trh;
2854 
2855         trh = &transit_list_head;
2856         ASSERT(MUTEX_HELD(&trh->trh_lock));
2857         tlp->trl_next = trh->trh_head;
2858         trh->trh_head = tlp;
2859 }
2860 
2861 static void
2862 transit_list_remove(struct transit_list *tlp)
2863 {
2864         struct transit_list_head *trh;
2865         struct transit_list **tlpp;
2866 
2867         trh = &transit_list_head;
2868         tlpp = &trh->trh_head;
2869         ASSERT(MUTEX_HELD(&trh->trh_lock));
2870         while (*tlpp != NULL && *tlpp != tlp)
2871                 tlpp = &(*tlpp)->trl_next;
2872         ASSERT(*tlpp != NULL);
2873         if (*tlpp == tlp)
2874                 *tlpp = tlp->trl_next;
2875         tlp->trl_next = NULL;
2876 }
2877 
2878 static struct transit_list *
2879 pfnum_to_transit_list(struct transit_list_head *trh, pfn_t pfnum)
2880 {
2881         struct transit_list *tlp;
2882 
2883         for (tlp = trh->trh_head; tlp != NULL; tlp = tlp->trl_next) {
2884                 struct memdelspan *mdsp;
2885 
2886                 for (mdsp = tlp->trl_spans; mdsp != NULL;
2887                     mdsp = mdsp->mds_next) {
2888                         if (pfnum >= mdsp->mds_base &&
2889                             pfnum < (mdsp->mds_base + mdsp->mds_npgs)) {
2890                                 return (tlp);
2891                         }
2892                 }
2893         }
2894         return (NULL);
2895 }
2896 
2897 int
2898 pfn_is_being_deleted(pfn_t pfnum)
2899 {
2900         struct transit_list_head *trh;
2901         struct transit_list *tlp;
2902         int ret;
2903 
2904         trh = &transit_list_head;
2905         if (trh->trh_head == NULL)
2906                 return (0);
2907 
2908         mutex_enter(&trh->trh_lock);
2909         tlp = pfnum_to_transit_list(trh, pfnum);
2910         ret = (tlp != NULL && tlp->trl_collect);
2911         mutex_exit(&trh->trh_lock);
2912 
2913         return (ret);
2914 }
2915 
2916 #ifdef MEM_DEL_STATS
2917 extern int hz;
2918 static void
2919 mem_del_stat_print_func(struct mem_handle *mhp)
2920 {
2921         uint64_t tmp;
2922 
2923         if (mem_del_stat_print) {
2924                 printf("memory delete loop %x/%x, statistics%s\n",
2925                     (uint_t)mhp->mh_transit.trl_spans->mds_base,
2926                     (uint_t)mhp->mh_transit.trl_spans->mds_npgs,
2927                     (mhp->mh_cancel ? " (cancelled)" : ""));
2928                 printf("\t%8u nloop\n", mhp->mh_delstat.nloop);
2929                 printf("\t%8u need_free\n", mhp->mh_delstat.need_free);
2930                 printf("\t%8u free_loop\n", mhp->mh_delstat.free_loop);
2931                 printf("\t%8u free_low\n", mhp->mh_delstat.free_low);
2932                 printf("\t%8u free_failed\n", mhp->mh_delstat.free_failed);
2933                 printf("\t%8u ncheck\n", mhp->mh_delstat.ncheck);
2934                 printf("\t%8u nopaget\n", mhp->mh_delstat.nopaget);
2935                 printf("\t%8u lockfail\n", mhp->mh_delstat.lockfail);
2936                 printf("\t%8u nfree\n", mhp->mh_delstat.nfree);
2937                 printf("\t%8u nreloc\n", mhp->mh_delstat.nreloc);
2938                 printf("\t%8u nrelocfail\n", mhp->mh_delstat.nrelocfail);
2939                 printf("\t%8u already_done\n", mhp->mh_delstat.already_done);
2940                 printf("\t%8u first_notfree\n", mhp->mh_delstat.first_notfree);
2941                 printf("\t%8u npplocked\n", mhp->mh_delstat.npplocked);
2942                 printf("\t%8u nlockreloc\n", mhp->mh_delstat.nlockreloc);
2943                 printf("\t%8u nnorepl\n", mhp->mh_delstat.nnorepl);
2944                 printf("\t%8u nmodreloc\n", mhp->mh_delstat.nmodreloc);
2945                 printf("\t%8u ndestroy\n", mhp->mh_delstat.ndestroy);
2946                 printf("\t%8u nputpage\n", mhp->mh_delstat.nputpage);
2947                 printf("\t%8u nnoreclaim\n", mhp->mh_delstat.nnoreclaim);
2948                 printf("\t%8u ndelay\n", mhp->mh_delstat.ndelay);
2949                 printf("\t%8u demotefail\n", mhp->mh_delstat.demotefail);
2950                 printf("\t%8u retired\n", mhp->mh_delstat.retired);
2951                 printf("\t%8u toxic\n", mhp->mh_delstat.toxic);
2952                 printf("\t%8u failing\n", mhp->mh_delstat.failing);
2953                 printf("\t%8u modtoxic\n", mhp->mh_delstat.modtoxic);
2954                 printf("\t%8u npplkdtoxic\n", mhp->mh_delstat.npplkdtoxic);
2955                 printf("\t%8u gptlmodfail\n", mhp->mh_delstat.gptlmodfail);
2956                 printf("\t%8u gptllckfail\n", mhp->mh_delstat.gptllckfail);
2957                 tmp = mhp->mh_delstat.nticks_total / hz;  /* seconds */
2958                 printf(
2959                     "\t%"PRIu64" nticks_total - %"PRIu64" min %"PRIu64" sec\n",
2960                     mhp->mh_delstat.nticks_total, tmp / 60, tmp % 60);
2961 
2962                 tmp = mhp->mh_delstat.nticks_pgrp / hz;  /* seconds */
2963                 printf(
2964                     "\t%"PRIu64" nticks_pgrp - %"PRIu64" min %"PRIu64" sec\n",
2965                     mhp->mh_delstat.nticks_pgrp, tmp / 60, tmp % 60);
2966         }
2967 }
2968 #endif /* MEM_DEL_STATS */
2969 
2970 struct mem_callback {
2971         kphysm_setup_vector_t   *vec;
2972         void                    *arg;
2973 };
2974 
2975 #define NMEMCALLBACKS           100
2976 
2977 static struct mem_callback mem_callbacks[NMEMCALLBACKS];
2978 static uint_t nmemcallbacks;
2979 static krwlock_t mem_callback_rwlock;
2980 
2981 int
2982 kphysm_setup_func_register(kphysm_setup_vector_t *vec, void *arg)
2983 {
2984         uint_t i, found;
2985 
2986         /*
2987          * This test will become more complicated when the version must
2988          * change.
2989          */
2990         if (vec->version != KPHYSM_SETUP_VECTOR_VERSION)
2991                 return (EINVAL);
2992 
2993         if (vec->post_add == NULL || vec->pre_del == NULL ||
2994             vec->post_del == NULL)
2995                 return (EINVAL);
2996 
2997         rw_enter(&mem_callback_rwlock, RW_WRITER);
2998         for (i = 0, found = 0; i < nmemcallbacks; i++) {
2999                 if (mem_callbacks[i].vec == NULL && found == 0)
3000                         found = i + 1;
3001                 if (mem_callbacks[i].vec == vec &&
3002                     mem_callbacks[i].arg == arg) {
3003 #ifdef DEBUG
3004                         /* Catch this in DEBUG kernels. */
3005                         cmn_err(CE_WARN, "kphysm_setup_func_register"
3006                             "(0x%p, 0x%p) duplicate registration from 0x%p",
3007                             (void *)vec, arg, (void *)caller());
3008 #endif /* DEBUG */
3009                         rw_exit(&mem_callback_rwlock);
3010                         return (EEXIST);
3011                 }
3012         }
3013         if (found != 0) {
3014                 i = found - 1;
3015         } else {
3016                 ASSERT(nmemcallbacks < NMEMCALLBACKS);
3017                 if (nmemcallbacks == NMEMCALLBACKS) {
3018                         rw_exit(&mem_callback_rwlock);
3019                         return (ENOMEM);
3020                 }
3021                 i = nmemcallbacks++;
3022         }
3023         mem_callbacks[i].vec = vec;
3024         mem_callbacks[i].arg = arg;
3025         rw_exit(&mem_callback_rwlock);
3026         return (0);
3027 }
3028 
3029 void
3030 kphysm_setup_func_unregister(kphysm_setup_vector_t *vec, void *arg)
3031 {
3032         uint_t i;
3033 
3034         rw_enter(&mem_callback_rwlock, RW_WRITER);
3035         for (i = 0; i < nmemcallbacks; i++) {
3036                 if (mem_callbacks[i].vec == vec &&
3037                     mem_callbacks[i].arg == arg) {
3038                         mem_callbacks[i].vec = NULL;
3039                         mem_callbacks[i].arg = NULL;
3040                         if (i == (nmemcallbacks - 1))
3041                                 nmemcallbacks--;
3042                         break;
3043                 }
3044         }
3045         rw_exit(&mem_callback_rwlock);
3046 }
3047 
3048 static void
3049 kphysm_setup_post_add(pgcnt_t delta_pages)
3050 {
3051         uint_t i;
3052 
3053         rw_enter(&mem_callback_rwlock, RW_READER);
3054         for (i = 0; i < nmemcallbacks; i++) {
3055                 if (mem_callbacks[i].vec != NULL) {
3056                         (*mem_callbacks[i].vec->post_add)
3057                             (mem_callbacks[i].arg, delta_pages);
3058                 }
3059         }
3060         rw_exit(&mem_callback_rwlock);
3061 }
3062 
3063 /*
3064  * Note the locking between pre_del and post_del: The reader lock is held
3065  * between the two calls to stop the set of functions from changing.
3066  */
3067 
3068 static int
3069 kphysm_setup_pre_del(pgcnt_t delta_pages)
3070 {
3071         uint_t i;
3072         int ret;
3073         int aret;
3074 
3075         ret = 0;
3076         rw_enter(&mem_callback_rwlock, RW_READER);
3077         for (i = 0; i < nmemcallbacks; i++) {
3078                 if (mem_callbacks[i].vec != NULL) {
3079                         aret = (*mem_callbacks[i].vec->pre_del)
3080                             (mem_callbacks[i].arg, delta_pages);
3081                         ret |= aret;
3082                 }
3083         }
3084 
3085         return (ret);
3086 }
3087 
3088 static void
3089 kphysm_setup_post_del(pgcnt_t delta_pages, int cancelled)
3090 {
3091         uint_t i;
3092 
3093         for (i = 0; i < nmemcallbacks; i++) {
3094                 if (mem_callbacks[i].vec != NULL) {
3095                         (*mem_callbacks[i].vec->post_del)
3096                             (mem_callbacks[i].arg, delta_pages, cancelled);
3097                 }
3098         }
3099         rw_exit(&mem_callback_rwlock);
3100 }
3101 
3102 static int
3103 kphysm_split_memseg(
3104         pfn_t base,
3105         pgcnt_t npgs)
3106 {
3107         struct memseg *seg;
3108         struct memseg **segpp;
3109         pgcnt_t size_low, size_high;
3110         struct memseg *seg_low, *seg_mid, *seg_high;
3111 
3112         /*
3113          * Lock the memsegs list against other updates now
3114          */
3115         memsegs_lock(1);
3116 
3117         /*
3118          * Find boot time memseg that wholly covers this area.
3119          */
3120 
3121         /* First find the memseg with page 'base' in it. */
3122         for (segpp = &memsegs; (seg = *segpp) != NULL;
3123             segpp = &((*segpp)->next)) {
3124                 if (base >= seg->pages_base && base < seg->pages_end)
3125                         break;
3126         }
3127         if (seg == NULL) {
3128                 memsegs_unlock(1);
3129                 return (0);
3130         }
3131         if (memseg_includes_meta(seg)) {
3132                 memsegs_unlock(1);
3133                 return (0);
3134         }
3135         if ((base + npgs) > seg->pages_end) {
3136                 memsegs_unlock(1);
3137                 return (0);
3138         }
3139 
3140         /*
3141          * Work out the size of the two segments that will
3142          * surround the new segment, one for low address
3143          * and one for high.
3144          */
3145         ASSERT(base >= seg->pages_base);
3146         size_low = base - seg->pages_base;
3147         ASSERT(seg->pages_end >= (base + npgs));
3148         size_high = seg->pages_end - (base + npgs);
3149 
3150         /*
3151          * Sanity check.
3152          */
3153         if ((size_low + size_high) == 0) {
3154                 memsegs_unlock(1);
3155                 return (0);
3156         }
3157 
3158         /*
3159          * Allocate the new structures. The old memseg will not be freed
3160          * as there may be a reference to it.
3161          */
3162         seg_low = NULL;
3163         seg_high = NULL;
3164 
3165         if (size_low != 0)
3166                 seg_low = memseg_alloc();
3167 
3168         seg_mid = memseg_alloc();
3169 
3170         if (size_high != 0)
3171                 seg_high = memseg_alloc();
3172 
3173         /*
3174          * All allocation done now.
3175          */
3176         if (size_low != 0) {
3177                 seg_low->pages = seg->pages;
3178                 seg_low->epages = seg_low->pages + size_low;
3179                 seg_low->pages_base = seg->pages_base;
3180                 seg_low->pages_end = seg_low->pages_base + size_low;
3181                 seg_low->next = seg_mid;
3182                 seg_low->msegflags = seg->msegflags;
3183         }
3184         if (size_high != 0) {
3185                 seg_high->pages = seg->epages - size_high;
3186                 seg_high->epages = seg_high->pages + size_high;
3187                 seg_high->pages_base = seg->pages_end - size_high;
3188                 seg_high->pages_end = seg_high->pages_base + size_high;
3189                 seg_high->next = seg->next;
3190                 seg_high->msegflags = seg->msegflags;
3191         }
3192 
3193         seg_mid->pages = seg->pages + size_low;
3194         seg_mid->pages_base = seg->pages_base + size_low;
3195         seg_mid->epages = seg->epages - size_high;
3196         seg_mid->pages_end = seg->pages_end - size_high;
3197         seg_mid->next = (seg_high != NULL) ? seg_high : seg->next;
3198         seg_mid->msegflags = seg->msegflags;
3199 
3200         /*
3201          * Update hat_kpm specific info of all involved memsegs and
3202          * allow hat_kpm specific global chain updates.
3203          */
3204         hat_kpm_split_mseg_update(seg, segpp, seg_low, seg_mid, seg_high);
3205 
3206         /*
3207          * At this point we have two equivalent memseg sub-chains,
3208          * seg and seg_low/seg_mid/seg_high, which both chain on to
3209          * the same place in the global chain. By re-writing the pointer
3210          * in the previous element we switch atomically from using the old
3211          * (seg) to the new.
3212          */
3213         *segpp = (seg_low != NULL) ? seg_low : seg_mid;
3214 
3215         membar_enter();
3216 
3217         build_pfn_hash();
3218         memsegs_unlock(1);
3219 
3220         /*
3221          * We leave the old segment, 'seg', intact as there may be
3222          * references to it. Also, as the value of total_pages has not
3223          * changed and the memsegs list is effectively the same when
3224          * accessed via the old or the new pointer, we do not have to
3225          * cause pageout_scanner() to re-evaluate its hand pointers.
3226          *
3227          * We currently do not re-use or reclaim the page_t memory.
3228          * If we do, then this may have to change.
3229          */
3230 
3231         mutex_enter(&memseg_lists_lock);
3232         seg->lnext = memseg_edit_junk;
3233         memseg_edit_junk = seg;
3234         mutex_exit(&memseg_lists_lock);
3235 
3236         return (1);
3237 }
3238 
3239 /*
3240  * The sfmmu hat layer (e.g.) accesses some parts of the memseg
3241  * structure using physical addresses. Therefore a kmem_cache is
3242  * used with KMC_NOHASH to avoid page crossings within a memseg
3243  * structure. KMC_NOHASH requires that no external (outside of
3244  * slab) information is allowed. This, in turn, implies that the
3245  * cache's slabsize must be exactly a single page, since per-slab
3246  * information (e.g. the freelist for the slab) is kept at the
3247  * end of the slab, where it is easy to locate. Should be changed
3248  * when a more obvious kmem_cache interface/flag will become
3249  * available.
3250  */
3251 void
3252 mem_config_init()
3253 {
3254         memseg_cache = kmem_cache_create("memseg_cache", sizeof (struct memseg),
3255             0, NULL, NULL, NULL, NULL, static_arena, KMC_NOHASH);
3256 }
3257 
3258 struct memseg *
3259 memseg_alloc()
3260 {
3261         struct memseg *seg;
3262 
3263         seg = kmem_cache_alloc(memseg_cache, KM_SLEEP);
3264         bzero(seg, sizeof (struct memseg));
3265 
3266         return (seg);
3267 }
3268 
3269 /*
3270  * Return whether the page_t memory for this memseg
3271  * is included in the memseg itself.
3272  */
3273 static int
3274 memseg_includes_meta(struct memseg *seg)
3275 {
3276         return (seg->msegflags & MEMSEG_META_INCL);
3277 }
3278 
3279 pfn_t
3280 memseg_get_start(struct memseg *seg)
3281 {
3282         pfn_t           pt_start;
3283 
3284         if (memseg_includes_meta(seg)) {
3285                 pt_start = hat_getpfnum(kas.a_hat, (caddr_t)seg->pages);
3286 
3287                 /* Meta data is required to be at the beginning */
3288                 ASSERT(pt_start < seg->pages_base);
3289         } else
3290                 pt_start = seg->pages_base;
3291 
3292         return (pt_start);
3293 }
3294 
3295 /*
3296  * Invalidate memseg pointers in cpu private vm data caches.
3297  */
3298 static void
3299 memseg_cpu_vm_flush()
3300 {
3301         cpu_t *cp;
3302         vm_cpu_data_t *vc;
3303 
3304         mutex_enter(&cpu_lock);
3305         pause_cpus(NULL);
3306 
3307         cp = cpu_list;
3308         do {
3309                 vc = cp->cpu_vm_data;
3310                 vc->vc_pnum_memseg = NULL;
3311                 vc->vc_pnext_memseg = NULL;
3312 
3313         } while ((cp = cp->cpu_next) != cpu_list);
3314 
3315         start_cpus();
3316         mutex_exit(&cpu_lock);
3317 }