1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 /*
  25  * Copyright (c) 2010, Intel Corporation.
  26  * All rights reserved.
  27  */
  28 
  29 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
  30 /*      All Rights Reserved   */
  31 
  32 /*
  33  * Portions of this source code were derived from Berkeley 4.3 BSD
  34  * under license from the Regents of the University of California.
  35  */
  36 
  37 /*
  38  * UNIX machine dependent virtual memory support.
  39  */
  40 
  41 #include <sys/types.h>
  42 #include <sys/param.h>
  43 #include <sys/systm.h>
  44 #include <sys/user.h>
  45 #include <sys/proc.h>
  46 #include <sys/kmem.h>
  47 #include <sys/vmem.h>
  48 #include <sys/buf.h>
  49 #include <sys/cpuvar.h>
  50 #include <sys/lgrp.h>
  51 #include <sys/disp.h>
  52 #include <sys/vm.h>
  53 #include <sys/mman.h>
  54 #include <sys/vnode.h>
  55 #include <sys/cred.h>
  56 #include <sys/exec.h>
  57 #include <sys/exechdr.h>
  58 #include <sys/debug.h>
  59 #include <sys/vmsystm.h>
  60 #include <sys/swap.h>
  61 #include <sys/dumphdr.h>
  62 
  63 #include <vm/hat.h>
  64 #include <vm/as.h>
  65 #include <vm/seg.h>
  66 #include <vm/seg_kp.h>
  67 #include <vm/seg_vn.h>
  68 #include <vm/page.h>
  69 #include <vm/seg_kmem.h>
  70 #include <vm/seg_kpm.h>
  71 #include <vm/vm_dep.h>
  72 
  73 #include <sys/cpu.h>
  74 #include <sys/vm_machparam.h>
  75 #include <sys/memlist.h>
  76 #include <sys/bootconf.h> /* XXX the memlist stuff belongs in memlist_plat.h */
  77 #include <vm/hat_i86.h>
  78 #include <sys/x86_archext.h>
  79 #include <sys/elf_386.h>
  80 #include <sys/cmn_err.h>
  81 #include <sys/archsystm.h>
  82 #include <sys/machsystm.h>
  83 
  84 #include <sys/vtrace.h>
  85 #include <sys/ddidmareq.h>
  86 #include <sys/promif.h>
  87 #include <sys/memnode.h>
  88 #include <sys/stack.h>
  89 #include <util/qsort.h>
  90 #include <sys/taskq.h>
  91 
  92 #ifdef __xpv
  93 
  94 #include <sys/hypervisor.h>
  95 #include <sys/xen_mmu.h>
  96 #include <sys/balloon_impl.h>
  97 
  98 /*
  99  * domain 0 pages usable for DMA are kept pre-allocated and kept in
 100  * distinct lists, ordered by increasing mfn.
 101  */
 102 static kmutex_t io_pool_lock;
 103 static kmutex_t contig_list_lock;
 104 static page_t *io_pool_4g;      /* pool for 32 bit dma limited devices */
 105 static page_t *io_pool_16m;     /* pool for 24 bit dma limited legacy devices */
 106 static long io_pool_cnt;
 107 static long io_pool_cnt_max = 0;
 108 #define DEFAULT_IO_POOL_MIN     128
 109 static long io_pool_cnt_min = DEFAULT_IO_POOL_MIN;
 110 static long io_pool_cnt_lowater = 0;
 111 static long io_pool_shrink_attempts; /* how many times did we try to shrink */
 112 static long io_pool_shrinks;    /* how many times did we really shrink */
 113 static long io_pool_grows;      /* how many times did we grow */
 114 static mfn_t start_mfn = 1;
 115 static caddr_t io_pool_kva;     /* use to alloc pages when needed */
 116 
 117 static int create_contig_pfnlist(uint_t);
 118 
 119 /*
 120  * percentage of phys mem to hold in the i/o pool
 121  */
 122 #define DEFAULT_IO_POOL_PCT     2
 123 static long io_pool_physmem_pct = DEFAULT_IO_POOL_PCT;
 124 static void page_io_pool_sub(page_t **, page_t *, page_t *);
 125 int ioalloc_dbg = 0;
 126 
 127 #endif /* __xpv */
 128 
 129 uint_t vac_colors = 1;
 130 
 131 int largepagesupport = 0;
 132 extern uint_t page_create_new;
 133 extern uint_t page_create_exists;
 134 extern uint_t page_create_putbacks;
 135 /*
 136  * Allow users to disable the kernel's use of SSE.
 137  */
 138 extern int use_sse_pagecopy, use_sse_pagezero;
 139 
 140 /*
 141  * combined memory ranges from mnode and memranges[] to manage single
 142  * mnode/mtype dimension in the page lists.
 143  */
 144 typedef struct {
 145         pfn_t   mnr_pfnlo;
 146         pfn_t   mnr_pfnhi;
 147         int     mnr_mnode;
 148         int     mnr_memrange;           /* index into memranges[] */
 149         int     mnr_next;               /* next lower PA mnoderange */
 150         int     mnr_exists;
 151         /* maintain page list stats */
 152         pgcnt_t mnr_mt_clpgcnt;         /* cache list cnt */
 153         pgcnt_t mnr_mt_flpgcnt[MMU_PAGE_SIZES]; /* free list cnt per szc */
 154         pgcnt_t mnr_mt_totcnt;          /* sum of cache and free lists */
 155 #ifdef DEBUG
 156         struct mnr_mts {                /* mnode/mtype szc stats */
 157                 pgcnt_t mnr_mts_pgcnt;
 158                 int     mnr_mts_colors;
 159                 pgcnt_t *mnr_mtsc_pgcnt;
 160         }       *mnr_mts;
 161 #endif
 162 } mnoderange_t;
 163 
 164 #define MEMRANGEHI(mtype)                                               \
 165         ((mtype > 0) ? memranges[mtype - 1] - 1: physmax)
 166 #define MEMRANGELO(mtype)       (memranges[mtype])
 167 
 168 #define MTYPE_FREEMEM(mt)       (mnoderanges[mt].mnr_mt_totcnt)
 169 
 170 /*
 171  * As the PC architecture evolved memory up was clumped into several
 172  * ranges for various historical I/O devices to do DMA.
 173  * < 16Meg - ISA bus
 174  * < 2Gig - ???
 175  * < 4Gig - PCI bus or drivers that don't understand PAE mode
 176  *
 177  * These are listed in reverse order, so that we can skip over unused
 178  * ranges on machines with small memories.
 179  *
 180  * For now under the Hypervisor, we'll only ever have one memrange.
 181  */
 182 #define PFN_4GIG        0x100000
 183 #define PFN_16MEG       0x1000
 184 /* Indices into the memory range (arch_memranges) array. */
 185 #define MRI_4G          0
 186 #define MRI_2G          1
 187 #define MRI_16M         2
 188 #define MRI_0           3
 189 static pfn_t arch_memranges[NUM_MEM_RANGES] = {
 190     PFN_4GIG,   /* pfn range for 4G and above */
 191     0x80000,    /* pfn range for 2G-4G */
 192     PFN_16MEG,  /* pfn range for 16M-2G */
 193     0x00000,    /* pfn range for 0-16M */
 194 };
 195 pfn_t *memranges = &arch_memranges[0];
 196 int nranges = NUM_MEM_RANGES;
 197 
 198 /*
 199  * This combines mem_node_config and memranges into one data
 200  * structure to be used for page list management.
 201  */
 202 mnoderange_t    *mnoderanges;
 203 int             mnoderangecnt;
 204 int             mtype4g;
 205 int             mtype16m;
 206 int             mtypetop;       /* index of highest pfn'ed mnoderange */
 207 
 208 /*
 209  * 4g memory management variables for systems with more than 4g of memory:
 210  *
 211  * physical memory below 4g is required for 32bit dma devices and, currently,
 212  * for kmem memory. On systems with more than 4g of memory, the pool of memory
 213  * below 4g can be depleted without any paging activity given that there is
 214  * likely to be sufficient memory above 4g.
 215  *
 216  * physmax4g is set true if the largest pfn is over 4g. The rest of the
 217  * 4g memory management code is enabled only when physmax4g is true.
 218  *
 219  * maxmem4g is the count of the maximum number of pages on the page lists
 220  * with physical addresses below 4g. It can be a lot less then 4g given that
 221  * BIOS may reserve large chunks of space below 4g for hot plug pci devices,
 222  * agp aperture etc.
 223  *
 224  * freemem4g maintains the count of the number of available pages on the
 225  * page lists with physical addresses below 4g.
 226  *
 227  * DESFREE4G specifies the desired amount of below 4g memory. It defaults to
 228  * 6% (desfree4gshift = 4) of maxmem4g.
 229  *
 230  * RESTRICT4G_ALLOC returns true if freemem4g falls below DESFREE4G
 231  * and the amount of physical memory above 4g is greater than freemem4g.
 232  * In this case, page_get_* routines will restrict below 4g allocations
 233  * for requests that don't specifically require it.
 234  */
 235 
 236 #define DESFREE4G       (maxmem4g >> desfree4gshift)
 237 
 238 #define RESTRICT4G_ALLOC                                        \
 239         (physmax4g && (freemem4g < DESFREE4G) && ((freemem4g << 1) < freemem))
 240 
 241 static pgcnt_t  maxmem4g;
 242 static pgcnt_t  freemem4g;
 243 static int      physmax4g;
 244 static int      desfree4gshift = 4;     /* maxmem4g shift to derive DESFREE4G */
 245 
 246 /*
 247  * 16m memory management:
 248  *
 249  * reserve some amount of physical memory below 16m for legacy devices.
 250  *
 251  * RESTRICT16M_ALLOC returns true if an there are sufficient free pages above
 252  * 16m or if the 16m pool drops below DESFREE16M.
 253  *
 254  * In this case, general page allocations via page_get_{free,cache}list
 255  * routines will be restricted from allocating from the 16m pool. Allocations
 256  * that require specific pfn ranges (page_get_anylist) and PG_PANIC allocations
 257  * are not restricted.
 258  */
 259 
 260 #define FREEMEM16M      MTYPE_FREEMEM(mtype16m)
 261 #define DESFREE16M      desfree16m
 262 #define RESTRICT16M_ALLOC(freemem, pgcnt, flags)                \
 263         ((freemem != 0) && ((flags & PG_PANIC) == 0) &&             \
 264             ((freemem >= (FREEMEM16M)) ||                    \
 265             (FREEMEM16M  < (DESFREE16M + pgcnt))))
 266 
 267 static pgcnt_t  desfree16m = 0x380;
 268 
 269 /*
 270  * This can be patched via /etc/system to allow old non-PAE aware device
 271  * drivers to use kmem_alloc'd memory on 32 bit systems with > 4Gig RAM.
 272  */
 273 int restricted_kmemalloc = 0;
 274 
 275 #ifdef VM_STATS
 276 struct {
 277         ulong_t pga_alloc;
 278         ulong_t pga_notfullrange;
 279         ulong_t pga_nulldmaattr;
 280         ulong_t pga_allocok;
 281         ulong_t pga_allocfailed;
 282         ulong_t pgma_alloc;
 283         ulong_t pgma_allocok;
 284         ulong_t pgma_allocfailed;
 285         ulong_t pgma_allocempty;
 286 } pga_vmstats;
 287 #endif
 288 
 289 uint_t mmu_page_sizes;
 290 
 291 /* How many page sizes the users can see */
 292 uint_t mmu_exported_page_sizes;
 293 
 294 /* page sizes that legacy applications can see */
 295 uint_t mmu_legacy_page_sizes;
 296 
 297 /*
 298  * Number of pages in 1 GB.  Don't enable automatic large pages if we have
 299  * fewer than this many pages.
 300  */
 301 pgcnt_t shm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
 302 pgcnt_t privm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
 303 
 304 /*
 305  * Maximum and default segment size tunables for user private
 306  * and shared anon memory, and user text and initialized data.
 307  * These can be patched via /etc/system to allow large pages
 308  * to be used for mapping application private and shared anon memory.
 309  */
 310 size_t mcntl0_lpsize = MMU_PAGESIZE;
 311 size_t max_uheap_lpsize = MMU_PAGESIZE;
 312 size_t default_uheap_lpsize = MMU_PAGESIZE;
 313 size_t max_ustack_lpsize = MMU_PAGESIZE;
 314 size_t default_ustack_lpsize = MMU_PAGESIZE;
 315 size_t max_privmap_lpsize = MMU_PAGESIZE;
 316 size_t max_uidata_lpsize = MMU_PAGESIZE;
 317 size_t max_utext_lpsize = MMU_PAGESIZE;
 318 size_t max_shm_lpsize = MMU_PAGESIZE;
 319 
 320 
 321 /*
 322  * initialized by page_coloring_init().
 323  */
 324 uint_t  page_colors;
 325 uint_t  page_colors_mask;
 326 uint_t  page_coloring_shift;
 327 int     cpu_page_colors;
 328 static uint_t   l2_colors;
 329 
 330 /*
 331  * Page freelists and cachelists are dynamically allocated once mnoderangecnt
 332  * and page_colors are calculated from the l2 cache n-way set size.  Within a
 333  * mnode range, the page freelist and cachelist are hashed into bins based on
 334  * color. This makes it easier to search for a page within a specific memory
 335  * range.
 336  */
 337 #define PAGE_COLORS_MIN 16
 338 
 339 page_t ****page_freelists;
 340 page_t ***page_cachelists;
 341 
 342 
 343 /*
 344  * Used by page layer to know about page sizes
 345  */
 346 hw_pagesize_t hw_page_array[MAX_NUM_LEVEL + 1];
 347 
 348 kmutex_t        *fpc_mutex[NPC_MUTEX];
 349 kmutex_t        *cpc_mutex[NPC_MUTEX];
 350 
 351 /* Lock to protect mnoderanges array for memory DR operations. */
 352 static kmutex_t mnoderange_lock;
 353 
 354 /*
 355  * Only let one thread at a time try to coalesce large pages, to
 356  * prevent them from working against each other.
 357  */
 358 static kmutex_t contig_lock;
 359 #define CONTIG_LOCK()   mutex_enter(&contig_lock);
 360 #define CONTIG_UNLOCK() mutex_exit(&contig_lock);
 361 
 362 #define PFN_16M         (mmu_btop((uint64_t)0x1000000))
 363 
 364 /*
 365  * Return the optimum page size for a given mapping
 366  */
 367 /*ARGSUSED*/
 368 size_t
 369 map_pgsz(int maptype, struct proc *p, caddr_t addr, size_t len, int memcntl)
 370 {
 371         level_t l = 0;
 372         size_t pgsz = MMU_PAGESIZE;
 373         size_t max_lpsize;
 374         uint_t mszc;
 375 
 376         ASSERT(maptype != MAPPGSZ_VA);
 377 
 378         if (maptype != MAPPGSZ_ISM && physmem < privm_lpg_min_physmem) {
 379                 return (MMU_PAGESIZE);
 380         }
 381 
 382         switch (maptype) {
 383         case MAPPGSZ_HEAP:
 384         case MAPPGSZ_STK:
 385                 max_lpsize = memcntl ? mcntl0_lpsize : (maptype ==
 386                     MAPPGSZ_HEAP ? max_uheap_lpsize : max_ustack_lpsize);
 387                 if (max_lpsize == MMU_PAGESIZE) {
 388                         return (MMU_PAGESIZE);
 389                 }
 390                 if (len == 0) {
 391                         len = (maptype == MAPPGSZ_HEAP) ? p->p_brkbase +
 392                             p->p_brksize - p->p_bssbase : p->p_stksize;
 393                 }
 394                 len = (maptype == MAPPGSZ_HEAP) ? MAX(len,
 395                     default_uheap_lpsize) : MAX(len, default_ustack_lpsize);
 396 
 397                 /*
 398                  * use the pages size that best fits len
 399                  */
 400                 for (l = mmu.umax_page_level; l > 0; --l) {
 401                         if (LEVEL_SIZE(l) > max_lpsize || len < LEVEL_SIZE(l)) {
 402                                 continue;
 403                         } else {
 404                                 pgsz = LEVEL_SIZE(l);
 405                         }
 406                         break;
 407                 }
 408 
 409                 mszc = (maptype == MAPPGSZ_HEAP ? p->p_brkpageszc :
 410                     p->p_stkpageszc);
 411                 if (addr == 0 && (pgsz < hw_page_array[mszc].hp_size)) {
 412                         pgsz = hw_page_array[mszc].hp_size;
 413                 }
 414                 return (pgsz);
 415 
 416         case MAPPGSZ_ISM:
 417                 for (l = mmu.umax_page_level; l > 0; --l) {
 418                         if (len >= LEVEL_SIZE(l))
 419                                 return (LEVEL_SIZE(l));
 420                 }
 421                 return (LEVEL_SIZE(0));
 422         }
 423         return (pgsz);
 424 }
 425 
 426 static uint_t
 427 map_szcvec(caddr_t addr, size_t size, uintptr_t off, size_t max_lpsize,
 428     size_t min_physmem)
 429 {
 430         caddr_t eaddr = addr + size;
 431         uint_t szcvec = 0;
 432         caddr_t raddr;
 433         caddr_t readdr;
 434         size_t  pgsz;
 435         int i;
 436 
 437         if (physmem < min_physmem || max_lpsize <= MMU_PAGESIZE) {
 438                 return (0);
 439         }
 440 
 441         for (i = mmu_exported_page_sizes - 1; i > 0; i--) {
 442                 pgsz = page_get_pagesize(i);
 443                 if (pgsz > max_lpsize) {
 444                         continue;
 445                 }
 446                 raddr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
 447                 readdr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
 448                 if (raddr < addr || raddr >= readdr) {
 449                         continue;
 450                 }
 451                 if (P2PHASE((uintptr_t)addr ^ off, pgsz)) {
 452                         continue;
 453                 }
 454                 /*
 455                  * Set szcvec to the remaining page sizes.
 456                  */
 457                 szcvec = ((1 << (i + 1)) - 1) & ~1;
 458                 break;
 459         }
 460         return (szcvec);
 461 }
 462 
 463 /*
 464  * Return a bit vector of large page size codes that
 465  * can be used to map [addr, addr + len) region.
 466  */
 467 /*ARGSUSED*/
 468 uint_t
 469 map_pgszcvec(caddr_t addr, size_t size, uintptr_t off, int flags, int type,
 470     int memcntl)
 471 {
 472         size_t max_lpsize = mcntl0_lpsize;
 473 
 474         if (mmu.max_page_level == 0)
 475                 return (0);
 476 
 477         if (flags & MAP_TEXT) {
 478                 if (!memcntl)
 479                         max_lpsize = max_utext_lpsize;
 480                 return (map_szcvec(addr, size, off, max_lpsize,
 481                     shm_lpg_min_physmem));
 482 
 483         } else if (flags & MAP_INITDATA) {
 484                 if (!memcntl)
 485                         max_lpsize = max_uidata_lpsize;
 486                 return (map_szcvec(addr, size, off, max_lpsize,
 487                     privm_lpg_min_physmem));
 488 
 489         } else if (type == MAPPGSZC_SHM) {
 490                 if (!memcntl)
 491                         max_lpsize = max_shm_lpsize;
 492                 return (map_szcvec(addr, size, off, max_lpsize,
 493                     shm_lpg_min_physmem));
 494 
 495         } else if (type == MAPPGSZC_HEAP) {
 496                 if (!memcntl)
 497                         max_lpsize = max_uheap_lpsize;
 498                 return (map_szcvec(addr, size, off, max_lpsize,
 499                     privm_lpg_min_physmem));
 500 
 501         } else if (type == MAPPGSZC_STACK) {
 502                 if (!memcntl)
 503                         max_lpsize = max_ustack_lpsize;
 504                 return (map_szcvec(addr, size, off, max_lpsize,
 505                     privm_lpg_min_physmem));
 506 
 507         } else {
 508                 if (!memcntl)
 509                         max_lpsize = max_privmap_lpsize;
 510                 return (map_szcvec(addr, size, off, max_lpsize,
 511                     privm_lpg_min_physmem));
 512         }
 513 }
 514 
 515 /*
 516  * Handle a pagefault.
 517  */
 518 faultcode_t
 519 pagefault(
 520         caddr_t addr,
 521         enum fault_type type,
 522         enum seg_rw rw,
 523         int iskernel)
 524 {
 525         struct as *as;
 526         struct hat *hat;
 527         struct proc *p;
 528         kthread_t *t;
 529         faultcode_t res;
 530         caddr_t base;
 531         size_t len;
 532         int err;
 533         int mapped_red;
 534         uintptr_t ea;
 535 
 536         ASSERT_STACK_ALIGNED();
 537 
 538         if (INVALID_VADDR(addr))
 539                 return (FC_NOMAP);
 540 
 541         mapped_red = segkp_map_red();
 542 
 543         if (iskernel) {
 544                 as = &kas;
 545                 hat = as->a_hat;
 546         } else {
 547                 t = curthread;
 548                 p = ttoproc(t);
 549                 as = p->p_as;
 550                 hat = as->a_hat;
 551         }
 552 
 553         /*
 554          * Dispatch pagefault.
 555          */
 556         res = as_fault(hat, as, addr, 1, type, rw);
 557 
 558         /*
 559          * If this isn't a potential unmapped hole in the user's
 560          * UNIX data or stack segments, just return status info.
 561          */
 562         if (res != FC_NOMAP || iskernel)
 563                 goto out;
 564 
 565         /*
 566          * Check to see if we happened to faulted on a currently unmapped
 567          * part of the UNIX data or stack segments.  If so, create a zfod
 568          * mapping there and then try calling the fault routine again.
 569          */
 570         base = p->p_brkbase;
 571         len = p->p_brksize;
 572 
 573         if (addr < base || addr >= base + len) {          /* data seg? */
 574                 base = (caddr_t)p->p_usrstack - p->p_stksize;
 575                 len = p->p_stksize;
 576                 if (addr < base || addr >= p->p_usrstack) {    /* stack seg? */
 577                         /* not in either UNIX data or stack segments */
 578                         res = FC_NOMAP;
 579                         goto out;
 580                 }
 581         }
 582 
 583         /*
 584          * the rest of this function implements a 3.X 4.X 5.X compatibility
 585          * This code is probably not needed anymore
 586          */
 587         if (p->p_model == DATAMODEL_ILP32) {
 588 
 589                 /* expand the gap to the page boundaries on each side */
 590                 ea = P2ROUNDUP((uintptr_t)base + len, MMU_PAGESIZE);
 591                 base = (caddr_t)P2ALIGN((uintptr_t)base, MMU_PAGESIZE);
 592                 len = ea - (uintptr_t)base;
 593 
 594                 as_rangelock(as);
 595                 if (as_gap(as, MMU_PAGESIZE, &base, &len, AH_CONTAIN, addr) ==
 596                     0) {
 597                         err = as_map(as, base, len, segvn_create, zfod_argsp);
 598                         as_rangeunlock(as);
 599                         if (err) {
 600                                 res = FC_MAKE_ERR(err);
 601                                 goto out;
 602                         }
 603                 } else {
 604                         /*
 605                          * This page is already mapped by another thread after
 606                          * we returned from as_fault() above.  We just fall
 607                          * through as_fault() below.
 608                          */
 609                         as_rangeunlock(as);
 610                 }
 611 
 612                 res = as_fault(hat, as, addr, 1, F_INVAL, rw);
 613         }
 614 
 615 out:
 616         if (mapped_red)
 617                 segkp_unmap_red();
 618 
 619         return (res);
 620 }
 621 
 622 void
 623 map_addr(caddr_t *addrp, size_t len, offset_t off, int vacalign, uint_t flags)
 624 {
 625         struct proc *p = curproc;
 626         caddr_t userlimit = (flags & _MAP_LOW32) ?
 627             (caddr_t)_userlimit32 : p->p_as->a_userlimit;
 628 
 629         map_addr_proc(addrp, len, off, vacalign, userlimit, curproc, flags);
 630 }
 631 
 632 /*ARGSUSED*/
 633 int
 634 map_addr_vacalign_check(caddr_t addr, u_offset_t off)
 635 {
 636         return (0);
 637 }
 638 
 639 /*
 640  * map_addr_proc() is the routine called when the system is to
 641  * choose an address for the user.  We will pick an address
 642  * range which is the highest available below userlimit.
 643  *
 644  * Every mapping will have a redzone of a single page on either side of
 645  * the request. This is done to leave one page unmapped between segments.
 646  * This is not required, but it's useful for the user because if their
 647  * program strays across a segment boundary, it will catch a fault
 648  * immediately making debugging a little easier.  Currently the redzone
 649  * is mandatory.
 650  *
 651  * addrp is a value/result parameter.
 652  *      On input it is a hint from the user to be used in a completely
 653  *      machine dependent fashion.  We decide to completely ignore this hint.
 654  *      If MAP_ALIGN was specified, addrp contains the minimal alignment, which
 655  *      must be some "power of two" multiple of pagesize.
 656  *
 657  *      On output it is NULL if no address can be found in the current
 658  *      processes address space or else an address that is currently
 659  *      not mapped for len bytes with a page of red zone on either side.
 660  *
 661  *      vacalign is not needed on x86 (it's for viturally addressed caches)
 662  */
 663 /*ARGSUSED*/
 664 void
 665 map_addr_proc(
 666         caddr_t *addrp,
 667         size_t len,
 668         offset_t off,
 669         int vacalign,
 670         caddr_t userlimit,
 671         struct proc *p,
 672         uint_t flags)
 673 {
 674         struct as *as = p->p_as;
 675         caddr_t addr;
 676         caddr_t base;
 677         size_t slen;
 678         size_t align_amount;
 679 
 680         ASSERT32(userlimit == as->a_userlimit);
 681 
 682         base = p->p_brkbase;
 683 #if defined(__amd64)
 684         /*
 685          * XX64 Yes, this needs more work.
 686          */
 687         if (p->p_model == DATAMODEL_NATIVE) {
 688                 if (userlimit < as->a_userlimit) {
 689                         /*
 690                          * This happens when a program wants to map
 691                          * something in a range that's accessible to a
 692                          * program in a smaller address space.  For example,
 693                          * a 64-bit program calling mmap32(2) to guarantee
 694                          * that the returned address is below 4Gbytes.
 695                          */
 696                         ASSERT((uintptr_t)userlimit < ADDRESS_C(0xffffffff));
 697 
 698                         if (userlimit > base)
 699                                 slen = userlimit - base;
 700                         else {
 701                                 *addrp = NULL;
 702                                 return;
 703                         }
 704                 } else {
 705                         /*
 706                          * XX64 This layout is probably wrong .. but in
 707                          * the event we make the amd64 address space look
 708                          * like sparcv9 i.e. with the stack -above- the
 709                          * heap, this bit of code might even be correct.
 710                          */
 711                         slen = p->p_usrstack - base -
 712                             ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK);
 713                 }
 714         } else
 715 #endif
 716                 slen = userlimit - base;
 717 
 718         /* Make len be a multiple of PAGESIZE */
 719         len = (len + PAGEOFFSET) & PAGEMASK;
 720 
 721         /*
 722          * figure out what the alignment should be
 723          *
 724          * XX64 -- is there an ELF_AMD64_MAXPGSZ or is it the same????
 725          */
 726         if (len <= ELF_386_MAXPGSZ) {
 727                 /*
 728                  * Align virtual addresses to ensure that ELF shared libraries
 729                  * are mapped with the appropriate alignment constraints by
 730                  * the run-time linker.
 731                  */
 732                 align_amount = ELF_386_MAXPGSZ;
 733         } else {
 734                 /*
 735                  * For 32-bit processes, only those which have specified
 736                  * MAP_ALIGN and an addr will be aligned on a larger page size.
 737                  * Not doing so can potentially waste up to 1G of process
 738                  * address space.
 739                  */
 740                 int lvl = (p->p_model == DATAMODEL_ILP32) ? 1 :
 741                     mmu.umax_page_level;
 742 
 743                 while (lvl && len < LEVEL_SIZE(lvl))
 744                         --lvl;
 745 
 746                 align_amount = LEVEL_SIZE(lvl);
 747         }
 748         if ((flags & MAP_ALIGN) && ((uintptr_t)*addrp > align_amount))
 749                 align_amount = (uintptr_t)*addrp;
 750 
 751         ASSERT(ISP2(align_amount));
 752         ASSERT(align_amount == 0 || align_amount >= PAGESIZE);
 753 
 754         off = off & (align_amount - 1);
 755         /*
 756          * Look for a large enough hole starting below userlimit.
 757          * After finding it, use the upper part.
 758          */
 759         if (as_gap_aligned(as, len, &base, &slen, AH_HI, NULL, align_amount,
 760             PAGESIZE, off) == 0) {
 761                 caddr_t as_addr;
 762 
 763                 /*
 764                  * addr is the highest possible address to use since we have
 765                  * a PAGESIZE redzone at the beginning and end.
 766                  */
 767                 addr = base + slen - (PAGESIZE + len);
 768                 as_addr = addr;
 769                 /*
 770                  * Round address DOWN to the alignment amount and
 771                  * add the offset in.
 772                  * If addr is greater than as_addr, len would not be large
 773                  * enough to include the redzone, so we must adjust down
 774                  * by the alignment amount.
 775                  */
 776                 addr = (caddr_t)((uintptr_t)addr & (~(align_amount - 1)));
 777                 addr += (uintptr_t)off;
 778                 if (addr > as_addr) {
 779                         addr -= align_amount;
 780                 }
 781 
 782                 ASSERT(addr > base);
 783                 ASSERT(addr + len < base + slen);
 784                 ASSERT(((uintptr_t)addr & (align_amount - 1)) ==
 785                     ((uintptr_t)(off)));
 786                 *addrp = addr;
 787         } else {
 788                 *addrp = NULL;  /* no more virtual space */
 789         }
 790 }
 791 
 792 int valid_va_range_aligned_wraparound;
 793 
 794 /*
 795  * Determine whether [*basep, *basep + *lenp) contains a mappable range of
 796  * addresses at least "minlen" long, where the base of the range is at "off"
 797  * phase from an "align" boundary and there is space for a "redzone"-sized
 798  * redzone on either side of the range.  On success, 1 is returned and *basep
 799  * and *lenp are adjusted to describe the acceptable range (including
 800  * the redzone).  On failure, 0 is returned.
 801  */
 802 /*ARGSUSED3*/
 803 int
 804 valid_va_range_aligned(caddr_t *basep, size_t *lenp, size_t minlen, int dir,
 805     size_t align, size_t redzone, size_t off)
 806 {
 807         uintptr_t hi, lo;
 808         size_t tot_len;
 809 
 810         ASSERT(align == 0 ? off == 0 : off < align);
 811         ASSERT(ISP2(align));
 812         ASSERT(align == 0 || align >= PAGESIZE);
 813 
 814         lo = (uintptr_t)*basep;
 815         hi = lo + *lenp;
 816         tot_len = minlen + 2 * redzone; /* need at least this much space */
 817 
 818         /*
 819          * If hi rolled over the top, try cutting back.
 820          */
 821         if (hi < lo) {
 822                 *lenp = 0UL - lo - 1UL;
 823                 /* See if this really happens. If so, then we figure out why */
 824                 valid_va_range_aligned_wraparound++;
 825                 hi = lo + *lenp;
 826         }
 827         if (*lenp < tot_len) {
 828                 return (0);
 829         }
 830 
 831 #if defined(__amd64)
 832         /*
 833          * Deal with a possible hole in the address range between
 834          * hole_start and hole_end that should never be mapped.
 835          */
 836         if (lo < hole_start) {
 837                 if (hi > hole_start) {
 838                         if (hi < hole_end) {
 839                                 hi = hole_start;
 840                         } else {
 841                                 /* lo < hole_start && hi >= hole_end */
 842                                 if (dir == AH_LO) {
 843                                         /*
 844                                          * prefer lowest range
 845                                          */
 846                                         if (hole_start - lo >= tot_len)
 847                                                 hi = hole_start;
 848                                         else if (hi - hole_end >= tot_len)
 849                                                 lo = hole_end;
 850                                         else
 851                                                 return (0);
 852                                 } else {
 853                                         /*
 854                                          * prefer highest range
 855                                          */
 856                                         if (hi - hole_end >= tot_len)
 857                                                 lo = hole_end;
 858                                         else if (hole_start - lo >= tot_len)
 859                                                 hi = hole_start;
 860                                         else
 861                                                 return (0);
 862                                 }
 863                         }
 864                 }
 865         } else {
 866                 /* lo >= hole_start */
 867                 if (hi < hole_end)
 868                         return (0);
 869                 if (lo < hole_end)
 870                         lo = hole_end;
 871         }
 872 #endif
 873 
 874         if (hi - lo < tot_len)
 875                 return (0);
 876 
 877         if (align > 1) {
 878                 uintptr_t tlo = lo + redzone;
 879                 uintptr_t thi = hi - redzone;
 880                 tlo = (uintptr_t)P2PHASEUP(tlo, align, off);
 881                 if (tlo < lo + redzone) {
 882                         return (0);
 883                 }
 884                 if (thi < tlo || thi - tlo < minlen) {
 885                         return (0);
 886                 }
 887         }
 888 
 889         *basep = (caddr_t)lo;
 890         *lenp = hi - lo;
 891         return (1);
 892 }
 893 
 894 /*
 895  * Determine whether [*basep, *basep + *lenp) contains a mappable range of
 896  * addresses at least "minlen" long.  On success, 1 is returned and *basep
 897  * and *lenp are adjusted to describe the acceptable range.  On failure, 0
 898  * is returned.
 899  */
 900 int
 901 valid_va_range(caddr_t *basep, size_t *lenp, size_t minlen, int dir)
 902 {
 903         return (valid_va_range_aligned(basep, lenp, minlen, dir, 0, 0, 0));
 904 }
 905 
 906 /*
 907  * Determine whether [addr, addr+len] are valid user addresses.
 908  */
 909 /*ARGSUSED*/
 910 int
 911 valid_usr_range(caddr_t addr, size_t len, uint_t prot, struct as *as,
 912     caddr_t userlimit)
 913 {
 914         caddr_t eaddr = addr + len;
 915 
 916         if (eaddr <= addr || addr >= userlimit || eaddr > userlimit)
 917                 return (RANGE_BADADDR);
 918 
 919 #if defined(__amd64)
 920         /*
 921          * Check for the VA hole
 922          */
 923         if (eaddr > (caddr_t)hole_start && addr < (caddr_t)hole_end)
 924                 return (RANGE_BADADDR);
 925 #endif
 926 
 927         return (RANGE_OKAY);
 928 }
 929 
 930 /*
 931  * Return 1 if the page frame is onboard memory, else 0.
 932  */
 933 int
 934 pf_is_memory(pfn_t pf)
 935 {
 936         if (pfn_is_foreign(pf))
 937                 return (0);
 938         return (address_in_memlist(phys_install, pfn_to_pa(pf), 1));
 939 }
 940 
 941 /*
 942  * return the memrange containing pfn
 943  */
 944 int
 945 memrange_num(pfn_t pfn)
 946 {
 947         int n;
 948 
 949         for (n = 0; n < nranges - 1; ++n) {
 950                 if (pfn >= memranges[n])
 951                         break;
 952         }
 953         return (n);
 954 }
 955 
 956 /*
 957  * return the mnoderange containing pfn
 958  */
 959 /*ARGSUSED*/
 960 int
 961 pfn_2_mtype(pfn_t pfn)
 962 {
 963 #if defined(__xpv)
 964         return (0);
 965 #else
 966         int     n;
 967 
 968         /* Always start from highest pfn and work our way down */
 969         for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
 970                 if (pfn >= mnoderanges[n].mnr_pfnlo) {
 971                         break;
 972                 }
 973         }
 974         return (n);
 975 #endif
 976 }
 977 
 978 #if !defined(__xpv)
 979 /*
 980  * is_contigpage_free:
 981  *      returns a page list of contiguous pages. It minimally has to return
 982  *      minctg pages. Caller determines minctg based on the scatter-gather
 983  *      list length.
 984  *
 985  *      pfnp is set to the next page frame to search on return.
 986  */
 987 static page_t *
 988 is_contigpage_free(
 989         pfn_t *pfnp,
 990         pgcnt_t *pgcnt,
 991         pgcnt_t minctg,
 992         uint64_t pfnseg,
 993         int iolock)
 994 {
 995         int     i = 0;
 996         pfn_t   pfn = *pfnp;
 997         page_t  *pp;
 998         page_t  *plist = NULL;
 999 
1000         /*
1001          * fail if pfn + minctg crosses a segment boundary.
1002          * Adjust for next starting pfn to begin at segment boundary.
1003          */
1004 
1005         if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg)) {
1006                 *pfnp = roundup(*pfnp, pfnseg + 1);
1007                 return (NULL);
1008         }
1009 
1010         do {
1011 retry:
1012                 pp = page_numtopp_nolock(pfn + i);
1013                 if ((pp == NULL) || IS_DUMP_PAGE(pp) ||
1014                     (page_trylock(pp, SE_EXCL) == 0)) {
1015                         (*pfnp)++;
1016                         break;
1017                 }
1018                 if (page_pptonum(pp) != pfn + i) {
1019                         page_unlock(pp);
1020                         goto retry;
1021                 }
1022 
1023                 if (!(PP_ISFREE(pp))) {
1024                         page_unlock(pp);
1025                         (*pfnp)++;
1026                         break;
1027                 }
1028 
1029                 if (!PP_ISAGED(pp)) {
1030                         page_list_sub(pp, PG_CACHE_LIST);
1031                         page_hashout(pp, (kmutex_t *)NULL);
1032                 } else {
1033                         page_list_sub(pp, PG_FREE_LIST);
1034                 }
1035 
1036                 if (iolock)
1037                         page_io_lock(pp);
1038                 page_list_concat(&plist, &pp);
1039 
1040                 /*
1041                  * exit loop when pgcnt satisfied or segment boundary reached.
1042                  */
1043 
1044         } while ((++i < *pgcnt) && ((pfn + i) & pfnseg));
1045 
1046         *pfnp += i;             /* set to next pfn to search */
1047 
1048         if (i >= minctg) {
1049                 *pgcnt -= i;
1050                 return (plist);
1051         }
1052 
1053         /*
1054          * failure: minctg not satisfied.
1055          *
1056          * if next request crosses segment boundary, set next pfn
1057          * to search from the segment boundary.
1058          */
1059         if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg))
1060                 *pfnp = roundup(*pfnp, pfnseg + 1);
1061 
1062         /* clean up any pages already allocated */
1063 
1064         while (plist) {
1065                 pp = plist;
1066                 page_sub(&plist, pp);
1067                 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
1068                 if (iolock)
1069                         page_io_unlock(pp);
1070                 page_unlock(pp);
1071         }
1072 
1073         return (NULL);
1074 }
1075 #endif  /* !__xpv */
1076 
1077 /*
1078  * verify that pages being returned from allocator have correct DMA attribute
1079  */
1080 #ifndef DEBUG
1081 #define check_dma(a, b, c) (void)(0)
1082 #else
1083 static void
1084 check_dma(ddi_dma_attr_t *dma_attr, page_t *pp, int cnt)
1085 {
1086         if (dma_attr == NULL)
1087                 return;
1088 
1089         while (cnt-- > 0) {
1090                 if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) <
1091                     dma_attr->dma_attr_addr_lo)
1092                         panic("PFN (pp=%p) below dma_attr_addr_lo", (void *)pp);
1093                 if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) >=
1094                     dma_attr->dma_attr_addr_hi)
1095                         panic("PFN (pp=%p) above dma_attr_addr_hi", (void *)pp);
1096                 pp = pp->p_next;
1097         }
1098 }
1099 #endif
1100 
1101 #if !defined(__xpv)
1102 static page_t *
1103 page_get_contigpage(pgcnt_t *pgcnt, ddi_dma_attr_t *mattr, int iolock)
1104 {
1105         pfn_t           pfn;
1106         int             sgllen;
1107         uint64_t        pfnseg;
1108         pgcnt_t         minctg;
1109         page_t          *pplist = NULL, *plist;
1110         uint64_t        lo, hi;
1111         pgcnt_t         pfnalign = 0;
1112         static pfn_t    startpfn;
1113         static pgcnt_t  lastctgcnt;
1114         uintptr_t       align;
1115 
1116         CONTIG_LOCK();
1117 
1118         if (mattr) {
1119                 lo = mmu_btop((mattr->dma_attr_addr_lo + MMU_PAGEOFFSET));
1120                 hi = mmu_btop(mattr->dma_attr_addr_hi);
1121                 if (hi >= physmax)
1122                         hi = physmax - 1;
1123                 sgllen = mattr->dma_attr_sgllen;
1124                 pfnseg = mmu_btop(mattr->dma_attr_seg);
1125 
1126                 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
1127                 if (align > MMU_PAGESIZE)
1128                         pfnalign = mmu_btop(align);
1129 
1130                 /*
1131                  * in order to satisfy the request, must minimally
1132                  * acquire minctg contiguous pages
1133                  */
1134                 minctg = howmany(*pgcnt, sgllen);
1135 
1136                 ASSERT(hi >= lo);
1137 
1138                 /*
1139                  * start from where last searched if the minctg >= lastctgcnt
1140                  */
1141                 if (minctg < lastctgcnt || startpfn < lo || startpfn > hi)
1142                         startpfn = lo;
1143         } else {
1144                 hi = physmax - 1;
1145                 lo = 0;
1146                 sgllen = 1;
1147                 pfnseg = mmu.highest_pfn;
1148                 minctg = *pgcnt;
1149 
1150                 if (minctg < lastctgcnt)
1151                         startpfn = lo;
1152         }
1153         lastctgcnt = minctg;
1154 
1155         ASSERT(pfnseg + 1 >= (uint64_t)minctg);
1156 
1157         /* conserve 16m memory - start search above 16m when possible */
1158         if (hi > PFN_16M && startpfn < PFN_16M)
1159                 startpfn = PFN_16M;
1160 
1161         pfn = startpfn;
1162         if (pfnalign)
1163                 pfn = P2ROUNDUP(pfn, pfnalign);
1164 
1165         while (pfn + minctg - 1 <= hi) {
1166 
1167                 plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
1168                 if (plist) {
1169                         page_list_concat(&pplist, &plist);
1170                         sgllen--;
1171                         /*
1172                          * return when contig pages no longer needed
1173                          */
1174                         if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
1175                                 startpfn = pfn;
1176                                 CONTIG_UNLOCK();
1177                                 check_dma(mattr, pplist, *pgcnt);
1178                                 return (pplist);
1179                         }
1180                         minctg = howmany(*pgcnt, sgllen);
1181                 }
1182                 if (pfnalign)
1183                         pfn = P2ROUNDUP(pfn, pfnalign);
1184         }
1185 
1186         /* cannot find contig pages in specified range */
1187         if (startpfn == lo) {
1188                 CONTIG_UNLOCK();
1189                 return (NULL);
1190         }
1191 
1192         /* did not start with lo previously */
1193         pfn = lo;
1194         if (pfnalign)
1195                 pfn = P2ROUNDUP(pfn, pfnalign);
1196 
1197         /* allow search to go above startpfn */
1198         while (pfn < startpfn) {
1199 
1200                 plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
1201                 if (plist != NULL) {
1202 
1203                         page_list_concat(&pplist, &plist);
1204                         sgllen--;
1205 
1206                         /*
1207                          * return when contig pages no longer needed
1208                          */
1209                         if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
1210                                 startpfn = pfn;
1211                                 CONTIG_UNLOCK();
1212                                 check_dma(mattr, pplist, *pgcnt);
1213                                 return (pplist);
1214                         }
1215                         minctg = howmany(*pgcnt, sgllen);
1216                 }
1217                 if (pfnalign)
1218                         pfn = P2ROUNDUP(pfn, pfnalign);
1219         }
1220         CONTIG_UNLOCK();
1221         return (NULL);
1222 }
1223 #endif  /* !__xpv */
1224 
1225 /*
1226  * mnode_range_cnt() calculates the number of memory ranges for mnode and
1227  * memranges[]. Used to determine the size of page lists and mnoderanges.
1228  */
1229 int
1230 mnode_range_cnt(int mnode)
1231 {
1232 #if defined(__xpv)
1233         ASSERT(mnode == 0);
1234         return (1);
1235 #else   /* __xpv */
1236         int     mri;
1237         int     mnrcnt = 0;
1238 
1239         if (mem_node_config[mnode].exists != 0) {
1240                 mri = nranges - 1;
1241 
1242                 /* find the memranges index below contained in mnode range */
1243 
1244                 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1245                         mri--;
1246 
1247                 /*
1248                  * increment mnode range counter when memranges or mnode
1249                  * boundary is reached.
1250                  */
1251                 while (mri >= 0 &&
1252                     mem_node_config[mnode].physmax >= MEMRANGELO(mri)) {
1253                         mnrcnt++;
1254                         if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1255                                 mri--;
1256                         else
1257                                 break;
1258                 }
1259         }
1260         ASSERT(mnrcnt <= MAX_MNODE_MRANGES);
1261         return (mnrcnt);
1262 #endif  /* __xpv */
1263 }
1264 
1265 /*
1266  * mnode_range_setup() initializes mnoderanges.
1267  */
1268 void
1269 mnode_range_setup(mnoderange_t *mnoderanges)
1270 {
1271         mnoderange_t *mp = mnoderanges;
1272         int     mnode, mri;
1273         int     mindex = 0;     /* current index into mnoderanges array */
1274         int     i, j;
1275         pfn_t   hipfn;
1276         int     last, hi;
1277 
1278         for (mnode = 0; mnode < max_mem_nodes; mnode++) {
1279                 if (mem_node_config[mnode].exists == 0)
1280                         continue;
1281 
1282                 mri = nranges - 1;
1283 
1284                 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1285                         mri--;
1286 
1287                 while (mri >= 0 && mem_node_config[mnode].physmax >=
1288                     MEMRANGELO(mri)) {
1289                         mnoderanges->mnr_pfnlo = MAX(MEMRANGELO(mri),
1290                             mem_node_config[mnode].physbase);
1291                         mnoderanges->mnr_pfnhi = MIN(MEMRANGEHI(mri),
1292                             mem_node_config[mnode].physmax);
1293                         mnoderanges->mnr_mnode = mnode;
1294                         mnoderanges->mnr_memrange = mri;
1295                         mnoderanges->mnr_exists = 1;
1296                         mnoderanges++;
1297                         mindex++;
1298                         if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1299                                 mri--;
1300                         else
1301                                 break;
1302                 }
1303         }
1304 
1305         /*
1306          * For now do a simple sort of the mnoderanges array to fill in
1307          * the mnr_next fields.  Since mindex is expected to be relatively
1308          * small, using a simple O(N^2) algorithm.
1309          */
1310         for (i = 0; i < mindex; i++) {
1311                 if (mp[i].mnr_pfnlo == 0)       /* find lowest */
1312                         break;
1313         }
1314         ASSERT(i < mindex);
1315         last = i;
1316         mtype16m = last;
1317         mp[last].mnr_next = -1;
1318         for (i = 0; i < mindex - 1; i++) {
1319                 hipfn = (pfn_t)(-1);
1320                 hi = -1;
1321                 /* find next highest mnode range */
1322                 for (j = 0; j < mindex; j++) {
1323                         if (mp[j].mnr_pfnlo > mp[last].mnr_pfnlo &&
1324                             mp[j].mnr_pfnlo < hipfn) {
1325                                 hipfn = mp[j].mnr_pfnlo;
1326                                 hi = j;
1327                         }
1328                 }
1329                 mp[hi].mnr_next = last;
1330                 last = hi;
1331         }
1332         mtypetop = last;
1333 }
1334 
1335 #ifndef __xpv
1336 /*
1337  * Update mnoderanges for memory hot-add DR operations.
1338  */
1339 static void
1340 mnode_range_add(int mnode)
1341 {
1342         int     *prev;
1343         int     n, mri;
1344         pfn_t   start, end;
1345         extern  void membar_sync(void);
1346 
1347         ASSERT(0 <= mnode && mnode < max_mem_nodes);
1348         ASSERT(mem_node_config[mnode].exists);
1349         start = mem_node_config[mnode].physbase;
1350         end = mem_node_config[mnode].physmax;
1351         ASSERT(start <= end);
1352         mutex_enter(&mnoderange_lock);
1353 
1354 #ifdef  DEBUG
1355         /* Check whether it interleaves with other memory nodes. */
1356         for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1357                 ASSERT(mnoderanges[n].mnr_exists);
1358                 if (mnoderanges[n].mnr_mnode == mnode)
1359                         continue;
1360                 ASSERT(start > mnoderanges[n].mnr_pfnhi ||
1361                     end < mnoderanges[n].mnr_pfnlo);
1362         }
1363 #endif  /* DEBUG */
1364 
1365         mri = nranges - 1;
1366         while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1367                 mri--;
1368         while (mri >= 0 && mem_node_config[mnode].physmax >= MEMRANGELO(mri)) {
1369                 /* Check whether mtype already exists. */
1370                 for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1371                         if (mnoderanges[n].mnr_mnode == mnode &&
1372                             mnoderanges[n].mnr_memrange == mri) {
1373                                 mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri),
1374                                     start);
1375                                 mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri),
1376                                     end);
1377                                 break;
1378                         }
1379                 }
1380 
1381                 /* Add a new entry if it doesn't exist yet. */
1382                 if (n == -1) {
1383                         /* Try to find an unused entry in mnoderanges array. */
1384                         for (n = 0; n < mnoderangecnt; n++) {
1385                                 if (mnoderanges[n].mnr_exists == 0)
1386                                         break;
1387                         }
1388                         ASSERT(n < mnoderangecnt);
1389                         mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri), start);
1390                         mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri), end);
1391                         mnoderanges[n].mnr_mnode = mnode;
1392                         mnoderanges[n].mnr_memrange = mri;
1393                         mnoderanges[n].mnr_exists = 1;
1394                         /* Page 0 should always be present. */
1395                         for (prev = &mtypetop;
1396                             mnoderanges[*prev].mnr_pfnlo > start;
1397                             prev = &mnoderanges[*prev].mnr_next) {
1398                                 ASSERT(mnoderanges[*prev].mnr_next >= 0);
1399                                 ASSERT(mnoderanges[*prev].mnr_pfnlo > end);
1400                         }
1401                         mnoderanges[n].mnr_next = *prev;
1402                         membar_sync();
1403                         *prev = n;
1404                 }
1405 
1406                 if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1407                         mri--;
1408                 else
1409                         break;
1410         }
1411 
1412         mutex_exit(&mnoderange_lock);
1413 }
1414 
1415 /*
1416  * Update mnoderanges for memory hot-removal DR operations.
1417  */
1418 static void
1419 mnode_range_del(int mnode)
1420 {
1421         _NOTE(ARGUNUSED(mnode));
1422         ASSERT(0 <= mnode && mnode < max_mem_nodes);
1423         /* TODO: support deletion operation. */
1424         ASSERT(0);
1425 }
1426 
1427 void
1428 plat_slice_add(pfn_t start, pfn_t end)
1429 {
1430         mem_node_add_slice(start, end);
1431         if (plat_dr_enabled()) {
1432                 mnode_range_add(PFN_2_MEM_NODE(start));
1433         }
1434 }
1435 
1436 void
1437 plat_slice_del(pfn_t start, pfn_t end)
1438 {
1439         ASSERT(PFN_2_MEM_NODE(start) == PFN_2_MEM_NODE(end));
1440         ASSERT(plat_dr_enabled());
1441         mnode_range_del(PFN_2_MEM_NODE(start));
1442         mem_node_del_slice(start, end);
1443 }
1444 #endif  /* __xpv */
1445 
1446 /*ARGSUSED*/
1447 int
1448 mtype_init(vnode_t *vp, caddr_t vaddr, uint_t *flags, size_t pgsz)
1449 {
1450         int mtype = mtypetop;
1451 
1452 #if !defined(__xpv)
1453 #if defined(__i386)
1454         /*
1455          * set the mtype range
1456          * - kmem requests need to be below 4g if restricted_kmemalloc is set.
1457          * - for non kmem requests, set range to above 4g if memory below 4g
1458          * runs low.
1459          */
1460         if (restricted_kmemalloc && VN_ISKAS(vp) &&
1461             (caddr_t)(vaddr) >= kernelheap &&
1462             (caddr_t)(vaddr) < ekernelheap) {
1463                 ASSERT(physmax4g);
1464                 mtype = mtype4g;
1465                 if (RESTRICT16M_ALLOC(freemem4g - btop(pgsz),
1466                     btop(pgsz), *flags)) {
1467                         *flags |= PGI_MT_RANGE16M;
1468                 } else {
1469                         VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1470                         VM_STAT_COND_ADD((*flags & PG_PANIC),
1471                             vmm_vmstats.pgpanicalloc);
1472                         *flags |= PGI_MT_RANGE0;
1473                 }
1474                 return (mtype);
1475         }
1476 #endif  /* __i386 */
1477 
1478         if (RESTRICT4G_ALLOC) {
1479                 VM_STAT_ADD(vmm_vmstats.restrict4gcnt);
1480                 /* here only for > 4g systems */
1481                 *flags |= PGI_MT_RANGE4G;
1482         } else if (RESTRICT16M_ALLOC(freemem, btop(pgsz), *flags)) {
1483                 *flags |= PGI_MT_RANGE16M;
1484         } else {
1485                 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1486                 VM_STAT_COND_ADD((*flags & PG_PANIC), vmm_vmstats.pgpanicalloc);
1487                 *flags |= PGI_MT_RANGE0;
1488         }
1489 #endif /* !__xpv */
1490         return (mtype);
1491 }
1492 
1493 
1494 /* mtype init for page_get_replacement_page */
1495 /*ARGSUSED*/
1496 int
1497 mtype_pgr_init(int *flags, page_t *pp, int mnode, pgcnt_t pgcnt)
1498 {
1499         int mtype = mtypetop;
1500 #if !defined(__xpv)
1501         if (RESTRICT16M_ALLOC(freemem, pgcnt, *flags)) {
1502                 *flags |= PGI_MT_RANGE16M;
1503         } else {
1504                 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1505                 *flags |= PGI_MT_RANGE0;
1506         }
1507 #endif
1508         return (mtype);
1509 }
1510 
1511 /*
1512  * Determine if the mnode range specified in mtype contains memory belonging
1513  * to memory node mnode.  If flags & PGI_MT_RANGE is set then mtype contains
1514  * the range from high pfn to 0, 16m or 4g.
1515  *
1516  * Return first mnode range type index found otherwise return -1 if none found.
1517  */
1518 int
1519 mtype_func(int mnode, int mtype, uint_t flags)
1520 {
1521         if (flags & PGI_MT_RANGE) {
1522                 int     mnr_lim = MRI_0;
1523 
1524                 if (flags & PGI_MT_NEXT) {
1525                         mtype = mnoderanges[mtype].mnr_next;
1526                 }
1527                 if (flags & PGI_MT_RANGE4G)
1528                         mnr_lim = MRI_4G;       /* exclude 0-4g range */
1529                 else if (flags & PGI_MT_RANGE16M)
1530                         mnr_lim = MRI_16M;      /* exclude 0-16m range */
1531                 while (mtype != -1 &&
1532                     mnoderanges[mtype].mnr_memrange <= mnr_lim) {
1533                         if (mnoderanges[mtype].mnr_mnode == mnode)
1534                                 return (mtype);
1535                         mtype = mnoderanges[mtype].mnr_next;
1536                 }
1537         } else if (mnoderanges[mtype].mnr_mnode == mnode) {
1538                 return (mtype);
1539         }
1540         return (-1);
1541 }
1542 
1543 /*
1544  * Update the page list max counts with the pfn range specified by the
1545  * input parameters.
1546  */
1547 void
1548 mtype_modify_max(pfn_t startpfn, long cnt)
1549 {
1550         int             mtype;
1551         pgcnt_t         inc;
1552         spgcnt_t        scnt = (spgcnt_t)(cnt);
1553         pgcnt_t         acnt = ABS(scnt);
1554         pfn_t           endpfn = startpfn + acnt;
1555         pfn_t           pfn, lo;
1556 
1557         if (!physmax4g)
1558                 return;
1559 
1560         mtype = mtypetop;
1561         for (pfn = endpfn; pfn > startpfn; ) {
1562                 ASSERT(mtype != -1);
1563                 lo = mnoderanges[mtype].mnr_pfnlo;
1564                 if (pfn > lo) {
1565                         if (startpfn >= lo) {
1566                                 inc = pfn - startpfn;
1567                         } else {
1568                                 inc = pfn - lo;
1569                         }
1570                         if (mnoderanges[mtype].mnr_memrange != MRI_4G) {
1571                                 if (scnt > 0)
1572                                         maxmem4g += inc;
1573                                 else
1574                                         maxmem4g -= inc;
1575                         }
1576                         pfn -= inc;
1577                 }
1578                 mtype = mnoderanges[mtype].mnr_next;
1579         }
1580 }
1581 
1582 int
1583 mtype_2_mrange(int mtype)
1584 {
1585         return (mnoderanges[mtype].mnr_memrange);
1586 }
1587 
1588 void
1589 mnodetype_2_pfn(int mnode, int mtype, pfn_t *pfnlo, pfn_t *pfnhi)
1590 {
1591         _NOTE(ARGUNUSED(mnode));
1592         ASSERT(mnoderanges[mtype].mnr_mnode == mnode);
1593         *pfnlo = mnoderanges[mtype].mnr_pfnlo;
1594         *pfnhi = mnoderanges[mtype].mnr_pfnhi;
1595 }
1596 
1597 size_t
1598 plcnt_sz(size_t ctrs_sz)
1599 {
1600 #ifdef DEBUG
1601         int     szc, colors;
1602 
1603         ctrs_sz += mnoderangecnt * sizeof (struct mnr_mts) * mmu_page_sizes;
1604         for (szc = 0; szc < mmu_page_sizes; szc++) {
1605                 colors = page_get_pagecolors(szc);
1606                 ctrs_sz += mnoderangecnt * sizeof (pgcnt_t) * colors;
1607         }
1608 #endif
1609         return (ctrs_sz);
1610 }
1611 
1612 caddr_t
1613 plcnt_init(caddr_t addr)
1614 {
1615 #ifdef DEBUG
1616         int     mt, szc, colors;
1617 
1618         for (mt = 0; mt < mnoderangecnt; mt++) {
1619                 mnoderanges[mt].mnr_mts = (struct mnr_mts *)addr;
1620                 addr += (sizeof (struct mnr_mts) * mmu_page_sizes);
1621                 for (szc = 0; szc < mmu_page_sizes; szc++) {
1622                         colors = page_get_pagecolors(szc);
1623                         mnoderanges[mt].mnr_mts[szc].mnr_mts_colors = colors;
1624                         mnoderanges[mt].mnr_mts[szc].mnr_mtsc_pgcnt =
1625                             (pgcnt_t *)addr;
1626                         addr += (sizeof (pgcnt_t) * colors);
1627                 }
1628         }
1629 #endif
1630         return (addr);
1631 }
1632 
1633 void
1634 plcnt_inc_dec(page_t *pp, int mtype, int szc, long cnt, int flags)
1635 {
1636         _NOTE(ARGUNUSED(pp));
1637 #ifdef DEBUG
1638         int     bin = PP_2_BIN(pp);
1639 
1640         atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mts_pgcnt, cnt);
1641         atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mtsc_pgcnt[bin],
1642             cnt);
1643 #endif
1644         ASSERT(mtype == PP_2_MTYPE(pp));
1645         if (physmax4g && mnoderanges[mtype].mnr_memrange != MRI_4G)
1646                 atomic_add_long(&freemem4g, cnt);
1647         if (flags & PG_CACHE_LIST)
1648                 atomic_add_long(&mnoderanges[mtype].mnr_mt_clpgcnt, cnt);
1649         else
1650                 atomic_add_long(&mnoderanges[mtype].mnr_mt_flpgcnt[szc], cnt);
1651         atomic_add_long(&mnoderanges[mtype].mnr_mt_totcnt, cnt);
1652 }
1653 
1654 /*
1655  * Returns the free page count for mnode
1656  */
1657 int
1658 mnode_pgcnt(int mnode)
1659 {
1660         int     mtype = mtypetop;
1661         int     flags = PGI_MT_RANGE0;
1662         pgcnt_t pgcnt = 0;
1663 
1664         mtype = mtype_func(mnode, mtype, flags);
1665 
1666         while (mtype != -1) {
1667                 pgcnt += MTYPE_FREEMEM(mtype);
1668                 mtype = mtype_func(mnode, mtype, flags | PGI_MT_NEXT);
1669         }
1670         return (pgcnt);
1671 }
1672 
1673 /*
1674  * Initialize page coloring variables based on the l2 cache parameters.
1675  * Calculate and return memory needed for page coloring data structures.
1676  */
1677 size_t
1678 page_coloring_init(uint_t l2_sz, int l2_linesz, int l2_assoc)
1679 {
1680         _NOTE(ARGUNUSED(l2_linesz));
1681         size_t  colorsz = 0;
1682         int     i;
1683         int     colors;
1684 
1685 #if defined(__xpv)
1686         /*
1687          * Hypervisor domains currently don't have any concept of NUMA.
1688          * Hence we'll act like there is only 1 memrange.
1689          */
1690         i = memrange_num(1);
1691 #else /* !__xpv */
1692         /*
1693          * Reduce the memory ranges lists if we don't have large amounts
1694          * of memory. This avoids searching known empty free lists.
1695          * To support memory DR operations, we need to keep memory ranges
1696          * for possible memory hot-add operations.
1697          */
1698         if (plat_dr_physmax > physmax)
1699                 i = memrange_num(plat_dr_physmax);
1700         else
1701                 i = memrange_num(physmax);
1702 #if defined(__i386)
1703         if (i > MRI_4G)
1704                 restricted_kmemalloc = 0;
1705 #endif
1706         /* physmax greater than 4g */
1707         if (i == MRI_4G)
1708                 physmax4g = 1;
1709 #endif /* !__xpv */
1710         memranges += i;
1711         nranges -= i;
1712 
1713         ASSERT(mmu_page_sizes <= MMU_PAGE_SIZES);
1714 
1715         ASSERT(ISP2(l2_linesz));
1716         ASSERT(l2_sz > MMU_PAGESIZE);
1717 
1718         /* l2_assoc is 0 for fully associative l2 cache */
1719         if (l2_assoc)
1720                 l2_colors = MAX(1, l2_sz / (l2_assoc * MMU_PAGESIZE));
1721         else
1722                 l2_colors = 1;
1723 
1724         ASSERT(ISP2(l2_colors));
1725 
1726         /* for scalability, configure at least PAGE_COLORS_MIN color bins */
1727         page_colors = MAX(l2_colors, PAGE_COLORS_MIN);
1728 
1729         /*
1730          * cpu_page_colors is non-zero when a page color may be spread across
1731          * multiple bins.
1732          */
1733         if (l2_colors < page_colors)
1734                 cpu_page_colors = l2_colors;
1735 
1736         ASSERT(ISP2(page_colors));
1737 
1738         page_colors_mask = page_colors - 1;
1739 
1740         ASSERT(ISP2(CPUSETSIZE()));
1741         page_coloring_shift = lowbit(CPUSETSIZE());
1742 
1743         /* initialize number of colors per page size */
1744         for (i = 0; i <= mmu.max_page_level; i++) {
1745                 hw_page_array[i].hp_size = LEVEL_SIZE(i);
1746                 hw_page_array[i].hp_shift = LEVEL_SHIFT(i);
1747                 hw_page_array[i].hp_pgcnt = LEVEL_SIZE(i) >> LEVEL_SHIFT(0);
1748                 hw_page_array[i].hp_colors = (page_colors_mask >>
1749                     (hw_page_array[i].hp_shift - hw_page_array[0].hp_shift))
1750                     + 1;
1751                 colorequivszc[i] = 0;
1752         }
1753 
1754         /*
1755          * The value of cpu_page_colors determines if additional color bins
1756          * need to be checked for a particular color in the page_get routines.
1757          */
1758         if (cpu_page_colors != 0) {
1759 
1760                 int a = lowbit(page_colors) - lowbit(cpu_page_colors);
1761                 ASSERT(a > 0);
1762                 ASSERT(a < 16);
1763 
1764                 for (i = 0; i <= mmu.max_page_level; i++) {
1765                         if ((colors = hw_page_array[i].hp_colors) <= 1) {
1766                                 colorequivszc[i] = 0;
1767                                 continue;
1768                         }
1769                         while ((colors >> a) == 0)
1770                                 a--;
1771                         ASSERT(a >= 0);
1772 
1773                         /* higher 4 bits encodes color equiv mask */
1774                         colorequivszc[i] = (a << 4);
1775                 }
1776         }
1777 
1778         /* factor in colorequiv to check additional 'equivalent' bins. */
1779         if (colorequiv > 1) {
1780 
1781                 int a = lowbit(colorequiv) - 1;
1782                 if (a > 15)
1783                         a = 15;
1784 
1785                 for (i = 0; i <= mmu.max_page_level; i++) {
1786                         if ((colors = hw_page_array[i].hp_colors) <= 1) {
1787                                 continue;
1788                         }
1789                         while ((colors >> a) == 0)
1790                                 a--;
1791                         if ((a << 4) > colorequivszc[i]) {
1792                                 colorequivszc[i] = (a << 4);
1793                         }
1794                 }
1795         }
1796 
1797         /* size for mnoderanges */
1798         for (mnoderangecnt = 0, i = 0; i < max_mem_nodes; i++)
1799                 mnoderangecnt += mnode_range_cnt(i);
1800         if (plat_dr_support_memory()) {
1801                 /*
1802                  * Reserve enough space for memory DR operations.
1803                  * Two extra mnoderanges for possbile fragmentations,
1804                  * one for the 2G boundary and the other for the 4G boundary.
1805                  * We don't expect a memory board crossing the 16M boundary
1806                  * for memory hot-add operations on x86 platforms.
1807                  */
1808                 mnoderangecnt += 2 + max_mem_nodes - lgrp_plat_node_cnt;
1809         }
1810         colorsz = mnoderangecnt * sizeof (mnoderange_t);
1811 
1812         /* size for fpc_mutex and cpc_mutex */
1813         colorsz += (2 * max_mem_nodes * sizeof (kmutex_t) * NPC_MUTEX);
1814 
1815         /* size of page_freelists */
1816         colorsz += mnoderangecnt * sizeof (page_t ***);
1817         colorsz += mnoderangecnt * mmu_page_sizes * sizeof (page_t **);
1818 
1819         for (i = 0; i < mmu_page_sizes; i++) {
1820                 colors = page_get_pagecolors(i);
1821                 colorsz += mnoderangecnt * colors * sizeof (page_t *);
1822         }
1823 
1824         /* size of page_cachelists */
1825         colorsz += mnoderangecnt * sizeof (page_t **);
1826         colorsz += mnoderangecnt * page_colors * sizeof (page_t *);
1827 
1828         return (colorsz);
1829 }
1830 
1831 /*
1832  * Called once at startup to configure page_coloring data structures and
1833  * does the 1st page_free()/page_freelist_add().
1834  */
1835 void
1836 page_coloring_setup(caddr_t pcmemaddr)
1837 {
1838         int     i;
1839         int     j;
1840         int     k;
1841         caddr_t addr;
1842         int     colors;
1843 
1844         /*
1845          * do page coloring setup
1846          */
1847         addr = pcmemaddr;
1848 
1849         mnoderanges = (mnoderange_t *)addr;
1850         addr += (mnoderangecnt * sizeof (mnoderange_t));
1851 
1852         mnode_range_setup(mnoderanges);
1853 
1854         if (physmax4g)
1855                 mtype4g = pfn_2_mtype(0xfffff);
1856 
1857         for (k = 0; k < NPC_MUTEX; k++) {
1858                 fpc_mutex[k] = (kmutex_t *)addr;
1859                 addr += (max_mem_nodes * sizeof (kmutex_t));
1860         }
1861         for (k = 0; k < NPC_MUTEX; k++) {
1862                 cpc_mutex[k] = (kmutex_t *)addr;
1863                 addr += (max_mem_nodes * sizeof (kmutex_t));
1864         }
1865         page_freelists = (page_t ****)addr;
1866         addr += (mnoderangecnt * sizeof (page_t ***));
1867 
1868         page_cachelists = (page_t ***)addr;
1869         addr += (mnoderangecnt * sizeof (page_t **));
1870 
1871         for (i = 0; i < mnoderangecnt; i++) {
1872                 page_freelists[i] = (page_t ***)addr;
1873                 addr += (mmu_page_sizes * sizeof (page_t **));
1874 
1875                 for (j = 0; j < mmu_page_sizes; j++) {
1876                         colors = page_get_pagecolors(j);
1877                         page_freelists[i][j] = (page_t **)addr;
1878                         addr += (colors * sizeof (page_t *));
1879                 }
1880                 page_cachelists[i] = (page_t **)addr;
1881                 addr += (page_colors * sizeof (page_t *));
1882         }
1883 }
1884 
1885 #if defined(__xpv)
1886 /*
1887  * Give back 10% of the io_pool pages to the free list.
1888  * Don't shrink the pool below some absolute minimum.
1889  */
1890 static void
1891 page_io_pool_shrink()
1892 {
1893         int retcnt;
1894         page_t *pp, *pp_first, *pp_last, **curpool;
1895         mfn_t mfn;
1896         int bothpools = 0;
1897 
1898         mutex_enter(&io_pool_lock);
1899         io_pool_shrink_attempts++;      /* should be a kstat? */
1900         retcnt = io_pool_cnt / 10;
1901         if (io_pool_cnt - retcnt < io_pool_cnt_min)
1902                 retcnt = io_pool_cnt - io_pool_cnt_min;
1903         if (retcnt <= 0)
1904                 goto done;
1905         io_pool_shrinks++;      /* should be a kstat? */
1906         curpool = &io_pool_4g;
1907 domore:
1908         /*
1909          * Loop through taking pages from the end of the list
1910          * (highest mfns) till amount to return reached.
1911          */
1912         for (pp = *curpool; pp && retcnt > 0; ) {
1913                 pp_first = pp_last = pp->p_prev;
1914                 if (pp_first == *curpool)
1915                         break;
1916                 retcnt--;
1917                 io_pool_cnt--;
1918                 page_io_pool_sub(curpool, pp_first, pp_last);
1919                 if ((mfn = pfn_to_mfn(pp->p_pagenum)) < start_mfn)
1920                         start_mfn = mfn;
1921                 page_free(pp_first, 1);
1922                 pp = *curpool;
1923         }
1924         if (retcnt != 0 && !bothpools) {
1925                 /*
1926                  * If not enough found in less constrained pool try the
1927                  * more constrained one.
1928                  */
1929                 curpool = &io_pool_16m;
1930                 bothpools = 1;
1931                 goto domore;
1932         }
1933 done:
1934         mutex_exit(&io_pool_lock);
1935 }
1936 
1937 #endif  /* __xpv */
1938 
1939 uint_t
1940 page_create_update_flags_x86(uint_t flags)
1941 {
1942 #if defined(__xpv)
1943         /*
1944          * Check this is an urgent allocation and free pages are depleted.
1945          */
1946         if (!(flags & PG_WAIT) && freemem < desfree)
1947                 page_io_pool_shrink();
1948 #else /* !__xpv */
1949         /*
1950          * page_create_get_something may call this because 4g memory may be
1951          * depleted. Set flags to allow for relocation of base page below
1952          * 4g if necessary.
1953          */
1954         if (physmax4g)
1955                 flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI);
1956 #endif /* __xpv */
1957         return (flags);
1958 }
1959 
1960 /*ARGSUSED*/
1961 int
1962 bp_color(struct buf *bp)
1963 {
1964         return (0);
1965 }
1966 
1967 #if defined(__xpv)
1968 
1969 /*
1970  * Take pages out of an io_pool
1971  */
1972 static void
1973 page_io_pool_sub(page_t **poolp, page_t *pp_first, page_t *pp_last)
1974 {
1975         if (*poolp == pp_first) {
1976                 *poolp = pp_last->p_next;
1977                 if (*poolp == pp_first)
1978                         *poolp = NULL;
1979         }
1980         pp_first->p_prev->p_next = pp_last->p_next;
1981         pp_last->p_next->p_prev = pp_first->p_prev;
1982         pp_first->p_prev = pp_last;
1983         pp_last->p_next = pp_first;
1984 }
1985 
1986 /*
1987  * Put a page on the io_pool list. The list is ordered by increasing MFN.
1988  */
1989 static void
1990 page_io_pool_add(page_t **poolp, page_t *pp)
1991 {
1992         page_t  *look;
1993         mfn_t   mfn = mfn_list[pp->p_pagenum];
1994 
1995         if (*poolp == NULL) {
1996                 *poolp = pp;
1997                 pp->p_next = pp;
1998                 pp->p_prev = pp;
1999                 return;
2000         }
2001 
2002         /*
2003          * Since we try to take pages from the high end of the pool
2004          * chances are good that the pages to be put on the list will
2005          * go at or near the end of the list. so start at the end and
2006          * work backwards.
2007          */
2008         look = (*poolp)->p_prev;
2009         while (mfn < mfn_list[look->p_pagenum]) {
2010                 look = look->p_prev;
2011                 if (look == (*poolp)->p_prev)
2012                         break; /* backed all the way to front of list */
2013         }
2014 
2015         /* insert after look */
2016         pp->p_prev = look;
2017         pp->p_next = look->p_next;
2018         pp->p_next->p_prev = pp;
2019         look->p_next = pp;
2020         if (mfn < mfn_list[(*poolp)->p_pagenum]) {
2021                 /*
2022                  * we inserted a new first list element
2023                  * adjust pool pointer to newly inserted element
2024                  */
2025                 *poolp = pp;
2026         }
2027 }
2028 
2029 /*
2030  * Add a page to the io_pool.  Setting the force flag will force the page
2031  * into the io_pool no matter what.
2032  */
2033 static void
2034 add_page_to_pool(page_t *pp, int force)
2035 {
2036         page_t *highest;
2037         page_t *freep = NULL;
2038 
2039         mutex_enter(&io_pool_lock);
2040         /*
2041          * Always keep the scarce low memory pages
2042          */
2043         if (mfn_list[pp->p_pagenum] < PFN_16MEG) {
2044                 ++io_pool_cnt;
2045                 page_io_pool_add(&io_pool_16m, pp);
2046                 goto done;
2047         }
2048         if (io_pool_cnt < io_pool_cnt_max || force || io_pool_4g == NULL) {
2049                 ++io_pool_cnt;
2050                 page_io_pool_add(&io_pool_4g, pp);
2051         } else {
2052                 highest = io_pool_4g->p_prev;
2053                 if (mfn_list[pp->p_pagenum] < mfn_list[highest->p_pagenum]) {
2054                         page_io_pool_sub(&io_pool_4g, highest, highest);
2055                         page_io_pool_add(&io_pool_4g, pp);
2056                         freep = highest;
2057                 } else {
2058                         freep = pp;
2059                 }
2060         }
2061 done:
2062         mutex_exit(&io_pool_lock);
2063         if (freep)
2064                 page_free(freep, 1);
2065 }
2066 
2067 
2068 int contig_pfn_cnt;     /* no of pfns in the contig pfn list */
2069 int contig_pfn_max;     /* capacity of the contig pfn list */
2070 int next_alloc_pfn;     /* next position in list to start a contig search */
2071 int contig_pfnlist_updates;     /* pfn list update count */
2072 int contig_pfnlist_builds;      /* how many times have we (re)built list */
2073 int contig_pfnlist_buildfailed; /* how many times has list build failed */
2074 int create_contig_pending;      /* nonzero means taskq creating contig list */
2075 pfn_t *contig_pfn_list = NULL;  /* list of contig pfns in ascending mfn order */
2076 
2077 /*
2078  * Function to use in sorting a list of pfns by their underlying mfns.
2079  */
2080 static int
2081 mfn_compare(const void *pfnp1, const void *pfnp2)
2082 {
2083         mfn_t mfn1 = mfn_list[*(pfn_t *)pfnp1];
2084         mfn_t mfn2 = mfn_list[*(pfn_t *)pfnp2];
2085 
2086         if (mfn1 > mfn2)
2087                 return (1);
2088         if (mfn1 < mfn2)
2089                 return (-1);
2090         return (0);
2091 }
2092 
2093 /*
2094  * Compact the contig_pfn_list by tossing all the non-contiguous
2095  * elements from the list.
2096  */
2097 static void
2098 compact_contig_pfn_list(void)
2099 {
2100         pfn_t pfn, lapfn, prev_lapfn;
2101         mfn_t mfn;
2102         int i, newcnt = 0;
2103 
2104         prev_lapfn = 0;
2105         for (i = 0; i < contig_pfn_cnt - 1; i++) {
2106                 pfn = contig_pfn_list[i];
2107                 lapfn = contig_pfn_list[i + 1];
2108                 mfn = mfn_list[pfn];
2109                 /*
2110                  * See if next pfn is for a contig mfn
2111                  */
2112                 if (mfn_list[lapfn] != mfn + 1)
2113                         continue;
2114                 /*
2115                  * pfn and lookahead are both put in list
2116                  * unless pfn is the previous lookahead.
2117                  */
2118                 if (pfn != prev_lapfn)
2119                         contig_pfn_list[newcnt++] = pfn;
2120                 contig_pfn_list[newcnt++] = lapfn;
2121                 prev_lapfn = lapfn;
2122         }
2123         for (i = newcnt; i < contig_pfn_cnt; i++)
2124                 contig_pfn_list[i] = 0;
2125         contig_pfn_cnt = newcnt;
2126 }
2127 
2128 /*ARGSUSED*/
2129 static void
2130 call_create_contiglist(void *arg)
2131 {
2132         (void) create_contig_pfnlist(PG_WAIT);
2133 }
2134 
2135 /*
2136  * Create list of freelist pfns that have underlying
2137  * contiguous mfns.  The list is kept in ascending mfn order.
2138  * returns 1 if list created else 0.
2139  */
2140 static int
2141 create_contig_pfnlist(uint_t flags)
2142 {
2143         pfn_t pfn;
2144         page_t *pp;
2145         int ret = 1;
2146 
2147         mutex_enter(&contig_list_lock);
2148         if (contig_pfn_list != NULL)
2149                 goto out;
2150         contig_pfn_max = freemem + (freemem / 10);
2151         contig_pfn_list = kmem_zalloc(contig_pfn_max * sizeof (pfn_t),
2152             (flags & PG_WAIT) ? KM_SLEEP : KM_NOSLEEP);
2153         if (contig_pfn_list == NULL) {
2154                 /*
2155                  * If we could not create the contig list (because
2156                  * we could not sleep for memory).  Dispatch a taskq that can
2157                  * sleep to get the memory.
2158                  */
2159                 if (!create_contig_pending) {
2160                         if (taskq_dispatch(system_taskq, call_create_contiglist,
2161                             NULL, TQ_NOSLEEP) != NULL)
2162                                 create_contig_pending = 1;
2163                 }
2164                 contig_pfnlist_buildfailed++;   /* count list build failures */
2165                 ret = 0;
2166                 goto out;
2167         }
2168         create_contig_pending = 0;
2169         ASSERT(contig_pfn_cnt == 0);
2170         for (pfn = 0; pfn < mfn_count; pfn++) {
2171                 pp = page_numtopp_nolock(pfn);
2172                 if (pp == NULL || !PP_ISFREE(pp))
2173                         continue;
2174                 contig_pfn_list[contig_pfn_cnt] = pfn;
2175                 if (++contig_pfn_cnt == contig_pfn_max)
2176                         break;
2177         }
2178         /*
2179          * Sanity check the new list.
2180          */
2181         if (contig_pfn_cnt < 2) { /* no contig pfns */
2182                 contig_pfn_cnt = 0;
2183                 contig_pfnlist_buildfailed++;
2184                 kmem_free(contig_pfn_list, contig_pfn_max * sizeof (pfn_t));
2185                 contig_pfn_list = NULL;
2186                 contig_pfn_max = 0;
2187                 ret = 0;
2188                 goto out;
2189         }
2190         qsort(contig_pfn_list, contig_pfn_cnt, sizeof (pfn_t), mfn_compare);
2191         compact_contig_pfn_list();
2192         /*
2193          * Make sure next search of the newly created contiguous pfn
2194          * list starts at the beginning of the list.
2195          */
2196         next_alloc_pfn = 0;
2197         contig_pfnlist_builds++;        /* count list builds */
2198 out:
2199         mutex_exit(&contig_list_lock);
2200         return (ret);
2201 }
2202 
2203 
2204 /*
2205  * Toss the current contig pfnlist.  Someone is about to do a massive
2206  * update to pfn<->mfn mappings.  So we have them destroy the list and lock
2207  * it till they are done with their update.
2208  */
2209 void
2210 clear_and_lock_contig_pfnlist()
2211 {
2212         pfn_t *listp = NULL;
2213         size_t listsize;
2214 
2215         mutex_enter(&contig_list_lock);
2216         if (contig_pfn_list != NULL) {
2217                 listp = contig_pfn_list;
2218                 listsize = contig_pfn_max * sizeof (pfn_t);
2219                 contig_pfn_list = NULL;
2220                 contig_pfn_max = contig_pfn_cnt = 0;
2221         }
2222         if (listp != NULL)
2223                 kmem_free(listp, listsize);
2224 }
2225 
2226 /*
2227  * Unlock the contig_pfn_list.  The next attempted use of it will cause
2228  * it to be re-created.
2229  */
2230 void
2231 unlock_contig_pfnlist()
2232 {
2233         mutex_exit(&contig_list_lock);
2234 }
2235 
2236 /*
2237  * Update the contiguous pfn list in response to a pfn <-> mfn reassignment
2238  */
2239 void
2240 update_contig_pfnlist(pfn_t pfn, mfn_t oldmfn, mfn_t newmfn)
2241 {
2242         int probe_hi, probe_lo, probe_pos, insert_after, insert_point;
2243         pfn_t probe_pfn;
2244         mfn_t probe_mfn;
2245         int drop_lock = 0;
2246 
2247         if (mutex_owner(&contig_list_lock) != curthread) {
2248                 drop_lock = 1;
2249                 mutex_enter(&contig_list_lock);
2250         }
2251         if (contig_pfn_list == NULL)
2252                 goto done;
2253         contig_pfnlist_updates++;
2254         /*
2255          * Find the pfn in the current list.  Use a binary chop to locate it.
2256          */
2257         probe_hi = contig_pfn_cnt - 1;
2258         probe_lo = 0;
2259         probe_pos = (probe_hi + probe_lo) / 2;
2260         while ((probe_pfn = contig_pfn_list[probe_pos]) != pfn) {
2261                 if (probe_pos == probe_lo) { /* pfn not in list */
2262                         probe_pos = -1;
2263                         break;
2264                 }
2265                 if (pfn_to_mfn(probe_pfn) <= oldmfn)
2266                         probe_lo = probe_pos;
2267                 else
2268                         probe_hi = probe_pos;
2269                 probe_pos = (probe_hi + probe_lo) / 2;
2270         }
2271         if (probe_pos >= 0) {
2272                 /*
2273                  * Remove pfn from list and ensure next alloc
2274                  * position stays in bounds.
2275                  */
2276                 if (--contig_pfn_cnt <= next_alloc_pfn)
2277                         next_alloc_pfn = 0;
2278                 if (contig_pfn_cnt < 2) { /* no contig pfns */
2279                         contig_pfn_cnt = 0;
2280                         kmem_free(contig_pfn_list,
2281                             contig_pfn_max * sizeof (pfn_t));
2282                         contig_pfn_list = NULL;
2283                         contig_pfn_max = 0;
2284                         goto done;
2285                 }
2286                 ovbcopy(&contig_pfn_list[probe_pos + 1],
2287                     &contig_pfn_list[probe_pos],
2288                     (contig_pfn_cnt - probe_pos) * sizeof (pfn_t));
2289         }
2290         if (newmfn == MFN_INVALID)
2291                 goto done;
2292         /*
2293          * Check if new mfn has adjacent mfns in the list
2294          */
2295         probe_hi = contig_pfn_cnt - 1;
2296         probe_lo = 0;
2297         insert_after = -2;
2298         do {
2299                 probe_pos = (probe_hi + probe_lo) / 2;
2300                 probe_mfn = pfn_to_mfn(contig_pfn_list[probe_pos]);
2301                 if (newmfn == probe_mfn + 1)
2302                         insert_after = probe_pos;
2303                 else if (newmfn == probe_mfn - 1)
2304                         insert_after = probe_pos - 1;
2305                 if (probe_pos == probe_lo)
2306                         break;
2307                 if (probe_mfn <= newmfn)
2308                         probe_lo = probe_pos;
2309                 else
2310                         probe_hi = probe_pos;
2311         } while (insert_after == -2);
2312         /*
2313          * If there is space in the list and there are adjacent mfns
2314          * insert the pfn in to its proper place in the list.
2315          */
2316         if (insert_after != -2 && contig_pfn_cnt + 1 <= contig_pfn_max) {
2317                 insert_point = insert_after + 1;
2318                 ovbcopy(&contig_pfn_list[insert_point],
2319                     &contig_pfn_list[insert_point + 1],
2320                     (contig_pfn_cnt - insert_point) * sizeof (pfn_t));
2321                 contig_pfn_list[insert_point] = pfn;
2322                 contig_pfn_cnt++;
2323         }
2324 done:
2325         if (drop_lock)
2326                 mutex_exit(&contig_list_lock);
2327 }
2328 
2329 /*
2330  * Called to (re-)populate the io_pool from the free page lists.
2331  */
2332 long
2333 populate_io_pool(void)
2334 {
2335         pfn_t pfn;
2336         mfn_t mfn, max_mfn;
2337         page_t *pp;
2338 
2339         /*
2340          * Figure out the bounds of the pool on first invocation.
2341          * We use a percentage of memory for the io pool size.
2342          * we allow that to shrink, but not to less than a fixed minimum
2343          */
2344         if (io_pool_cnt_max == 0) {
2345                 io_pool_cnt_max = physmem / (100 / io_pool_physmem_pct);
2346                 io_pool_cnt_lowater = io_pool_cnt_max;
2347                 /*
2348                  * This is the first time in populate_io_pool, grab a va to use
2349                  * when we need to allocate pages.
2350                  */
2351                 io_pool_kva = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
2352         }
2353         /*
2354          * If we are out of pages in the pool, then grow the size of the pool
2355          */
2356         if (io_pool_cnt == 0) {
2357                 /*
2358                  * Grow the max size of the io pool by 5%, but never more than
2359                  * 25% of physical memory.
2360                  */
2361                 if (io_pool_cnt_max < physmem / 4)
2362                         io_pool_cnt_max += io_pool_cnt_max / 20;
2363         }
2364         io_pool_grows++;        /* should be a kstat? */
2365 
2366         /*
2367          * Get highest mfn on this platform, but limit to the 32 bit DMA max.
2368          */
2369         (void) mfn_to_pfn(start_mfn);
2370         max_mfn = MIN(cached_max_mfn, PFN_4GIG);
2371         for (mfn = start_mfn; mfn < max_mfn; start_mfn = ++mfn) {
2372                 pfn = mfn_to_pfn(mfn);
2373                 if (pfn & PFN_IS_FOREIGN_MFN)
2374                         continue;
2375                 /*
2376                  * try to allocate it from free pages
2377                  */
2378                 pp = page_numtopp_alloc(pfn);
2379                 if (pp == NULL)
2380                         continue;
2381                 PP_CLRFREE(pp);
2382                 add_page_to_pool(pp, 1);
2383                 if (io_pool_cnt >= io_pool_cnt_max)
2384                         break;
2385         }
2386 
2387         return (io_pool_cnt);
2388 }
2389 
2390 /*
2391  * Destroy a page that was being used for DMA I/O. It may or
2392  * may not actually go back to the io_pool.
2393  */
2394 void
2395 page_destroy_io(page_t *pp)
2396 {
2397         mfn_t mfn = mfn_list[pp->p_pagenum];
2398 
2399         /*
2400          * When the page was alloc'd a reservation was made, release it now
2401          */
2402         page_unresv(1);
2403         /*
2404          * Unload translations, if any, then hash out the
2405          * page to erase its identity.
2406          */
2407         (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
2408         page_hashout(pp, NULL);
2409 
2410         /*
2411          * If the page came from the free lists, just put it back to them.
2412          * DomU pages always go on the free lists as well.
2413          */
2414         if (!DOMAIN_IS_INITDOMAIN(xen_info) || mfn >= PFN_4GIG) {
2415                 page_free(pp, 1);
2416                 return;
2417         }
2418 
2419         add_page_to_pool(pp, 0);
2420 }
2421 
2422 
2423 long contig_searches;           /* count of times contig pages requested */
2424 long contig_search_restarts;    /* count of contig ranges tried */
2425 long contig_search_failed;      /* count of contig alloc failures */
2426 
2427 /*
2428  * Free partial page list
2429  */
2430 static void
2431 free_partial_list(page_t **pplist)
2432 {
2433         page_t *pp;
2434 
2435         while (*pplist != NULL) {
2436                 pp = *pplist;
2437                 page_io_pool_sub(pplist, pp, pp);
2438                 page_free(pp, 1);
2439         }
2440 }
2441 
2442 /*
2443  * Look thru the contiguous pfns that are not part of the io_pool for
2444  * contiguous free pages.  Return a list of the found pages or NULL.
2445  */
2446 page_t *
2447 find_contig_free(uint_t npages, uint_t flags, uint64_t pfnseg,
2448     pgcnt_t pfnalign)
2449 {
2450         page_t *pp, *plist = NULL;
2451         mfn_t mfn, prev_mfn, start_mfn;
2452         pfn_t pfn;
2453         int pages_needed, pages_requested;
2454         int search_start;
2455 
2456         /*
2457          * create the contig pfn list if not already done
2458          */
2459 retry:
2460         mutex_enter(&contig_list_lock);
2461         if (contig_pfn_list == NULL) {
2462                 mutex_exit(&contig_list_lock);
2463                 if (!create_contig_pfnlist(flags)) {
2464                         return (NULL);
2465                 }
2466                 goto retry;
2467         }
2468         contig_searches++;
2469         /*
2470          * Search contiguous pfn list for physically contiguous pages not in
2471          * the io_pool.  Start the search where the last search left off.
2472          */
2473         pages_requested = pages_needed = npages;
2474         search_start = next_alloc_pfn;
2475         start_mfn = prev_mfn = 0;
2476         while (pages_needed) {
2477                 pfn = contig_pfn_list[next_alloc_pfn];
2478                 mfn = pfn_to_mfn(pfn);
2479                 /*
2480                  * Check if mfn is first one or contig to previous one and
2481                  * if page corresponding to mfn is free and that mfn
2482                  * range is not crossing a segment boundary.
2483                  */
2484                 if ((prev_mfn == 0 || mfn == prev_mfn + 1) &&
2485                     (pp = page_numtopp_alloc(pfn)) != NULL &&
2486                     !((mfn & pfnseg) < (start_mfn & pfnseg))) {
2487                         PP_CLRFREE(pp);
2488                         page_io_pool_add(&plist, pp);
2489                         pages_needed--;
2490                         if (prev_mfn == 0) {
2491                                 if (pfnalign &&
2492                                     mfn != P2ROUNDUP(mfn, pfnalign)) {
2493                                         /*
2494                                          * not properly aligned
2495                                          */
2496                                         contig_search_restarts++;
2497                                         free_partial_list(&plist);
2498                                         pages_needed = pages_requested;
2499                                         start_mfn = prev_mfn = 0;
2500                                         goto skip;
2501                                 }
2502                                 start_mfn = mfn;
2503                         }
2504                         prev_mfn = mfn;
2505                 } else {
2506                         contig_search_restarts++;
2507                         free_partial_list(&plist);
2508                         pages_needed = pages_requested;
2509                         start_mfn = prev_mfn = 0;
2510                 }
2511 skip:
2512                 if (++next_alloc_pfn == contig_pfn_cnt)
2513                         next_alloc_pfn = 0;
2514                 if (next_alloc_pfn == search_start)
2515                         break; /* all pfns searched */
2516         }
2517         mutex_exit(&contig_list_lock);
2518         if (pages_needed) {
2519                 contig_search_failed++;
2520                 /*
2521                  * Failed to find enough contig pages.
2522                  * free partial page list
2523                  */
2524                 free_partial_list(&plist);
2525         }
2526         return (plist);
2527 }
2528 
2529 /*
2530  * Search the reserved io pool pages for a page range with the
2531  * desired characteristics.
2532  */
2533 page_t *
2534 page_io_pool_alloc(ddi_dma_attr_t *mattr, int contig, pgcnt_t minctg)
2535 {
2536         page_t *pp_first, *pp_last;
2537         page_t *pp, **poolp;
2538         pgcnt_t nwanted, pfnalign;
2539         uint64_t pfnseg;
2540         mfn_t mfn, tmfn, hi_mfn, lo_mfn;
2541         int align, attempt = 0;
2542 
2543         if (minctg == 1)
2544                 contig = 0;
2545         lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2546         hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2547         pfnseg = mmu_btop(mattr->dma_attr_seg);
2548         align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2549         if (align > MMU_PAGESIZE)
2550                 pfnalign = mmu_btop(align);
2551         else
2552                 pfnalign = 0;
2553 
2554 try_again:
2555         /*
2556          * See if we want pages for a legacy device
2557          */
2558         if (hi_mfn < PFN_16MEG)
2559                 poolp = &io_pool_16m;
2560         else
2561                 poolp = &io_pool_4g;
2562 try_smaller:
2563         /*
2564          * Take pages from I/O pool. We'll use pages from the highest
2565          * MFN range possible.
2566          */
2567         pp_first = pp_last = NULL;
2568         mutex_enter(&io_pool_lock);
2569         nwanted = minctg;
2570         for (pp = *poolp; pp && nwanted > 0; ) {
2571                 pp = pp->p_prev;
2572 
2573                 /*
2574                  * skip pages above allowable range
2575                  */
2576                 mfn = mfn_list[pp->p_pagenum];
2577                 if (hi_mfn < mfn)
2578                         goto skip;
2579 
2580                 /*
2581                  * stop at pages below allowable range
2582                  */
2583                 if (lo_mfn > mfn)
2584                         break;
2585 restart:
2586                 if (pp_last == NULL) {
2587                         /*
2588                          * Check alignment
2589                          */
2590                         tmfn = mfn - (minctg - 1);
2591                         if (pfnalign && tmfn != P2ROUNDUP(tmfn, pfnalign))
2592                                 goto skip; /* not properly aligned */
2593                         /*
2594                          * Check segment
2595                          */
2596                         if ((mfn & pfnseg) < (tmfn & pfnseg))
2597                                 goto skip; /* crosses seg boundary */
2598                         /*
2599                          * Start building page list
2600                          */
2601                         pp_first = pp_last = pp;
2602                         nwanted--;
2603                 } else {
2604                         /*
2605                          * check physical contiguity if required
2606                          */
2607                         if (contig &&
2608                             mfn_list[pp_first->p_pagenum] != mfn + 1) {
2609                                 /*
2610                                  * not a contiguous page, restart list.
2611                                  */
2612                                 pp_last = NULL;
2613                                 nwanted = minctg;
2614                                 goto restart;
2615                         } else { /* add page to list */
2616                                 pp_first = pp;
2617                                 nwanted--;
2618                         }
2619                 }
2620 skip:
2621                 if (pp == *poolp)
2622                         break;
2623         }
2624 
2625         /*
2626          * If we didn't find memory. Try the more constrained pool, then
2627          * sweep free pages into the DMA pool and try again.
2628          */
2629         if (nwanted != 0) {
2630                 mutex_exit(&io_pool_lock);
2631                 /*
2632                  * If we were looking in the less constrained pool and
2633                  * didn't find pages, try the more constrained pool.
2634                  */
2635                 if (poolp == &io_pool_4g) {
2636                         poolp = &io_pool_16m;
2637                         goto try_smaller;
2638                 }
2639                 kmem_reap();
2640                 if (++attempt < 4) {
2641                         /*
2642                          * Grab some more io_pool pages
2643                          */
2644                         (void) populate_io_pool();
2645                         goto try_again; /* go around and retry */
2646                 }
2647                 return (NULL);
2648         }
2649         /*
2650          * Found the pages, now snip them from the list
2651          */
2652         page_io_pool_sub(poolp, pp_first, pp_last);
2653         io_pool_cnt -= minctg;
2654         /*
2655          * reset low water mark
2656          */
2657         if (io_pool_cnt < io_pool_cnt_lowater)
2658                 io_pool_cnt_lowater = io_pool_cnt;
2659         mutex_exit(&io_pool_lock);
2660         return (pp_first);
2661 }
2662 
2663 page_t *
2664 page_swap_with_hypervisor(struct vnode *vp, u_offset_t off, caddr_t vaddr,
2665     ddi_dma_attr_t *mattr, uint_t flags, pgcnt_t minctg)
2666 {
2667         uint_t kflags;
2668         int order, extra, extpages, i, contig, nbits, extents;
2669         page_t *pp, *expp, *pp_first, **pplist = NULL;
2670         mfn_t *mfnlist = NULL;
2671 
2672         contig = flags & PG_PHYSCONTIG;
2673         if (minctg == 1)
2674                 contig = 0;
2675         flags &= ~PG_PHYSCONTIG;
2676         kflags = flags & PG_WAIT ? KM_SLEEP : KM_NOSLEEP;
2677         /*
2678          * Hypervisor will allocate extents, if we want contig
2679          * pages extent must be >= minctg
2680          */
2681         if (contig) {
2682                 order = highbit(minctg) - 1;
2683                 if (minctg & ((1 << order) - 1))
2684                         order++;
2685                 extpages = 1 << order;
2686         } else {
2687                 order = 0;
2688                 extpages = minctg;
2689         }
2690         if (extpages > minctg) {
2691                 extra = extpages - minctg;
2692                 if (!page_resv(extra, kflags))
2693                         return (NULL);
2694         }
2695         pp_first = NULL;
2696         pplist = kmem_alloc(extpages * sizeof (page_t *), kflags);
2697         if (pplist == NULL)
2698                 goto balloon_fail;
2699         mfnlist = kmem_alloc(extpages * sizeof (mfn_t), kflags);
2700         if (mfnlist == NULL)
2701                 goto balloon_fail;
2702         pp = page_create_va(vp, off, minctg * PAGESIZE, flags, &kvseg, vaddr);
2703         if (pp == NULL)
2704                 goto balloon_fail;
2705         pp_first = pp;
2706         if (extpages > minctg) {
2707                 /*
2708                  * fill out the rest of extent pages to swap
2709                  * with the hypervisor
2710                  */
2711                 for (i = 0; i < extra; i++) {
2712                         expp = page_create_va(vp,
2713                             (u_offset_t)(uintptr_t)io_pool_kva,
2714                             PAGESIZE, flags, &kvseg, io_pool_kva);
2715                         if (expp == NULL)
2716                                 goto balloon_fail;
2717                         (void) hat_pageunload(expp, HAT_FORCE_PGUNLOAD);
2718                         page_io_unlock(expp);
2719                         page_hashout(expp, NULL);
2720                         page_io_lock(expp);
2721                         /*
2722                          * add page to end of list
2723                          */
2724                         expp->p_prev = pp_first->p_prev;
2725                         expp->p_next = pp_first;
2726                         expp->p_prev->p_next = expp;
2727                         pp_first->p_prev = expp;
2728                 }
2729 
2730         }
2731         for (i = 0; i < extpages; i++) {
2732                 pplist[i] = pp;
2733                 pp = pp->p_next;
2734         }
2735         nbits = highbit(mattr->dma_attr_addr_hi);
2736         extents = contig ? 1 : minctg;
2737         if (balloon_replace_pages(extents, pplist, nbits, order,
2738             mfnlist) != extents) {
2739                 if (ioalloc_dbg)
2740                         cmn_err(CE_NOTE, "request to hypervisor"
2741                             " for %d pages, maxaddr %" PRIx64 " failed",
2742                             extpages, mattr->dma_attr_addr_hi);
2743                 goto balloon_fail;
2744         }
2745 
2746         kmem_free(pplist, extpages * sizeof (page_t *));
2747         kmem_free(mfnlist, extpages * sizeof (mfn_t));
2748         /*
2749          * Return any excess pages to free list
2750          */
2751         if (extpages > minctg) {
2752                 for (i = 0; i < extra; i++) {
2753                         pp = pp_first->p_prev;
2754                         page_sub(&pp_first, pp);
2755                         page_io_unlock(pp);
2756                         page_unresv(1);
2757                         page_free(pp, 1);
2758                 }
2759         }
2760         return (pp_first);
2761 balloon_fail:
2762         /*
2763          * Return pages to free list and return failure
2764          */
2765         while (pp_first != NULL) {
2766                 pp = pp_first;
2767                 page_sub(&pp_first, pp);
2768                 page_io_unlock(pp);
2769                 if (pp->p_vnode != NULL)
2770                         page_hashout(pp, NULL);
2771                 page_free(pp, 1);
2772         }
2773         if (pplist)
2774                 kmem_free(pplist, extpages * sizeof (page_t *));
2775         if (mfnlist)
2776                 kmem_free(mfnlist, extpages * sizeof (mfn_t));
2777         page_unresv(extpages - minctg);
2778         return (NULL);
2779 }
2780 
2781 static void
2782 return_partial_alloc(page_t *plist)
2783 {
2784         page_t *pp;
2785 
2786         while (plist != NULL) {
2787                 pp = plist;
2788                 page_sub(&plist, pp);
2789                 page_io_unlock(pp);
2790                 page_destroy_io(pp);
2791         }
2792 }
2793 
2794 static page_t *
2795 page_get_contigpages(
2796         struct vnode    *vp,
2797         u_offset_t      off,
2798         int             *npagesp,
2799         uint_t          flags,
2800         caddr_t         vaddr,
2801         ddi_dma_attr_t  *mattr)
2802 {
2803         mfn_t   max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
2804         page_t  *plist; /* list to return */
2805         page_t  *pp, *mcpl;
2806         int     contig, anyaddr, npages, getone = 0;
2807         mfn_t   lo_mfn;
2808         mfn_t   hi_mfn;
2809         pgcnt_t pfnalign = 0;
2810         int     align, sgllen;
2811         uint64_t pfnseg;
2812         pgcnt_t minctg;
2813 
2814         npages = *npagesp;
2815         ASSERT(mattr != NULL);
2816         lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2817         hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2818         sgllen = mattr->dma_attr_sgllen;
2819         pfnseg = mmu_btop(mattr->dma_attr_seg);
2820         align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2821         if (align > MMU_PAGESIZE)
2822                 pfnalign = mmu_btop(align);
2823 
2824         contig = flags & PG_PHYSCONTIG;
2825         if (npages == -1) {
2826                 npages = 1;
2827                 pfnalign = 0;
2828         }
2829         /*
2830          * Clear the contig flag if only one page is needed.
2831          */
2832         if (npages == 1) {
2833                 getone = 1;
2834                 contig = 0;
2835         }
2836 
2837         /*
2838          * Check if any page in the system is fine.
2839          */
2840         anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn;
2841         if (!contig && anyaddr && !pfnalign) {
2842                 flags &= ~PG_PHYSCONTIG;
2843                 plist = page_create_va(vp, off, npages * MMU_PAGESIZE,
2844                     flags, &kvseg, vaddr);
2845                 if (plist != NULL) {
2846                         *npagesp = 0;
2847                         return (plist);
2848                 }
2849         }
2850         plist = NULL;
2851         minctg = howmany(npages, sgllen);
2852         while (npages > sgllen || getone) {
2853                 if (minctg > npages)
2854                         minctg = npages;
2855                 mcpl = NULL;
2856                 /*
2857                  * We could want contig pages with no address range limits.
2858                  */
2859                 if (anyaddr && contig) {
2860                         /*
2861                          * Look for free contig pages to satisfy the request.
2862                          */
2863                         mcpl = find_contig_free(minctg, flags, pfnseg,
2864                             pfnalign);
2865                 }
2866                 /*
2867                  * Try the reserved io pools next
2868                  */
2869                 if (mcpl == NULL)
2870                         mcpl = page_io_pool_alloc(mattr, contig, minctg);
2871                 if (mcpl != NULL) {
2872                         pp = mcpl;
2873                         do {
2874                                 if (!page_hashin(pp, vp, off, NULL)) {
2875                                         panic("page_get_contigpages:"
2876                                             " hashin failed"
2877                                             " pp %p, vp %p, off %llx",
2878                                             (void *)pp, (void *)vp, off);
2879                                 }
2880                                 off += MMU_PAGESIZE;
2881                                 PP_CLRFREE(pp);
2882                                 PP_CLRAGED(pp);
2883                                 page_set_props(pp, P_REF);
2884                                 page_io_lock(pp);
2885                                 pp = pp->p_next;
2886                         } while (pp != mcpl);
2887                 } else {
2888                         /*
2889                          * Hypervisor exchange doesn't handle segment or
2890                          * alignment constraints
2891                          */
2892                         if (mattr->dma_attr_seg < mattr->dma_attr_addr_hi ||
2893                             pfnalign)
2894                                 goto fail;
2895                         /*
2896                          * Try exchanging pages with the hypervisor
2897                          */
2898                         mcpl = page_swap_with_hypervisor(vp, off, vaddr, mattr,
2899                             flags, minctg);
2900                         if (mcpl == NULL)
2901                                 goto fail;
2902                         off += minctg * MMU_PAGESIZE;
2903                 }
2904                 check_dma(mattr, mcpl, minctg);
2905                 /*
2906                  * Here with a minctg run of contiguous pages, add them to the
2907                  * list we will return for this request.
2908                  */
2909                 page_list_concat(&plist, &mcpl);
2910                 npages -= minctg;
2911                 *npagesp = npages;
2912                 sgllen--;
2913                 if (getone)
2914                         break;
2915         }
2916         return (plist);
2917 fail:
2918         return_partial_alloc(plist);
2919         return (NULL);
2920 }
2921 
2922 /*
2923  * Allocator for domain 0 I/O pages. We match the required
2924  * DMA attributes and contiguity constraints.
2925  */
2926 /*ARGSUSED*/
2927 page_t *
2928 page_create_io(
2929         struct vnode    *vp,
2930         u_offset_t      off,
2931         uint_t          bytes,
2932         uint_t          flags,
2933         struct as       *as,
2934         caddr_t         vaddr,
2935         ddi_dma_attr_t  *mattr)
2936 {
2937         page_t  *plist = NULL, *pp;
2938         int     npages = 0, contig, anyaddr, pages_req;
2939         mfn_t   lo_mfn;
2940         mfn_t   hi_mfn;
2941         pgcnt_t pfnalign = 0;
2942         int     align;
2943         int     is_domu = 0;
2944         int     dummy, bytes_got;
2945         mfn_t   max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
2946 
2947         ASSERT(mattr != NULL);
2948         lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2949         hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2950         align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2951         if (align > MMU_PAGESIZE)
2952                 pfnalign = mmu_btop(align);
2953 
2954         /*
2955          * Clear the contig flag if only one page is needed or the scatter
2956          * gather list length is >= npages.
2957          */
2958         pages_req = npages = mmu_btopr(bytes);
2959         contig = (flags & PG_PHYSCONTIG);
2960         bytes = P2ROUNDUP(bytes, MMU_PAGESIZE);
2961         if (bytes == MMU_PAGESIZE || mattr->dma_attr_sgllen >= npages)
2962                 contig = 0;
2963 
2964         /*
2965          * Check if any old page in the system is fine.
2966          * DomU should always go down this path.
2967          */
2968         is_domu = !DOMAIN_IS_INITDOMAIN(xen_info);
2969         anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn && !pfnalign;
2970         if ((!contig && anyaddr) || is_domu) {
2971                 flags &= ~PG_PHYSCONTIG;
2972                 plist = page_create_va(vp, off, bytes, flags, &kvseg, vaddr);
2973                 if (plist != NULL)
2974                         return (plist);
2975                 else if (is_domu)
2976                         return (NULL); /* no memory available */
2977         }
2978         /*
2979          * DomU should never reach here
2980          */
2981         if (contig) {
2982                 plist = page_get_contigpages(vp, off, &npages, flags, vaddr,
2983                     mattr);
2984                 if (plist == NULL)
2985                         goto fail;
2986                 bytes_got = (pages_req - npages) << MMU_PAGESHIFT;
2987                 vaddr += bytes_got;
2988                 off += bytes_got;
2989                 /*
2990                  * We now have all the contiguous pages we need, but
2991                  * we may still need additional non-contiguous pages.
2992                  */
2993         }
2994         /*
2995          * now loop collecting the requested number of pages, these do
2996          * not have to be contiguous pages but we will use the contig
2997          * page alloc code to get the pages since it will honor any
2998          * other constraints the pages may have.
2999          */
3000         while (npages--) {
3001                 dummy = -1;
3002                 pp = page_get_contigpages(vp, off, &dummy, flags, vaddr, mattr);
3003                 if (pp == NULL)
3004                         goto fail;
3005                 page_add(&plist, pp);
3006                 vaddr += MMU_PAGESIZE;
3007                 off += MMU_PAGESIZE;
3008         }
3009         return (plist);
3010 fail:
3011         /*
3012          * Failed to get enough pages, return ones we did get
3013          */
3014         return_partial_alloc(plist);
3015         return (NULL);
3016 }
3017 
3018 /*
3019  * Lock and return the page with the highest mfn that we can find.  last_mfn
3020  * holds the last one found, so the next search can start from there.  We
3021  * also keep a counter so that we don't loop forever if the machine has no
3022  * free pages.
3023  *
3024  * This is called from the balloon thread to find pages to give away.  new_high
3025  * is used when new mfn's have been added to the system - we will reset our
3026  * search if the new mfn's are higher than our current search position.
3027  */
3028 page_t *
3029 page_get_high_mfn(mfn_t new_high)
3030 {
3031         static mfn_t last_mfn = 0;
3032         pfn_t pfn;
3033         page_t *pp;
3034         ulong_t loop_count = 0;
3035 
3036         if (new_high > last_mfn)
3037                 last_mfn = new_high;
3038 
3039         for (; loop_count < mfn_count; loop_count++, last_mfn--) {
3040                 if (last_mfn == 0) {
3041                         last_mfn = cached_max_mfn;
3042                 }
3043 
3044                 pfn = mfn_to_pfn(last_mfn);
3045                 if (pfn & PFN_IS_FOREIGN_MFN)
3046                         continue;
3047 
3048                 /* See if the page is free.  If so, lock it. */
3049                 pp = page_numtopp_alloc(pfn);
3050                 if (pp == NULL)
3051                         continue;
3052                 PP_CLRFREE(pp);
3053 
3054                 ASSERT(PAGE_EXCL(pp));
3055                 ASSERT(pp->p_vnode == NULL);
3056                 ASSERT(!hat_page_is_mapped(pp));
3057                 last_mfn--;
3058                 return (pp);
3059         }
3060         return (NULL);
3061 }
3062 
3063 #else /* !__xpv */
3064 
3065 /*
3066  * get a page from any list with the given mnode
3067  */
3068 static page_t *
3069 page_get_mnode_anylist(ulong_t origbin, uchar_t szc, uint_t flags,
3070     int mnode, int mtype, ddi_dma_attr_t *dma_attr)
3071 {
3072         kmutex_t                *pcm;
3073         int                     i;
3074         page_t                  *pp;
3075         page_t                  *first_pp;
3076         uint64_t                pgaddr;
3077         ulong_t                 bin;
3078         int                     mtypestart;
3079         int                     plw_initialized;
3080         page_list_walker_t      plw;
3081 
3082         VM_STAT_ADD(pga_vmstats.pgma_alloc);
3083 
3084         ASSERT((flags & PG_MATCH_COLOR) == 0);
3085         ASSERT(szc == 0);
3086         ASSERT(dma_attr != NULL);
3087 
3088         MTYPE_START(mnode, mtype, flags);
3089         if (mtype < 0) {
3090                 VM_STAT_ADD(pga_vmstats.pgma_allocempty);
3091                 return (NULL);
3092         }
3093 
3094         mtypestart = mtype;
3095 
3096         bin = origbin;
3097 
3098         /*
3099          * check up to page_colors + 1 bins - origbin may be checked twice
3100          * because of BIN_STEP skip
3101          */
3102         do {
3103                 plw_initialized = 0;
3104 
3105                 for (plw.plw_count = 0;
3106                     plw.plw_count < page_colors; plw.plw_count++) {
3107 
3108                         if (PAGE_FREELISTS(mnode, szc, bin, mtype) == NULL)
3109                                 goto nextfreebin;
3110 
3111                         pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST);
3112                         mutex_enter(pcm);
3113                         pp = PAGE_FREELISTS(mnode, szc, bin, mtype);
3114                         first_pp = pp;
3115                         while (pp != NULL) {
3116                                 if (IS_DUMP_PAGE(pp) || page_trylock(pp,
3117                                     SE_EXCL) == 0) {
3118                                         pp = pp->p_next;
3119                                         if (pp == first_pp) {
3120                                                 pp = NULL;
3121                                         }
3122                                         continue;
3123                                 }
3124 
3125                                 ASSERT(PP_ISFREE(pp));
3126                                 ASSERT(PP_ISAGED(pp));
3127                                 ASSERT(pp->p_vnode == NULL);
3128                                 ASSERT(pp->p_hash == NULL);
3129                                 ASSERT(pp->p_offset == (u_offset_t)-1);
3130                                 ASSERT(pp->p_szc == szc);
3131                                 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
3132                                 /* check if page within DMA attributes */
3133                                 pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
3134                                 if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
3135                                     (pgaddr + MMU_PAGESIZE - 1 <=
3136                                     dma_attr->dma_attr_addr_hi)) {
3137                                         break;
3138                                 }
3139 
3140                                 /* continue looking */
3141                                 page_unlock(pp);
3142                                 pp = pp->p_next;
3143                                 if (pp == first_pp)
3144                                         pp = NULL;
3145 
3146                         }
3147                         if (pp != NULL) {
3148                                 ASSERT(mtype == PP_2_MTYPE(pp));
3149                                 ASSERT(pp->p_szc == 0);
3150 
3151                                 /* found a page with specified DMA attributes */
3152                                 page_sub(&PAGE_FREELISTS(mnode, szc, bin,
3153                                     mtype), pp);
3154                                 page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST);
3155 
3156                                 if ((PP_ISFREE(pp) == 0) ||
3157                                     (PP_ISAGED(pp) == 0)) {
3158                                         cmn_err(CE_PANIC, "page %p is not free",
3159                                             (void *)pp);
3160                                 }
3161 
3162                                 mutex_exit(pcm);
3163                                 check_dma(dma_attr, pp, 1);
3164                                 VM_STAT_ADD(pga_vmstats.pgma_allocok);
3165                                 return (pp);
3166                         }
3167                         mutex_exit(pcm);
3168 nextfreebin:
3169                         if (plw_initialized == 0) {
3170                                 page_list_walk_init(szc, 0, bin, 1, 0, &plw);
3171                                 ASSERT(plw.plw_ceq_dif == page_colors);
3172                                 plw_initialized = 1;
3173                         }
3174 
3175                         if (plw.plw_do_split) {
3176                                 pp = page_freelist_split(szc, bin, mnode,
3177                                     mtype,
3178                                     mmu_btop(dma_attr->dma_attr_addr_lo),
3179                                     mmu_btop(dma_attr->dma_attr_addr_hi + 1),
3180                                     &plw);
3181                                 if (pp != NULL) {
3182                                         check_dma(dma_attr, pp, 1);
3183                                         return (pp);
3184                                 }
3185                         }
3186 
3187                         bin = page_list_walk_next_bin(szc, bin, &plw);
3188                 }
3189 
3190                 MTYPE_NEXT(mnode, mtype, flags);
3191         } while (mtype >= 0);
3192 
3193         /* failed to find a page in the freelist; try it in the cachelist */
3194 
3195         /* reset mtype start for cachelist search */
3196         mtype = mtypestart;
3197         ASSERT(mtype >= 0);
3198 
3199         /* start with the bin of matching color */
3200         bin = origbin;
3201 
3202         do {
3203                 for (i = 0; i <= page_colors; i++) {
3204                         if (PAGE_CACHELISTS(mnode, bin, mtype) == NULL)
3205                                 goto nextcachebin;
3206                         pcm = PC_BIN_MUTEX(mnode, bin, PG_CACHE_LIST);
3207                         mutex_enter(pcm);
3208                         pp = PAGE_CACHELISTS(mnode, bin, mtype);
3209                         first_pp = pp;
3210                         while (pp != NULL) {
3211                                 if (IS_DUMP_PAGE(pp) || page_trylock(pp,
3212                                     SE_EXCL) == 0) {
3213                                         pp = pp->p_next;
3214                                         if (pp == first_pp)
3215                                                 pp = NULL;
3216                                         continue;
3217                                 }
3218                                 ASSERT(pp->p_vnode);
3219                                 ASSERT(PP_ISAGED(pp) == 0);
3220                                 ASSERT(pp->p_szc == 0);
3221                                 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
3222 
3223                                 /* check if page within DMA attributes */
3224 
3225                                 pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
3226                                 if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
3227                                     (pgaddr + MMU_PAGESIZE - 1 <=
3228                                     dma_attr->dma_attr_addr_hi)) {
3229                                         break;
3230                                 }
3231 
3232                                 /* continue looking */
3233                                 page_unlock(pp);
3234                                 pp = pp->p_next;
3235                                 if (pp == first_pp)
3236                                         pp = NULL;
3237                         }
3238 
3239                         if (pp != NULL) {
3240                                 ASSERT(mtype == PP_2_MTYPE(pp));
3241                                 ASSERT(pp->p_szc == 0);
3242 
3243                                 /* found a page with specified DMA attributes */
3244                                 page_sub(&PAGE_CACHELISTS(mnode, bin,
3245                                     mtype), pp);
3246                                 page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST);
3247 
3248                                 mutex_exit(pcm);
3249                                 ASSERT(pp->p_vnode);
3250                                 ASSERT(PP_ISAGED(pp) == 0);
3251                                 check_dma(dma_attr, pp, 1);
3252                                 VM_STAT_ADD(pga_vmstats.pgma_allocok);
3253                                 return (pp);
3254                         }
3255                         mutex_exit(pcm);
3256 nextcachebin:
3257                         bin += (i == 0) ? BIN_STEP : 1;
3258                         bin &= page_colors_mask;
3259                 }
3260                 MTYPE_NEXT(mnode, mtype, flags);
3261         } while (mtype >= 0);
3262 
3263         VM_STAT_ADD(pga_vmstats.pgma_allocfailed);
3264         return (NULL);
3265 }
3266 
3267 /*
3268  * This function is similar to page_get_freelist()/page_get_cachelist()
3269  * but it searches both the lists to find a page with the specified
3270  * color (or no color) and DMA attributes. The search is done in the
3271  * freelist first and then in the cache list within the highest memory
3272  * range (based on DMA attributes) before searching in the lower
3273  * memory ranges.
3274  *
3275  * Note: This function is called only by page_create_io().
3276  */
3277 /*ARGSUSED*/
3278 static page_t *
3279 page_get_anylist(struct vnode *vp, u_offset_t off, struct as *as, caddr_t vaddr,
3280     size_t size, uint_t flags, ddi_dma_attr_t *dma_attr, lgrp_t *lgrp)
3281 {
3282         uint_t          bin;
3283         int             mtype;
3284         page_t          *pp;
3285         int             n;
3286         int             m;
3287         int             szc;
3288         int             fullrange;
3289         int             mnode;
3290         int             local_failed_stat = 0;
3291         lgrp_mnode_cookie_t     lgrp_cookie;
3292 
3293         VM_STAT_ADD(pga_vmstats.pga_alloc);
3294 
3295         /* only base pagesize currently supported */
3296         if (size != MMU_PAGESIZE)
3297                 return (NULL);
3298 
3299         /*
3300          * If we're passed a specific lgroup, we use it.  Otherwise,
3301          * assume first-touch placement is desired.
3302          */
3303         if (!LGRP_EXISTS(lgrp))
3304                 lgrp = lgrp_home_lgrp();
3305 
3306         /* LINTED */
3307         AS_2_BIN(as, seg, vp, vaddr, bin, 0);
3308 
3309         /*
3310          * Only hold one freelist or cachelist lock at a time, that way we
3311          * can start anywhere and not have to worry about lock
3312          * ordering.
3313          */
3314         if (dma_attr == NULL) {
3315                 n = mtype16m;
3316                 m = mtypetop;
3317                 fullrange = 1;
3318                 VM_STAT_ADD(pga_vmstats.pga_nulldmaattr);
3319         } else {
3320                 pfn_t pfnlo = mmu_btop(dma_attr->dma_attr_addr_lo);
3321                 pfn_t pfnhi = mmu_btop(dma_attr->dma_attr_addr_hi);
3322 
3323                 /*
3324                  * We can guarantee alignment only for page boundary.
3325                  */
3326                 if (dma_attr->dma_attr_align > MMU_PAGESIZE)
3327                         return (NULL);
3328 
3329                 /* Sanity check the dma_attr */
3330                 if (pfnlo > pfnhi)
3331                         return (NULL);
3332 
3333                 n = pfn_2_mtype(pfnlo);
3334                 m = pfn_2_mtype(pfnhi);
3335 
3336                 fullrange = ((pfnlo == mnoderanges[n].mnr_pfnlo) &&
3337                     (pfnhi >= mnoderanges[m].mnr_pfnhi));
3338         }
3339         VM_STAT_COND_ADD(fullrange == 0, pga_vmstats.pga_notfullrange);
3340 
3341         szc = 0;
3342 
3343         /* cylcing thru mtype handled by RANGE0 if n == mtype16m */
3344         if (n == mtype16m) {
3345                 flags |= PGI_MT_RANGE0;
3346                 n = m;
3347         }
3348 
3349         /*
3350          * Try local memory node first, but try remote if we can't
3351          * get a page of the right color.
3352          */
3353         LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_HIER);
3354         while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) {
3355                 /*
3356                  * allocate pages from high pfn to low.
3357                  */
3358                 mtype = m;
3359                 do {
3360                         if (fullrange != 0) {
3361                                 pp = page_get_mnode_freelist(mnode,
3362                                     bin, mtype, szc, flags);
3363                                 if (pp == NULL) {
3364                                         pp = page_get_mnode_cachelist(
3365                                             bin, flags, mnode, mtype);
3366                                 }
3367                         } else {
3368                                 pp = page_get_mnode_anylist(bin, szc,
3369                                     flags, mnode, mtype, dma_attr);
3370                         }
3371                         if (pp != NULL) {
3372                                 VM_STAT_ADD(pga_vmstats.pga_allocok);
3373                                 check_dma(dma_attr, pp, 1);
3374                                 return (pp);
3375                         }
3376                 } while (mtype != n &&
3377                     (mtype = mnoderanges[mtype].mnr_next) != -1);
3378                 if (!local_failed_stat) {
3379                         lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1);
3380                         local_failed_stat = 1;
3381                 }
3382         }
3383         VM_STAT_ADD(pga_vmstats.pga_allocfailed);
3384 
3385         return (NULL);
3386 }
3387 
3388 /*
3389  * page_create_io()
3390  *
3391  * This function is a copy of page_create_va() with an additional
3392  * argument 'mattr' that specifies DMA memory requirements to
3393  * the page list functions. This function is used by the segkmem
3394  * allocator so it is only to create new pages (i.e PG_EXCL is
3395  * set).
3396  *
3397  * Note: This interface is currently used by x86 PSM only and is
3398  *       not fully specified so the commitment level is only for
3399  *       private interface specific to x86. This interface uses PSM
3400  *       specific page_get_anylist() interface.
3401  */
3402 
3403 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \
3404         for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \
3405                 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
3406                         break; \
3407         } \
3408 }
3409 
3410 
3411 page_t *
3412 page_create_io(
3413         struct vnode    *vp,
3414         u_offset_t      off,
3415         uint_t          bytes,
3416         uint_t          flags,
3417         struct as       *as,
3418         caddr_t         vaddr,
3419         ddi_dma_attr_t  *mattr) /* DMA memory attributes if any */
3420 {
3421         page_t          *plist = NULL;
3422         uint_t          plist_len = 0;
3423         pgcnt_t         npages;
3424         page_t          *npp = NULL;
3425         uint_t          pages_req;
3426         page_t          *pp;
3427         kmutex_t        *phm = NULL;
3428         uint_t          index;
3429 
3430         TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START,
3431             "page_create_start:vp %p off %llx bytes %u flags %x",
3432             vp, off, bytes, flags);
3433 
3434         ASSERT((flags & ~(PG_EXCL | PG_WAIT | PG_PHYSCONTIG)) == 0);
3435 
3436         pages_req = npages = mmu_btopr(bytes);
3437 
3438         /*
3439          * Do the freemem and pcf accounting.
3440          */
3441         if (!page_create_wait(npages, flags)) {
3442                 return (NULL);
3443         }
3444 
3445         TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS,
3446             "page_create_success:vp %p off %llx", vp, off);
3447 
3448         /*
3449          * If satisfying this request has left us with too little
3450          * memory, start the wheels turning to get some back.  The
3451          * first clause of the test prevents waking up the pageout
3452          * daemon in situations where it would decide that there's
3453          * nothing to do.
3454          */
3455         if (nscan < desscan && freemem < minfree) {
3456                 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
3457                     "pageout_cv_signal:freemem %ld", freemem);
3458                 cv_signal(&proc_pageout->p_cv);
3459         }
3460 
3461         if (flags & PG_PHYSCONTIG) {
3462 
3463                 plist = page_get_contigpage(&npages, mattr, 1);
3464                 if (plist == NULL) {
3465                         page_create_putback(npages);
3466                         return (NULL);
3467                 }
3468 
3469                 pp = plist;
3470 
3471                 do {
3472                         if (!page_hashin(pp, vp, off, NULL)) {
3473                                 panic("pg_creat_io: hashin failed %p %p %llx",
3474                                     (void *)pp, (void *)vp, off);
3475                         }
3476                         VM_STAT_ADD(page_create_new);
3477                         off += MMU_PAGESIZE;
3478                         PP_CLRFREE(pp);
3479                         PP_CLRAGED(pp);
3480                         page_set_props(pp, P_REF);
3481                         pp = pp->p_next;
3482                 } while (pp != plist);
3483 
3484                 if (!npages) {
3485                         check_dma(mattr, plist, pages_req);
3486                         return (plist);
3487                 } else {
3488                         vaddr += (pages_req - npages) << MMU_PAGESHIFT;
3489                 }
3490 
3491                 /*
3492                  * fall-thru:
3493                  *
3494                  * page_get_contigpage returns when npages <= sgllen.
3495                  * Grab the rest of the non-contig pages below from anylist.
3496                  */
3497         }
3498 
3499         /*
3500          * Loop around collecting the requested number of pages.
3501          * Most of the time, we have to `create' a new page. With
3502          * this in mind, pull the page off the free list before
3503          * getting the hash lock.  This will minimize the hash
3504          * lock hold time, nesting, and the like.  If it turns
3505          * out we don't need the page, we put it back at the end.
3506          */
3507         while (npages--) {
3508                 phm = NULL;
3509 
3510                 index = PAGE_HASH_FUNC(vp, off);
3511 top:
3512                 ASSERT(phm == NULL);
3513                 ASSERT(index == PAGE_HASH_FUNC(vp, off));
3514                 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3515 
3516                 if (npp == NULL) {
3517                         /*
3518                          * Try to get the page of any color either from
3519                          * the freelist or from the cache list.
3520                          */
3521                         npp = page_get_anylist(vp, off, as, vaddr, MMU_PAGESIZE,
3522                             flags & ~PG_MATCH_COLOR, mattr, NULL);
3523                         if (npp == NULL) {
3524                                 if (mattr == NULL) {
3525                                         /*
3526                                          * Not looking for a special page;
3527                                          * panic!
3528                                          */
3529                                         panic("no page found %d", (int)npages);
3530                                 }
3531                                 /*
3532                                  * No page found! This can happen
3533                                  * if we are looking for a page
3534                                  * within a specific memory range
3535                                  * for DMA purposes. If PG_WAIT is
3536                                  * specified then we wait for a
3537                                  * while and then try again. The
3538                                  * wait could be forever if we
3539                                  * don't get the page(s) we need.
3540                                  *
3541                                  * Note: XXX We really need a mechanism
3542                                  * to wait for pages in the desired
3543                                  * range. For now, we wait for any
3544                                  * pages and see if we can use it.
3545                                  */
3546 
3547                                 if ((mattr != NULL) && (flags & PG_WAIT)) {
3548                                         delay(10);
3549                                         goto top;
3550                                 }
3551                                 goto fail; /* undo accounting stuff */
3552                         }
3553 
3554                         if (PP_ISAGED(npp) == 0) {
3555                                 /*
3556                                  * Since this page came from the
3557                                  * cachelist, we must destroy the
3558                                  * old vnode association.
3559                                  */
3560                                 page_hashout(npp, (kmutex_t *)NULL);
3561                         }
3562                 }
3563 
3564                 /*
3565                  * We own this page!
3566                  */
3567                 ASSERT(PAGE_EXCL(npp));
3568                 ASSERT(npp->p_vnode == NULL);
3569                 ASSERT(!hat_page_is_mapped(npp));
3570                 PP_CLRFREE(npp);
3571                 PP_CLRAGED(npp);
3572 
3573                 /*
3574                  * Here we have a page in our hot little mits and are
3575                  * just waiting to stuff it on the appropriate lists.
3576                  * Get the mutex and check to see if it really does
3577                  * not exist.
3578                  */
3579                 phm = PAGE_HASH_MUTEX(index);
3580                 mutex_enter(phm);
3581                 PAGE_HASH_SEARCH(index, pp, vp, off);
3582                 if (pp == NULL) {
3583                         VM_STAT_ADD(page_create_new);
3584                         pp = npp;
3585                         npp = NULL;
3586                         if (!page_hashin(pp, vp, off, phm)) {
3587                                 /*
3588                                  * Since we hold the page hash mutex and
3589                                  * just searched for this page, page_hashin
3590                                  * had better not fail.  If it does, that
3591                                  * means somethread did not follow the
3592                                  * page hash mutex rules.  Panic now and
3593                                  * get it over with.  As usual, go down
3594                                  * holding all the locks.
3595                                  */
3596                                 ASSERT(MUTEX_HELD(phm));
3597                                 panic("page_create: hashin fail %p %p %llx %p",
3598                                     (void *)pp, (void *)vp, off, (void *)phm);
3599 
3600                         }
3601                         ASSERT(MUTEX_HELD(phm));
3602                         mutex_exit(phm);
3603                         phm = NULL;
3604 
3605                         /*
3606                          * Hat layer locking need not be done to set
3607                          * the following bits since the page is not hashed
3608                          * and was on the free list (i.e., had no mappings).
3609                          *
3610                          * Set the reference bit to protect
3611                          * against immediate pageout
3612                          *
3613                          * XXXmh modify freelist code to set reference
3614                          * bit so we don't have to do it here.
3615                          */
3616                         page_set_props(pp, P_REF);
3617                 } else {
3618                         ASSERT(MUTEX_HELD(phm));
3619                         mutex_exit(phm);
3620                         phm = NULL;
3621                         /*
3622                          * NOTE: This should not happen for pages associated
3623                          *       with kernel vnode 'kvp'.
3624                          */
3625                         /* XX64 - to debug why this happens! */
3626                         ASSERT(!VN_ISKAS(vp));
3627                         if (VN_ISKAS(vp))
3628                                 cmn_err(CE_NOTE,
3629                                     "page_create: page not expected "
3630                                     "in hash list for kernel vnode - pp 0x%p",
3631                                     (void *)pp);
3632                         VM_STAT_ADD(page_create_exists);
3633                         goto fail;
3634                 }
3635 
3636                 /*
3637                  * Got a page!  It is locked.  Acquire the i/o
3638                  * lock since we are going to use the p_next and
3639                  * p_prev fields to link the requested pages together.
3640                  */
3641                 page_io_lock(pp);
3642                 page_add(&plist, pp);
3643                 plist = plist->p_next;
3644                 off += MMU_PAGESIZE;
3645                 vaddr += MMU_PAGESIZE;
3646         }
3647 
3648         check_dma(mattr, plist, pages_req);
3649         return (plist);
3650 
3651 fail:
3652         if (npp != NULL) {
3653                 /*
3654                  * Did not need this page after all.
3655                  * Put it back on the free list.
3656                  */
3657                 VM_STAT_ADD(page_create_putbacks);
3658                 PP_SETFREE(npp);
3659                 PP_SETAGED(npp);
3660                 npp->p_offset = (u_offset_t)-1;
3661                 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL);
3662                 page_unlock(npp);
3663         }
3664 
3665         /*
3666          * Give up the pages we already got.
3667          */
3668         while (plist != NULL) {
3669                 pp = plist;
3670                 page_sub(&plist, pp);
3671                 page_io_unlock(pp);
3672                 plist_len++;
3673                 /*LINTED: constant in conditional ctx*/
3674                 VN_DISPOSE(pp, B_INVAL, 0, kcred);
3675         }
3676 
3677         /*
3678          * VN_DISPOSE does freemem accounting for the pages in plist
3679          * by calling page_free. So, we need to undo the pcf accounting
3680          * for only the remaining pages.
3681          */
3682         VM_STAT_ADD(page_create_putbacks);
3683         page_create_putback(pages_req - plist_len);
3684 
3685         return (NULL);
3686 }
3687 #endif /* !__xpv */
3688 
3689 
3690 /*
3691  * Copy the data from the physical page represented by "frompp" to
3692  * that represented by "topp". ppcopy uses CPU->cpu_caddr1 and
3693  * CPU->cpu_caddr2.  It assumes that no one uses either map at interrupt
3694  * level and no one sleeps with an active mapping there.
3695  *
3696  * Note that the ref/mod bits in the page_t's are not affected by
3697  * this operation, hence it is up to the caller to update them appropriately.
3698  */
3699 int
3700 ppcopy(page_t *frompp, page_t *topp)
3701 {
3702         caddr_t         pp_addr1;
3703         caddr_t         pp_addr2;
3704         hat_mempte_t    pte1;
3705         hat_mempte_t    pte2;
3706         kmutex_t        *ppaddr_mutex;
3707         label_t         ljb;
3708         int             ret = 1;
3709 
3710         ASSERT_STACK_ALIGNED();
3711         ASSERT(PAGE_LOCKED(frompp));
3712         ASSERT(PAGE_LOCKED(topp));
3713 
3714         if (kpm_enable) {
3715                 pp_addr1 = hat_kpm_page2va(frompp, 0);
3716                 pp_addr2 = hat_kpm_page2va(topp, 0);
3717                 kpreempt_disable();
3718         } else {
3719                 /*
3720                  * disable pre-emption so that CPU can't change
3721                  */
3722                 kpreempt_disable();
3723 
3724                 pp_addr1 = CPU->cpu_caddr1;
3725                 pp_addr2 = CPU->cpu_caddr2;
3726                 pte1 = CPU->cpu_caddr1pte;
3727                 pte2 = CPU->cpu_caddr2pte;
3728 
3729                 ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
3730                 mutex_enter(ppaddr_mutex);
3731 
3732                 hat_mempte_remap(page_pptonum(frompp), pp_addr1, pte1,
3733                     PROT_READ | HAT_STORECACHING_OK, HAT_LOAD_NOCONSIST);
3734                 hat_mempte_remap(page_pptonum(topp), pp_addr2, pte2,
3735                     PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
3736                     HAT_LOAD_NOCONSIST);
3737         }
3738 
3739         if (on_fault(&ljb)) {
3740                 ret = 0;
3741                 goto faulted;
3742         }
3743         if (use_sse_pagecopy)
3744 #ifdef __xpv
3745                 page_copy_no_xmm(pp_addr2, pp_addr1);
3746 #else
3747                 hwblkpagecopy(pp_addr1, pp_addr2);
3748 #endif
3749         else
3750                 bcopy(pp_addr1, pp_addr2, PAGESIZE);
3751 
3752         no_fault();
3753 faulted:
3754         if (!kpm_enable) {
3755 #ifdef __xpv
3756                 /*
3757                  * We can't leave unused mappings laying about under the
3758                  * hypervisor, so blow them away.
3759                  */
3760                 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr1, 0,
3761                     UVMF_INVLPG | UVMF_LOCAL) < 0)
3762                         panic("HYPERVISOR_update_va_mapping() failed");
3763                 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3764                     UVMF_INVLPG | UVMF_LOCAL) < 0)
3765                         panic("HYPERVISOR_update_va_mapping() failed");
3766 #endif
3767                 mutex_exit(ppaddr_mutex);
3768         }
3769         kpreempt_enable();
3770         return (ret);
3771 }
3772 
3773 void
3774 pagezero(page_t *pp, uint_t off, uint_t len)
3775 {
3776         ASSERT(PAGE_LOCKED(pp));
3777         pfnzero(page_pptonum(pp), off, len);
3778 }
3779 
3780 /*
3781  * Zero the physical page from off to off + len given by pfn
3782  * without changing the reference and modified bits of page.
3783  *
3784  * We use this using CPU private page address #2, see ppcopy() for more info.
3785  * pfnzero() must not be called at interrupt level.
3786  */
3787 void
3788 pfnzero(pfn_t pfn, uint_t off, uint_t len)
3789 {
3790         caddr_t         pp_addr2;
3791         hat_mempte_t    pte2;
3792         kmutex_t        *ppaddr_mutex = NULL;
3793 
3794         ASSERT_STACK_ALIGNED();
3795         ASSERT(len <= MMU_PAGESIZE);
3796         ASSERT(off <= MMU_PAGESIZE);
3797         ASSERT(off + len <= MMU_PAGESIZE);
3798 
3799         if (kpm_enable && !pfn_is_foreign(pfn)) {
3800                 pp_addr2 = hat_kpm_pfn2va(pfn);
3801                 kpreempt_disable();
3802         } else {
3803                 kpreempt_disable();
3804 
3805                 pp_addr2 = CPU->cpu_caddr2;
3806                 pte2 = CPU->cpu_caddr2pte;
3807 
3808                 ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
3809                 mutex_enter(ppaddr_mutex);
3810 
3811                 hat_mempte_remap(pfn, pp_addr2, pte2,
3812                     PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
3813                     HAT_LOAD_NOCONSIST);
3814         }
3815 
3816         if (use_sse_pagezero) {
3817 #ifdef __xpv
3818                 uint_t rem;
3819 
3820                 /*
3821                  * zero a byte at a time until properly aligned for
3822                  * block_zero_no_xmm().
3823                  */
3824                 while (!P2NPHASE(off, ((uint_t)BLOCKZEROALIGN)) && len-- > 0)
3825                         pp_addr2[off++] = 0;
3826 
3827                 /*
3828                  * Now use faster block_zero_no_xmm() for any range
3829                  * that is properly aligned and sized.
3830                  */
3831                 rem = P2PHASE(len, ((uint_t)BLOCKZEROALIGN));
3832                 len -= rem;
3833                 if (len != 0) {
3834                         block_zero_no_xmm(pp_addr2 + off, len);
3835                         off += len;
3836                 }
3837 
3838                 /*
3839                  * zero remainder with byte stores.
3840                  */
3841                 while (rem-- > 0)
3842                         pp_addr2[off++] = 0;
3843 #else
3844                 hwblkclr(pp_addr2 + off, len);
3845 #endif
3846         } else {
3847                 bzero(pp_addr2 + off, len);
3848         }
3849 
3850         if (!kpm_enable || pfn_is_foreign(pfn)) {
3851 #ifdef __xpv
3852                 /*
3853                  * On the hypervisor this page might get used for a page
3854                  * table before any intervening change to this mapping,
3855                  * so blow it away.
3856                  */
3857                 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3858                     UVMF_INVLPG) < 0)
3859                         panic("HYPERVISOR_update_va_mapping() failed");
3860 #endif
3861                 mutex_exit(ppaddr_mutex);
3862         }
3863 
3864         kpreempt_enable();
3865 }
3866 
3867 /*
3868  * Platform-dependent page scrub call.
3869  */
3870 void
3871 pagescrub(page_t *pp, uint_t off, uint_t len)
3872 {
3873         /*
3874          * For now, we rely on the fact that pagezero() will
3875          * always clear UEs.
3876          */
3877         pagezero(pp, off, len);
3878 }
3879 
3880 /*
3881  * set up two private addresses for use on a given CPU for use in ppcopy()
3882  */
3883 void
3884 setup_vaddr_for_ppcopy(struct cpu *cpup)
3885 {
3886         void *addr;
3887         hat_mempte_t pte_pa;
3888 
3889         addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
3890         pte_pa = hat_mempte_setup(addr);
3891         cpup->cpu_caddr1 = addr;
3892         cpup->cpu_caddr1pte = pte_pa;
3893 
3894         addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
3895         pte_pa = hat_mempte_setup(addr);
3896         cpup->cpu_caddr2 = addr;
3897         cpup->cpu_caddr2pte = pte_pa;
3898 
3899         mutex_init(&cpup->cpu_ppaddr_mutex, NULL, MUTEX_DEFAULT, NULL);
3900 }
3901 
3902 /*
3903  * Undo setup_vaddr_for_ppcopy
3904  */
3905 void
3906 teardown_vaddr_for_ppcopy(struct cpu *cpup)
3907 {
3908         mutex_destroy(&cpup->cpu_ppaddr_mutex);
3909 
3910         hat_mempte_release(cpup->cpu_caddr2, cpup->cpu_caddr2pte);
3911         cpup->cpu_caddr2pte = 0;
3912         vmem_free(heap_arena, cpup->cpu_caddr2, mmu_ptob(1));
3913         cpup->cpu_caddr2 = 0;
3914 
3915         hat_mempte_release(cpup->cpu_caddr1, cpup->cpu_caddr1pte);
3916         cpup->cpu_caddr1pte = 0;
3917         vmem_free(heap_arena, cpup->cpu_caddr1, mmu_ptob(1));
3918         cpup->cpu_caddr1 = 0;
3919 }
3920 
3921 /*
3922  * Function for flushing D-cache when performing module relocations
3923  * to an alternate mapping.  Unnecessary on Intel / AMD platforms.
3924  */
3925 void
3926 dcache_flushall()
3927 {}
3928 
3929 size_t
3930 exec_get_spslew(void)
3931 {
3932         return (0);
3933 }
3934 
3935 /*
3936  * Allocate a memory page.  The argument 'seed' can be any pseudo-random
3937  * number to vary where the pages come from.  This is quite a hacked up
3938  * method -- it works for now, but really needs to be fixed up a bit.
3939  *
3940  * We currently use page_create_va() on the kvp with fake offsets,
3941  * segments and virt address.  This is pretty bogus, but was copied from the
3942  * old hat_i86.c code.  A better approach would be to specify either mnode
3943  * random or mnode local and takes a page from whatever color has the MOST
3944  * available - this would have a minimal impact on page coloring.
3945  */
3946 page_t *
3947 page_get_physical(uintptr_t seed)
3948 {
3949         page_t *pp;
3950         u_offset_t offset;
3951         static struct seg tmpseg;
3952         static uintptr_t ctr = 0;
3953 
3954         /*
3955          * This code is gross, we really need a simpler page allocator.
3956          *
3957          * We need to assign an offset for the page to call page_create_va()
3958          * To avoid conflicts with other pages, we get creative with the offset.
3959          * For 32 bits, we need an offset > 4Gig
3960          * For 64 bits, need an offset somewhere in the VA hole.
3961          */
3962         offset = seed;
3963         if (offset > kernelbase)
3964                 offset -= kernelbase;
3965         offset <<= MMU_PAGESHIFT;
3966 #if defined(__amd64)
3967         offset += mmu.hole_start;       /* something in VA hole */
3968 #else
3969         offset += 1ULL << 40;     /* something > 4 Gig */
3970 #endif
3971 
3972         if (page_resv(1, KM_NOSLEEP) == 0)
3973                 return (NULL);
3974 
3975 #ifdef  DEBUG
3976         pp = page_exists(&kvp, offset);
3977         if (pp != NULL)
3978                 panic("page already exists %p", (void *)pp);
3979 #endif
3980 
3981         pp = page_create_va(&kvp, offset, MMU_PAGESIZE, PG_EXCL,
3982             &tmpseg, (caddr_t)(ctr += MMU_PAGESIZE));       /* changing VA usage */
3983         if (pp != NULL) {
3984                 page_io_unlock(pp);
3985                 page_downgrade(pp);
3986         }
3987         return (pp);
3988 }