1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 
  25 #include <sys/types.h>
  26 #include <sys/t_lock.h>
  27 #include <sys/param.h>
  28 #include <sys/sysmacros.h>
  29 #include <sys/tuneable.h>
  30 #include <sys/systm.h>
  31 #include <sys/vm.h>
  32 #include <sys/kmem.h>
  33 #include <sys/vmem.h>
  34 #include <sys/mman.h>
  35 #include <sys/cmn_err.h>
  36 #include <sys/debug.h>
  37 #include <sys/dumphdr.h>
  38 #include <sys/bootconf.h>
  39 #include <sys/lgrp.h>
  40 #include <vm/seg_kmem.h>
  41 #include <vm/hat.h>
  42 #include <vm/page.h>
  43 #include <vm/vm_dep.h>
  44 #include <vm/faultcode.h>
  45 #include <sys/promif.h>
  46 #include <vm/seg_kp.h>
  47 #include <sys/bitmap.h>
  48 #include <sys/mem_cage.h>
  49 
  50 #ifdef __sparc
  51 #include <sys/ivintr.h>
  52 #include <sys/panic.h>
  53 #endif
  54 
  55 /*
  56  * seg_kmem is the primary kernel memory segment driver.  It
  57  * maps the kernel heap [kernelheap, ekernelheap), module text,
  58  * and all memory which was allocated before the VM was initialized
  59  * into kas.
  60  *
  61  * Pages which belong to seg_kmem are hashed into &kvp vnode at
  62  * an offset equal to (u_offset_t)virt_addr, and have p_lckcnt >= 1.
  63  * They must never be paged out since segkmem_fault() is a no-op to
  64  * prevent recursive faults.
  65  *
  66  * Currently, seg_kmem pages are sharelocked (p_sharelock == 1) on
  67  * __x86 and are unlocked (p_sharelock == 0) on __sparc.  Once __x86
  68  * supports relocation the #ifdef kludges can be removed.
  69  *
  70  * seg_kmem pages may be subject to relocation by page_relocate(),
  71  * provided that the HAT supports it; if this is so, segkmem_reloc
  72  * will be set to a nonzero value. All boot time allocated memory as
  73  * well as static memory is considered off limits to relocation.
  74  * Pages are "relocatable" if p_state does not have P_NORELOC set, so
  75  * we request P_NORELOC pages for memory that isn't safe to relocate.
  76  *
  77  * The kernel heap is logically divided up into four pieces:
  78  *
  79  *   heap32_arena is for allocations that require 32-bit absolute
  80  *   virtual addresses (e.g. code that uses 32-bit pointers/offsets).
  81  *
  82  *   heap_core is for allocations that require 2GB *relative*
  83  *   offsets; in other words all memory from heap_core is within
  84  *   2GB of all other memory from the same arena. This is a requirement
  85  *   of the addressing modes of some processors in supervisor code.
  86  *
  87  *   heap_arena is the general heap arena.
  88  *
  89  *   static_arena is the static memory arena.  Allocations from it
  90  *   are not subject to relocation so it is safe to use the memory
  91  *   physical address as well as the virtual address (e.g. the VA to
  92  *   PA translations are static).  Caches may import from static_arena;
  93  *   all other static memory allocations should use static_alloc_arena.
  94  *
  95  * On some platforms which have limited virtual address space, seg_kmem
  96  * may share [kernelheap, ekernelheap) with seg_kp; if this is so,
  97  * segkp_bitmap is non-NULL, and each bit represents a page of virtual
  98  * address space which is actually seg_kp mapped.
  99  */
 100 
 101 extern ulong_t *segkp_bitmap;   /* Is set if segkp is from the kernel heap */
 102 
 103 char *kernelheap;               /* start of primary kernel heap */
 104 char *ekernelheap;              /* end of primary kernel heap */
 105 struct seg kvseg;               /* primary kernel heap segment */
 106 struct seg kvseg_core;          /* "core" kernel heap segment */
 107 struct seg kzioseg;             /* Segment for zio mappings */
 108 vmem_t *heap_arena;             /* primary kernel heap arena */
 109 vmem_t *heap_core_arena;        /* core kernel heap arena */
 110 char *heap_core_base;           /* start of core kernel heap arena */
 111 char *heap_lp_base;             /* start of kernel large page heap arena */
 112 char *heap_lp_end;              /* end of kernel large page heap arena */
 113 vmem_t *hat_memload_arena;      /* HAT translation data */
 114 struct seg kvseg32;             /* 32-bit kernel heap segment */
 115 vmem_t *heap32_arena;           /* 32-bit kernel heap arena */
 116 vmem_t *heaptext_arena;         /* heaptext arena */
 117 struct as kas;                  /* kernel address space */
 118 int segkmem_reloc;              /* enable/disable relocatable segkmem pages */
 119 vmem_t *static_arena;           /* arena for caches to import static memory */
 120 vmem_t *static_alloc_arena;     /* arena for allocating static memory */
 121 vmem_t *zio_arena = NULL;       /* arena for allocating zio memory */
 122 vmem_t *zio_alloc_arena = NULL; /* arena for allocating zio memory */
 123 
 124 /*
 125  * seg_kmem driver can map part of the kernel heap with large pages.
 126  * Currently this functionality is implemented for sparc platforms only.
 127  *
 128  * The large page size "segkmem_lpsize" for kernel heap is selected in the
 129  * platform specific code. It can also be modified via /etc/system file.
 130  * Setting segkmem_lpsize to PAGESIZE in /etc/system disables usage of large
 131  * pages for kernel heap. "segkmem_lpshift" is adjusted appropriately to
 132  * match segkmem_lpsize.
 133  *
 134  * At boot time we carve from kernel heap arena a range of virtual addresses
 135  * that will be used for large page mappings. This range [heap_lp_base,
 136  * heap_lp_end) is set up as a separate vmem arena - "heap_lp_arena". We also
 137  * create "kmem_lp_arena" that caches memory already backed up by large
 138  * pages. kmem_lp_arena imports virtual segments from heap_lp_arena.
 139  */
 140 
 141 size_t  segkmem_lpsize;
 142 static  uint_t  segkmem_lpshift = PAGESHIFT;
 143 int     segkmem_lpszc = 0;
 144 
 145 size_t  segkmem_kmemlp_quantum = 0x400000;      /* 4MB */
 146 size_t  segkmem_heaplp_quantum;
 147 vmem_t *heap_lp_arena;
 148 static  vmem_t *kmem_lp_arena;
 149 static  vmem_t *segkmem_ppa_arena;
 150 static  segkmem_lpcb_t segkmem_lpcb;
 151 
 152 /*
 153  * We use "segkmem_kmemlp_max" to limit the total amount of physical memory
 154  * consumed by the large page heap. By default this parameter is set to 1/8 of
 155  * physmem but can be adjusted through /etc/system either directly or
 156  * indirectly by setting "segkmem_kmemlp_pcnt" to the percent of physmem
 157  * we allow for large page heap.
 158  */
 159 size_t  segkmem_kmemlp_max;
 160 static  uint_t  segkmem_kmemlp_pcnt;
 161 
 162 /*
 163  * Getting large pages for kernel heap could be problematic due to
 164  * physical memory fragmentation. That's why we allow to preallocate
 165  * "segkmem_kmemlp_min" bytes at boot time.
 166  */
 167 static  size_t  segkmem_kmemlp_min;
 168 
 169 /*
 170  * Throttling is used to avoid expensive tries to allocate large pages
 171  * for kernel heap when a lot of succesive attempts to do so fail.
 172  */
 173 static  ulong_t segkmem_lpthrottle_max = 0x400000;
 174 static  ulong_t segkmem_lpthrottle_start = 0x40;
 175 static  ulong_t segkmem_use_lpthrottle = 1;
 176 
 177 /*
 178  * Freed pages accumulate on a garbage list until segkmem is ready,
 179  * at which point we call segkmem_gc() to free it all.
 180  */
 181 typedef struct segkmem_gc_list {
 182         struct segkmem_gc_list  *gc_next;
 183         vmem_t                  *gc_arena;
 184         size_t                  gc_size;
 185 } segkmem_gc_list_t;
 186 
 187 static segkmem_gc_list_t *segkmem_gc_list;
 188 
 189 /*
 190  * Allocations from the hat_memload arena add VM_MEMLOAD to their
 191  * vmflags so that segkmem_xalloc() can inform the hat layer that it needs
 192  * to take steps to prevent infinite recursion.  HAT allocations also
 193  * must be non-relocatable to prevent recursive page faults.
 194  */
 195 static void *
 196 hat_memload_alloc(vmem_t *vmp, size_t size, int flags)
 197 {
 198         flags |= (VM_MEMLOAD | VM_NORELOC);
 199         return (segkmem_alloc(vmp, size, flags));
 200 }
 201 
 202 /*
 203  * Allocations from static_arena arena (or any other arena that uses
 204  * segkmem_alloc_permanent()) require non-relocatable (permanently
 205  * wired) memory pages, since these pages are referenced by physical
 206  * as well as virtual address.
 207  */
 208 void *
 209 segkmem_alloc_permanent(vmem_t *vmp, size_t size, int flags)
 210 {
 211         return (segkmem_alloc(vmp, size, flags | VM_NORELOC));
 212 }
 213 
 214 /*
 215  * Initialize kernel heap boundaries.
 216  */
 217 void
 218 kernelheap_init(
 219         void *heap_start,
 220         void *heap_end,
 221         char *first_avail,
 222         void *core_start,
 223         void *core_end)
 224 {
 225         uintptr_t textbase;
 226         size_t core_size;
 227         size_t heap_size;
 228         vmem_t *heaptext_parent;
 229         size_t  heap_lp_size = 0;
 230 #ifdef __sparc
 231         size_t kmem64_sz = kmem64_aligned_end - kmem64_base;
 232 #endif  /* __sparc */
 233 
 234         kernelheap = heap_start;
 235         ekernelheap = heap_end;
 236 
 237 #ifdef __sparc
 238         heap_lp_size = (((uintptr_t)heap_end - (uintptr_t)heap_start) / 4);
 239         /*
 240          * Bias heap_lp start address by kmem64_sz to reduce collisions
 241          * in 4M kernel TSB between kmem64 area and heap_lp
 242          */
 243         kmem64_sz = P2ROUNDUP(kmem64_sz, MMU_PAGESIZE256M);
 244         if (kmem64_sz <= heap_lp_size / 2)
 245                 heap_lp_size -= kmem64_sz;
 246         heap_lp_base = ekernelheap - heap_lp_size;
 247         heap_lp_end = heap_lp_base + heap_lp_size;
 248 #endif  /* __sparc */
 249 
 250         /*
 251          * If this platform has a 'core' heap area, then the space for
 252          * overflow module text should be carved out of the end of that
 253          * heap.  Otherwise, it gets carved out of the general purpose
 254          * heap.
 255          */
 256         core_size = (uintptr_t)core_end - (uintptr_t)core_start;
 257         if (core_size > 0) {
 258                 ASSERT(core_size >= HEAPTEXT_SIZE);
 259                 textbase = (uintptr_t)core_end - HEAPTEXT_SIZE;
 260                 core_size -= HEAPTEXT_SIZE;
 261         }
 262 #ifndef __sparc
 263         else {
 264                 ekernelheap -= HEAPTEXT_SIZE;
 265                 textbase = (uintptr_t)ekernelheap;
 266         }
 267 #endif
 268 
 269         heap_size = (uintptr_t)ekernelheap - (uintptr_t)kernelheap;
 270         heap_arena = vmem_init("heap", kernelheap, heap_size, PAGESIZE,
 271             segkmem_alloc, segkmem_free);
 272 
 273         if (core_size > 0) {
 274                 heap_core_arena = vmem_create("heap_core", core_start,
 275                     core_size, PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP);
 276                 heap_core_base = core_start;
 277         } else {
 278                 heap_core_arena = heap_arena;
 279                 heap_core_base = kernelheap;
 280         }
 281 
 282         /*
 283          * reserve space for the large page heap. If large pages for kernel
 284          * heap is enabled large page heap arean will be created later in the
 285          * boot sequence in segkmem_heap_lp_init(). Otherwise the allocated
 286          * range will be returned back to the heap_arena.
 287          */
 288         if (heap_lp_size) {
 289                 (void) vmem_xalloc(heap_arena, heap_lp_size, PAGESIZE, 0, 0,
 290                     heap_lp_base, heap_lp_end,
 291                     VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
 292         }
 293 
 294         /*
 295          * Remove the already-spoken-for memory range [kernelheap, first_avail).
 296          */
 297         (void) vmem_xalloc(heap_arena, first_avail - kernelheap, PAGESIZE,
 298             0, 0, kernelheap, first_avail, VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
 299 
 300 #ifdef __sparc
 301         heap32_arena = vmem_create("heap32", (void *)SYSBASE32,
 302             SYSLIMIT32 - SYSBASE32 - HEAPTEXT_SIZE, PAGESIZE, NULL,
 303             NULL, NULL, 0, VM_SLEEP);
 304         /*
 305          * Prom claims the physical and virtual resources used by panicbuf
 306          * and inter_vec_table. So reserve space for panicbuf, intr_vec_table,
 307          * reserved interrupt vector data structures from 32-bit heap.
 308          */
 309         (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0,
 310             panicbuf, panicbuf + PANICBUFSIZE,
 311             VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
 312 
 313         (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0,
 314             intr_vec_table, (caddr_t)intr_vec_table + IVSIZE,
 315             VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
 316 
 317         textbase = SYSLIMIT32 - HEAPTEXT_SIZE;
 318         heaptext_parent = NULL;
 319 #else   /* __sparc */
 320         heap32_arena = heap_core_arena;
 321         heaptext_parent = heap_core_arena;
 322 #endif  /* __sparc */
 323 
 324         heaptext_arena = vmem_create("heaptext", (void *)textbase,
 325             HEAPTEXT_SIZE, PAGESIZE, NULL, NULL, heaptext_parent, 0, VM_SLEEP);
 326 
 327         /*
 328          * Create a set of arenas for memory with static translations
 329          * (e.g. VA -> PA translations cannot change).  Since using
 330          * kernel pages by physical address implies it isn't safe to
 331          * walk across page boundaries, the static_arena quantum must
 332          * be PAGESIZE.  Any kmem caches that require static memory
 333          * should source from static_arena, while direct allocations
 334          * should only use static_alloc_arena.
 335          */
 336         static_arena = vmem_create("static", NULL, 0, PAGESIZE,
 337             segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP);
 338         static_alloc_arena = vmem_create("static_alloc", NULL, 0,
 339             sizeof (uint64_t), vmem_alloc, vmem_free, static_arena,
 340             0, VM_SLEEP);
 341 
 342         /*
 343          * Create an arena for translation data (ptes, hmes, or hblks).
 344          * We need an arena for this because hat_memload() is essential
 345          * to vmem_populate() (see comments in common/os/vmem.c).
 346          *
 347          * Note: any kmem cache that allocates from hat_memload_arena
 348          * must be created as a KMC_NOHASH cache (i.e. no external slab
 349          * and bufctl structures to allocate) so that slab creation doesn't
 350          * require anything more than a single vmem_alloc().
 351          */
 352         hat_memload_arena = vmem_create("hat_memload", NULL, 0, PAGESIZE,
 353             hat_memload_alloc, segkmem_free, heap_arena, 0,
 354             VM_SLEEP | VMC_POPULATOR | VMC_DUMPSAFE);
 355 }
 356 
 357 void
 358 boot_mapin(caddr_t addr, size_t size)
 359 {
 360         caddr_t  eaddr;
 361         page_t  *pp;
 362         pfn_t    pfnum;
 363 
 364         if (page_resv(btop(size), KM_NOSLEEP) == 0)
 365                 panic("boot_mapin: page_resv failed");
 366 
 367         for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
 368                 pfnum = va_to_pfn(addr);
 369                 if (pfnum == PFN_INVALID)
 370                         continue;
 371                 if ((pp = page_numtopp_nolock(pfnum)) == NULL)
 372                         panic("boot_mapin(): No pp for pfnum = %lx", pfnum);
 373 
 374                 /*
 375                  * must break up any large pages that may have constituent
 376                  * pages being utilized for BOP_ALLOC()'s before calling
 377                  * page_numtopp().The locking code (ie. page_reclaim())
 378                  * can't handle them
 379                  */
 380                 if (pp->p_szc != 0)
 381                         page_boot_demote(pp);
 382 
 383                 pp = page_numtopp(pfnum, SE_EXCL);
 384                 if (pp == NULL || PP_ISFREE(pp))
 385                         panic("boot_alloc: pp is NULL or free");
 386 
 387                 /*
 388                  * If the cage is on but doesn't yet contain this page,
 389                  * mark it as non-relocatable.
 390                  */
 391                 if (kcage_on && !PP_ISNORELOC(pp)) {
 392                         PP_SETNORELOC(pp);
 393                         PLCNT_XFER_NORELOC(pp);
 394                 }
 395 
 396                 (void) page_hashin(pp, &kvp, (u_offset_t)(uintptr_t)addr, NULL);
 397                 pp->p_lckcnt = 1;
 398 #if defined(__x86)
 399                 page_downgrade(pp);
 400 #else
 401                 page_unlock(pp);
 402 #endif
 403         }
 404 }
 405 
 406 /*
 407  * Get pages from boot and hash them into the kernel's vp.
 408  * Used after page structs have been allocated, but before segkmem is ready.
 409  */
 410 void *
 411 boot_alloc(void *inaddr, size_t size, uint_t align)
 412 {
 413         caddr_t addr = inaddr;
 414 
 415         if (bootops == NULL)
 416                 prom_panic("boot_alloc: attempt to allocate memory after "
 417                     "BOP_GONE");
 418 
 419         size = ptob(btopr(size));
 420 #ifdef __sparc
 421         if (bop_alloc_chunk(addr, size, align) != (caddr_t)addr)
 422                 panic("boot_alloc: bop_alloc_chunk failed");
 423 #else
 424         if (BOP_ALLOC(bootops, addr, size, align) != addr)
 425                 panic("boot_alloc: BOP_ALLOC failed");
 426 #endif
 427         boot_mapin((caddr_t)addr, size);
 428         return (addr);
 429 }
 430 
 431 static void
 432 segkmem_badop()
 433 {
 434         panic("segkmem_badop");
 435 }
 436 
 437 #define SEGKMEM_BADOP(t)        (t(*)())segkmem_badop
 438 
 439 /*ARGSUSED*/
 440 static faultcode_t
 441 segkmem_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t size,
 442         enum fault_type type, enum seg_rw rw)
 443 {
 444         pgcnt_t npages;
 445         spgcnt_t pg;
 446         page_t *pp;
 447         struct vnode *vp = seg->s_data;
 448 
 449         ASSERT(RW_READ_HELD(&seg->s_as->a_lock));
 450 
 451         if (seg->s_as != &kas || size > seg->s_size ||
 452             addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
 453                 panic("segkmem_fault: bad args");
 454 
 455         /*
 456          * If it is one of segkp pages, call segkp_fault.
 457          */
 458         if (segkp_bitmap && seg == &kvseg &&
 459             BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
 460                 return (segop_fault(hat, segkp, addr, size, type, rw));
 461 
 462         if (rw != S_READ && rw != S_WRITE && rw != S_OTHER)
 463                 return (FC_NOSUPPORT);
 464 
 465         npages = btopr(size);
 466 
 467         switch (type) {
 468         case F_SOFTLOCK:        /* lock down already-loaded translations */
 469                 for (pg = 0; pg < npages; pg++) {
 470                         pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr,
 471                             SE_SHARED);
 472                         if (pp == NULL) {
 473                                 /*
 474                                  * Hmm, no page. Does a kernel mapping
 475                                  * exist for it?
 476                                  */
 477                                 if (!hat_probe(kas.a_hat, addr)) {
 478                                         addr -= PAGESIZE;
 479                                         while (--pg >= 0) {
 480                                                 pp = page_find(vp, (u_offset_t)
 481                                                     (uintptr_t)addr);
 482                                                 if (pp)
 483                                                         page_unlock(pp);
 484                                                 addr -= PAGESIZE;
 485                                         }
 486                                         return (FC_NOMAP);
 487                                 }
 488                         }
 489                         addr += PAGESIZE;
 490                 }
 491                 if (rw == S_OTHER)
 492                         hat_reserve(seg->s_as, addr, size);
 493                 return (0);
 494         case F_SOFTUNLOCK:
 495                 while (npages--) {
 496                         pp = page_find(vp, (u_offset_t)(uintptr_t)addr);
 497                         if (pp)
 498                                 page_unlock(pp);
 499                         addr += PAGESIZE;
 500                 }
 501                 return (0);
 502         default:
 503                 return (FC_NOSUPPORT);
 504         }
 505         /*NOTREACHED*/
 506 }
 507 
 508 static int
 509 segkmem_setprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
 510 {
 511         ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
 512 
 513         if (seg->s_as != &kas || size > seg->s_size ||
 514             addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
 515                 panic("segkmem_setprot: bad args");
 516 
 517         /*
 518          * If it is one of segkp pages, call segkp.
 519          */
 520         if (segkp_bitmap && seg == &kvseg &&
 521             BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
 522                 return (segop_setprot(segkp, addr, size, prot));
 523 
 524         if (prot == 0)
 525                 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD);
 526         else
 527                 hat_chgprot(kas.a_hat, addr, size, prot);
 528         return (0);
 529 }
 530 
 531 /*
 532  * This is a dummy segkmem function overloaded to call segkp
 533  * when segkp is under the heap.
 534  */
 535 /* ARGSUSED */
 536 static int
 537 segkmem_checkprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
 538 {
 539         ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
 540 
 541         if (seg->s_as != &kas)
 542                 segkmem_badop();
 543 
 544         /*
 545          * If it is one of segkp pages, call into segkp.
 546          */
 547         if (segkp_bitmap && seg == &kvseg &&
 548             BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
 549                 return (segop_checkprot(segkp, addr, size, prot));
 550 
 551         segkmem_badop();
 552         return (0);
 553 }
 554 
 555 /*
 556  * This is a dummy segkmem function overloaded to call segkp
 557  * when segkp is under the heap.
 558  */
 559 /* ARGSUSED */
 560 static int
 561 segkmem_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
 562 {
 563         ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
 564 
 565         if (seg->s_as != &kas)
 566                 segkmem_badop();
 567 
 568         /*
 569          * If it is one of segkp pages, call into segkp.
 570          */
 571         if (segkp_bitmap && seg == &kvseg &&
 572             BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
 573                 return (segop_kluster(segkp, addr, delta));
 574 
 575         segkmem_badop();
 576         return (0);
 577 }
 578 
 579 static void
 580 segkmem_xdump_range(void *arg, void *start, size_t size)
 581 {
 582         struct as *as = arg;
 583         caddr_t addr = start;
 584         caddr_t addr_end = addr + size;
 585 
 586         while (addr < addr_end) {
 587                 pfn_t pfn = hat_getpfnum(kas.a_hat, addr);
 588                 if (pfn != PFN_INVALID && pfn <= physmax && pf_is_memory(pfn))
 589                         dump_addpage(as, addr, pfn);
 590                 addr += PAGESIZE;
 591                 dump_timeleft = dump_timeout;
 592         }
 593 }
 594 
 595 static void
 596 segkmem_dump_range(void *arg, void *start, size_t size)
 597 {
 598         caddr_t addr = start;
 599         caddr_t addr_end = addr + size;
 600 
 601         /*
 602          * If we are about to start dumping the range of addresses we
 603          * carved out of the kernel heap for the large page heap walk
 604          * heap_lp_arena to find what segments are actually populated
 605          */
 606         if (SEGKMEM_USE_LARGEPAGES &&
 607             addr == heap_lp_base && addr_end == heap_lp_end &&
 608             vmem_size(heap_lp_arena, VMEM_ALLOC) < size) {
 609                 vmem_walk(heap_lp_arena, VMEM_ALLOC | VMEM_REENTRANT,
 610                     segkmem_xdump_range, arg);
 611         } else {
 612                 segkmem_xdump_range(arg, start, size);
 613         }
 614 }
 615 
 616 static void
 617 segkmem_dump(struct seg *seg)
 618 {
 619         /*
 620          * The kernel's heap_arena (represented by kvseg) is a very large
 621          * VA space, most of which is typically unused.  To speed up dumping
 622          * we use vmem_walk() to quickly find the pieces of heap_arena that
 623          * are actually in use.  We do the same for heap32_arena and
 624          * heap_core.
 625          *
 626          * We specify VMEM_REENTRANT to vmem_walk() because dump_addpage()
 627          * may ultimately need to allocate memory.  Reentrant walks are
 628          * necessarily imperfect snapshots.  The kernel heap continues
 629          * to change during a live crash dump, for example.  For a normal
 630          * crash dump, however, we know that there won't be any other threads
 631          * messing with the heap.  Therefore, at worst, we may fail to dump
 632          * the pages that get allocated by the act of dumping; but we will
 633          * always dump every page that was allocated when the walk began.
 634          *
 635          * The other segkmem segments are dense (fully populated), so there's
 636          * no need to use this technique when dumping them.
 637          *
 638          * Note: when adding special dump handling for any new sparsely-
 639          * populated segments, be sure to add similar handling to the ::kgrep
 640          * code in mdb.
 641          */
 642         if (seg == &kvseg) {
 643                 vmem_walk(heap_arena, VMEM_ALLOC | VMEM_REENTRANT,
 644                     segkmem_dump_range, seg->s_as);
 645 #ifndef __sparc
 646                 vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
 647                     segkmem_dump_range, seg->s_as);
 648 #endif
 649         } else if (seg == &kvseg_core) {
 650                 vmem_walk(heap_core_arena, VMEM_ALLOC | VMEM_REENTRANT,
 651                     segkmem_dump_range, seg->s_as);
 652         } else if (seg == &kvseg32) {
 653                 vmem_walk(heap32_arena, VMEM_ALLOC | VMEM_REENTRANT,
 654                     segkmem_dump_range, seg->s_as);
 655                 vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
 656                     segkmem_dump_range, seg->s_as);
 657         } else if (seg == &kzioseg) {
 658                 /*
 659                  * We don't want to dump pages attached to kzioseg since they
 660                  * contain file data from ZFS.  If this page's segment is
 661                  * kzioseg return instead of writing it to the dump device.
 662                  */
 663                 return;
 664         } else {
 665                 segkmem_dump_range(seg->s_as, seg->s_base, seg->s_size);
 666         }
 667 }
 668 
 669 /*
 670  * lock/unlock kmem pages over a given range [addr, addr+len).
 671  * Returns a shadow list of pages in ppp. If there are holes
 672  * in the range (e.g. some of the kernel mappings do not have
 673  * underlying page_ts) returns ENOTSUP so that as_pagelock()
 674  * will handle the range via as_fault(F_SOFTLOCK).
 675  */
 676 /*ARGSUSED*/
 677 static int
 678 segkmem_pagelock(struct seg *seg, caddr_t addr, size_t len,
 679         page_t ***ppp, enum lock_type type, enum seg_rw rw)
 680 {
 681         page_t **pplist, *pp;
 682         pgcnt_t npages;
 683         spgcnt_t pg;
 684         size_t nb;
 685         struct vnode *vp = seg->s_data;
 686 
 687         ASSERT(ppp != NULL);
 688 
 689         /*
 690          * If it is one of segkp pages, call into segkp.
 691          */
 692         if (segkp_bitmap && seg == &kvseg &&
 693             BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
 694                 return (segop_pagelock(segkp, addr, len, ppp, type, rw));
 695 
 696         npages = btopr(len);
 697         nb = sizeof (page_t *) * npages;
 698 
 699         if (type == L_PAGEUNLOCK) {
 700                 pplist = *ppp;
 701                 ASSERT(pplist != NULL);
 702 
 703                 for (pg = 0; pg < npages; pg++) {
 704                         pp = pplist[pg];
 705                         page_unlock(pp);
 706                 }
 707                 kmem_free(pplist, nb);
 708                 return (0);
 709         }
 710 
 711         ASSERT(type == L_PAGELOCK);
 712 
 713         pplist = kmem_alloc(nb, KM_NOSLEEP);
 714         if (pplist == NULL) {
 715                 *ppp = NULL;
 716                 return (ENOTSUP);       /* take the slow path */
 717         }
 718 
 719         for (pg = 0; pg < npages; pg++) {
 720                 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_SHARED);
 721                 if (pp == NULL) {
 722                         while (--pg >= 0)
 723                                 page_unlock(pplist[pg]);
 724                         kmem_free(pplist, nb);
 725                         *ppp = NULL;
 726                         return (ENOTSUP);
 727                 }
 728                 pplist[pg] = pp;
 729                 addr += PAGESIZE;
 730         }
 731 
 732         *ppp = pplist;
 733         return (0);
 734 }
 735 
 736 /*
 737  * This is a dummy segkmem function overloaded to call segkp
 738  * when segkp is under the heap.
 739  */
 740 /* ARGSUSED */
 741 static int
 742 segkmem_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
 743 {
 744         ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
 745 
 746         if (seg->s_as != &kas)
 747                 segkmem_badop();
 748 
 749         /*
 750          * If it is one of segkp pages, call into segkp.
 751          */
 752         if (segkp_bitmap && seg == &kvseg &&
 753             BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
 754                 return (segop_getmemid(segkp, addr, memidp));
 755 
 756         segkmem_badop();
 757         return (0);
 758 }
 759 
 760 /*ARGSUSED*/
 761 static lgrp_mem_policy_info_t *
 762 segkmem_getpolicy(struct seg *seg, caddr_t addr)
 763 {
 764         return (NULL);
 765 }
 766 
 767 /*ARGSUSED*/
 768 static int
 769 segkmem_capable(struct seg *seg, segcapability_t capability)
 770 {
 771         if (capability == S_CAPABILITY_NOMINFLT)
 772                 return (1);
 773         return (0);
 774 }
 775 
 776 static struct seg_ops segkmem_ops = {
 777         .dup            = SEGKMEM_BADOP(int),
 778         .unmap          = SEGKMEM_BADOP(int),
 779         .free           = SEGKMEM_BADOP(void),
 780         .fault          = segkmem_fault,
 781         .faulta         = SEGKMEM_BADOP(faultcode_t),
 782         .setprot        = segkmem_setprot,
 783         .checkprot      = segkmem_checkprot,
 784         .kluster        = segkmem_kluster,
 785         .swapout        = SEGKMEM_BADOP(size_t),
 786         .sync           = SEGKMEM_BADOP(int),
 787         .incore         = SEGKMEM_BADOP(size_t),
 788         .lockop         = SEGKMEM_BADOP(int),
 789         .getprot        = SEGKMEM_BADOP(int),
 790         .getoffset      = SEGKMEM_BADOP(u_offset_t),
 791         .gettype        = SEGKMEM_BADOP(int),
 792         .getvp          = SEGKMEM_BADOP(int),
 793         .advise         = SEGKMEM_BADOP(int),
 794         .dump           = segkmem_dump,
 795         .pagelock       = segkmem_pagelock,
 796         .setpagesize    = SEGKMEM_BADOP(int),
 797         .getmemid       = segkmem_getmemid,
 798         .getpolicy      = segkmem_getpolicy,
 799         .capable        = segkmem_capable,
 800         .inherit        = seg_inherit_notsup,
 801 };
 802 
 803 int
 804 segkmem_zio_create(struct seg *seg)
 805 {
 806         ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock));
 807         seg->s_ops = &segkmem_ops;
 808         seg->s_data = &zvp;
 809         kas.a_size += seg->s_size;
 810         return (0);
 811 }
 812 
 813 int
 814 segkmem_create(struct seg *seg)
 815 {
 816         ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock));
 817         seg->s_ops = &segkmem_ops;
 818         seg->s_data = &kvp;
 819         kas.a_size += seg->s_size;
 820         return (0);
 821 }
 822 
 823 /*ARGSUSED*/
 824 page_t *
 825 segkmem_page_create(void *addr, size_t size, int vmflag, void *arg)
 826 {
 827         struct seg kseg;
 828         int pgflags;
 829         struct vnode *vp = arg;
 830 
 831         if (vp == NULL)
 832                 vp = &kvp;
 833 
 834         kseg.s_as = &kas;
 835         pgflags = PG_EXCL;
 836 
 837         if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
 838                 pgflags |= PG_NORELOC;
 839         if ((vmflag & VM_NOSLEEP) == 0)
 840                 pgflags |= PG_WAIT;
 841         if (vmflag & VM_PANIC)
 842                 pgflags |= PG_PANIC;
 843         if (vmflag & VM_PUSHPAGE)
 844                 pgflags |= PG_PUSHPAGE;
 845         if (vmflag & VM_NORMALPRI) {
 846                 ASSERT(vmflag & VM_NOSLEEP);
 847                 pgflags |= PG_NORMALPRI;
 848         }
 849 
 850         return (page_create_va(vp, (u_offset_t)(uintptr_t)addr, size,
 851             pgflags, &kseg, addr));
 852 }
 853 
 854 /*
 855  * Allocate pages to back the virtual address range [addr, addr + size).
 856  * If addr is NULL, allocate the virtual address space as well.
 857  */
 858 void *
 859 segkmem_xalloc(vmem_t *vmp, void *inaddr, size_t size, int vmflag, uint_t attr,
 860         page_t *(*page_create_func)(void *, size_t, int, void *), void *pcarg)
 861 {
 862         page_t *ppl;
 863         caddr_t addr = inaddr;
 864         pgcnt_t npages = btopr(size);
 865         int allocflag;
 866 
 867         if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
 868                 return (NULL);
 869 
 870         ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
 871 
 872         if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
 873                 if (inaddr == NULL)
 874                         vmem_free(vmp, addr, size);
 875                 return (NULL);
 876         }
 877 
 878         ppl = page_create_func(addr, size, vmflag, pcarg);
 879         if (ppl == NULL) {
 880                 if (inaddr == NULL)
 881                         vmem_free(vmp, addr, size);
 882                 page_unresv(npages);
 883                 return (NULL);
 884         }
 885 
 886         /*
 887          * Under certain conditions, we need to let the HAT layer know
 888          * that it cannot safely allocate memory.  Allocations from
 889          * the hat_memload vmem arena always need this, to prevent
 890          * infinite recursion.
 891          *
 892          * In addition, the x86 hat cannot safely do memory
 893          * allocations while in vmem_populate(), because there
 894          * is no simple bound on its usage.
 895          */
 896         if (vmflag & VM_MEMLOAD)
 897                 allocflag = HAT_NO_KALLOC;
 898 #if defined(__x86)
 899         else if (vmem_is_populator())
 900                 allocflag = HAT_NO_KALLOC;
 901 #endif
 902         else
 903                 allocflag = 0;
 904 
 905         while (ppl != NULL) {
 906                 page_t *pp = ppl;
 907                 page_sub(&ppl, pp);
 908                 ASSERT(page_iolock_assert(pp));
 909                 ASSERT(PAGE_EXCL(pp));
 910                 page_io_unlock(pp);
 911                 hat_memload(kas.a_hat, (caddr_t)(uintptr_t)pp->p_offset, pp,
 912                     (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
 913                     HAT_LOAD_LOCK | allocflag);
 914                 pp->p_lckcnt = 1;
 915 #if defined(__x86)
 916                 page_downgrade(pp);
 917 #else
 918                 if (vmflag & SEGKMEM_SHARELOCKED)
 919                         page_downgrade(pp);
 920                 else
 921                         page_unlock(pp);
 922 #endif
 923         }
 924 
 925         return (addr);
 926 }
 927 
 928 static void *
 929 segkmem_alloc_vn(vmem_t *vmp, size_t size, int vmflag, struct vnode *vp)
 930 {
 931         void *addr;
 932         segkmem_gc_list_t *gcp, **prev_gcpp;
 933 
 934         ASSERT(vp != NULL);
 935 
 936         if (kvseg.s_base == NULL) {
 937 #ifndef __sparc
 938                 if (bootops->bsys_alloc == NULL)
 939                         halt("Memory allocation between bop_alloc() and "
 940                             "kmem_alloc().\n");
 941 #endif
 942 
 943                 /*
 944                  * There's not a lot of memory to go around during boot,
 945                  * so recycle it if we can.
 946                  */
 947                 for (prev_gcpp = &segkmem_gc_list; (gcp = *prev_gcpp) != NULL;
 948                     prev_gcpp = &gcp->gc_next) {
 949                         if (gcp->gc_arena == vmp && gcp->gc_size == size) {
 950                                 *prev_gcpp = gcp->gc_next;
 951                                 return (gcp);
 952                         }
 953                 }
 954 
 955                 addr = vmem_alloc(vmp, size, vmflag | VM_PANIC);
 956                 if (boot_alloc(addr, size, BO_NO_ALIGN) != addr)
 957                         panic("segkmem_alloc: boot_alloc failed");
 958                 return (addr);
 959         }
 960         return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
 961             segkmem_page_create, vp));
 962 }
 963 
 964 void *
 965 segkmem_alloc(vmem_t *vmp, size_t size, int vmflag)
 966 {
 967         return (segkmem_alloc_vn(vmp, size, vmflag, &kvp));
 968 }
 969 
 970 void *
 971 segkmem_zio_alloc(vmem_t *vmp, size_t size, int vmflag)
 972 {
 973         return (segkmem_alloc_vn(vmp, size, vmflag, &zvp));
 974 }
 975 
 976 /*
 977  * Any changes to this routine must also be carried over to
 978  * devmap_free_pages() in the seg_dev driver. This is because
 979  * we currently don't have a special kernel segment for non-paged
 980  * kernel memory that is exported by drivers to user space.
 981  */
 982 static void
 983 segkmem_free_vn(vmem_t *vmp, void *inaddr, size_t size, struct vnode *vp,
 984     void (*func)(page_t *))
 985 {
 986         page_t *pp;
 987         caddr_t addr = inaddr;
 988         caddr_t eaddr;
 989         pgcnt_t npages = btopr(size);
 990 
 991         ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
 992         ASSERT(vp != NULL);
 993 
 994         if (kvseg.s_base == NULL) {
 995                 segkmem_gc_list_t *gc = inaddr;
 996                 gc->gc_arena = vmp;
 997                 gc->gc_size = size;
 998                 gc->gc_next = segkmem_gc_list;
 999                 segkmem_gc_list = gc;
1000                 return;
1001         }
1002 
1003         hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1004 
1005         for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
1006 #if defined(__x86)
1007                 pp = page_find(vp, (u_offset_t)(uintptr_t)addr);
1008                 if (pp == NULL)
1009                         panic("segkmem_free: page not found");
1010                 if (!page_tryupgrade(pp)) {
1011                         /*
1012                          * Some other thread has a sharelock. Wait for
1013                          * it to drop the lock so we can free this page.
1014                          */
1015                         page_unlock(pp);
1016                         pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr,
1017                             SE_EXCL);
1018                 }
1019 #else
1020                 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
1021 #endif
1022                 if (pp == NULL)
1023                         panic("segkmem_free: page not found");
1024                 /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
1025                 pp->p_lckcnt = 0;
1026                 if (func)
1027                         func(pp);
1028                 else
1029                         page_destroy(pp, 0);
1030         }
1031         if (func == NULL)
1032                 page_unresv(npages);
1033 
1034         if (vmp != NULL)
1035                 vmem_free(vmp, inaddr, size);
1036 
1037 }
1038 
1039 void
1040 segkmem_xfree(vmem_t *vmp, void *inaddr, size_t size, void (*func)(page_t *))
1041 {
1042         segkmem_free_vn(vmp, inaddr, size, &kvp, func);
1043 }
1044 
1045 void
1046 segkmem_free(vmem_t *vmp, void *inaddr, size_t size)
1047 {
1048         segkmem_free_vn(vmp, inaddr, size, &kvp, NULL);
1049 }
1050 
1051 void
1052 segkmem_zio_free(vmem_t *vmp, void *inaddr, size_t size)
1053 {
1054         segkmem_free_vn(vmp, inaddr, size, &zvp, NULL);
1055 }
1056 
1057 void
1058 segkmem_gc(void)
1059 {
1060         ASSERT(kvseg.s_base != NULL);
1061         while (segkmem_gc_list != NULL) {
1062                 segkmem_gc_list_t *gc = segkmem_gc_list;
1063                 segkmem_gc_list = gc->gc_next;
1064                 segkmem_free(gc->gc_arena, gc, gc->gc_size);
1065         }
1066 }
1067 
1068 /*
1069  * Legacy entry points from here to end of file.
1070  */
1071 void
1072 segkmem_mapin(struct seg *seg, void *addr, size_t size, uint_t vprot,
1073     pfn_t pfn, uint_t flags)
1074 {
1075         hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1076         hat_devload(seg->s_as->a_hat, addr, size, pfn, vprot,
1077             flags | HAT_LOAD_LOCK);
1078 }
1079 
1080 void
1081 segkmem_mapout(struct seg *seg, void *addr, size_t size)
1082 {
1083         hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1084 }
1085 
1086 void *
1087 kmem_getpages(pgcnt_t npages, int kmflag)
1088 {
1089         return (kmem_alloc(ptob(npages), kmflag));
1090 }
1091 
1092 void
1093 kmem_freepages(void *addr, pgcnt_t npages)
1094 {
1095         kmem_free(addr, ptob(npages));
1096 }
1097 
1098 /*
1099  * segkmem_page_create_large() allocates a large page to be used for the kmem
1100  * caches. If kpr is enabled we ask for a relocatable page unless requested
1101  * otherwise. If kpr is disabled we have to ask for a non-reloc page
1102  */
1103 static page_t *
1104 segkmem_page_create_large(void *addr, size_t size, int vmflag, void *arg)
1105 {
1106         int pgflags;
1107 
1108         pgflags = PG_EXCL;
1109 
1110         if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
1111                 pgflags |= PG_NORELOC;
1112         if (!(vmflag & VM_NOSLEEP))
1113                 pgflags |= PG_WAIT;
1114         if (vmflag & VM_PUSHPAGE)
1115                 pgflags |= PG_PUSHPAGE;
1116         if (vmflag & VM_NORMALPRI)
1117                 pgflags |= PG_NORMALPRI;
1118 
1119         return (page_create_va_large(&kvp, (u_offset_t)(uintptr_t)addr, size,
1120             pgflags, &kvseg, addr, arg));
1121 }
1122 
1123 /*
1124  * Allocate a large page to back the virtual address range
1125  * [addr, addr + size).  If addr is NULL, allocate the virtual address
1126  * space as well.
1127  */
1128 static void *
1129 segkmem_xalloc_lp(vmem_t *vmp, void *inaddr, size_t size, int vmflag,
1130     uint_t attr, page_t *(*page_create_func)(void *, size_t, int, void *),
1131     void *pcarg)
1132 {
1133         caddr_t addr = inaddr, pa;
1134         size_t  lpsize = segkmem_lpsize;
1135         pgcnt_t npages = btopr(size);
1136         pgcnt_t nbpages = btop(lpsize);
1137         pgcnt_t nlpages = size >> segkmem_lpshift;
1138         size_t  ppasize = nbpages * sizeof (page_t *);
1139         page_t *pp, *rootpp, **ppa, *pplist = NULL;
1140         int i;
1141 
1142         vmflag |= VM_NOSLEEP;
1143 
1144         if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
1145                 return (NULL);
1146         }
1147 
1148         /*
1149          * allocate an array we need for hat_memload_array.
1150          * we use a separate arena to avoid recursion.
1151          * we will not need this array when hat_memload_array learns pp++
1152          */
1153         if ((ppa = vmem_alloc(segkmem_ppa_arena, ppasize, vmflag)) == NULL) {
1154                 goto fail_array_alloc;
1155         }
1156 
1157         if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
1158                 goto fail_vmem_alloc;
1159 
1160         ASSERT(((uintptr_t)addr & (lpsize - 1)) == 0);
1161 
1162         /* create all the pages */
1163         for (pa = addr, i = 0; i < nlpages; i++, pa += lpsize) {
1164                 if ((pp = page_create_func(pa, lpsize, vmflag, pcarg)) == NULL)
1165                         goto fail_page_create;
1166                 page_list_concat(&pplist, &pp);
1167         }
1168 
1169         /* at this point we have all the resource to complete the request */
1170         while ((rootpp = pplist) != NULL) {
1171                 for (i = 0; i < nbpages; i++) {
1172                         ASSERT(pplist != NULL);
1173                         pp = pplist;
1174                         page_sub(&pplist, pp);
1175                         ASSERT(page_iolock_assert(pp));
1176                         page_io_unlock(pp);
1177                         ppa[i] = pp;
1178                 }
1179                 /*
1180                  * Load the locked entry. It's OK to preload the entry into the
1181                  * TSB since we now support large mappings in the kernel TSB.
1182                  */
1183                 hat_memload_array(kas.a_hat,
1184                     (caddr_t)(uintptr_t)rootpp->p_offset, lpsize,
1185                     ppa, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
1186                     HAT_LOAD_LOCK);
1187 
1188                 for (--i; i >= 0; --i) {
1189                         ppa[i]->p_lckcnt = 1;
1190                         page_unlock(ppa[i]);
1191                 }
1192         }
1193 
1194         vmem_free(segkmem_ppa_arena, ppa, ppasize);
1195         return (addr);
1196 
1197 fail_page_create:
1198         while ((rootpp = pplist) != NULL) {
1199                 for (i = 0, pp = pplist; i < nbpages; i++, pp = pplist) {
1200                         ASSERT(pp != NULL);
1201                         page_sub(&pplist, pp);
1202                         ASSERT(page_iolock_assert(pp));
1203                         page_io_unlock(pp);
1204                 }
1205                 page_destroy_pages(rootpp);
1206         }
1207 
1208         if (inaddr == NULL)
1209                 vmem_free(vmp, addr, size);
1210 
1211 fail_vmem_alloc:
1212         vmem_free(segkmem_ppa_arena, ppa, ppasize);
1213 
1214 fail_array_alloc:
1215         page_unresv(npages);
1216 
1217         return (NULL);
1218 }
1219 
1220 static void
1221 segkmem_free_one_lp(caddr_t addr, size_t size)
1222 {
1223         page_t          *pp, *rootpp = NULL;
1224         pgcnt_t         pgs_left = btopr(size);
1225 
1226         ASSERT(size == segkmem_lpsize);
1227 
1228         hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1229 
1230         for (; pgs_left > 0; addr += PAGESIZE, pgs_left--) {
1231                 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
1232                 if (pp == NULL)
1233                         panic("segkmem_free_one_lp: page not found");
1234                 ASSERT(PAGE_EXCL(pp));
1235                 pp->p_lckcnt = 0;
1236                 if (rootpp == NULL)
1237                         rootpp = pp;
1238         }
1239         ASSERT(rootpp != NULL);
1240         page_destroy_pages(rootpp);
1241 
1242         /* page_unresv() is done by the caller */
1243 }
1244 
1245 /*
1246  * This function is called to import new spans into the vmem arenas like
1247  * kmem_default_arena and kmem_oversize_arena. It first tries to import
1248  * spans from large page arena - kmem_lp_arena. In order to do this it might
1249  * have to "upgrade the requested size" to kmem_lp_arena quantum. If
1250  * it was not able to satisfy the upgraded request it then calls regular
1251  * segkmem_alloc() that satisfies the request by importing from "*vmp" arena
1252  */
1253 /*ARGSUSED*/
1254 void *
1255 segkmem_alloc_lp(vmem_t *vmp, size_t *sizep, size_t align, int vmflag)
1256 {
1257         size_t size;
1258         kthread_t *t = curthread;
1259         segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1260 
1261         ASSERT(sizep != NULL);
1262 
1263         size = *sizep;
1264 
1265         if (lpcb->lp_uselp && !(t->t_flag & T_PANIC) &&
1266             !(vmflag & SEGKMEM_SHARELOCKED)) {
1267 
1268                 size_t kmemlp_qnt = segkmem_kmemlp_quantum;
1269                 size_t asize = P2ROUNDUP(size, kmemlp_qnt);
1270                 void  *addr = NULL;
1271                 ulong_t *lpthrtp = &lpcb->lp_throttle;
1272                 ulong_t lpthrt = *lpthrtp;
1273                 int     dowakeup = 0;
1274                 int     doalloc = 1;
1275 
1276                 ASSERT(kmem_lp_arena != NULL);
1277                 ASSERT(asize >= size);
1278 
1279                 if (lpthrt != 0) {
1280                         /* try to update the throttle value */
1281                         lpthrt = atomic_inc_ulong_nv(lpthrtp);
1282                         if (lpthrt >= segkmem_lpthrottle_max) {
1283                                 lpthrt = atomic_cas_ulong(lpthrtp, lpthrt,
1284                                     segkmem_lpthrottle_max / 4);
1285                         }
1286 
1287                         /*
1288                          * when we get above throttle start do an exponential
1289                          * backoff at trying large pages and reaping
1290                          */
1291                         if (lpthrt > segkmem_lpthrottle_start &&
1292                             !ISP2(lpthrt)) {
1293                                 lpcb->allocs_throttled++;
1294                                 lpthrt--;
1295                                 if (ISP2(lpthrt))
1296                                         kmem_reap();
1297                                 return (segkmem_alloc(vmp, size, vmflag));
1298                         }
1299                 }
1300 
1301                 if (!(vmflag & VM_NOSLEEP) &&
1302                     segkmem_heaplp_quantum >= (8 * kmemlp_qnt) &&
1303                     vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt &&
1304                     asize < (segkmem_heaplp_quantum - kmemlp_qnt)) {
1305 
1306                         /*
1307                          * we are low on free memory in kmem_lp_arena
1308                          * we let only one guy to allocate heap_lp
1309                          * quantum size chunk that everybody is going to
1310                          * share
1311                          */
1312                         mutex_enter(&lpcb->lp_lock);
1313 
1314                         if (lpcb->lp_wait) {
1315 
1316                                 /* we are not the first one - wait */
1317                                 cv_wait(&lpcb->lp_cv, &lpcb->lp_lock);
1318                                 if (vmem_size(kmem_lp_arena, VMEM_FREE) <
1319                                     kmemlp_qnt)  {
1320                                         doalloc = 0;
1321                                 }
1322                         } else if (vmem_size(kmem_lp_arena, VMEM_FREE) <=
1323                             kmemlp_qnt) {
1324 
1325                                 /*
1326                                  * we are the first one, make sure we import
1327                                  * a large page
1328                                  */
1329                                 if (asize == kmemlp_qnt)
1330                                         asize += kmemlp_qnt;
1331                                 dowakeup = 1;
1332                                 lpcb->lp_wait = 1;
1333                         }
1334 
1335                         mutex_exit(&lpcb->lp_lock);
1336                 }
1337 
1338                 /*
1339                  * VM_ABORT flag prevents sleeps in vmem_xalloc when
1340                  * large pages are not available. In that case this allocation
1341                  * attempt will fail and we will retry allocation with small
1342                  * pages. We also do not want to panic if this allocation fails
1343                  * because we are going to retry.
1344                  */
1345                 if (doalloc) {
1346                         addr = vmem_alloc(kmem_lp_arena, asize,
1347                             (vmflag | VM_ABORT) & ~VM_PANIC);
1348 
1349                         if (dowakeup) {
1350                                 mutex_enter(&lpcb->lp_lock);
1351                                 ASSERT(lpcb->lp_wait != 0);
1352                                 lpcb->lp_wait = 0;
1353                                 cv_broadcast(&lpcb->lp_cv);
1354                                 mutex_exit(&lpcb->lp_lock);
1355                         }
1356                 }
1357 
1358                 if (addr != NULL) {
1359                         *sizep = asize;
1360                         *lpthrtp = 0;
1361                         return (addr);
1362                 }
1363 
1364                 if (vmflag & VM_NOSLEEP)
1365                         lpcb->nosleep_allocs_failed++;
1366                 else
1367                         lpcb->sleep_allocs_failed++;
1368                 lpcb->alloc_bytes_failed += size;
1369 
1370                 /* if large page throttling is not started yet do it */
1371                 if (segkmem_use_lpthrottle && lpthrt == 0) {
1372                         lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 1);
1373                 }
1374         }
1375         return (segkmem_alloc(vmp, size, vmflag));
1376 }
1377 
1378 void
1379 segkmem_free_lp(vmem_t *vmp, void *inaddr, size_t size)
1380 {
1381         if (kmem_lp_arena == NULL || !IS_KMEM_VA_LARGEPAGE((caddr_t)inaddr)) {
1382                 segkmem_free(vmp, inaddr, size);
1383         } else {
1384                 vmem_free(kmem_lp_arena, inaddr, size);
1385         }
1386 }
1387 
1388 /*
1389  * segkmem_alloc_lpi() imports virtual memory from large page heap arena
1390  * into kmem_lp arena. In the process it maps the imported segment with
1391  * large pages
1392  */
1393 static void *
1394 segkmem_alloc_lpi(vmem_t *vmp, size_t size, int vmflag)
1395 {
1396         segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1397         void  *addr;
1398 
1399         ASSERT(size != 0);
1400         ASSERT(vmp == heap_lp_arena);
1401 
1402         /* do not allow large page heap grow beyound limits */
1403         if (vmem_size(vmp, VMEM_ALLOC) >= segkmem_kmemlp_max) {
1404                 lpcb->allocs_limited++;
1405                 return (NULL);
1406         }
1407 
1408         addr = segkmem_xalloc_lp(vmp, NULL, size, vmflag, 0,
1409             segkmem_page_create_large, NULL);
1410         return (addr);
1411 }
1412 
1413 /*
1414  * segkmem_free_lpi() returns virtual memory back into large page heap arena
1415  * from kmem_lp arena. Beore doing this it unmaps the segment and frees
1416  * large pages used to map it.
1417  */
1418 static void
1419 segkmem_free_lpi(vmem_t *vmp, void *inaddr, size_t size)
1420 {
1421         pgcnt_t         nlpages = size >> segkmem_lpshift;
1422         size_t          lpsize = segkmem_lpsize;
1423         caddr_t         addr = inaddr;
1424         pgcnt_t         npages = btopr(size);
1425         int             i;
1426 
1427         ASSERT(vmp == heap_lp_arena);
1428         ASSERT(IS_KMEM_VA_LARGEPAGE(addr));
1429         ASSERT(((uintptr_t)inaddr & (lpsize - 1)) == 0);
1430 
1431         for (i = 0; i < nlpages; i++) {
1432                 segkmem_free_one_lp(addr, lpsize);
1433                 addr += lpsize;
1434         }
1435 
1436         page_unresv(npages);
1437 
1438         vmem_free(vmp, inaddr, size);
1439 }
1440 
1441 /*
1442  * This function is called at system boot time by kmem_init right after
1443  * /etc/system file has been read. It checks based on hardware configuration
1444  * and /etc/system settings if system is going to use large pages. The
1445  * initialiazation necessary to actually start using large pages
1446  * happens later in the process after segkmem_heap_lp_init() is called.
1447  */
1448 int
1449 segkmem_lpsetup()
1450 {
1451         int use_large_pages = 0;
1452 
1453 #ifdef __sparc
1454 
1455         size_t memtotal = physmem * PAGESIZE;
1456 
1457         if (heap_lp_base == NULL) {
1458                 segkmem_lpsize = PAGESIZE;
1459                 return (0);
1460         }
1461 
1462         /* get a platform dependent value of large page size for kernel heap */
1463         segkmem_lpsize = get_segkmem_lpsize(segkmem_lpsize);
1464 
1465         if (segkmem_lpsize <= PAGESIZE) {
1466                 /*
1467                  * put virtual space reserved for the large page kernel
1468                  * back to the regular heap
1469                  */
1470                 vmem_xfree(heap_arena, heap_lp_base,
1471                     heap_lp_end - heap_lp_base);
1472                 heap_lp_base = NULL;
1473                 heap_lp_end = NULL;
1474                 segkmem_lpsize = PAGESIZE;
1475                 return (0);
1476         }
1477 
1478         /* set heap_lp quantum if necessary */
1479         if (segkmem_heaplp_quantum == 0 || !ISP2(segkmem_heaplp_quantum) ||
1480             P2PHASE(segkmem_heaplp_quantum, segkmem_lpsize)) {
1481                 segkmem_heaplp_quantum = segkmem_lpsize;
1482         }
1483 
1484         /* set kmem_lp quantum if necessary */
1485         if (segkmem_kmemlp_quantum == 0 || !ISP2(segkmem_kmemlp_quantum) ||
1486             segkmem_kmemlp_quantum > segkmem_heaplp_quantum) {
1487                 segkmem_kmemlp_quantum = segkmem_heaplp_quantum;
1488         }
1489 
1490         /* set total amount of memory allowed for large page kernel heap */
1491         if (segkmem_kmemlp_max == 0) {
1492                 if (segkmem_kmemlp_pcnt == 0 || segkmem_kmemlp_pcnt > 100)
1493                         segkmem_kmemlp_pcnt = 12;
1494                 segkmem_kmemlp_max = (memtotal * segkmem_kmemlp_pcnt) / 100;
1495         }
1496         segkmem_kmemlp_max = P2ROUNDUP(segkmem_kmemlp_max,
1497             segkmem_heaplp_quantum);
1498 
1499         /* fix lp kmem preallocation request if necesssary */
1500         if (segkmem_kmemlp_min) {
1501                 segkmem_kmemlp_min = P2ROUNDUP(segkmem_kmemlp_min,
1502                     segkmem_heaplp_quantum);
1503                 if (segkmem_kmemlp_min > segkmem_kmemlp_max)
1504                         segkmem_kmemlp_min = segkmem_kmemlp_max;
1505         }
1506 
1507         use_large_pages = 1;
1508         segkmem_lpszc = page_szc(segkmem_lpsize);
1509         segkmem_lpshift = page_get_shift(segkmem_lpszc);
1510 
1511 #endif
1512         return (use_large_pages);
1513 }
1514 
1515 void
1516 segkmem_zio_init(void *zio_mem_base, size_t zio_mem_size)
1517 {
1518         ASSERT(zio_mem_base != NULL);
1519         ASSERT(zio_mem_size != 0);
1520 
1521         /*
1522          * To reduce VA space fragmentation, we set up quantum caches for the
1523          * smaller sizes;  we chose 32k because that translates to 128k VA
1524          * slabs, which matches nicely with the common 128k zio_data bufs.
1525          */
1526         zio_arena = vmem_create("zfs_file_data", zio_mem_base, zio_mem_size,
1527             PAGESIZE, NULL, NULL, NULL, 32 * 1024, VM_SLEEP);
1528 
1529         zio_alloc_arena = vmem_create("zfs_file_data_buf", NULL, 0, PAGESIZE,
1530             segkmem_zio_alloc, segkmem_zio_free, zio_arena, 0, VM_SLEEP);
1531 
1532         ASSERT(zio_arena != NULL);
1533         ASSERT(zio_alloc_arena != NULL);
1534 }
1535 
1536 #ifdef __sparc
1537 
1538 
1539 static void *
1540 segkmem_alloc_ppa(vmem_t *vmp, size_t size, int vmflag)
1541 {
1542         size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
1543         void   *addr;
1544 
1545         if (ppaquantum <= PAGESIZE)
1546                 return (segkmem_alloc(vmp, size, vmflag));
1547 
1548         ASSERT((size & (ppaquantum - 1)) == 0);
1549 
1550         addr = vmem_xalloc(vmp, size, ppaquantum, 0, 0, NULL, NULL, vmflag);
1551         if (addr != NULL && segkmem_xalloc(vmp, addr, size, vmflag, 0,
1552             segkmem_page_create, NULL) == NULL) {
1553                 vmem_xfree(vmp, addr, size);
1554                 addr = NULL;
1555         }
1556 
1557         return (addr);
1558 }
1559 
1560 static void
1561 segkmem_free_ppa(vmem_t *vmp, void *addr, size_t size)
1562 {
1563         size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
1564 
1565         ASSERT(addr != NULL);
1566 
1567         if (ppaquantum <= PAGESIZE) {
1568                 segkmem_free(vmp, addr, size);
1569         } else {
1570                 segkmem_free(NULL, addr, size);
1571                 vmem_xfree(vmp, addr, size);
1572         }
1573 }
1574 
1575 void
1576 segkmem_heap_lp_init()
1577 {
1578         segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1579         size_t heap_lp_size = heap_lp_end - heap_lp_base;
1580         size_t lpsize = segkmem_lpsize;
1581         size_t ppaquantum;
1582         void   *addr;
1583 
1584         if (segkmem_lpsize <= PAGESIZE) {
1585                 ASSERT(heap_lp_base == NULL);
1586                 ASSERT(heap_lp_end == NULL);
1587                 return;
1588         }
1589 
1590         ASSERT(segkmem_heaplp_quantum >= lpsize);
1591         ASSERT((segkmem_heaplp_quantum & (lpsize - 1)) == 0);
1592         ASSERT(lpcb->lp_uselp == 0);
1593         ASSERT(heap_lp_base != NULL);
1594         ASSERT(heap_lp_end != NULL);
1595         ASSERT(heap_lp_base < heap_lp_end);
1596         ASSERT(heap_lp_arena == NULL);
1597         ASSERT(((uintptr_t)heap_lp_base & (lpsize - 1)) == 0);
1598         ASSERT(((uintptr_t)heap_lp_end & (lpsize - 1)) == 0);
1599 
1600         /* create large page heap arena */
1601         heap_lp_arena = vmem_create("heap_lp", heap_lp_base, heap_lp_size,
1602             segkmem_heaplp_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1603 
1604         ASSERT(heap_lp_arena != NULL);
1605 
1606         /* This arena caches memory already mapped by large pages */
1607         kmem_lp_arena = vmem_create("kmem_lp", NULL, 0, segkmem_kmemlp_quantum,
1608             segkmem_alloc_lpi, segkmem_free_lpi, heap_lp_arena, 0, VM_SLEEP);
1609 
1610         ASSERT(kmem_lp_arena != NULL);
1611 
1612         mutex_init(&lpcb->lp_lock, NULL, MUTEX_DEFAULT, NULL);
1613         cv_init(&lpcb->lp_cv, NULL, CV_DEFAULT, NULL);
1614 
1615         /*
1616          * this arena is used for the array of page_t pointers necessary
1617          * to call hat_mem_load_array
1618          */
1619         ppaquantum = btopr(lpsize) * sizeof (page_t *);
1620         segkmem_ppa_arena = vmem_create("segkmem_ppa", NULL, 0, ppaquantum,
1621             segkmem_alloc_ppa, segkmem_free_ppa, heap_arena, ppaquantum,
1622             VM_SLEEP);
1623 
1624         ASSERT(segkmem_ppa_arena != NULL);
1625 
1626         /* prealloacate some memory for the lp kernel heap */
1627         if (segkmem_kmemlp_min) {
1628 
1629                 ASSERT(P2PHASE(segkmem_kmemlp_min,
1630                     segkmem_heaplp_quantum) == 0);
1631 
1632                 if ((addr = segkmem_alloc_lpi(heap_lp_arena,
1633                     segkmem_kmemlp_min, VM_SLEEP)) != NULL) {
1634 
1635                         addr = vmem_add(kmem_lp_arena, addr,
1636                             segkmem_kmemlp_min, VM_SLEEP);
1637                         ASSERT(addr != NULL);
1638                 }
1639         }
1640 
1641         lpcb->lp_uselp = 1;
1642 }
1643 
1644 #endif