1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 26 /* All Rights Reserved */ 27 28 /* 29 * Portions of this source code were derived from Berkeley 4.3 BSD 30 * under license from the Regents of the University of California. 31 */ 32 33 /* 34 * segkp is a segment driver that administers the allocation and deallocation 35 * of pageable variable size chunks of kernel virtual address space. Each 36 * allocated resource is page-aligned. 37 * 38 * The user may specify whether the resource should be initialized to 0, 39 * include a redzone, or locked in memory. 40 */ 41 42 #include <sys/types.h> 43 #include <sys/t_lock.h> 44 #include <sys/thread.h> 45 #include <sys/param.h> 46 #include <sys/errno.h> 47 #include <sys/sysmacros.h> 48 #include <sys/systm.h> 49 #include <sys/buf.h> 50 #include <sys/mman.h> 51 #include <sys/vnode.h> 52 #include <sys/cmn_err.h> 53 #include <sys/swap.h> 54 #include <sys/tuneable.h> 55 #include <sys/kmem.h> 56 #include <sys/vmem.h> 57 #include <sys/cred.h> 58 #include <sys/dumphdr.h> 59 #include <sys/debug.h> 60 #include <sys/vtrace.h> 61 #include <sys/stack.h> 62 #include <sys/atomic.h> 63 #include <sys/archsystm.h> 64 #include <sys/lgrp.h> 65 66 #include <vm/as.h> 67 #include <vm/seg.h> 68 #include <vm/seg_kp.h> 69 #include <vm/seg_kmem.h> 70 #include <vm/anon.h> 71 #include <vm/page.h> 72 #include <vm/hat.h> 73 #include <sys/bitmap.h> 74 75 /* 76 * Private seg op routines 77 */ 78 static void segkp_badop(void); 79 static void segkp_dump(struct seg *seg); 80 static int segkp_checkprot(struct seg *seg, caddr_t addr, size_t len, 81 uint_t prot); 82 static int segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta); 83 static int segkp_pagelock(struct seg *seg, caddr_t addr, size_t len, 84 struct page ***page, enum lock_type type, 85 enum seg_rw rw); 86 static void segkp_insert(struct seg *seg, struct segkp_data *kpd); 87 static void segkp_delete(struct seg *seg, struct segkp_data *kpd); 88 static caddr_t segkp_get_internal(struct seg *seg, size_t len, uint_t flags, 89 struct segkp_data **tkpd, struct anon_map *amp); 90 static void segkp_release_internal(struct seg *seg, 91 struct segkp_data *kpd, size_t len); 92 static int segkp_unlock(struct hat *hat, struct seg *seg, caddr_t vaddr, 93 size_t len, struct segkp_data *kpd, uint_t flags); 94 static int segkp_load(struct hat *hat, struct seg *seg, caddr_t vaddr, 95 size_t len, struct segkp_data *kpd, uint_t flags); 96 static struct segkp_data *segkp_find(struct seg *seg, caddr_t vaddr); 97 static int segkp_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp); 98 static lgrp_mem_policy_info_t *segkp_getpolicy(struct seg *seg, 99 caddr_t addr); 100 static int segkp_capable(struct seg *seg, segcapability_t capability); 101 102 /* 103 * Lock used to protect the hash table(s) and caches. 104 */ 105 static kmutex_t segkp_lock; 106 107 /* 108 * The segkp caches 109 */ 110 static struct segkp_cache segkp_cache[SEGKP_MAX_CACHE]; 111 112 #define SEGKP_BADOP(t) (t(*)())segkp_badop 113 114 /* 115 * When there are fewer than red_minavail bytes left on the stack, 116 * segkp_map_red() will map in the redzone (if called). 5000 seems 117 * to work reasonably well... 118 */ 119 long red_minavail = 5000; 120 121 /* 122 * will be set to 1 for 32 bit x86 systems only, in startup.c 123 */ 124 int segkp_fromheap = 0; 125 ulong_t *segkp_bitmap; 126 127 /* 128 * If segkp_map_red() is called with the redzone already mapped and 129 * with less than RED_DEEP_THRESHOLD bytes available on the stack, 130 * then the stack situation has become quite serious; if much more stack 131 * is consumed, we have the potential of scrogging the next thread/LWP 132 * structure. To help debug the "can't happen" panics which may 133 * result from this condition, we record hrestime and the calling thread 134 * in red_deep_hires and red_deep_thread respectively. 135 */ 136 #define RED_DEEP_THRESHOLD 2000 137 138 hrtime_t red_deep_hires; 139 kthread_t *red_deep_thread; 140 141 uint32_t red_nmapped; 142 uint32_t red_closest = UINT_MAX; 143 uint32_t red_ndoubles; 144 145 pgcnt_t anon_segkp_pages_locked; /* See vm/anon.h */ 146 pgcnt_t anon_segkp_pages_resv; /* anon reserved by seg_kp */ 147 148 static struct seg_ops segkp_ops = { 149 SEGKP_BADOP(int), /* dup */ 150 SEGKP_BADOP(int), /* unmap */ 151 SEGKP_BADOP(void), /* free */ 152 segkp_fault, 153 SEGKP_BADOP(faultcode_t), /* faulta */ 154 SEGKP_BADOP(int), /* setprot */ 155 segkp_checkprot, 156 segkp_kluster, 157 SEGKP_BADOP(int), /* sync */ 158 SEGKP_BADOP(size_t), /* incore */ 159 SEGKP_BADOP(int), /* lockop */ 160 SEGKP_BADOP(int), /* getprot */ 161 SEGKP_BADOP(u_offset_t), /* getoffset */ 162 SEGKP_BADOP(int), /* gettype */ 163 SEGKP_BADOP(int), /* getvp */ 164 SEGKP_BADOP(int), /* advise */ 165 segkp_dump, /* dump */ 166 segkp_pagelock, /* pagelock */ 167 SEGKP_BADOP(int), /* setpgsz */ 168 segkp_getmemid, /* getmemid */ 169 segkp_getpolicy, /* getpolicy */ 170 segkp_capable, /* capable */ 171 }; 172 173 174 static void 175 segkp_badop(void) 176 { 177 panic("segkp_badop"); 178 /*NOTREACHED*/ 179 } 180 181 static void segkpinit_mem_config(struct seg *); 182 183 static uint32_t segkp_indel; 184 185 /* 186 * Allocate the segment specific private data struct and fill it in 187 * with the per kp segment mutex, anon ptr. array and hash table. 188 */ 189 int 190 segkp_create(struct seg *seg) 191 { 192 struct segkp_segdata *kpsd; 193 size_t np; 194 195 ASSERT(seg != NULL && seg->s_as == &kas); 196 ASSERT(RW_WRITE_HELD(&seg->s_as->a_lock)); 197 198 if (seg->s_size & PAGEOFFSET) { 199 panic("Bad segkp size"); 200 /*NOTREACHED*/ 201 } 202 203 kpsd = kmem_zalloc(sizeof (struct segkp_segdata), KM_SLEEP); 204 205 /* 206 * Allocate the virtual memory for segkp and initialize it 207 */ 208 if (segkp_fromheap) { 209 np = btop(kvseg.s_size); 210 segkp_bitmap = kmem_zalloc(BT_SIZEOFMAP(np), KM_SLEEP); 211 kpsd->kpsd_arena = vmem_create("segkp", NULL, 0, PAGESIZE, 212 vmem_alloc, vmem_free, heap_arena, 5 * PAGESIZE, VM_SLEEP); 213 } else { 214 segkp_bitmap = NULL; 215 np = btop(seg->s_size); 216 kpsd->kpsd_arena = vmem_create("segkp", seg->s_base, 217 seg->s_size, PAGESIZE, NULL, NULL, NULL, 5 * PAGESIZE, 218 VM_SLEEP); 219 } 220 221 kpsd->kpsd_anon = anon_create(np, ANON_SLEEP | ANON_ALLOC_FORCE); 222 223 kpsd->kpsd_hash = kmem_zalloc(SEGKP_HASHSZ * sizeof (struct segkp *), 224 KM_SLEEP); 225 seg->s_data = (void *)kpsd; 226 seg->s_ops = &segkp_ops; 227 segkpinit_mem_config(seg); 228 return (0); 229 } 230 231 232 /* 233 * Find a free 'freelist' and initialize it with the appropriate attributes 234 */ 235 void * 236 segkp_cache_init(struct seg *seg, int maxsize, size_t len, uint_t flags) 237 { 238 int i; 239 240 if ((flags & KPD_NO_ANON) && !(flags & KPD_LOCKED)) 241 return ((void *)-1); 242 243 mutex_enter(&segkp_lock); 244 for (i = 0; i < SEGKP_MAX_CACHE; i++) { 245 if (segkp_cache[i].kpf_inuse) 246 continue; 247 segkp_cache[i].kpf_inuse = 1; 248 segkp_cache[i].kpf_max = maxsize; 249 segkp_cache[i].kpf_flags = flags; 250 segkp_cache[i].kpf_seg = seg; 251 segkp_cache[i].kpf_len = len; 252 mutex_exit(&segkp_lock); 253 return ((void *)(uintptr_t)i); 254 } 255 mutex_exit(&segkp_lock); 256 return ((void *)-1); 257 } 258 259 /* 260 * Free all the cache resources. 261 */ 262 void 263 segkp_cache_free(void) 264 { 265 struct segkp_data *kpd; 266 struct seg *seg; 267 int i; 268 269 mutex_enter(&segkp_lock); 270 for (i = 0; i < SEGKP_MAX_CACHE; i++) { 271 if (!segkp_cache[i].kpf_inuse) 272 continue; 273 /* 274 * Disconnect the freelist and process each element 275 */ 276 kpd = segkp_cache[i].kpf_list; 277 seg = segkp_cache[i].kpf_seg; 278 segkp_cache[i].kpf_list = NULL; 279 segkp_cache[i].kpf_count = 0; 280 mutex_exit(&segkp_lock); 281 282 while (kpd != NULL) { 283 struct segkp_data *next; 284 285 next = kpd->kp_next; 286 segkp_release_internal(seg, kpd, kpd->kp_len); 287 kpd = next; 288 } 289 mutex_enter(&segkp_lock); 290 } 291 mutex_exit(&segkp_lock); 292 } 293 294 /* 295 * There are 2 entries into segkp_get_internal. The first includes a cookie 296 * used to access a pool of cached segkp resources. The second does not 297 * use the cache. 298 */ 299 caddr_t 300 segkp_get(struct seg *seg, size_t len, uint_t flags) 301 { 302 struct segkp_data *kpd = NULL; 303 304 if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) { 305 kpd->kp_cookie = -1; 306 return (stom(kpd->kp_base, flags)); 307 } 308 return (NULL); 309 } 310 311 /* 312 * Return a 'cached' segkp address 313 */ 314 caddr_t 315 segkp_cache_get(void *cookie) 316 { 317 struct segkp_cache *freelist = NULL; 318 struct segkp_data *kpd = NULL; 319 int index = (int)(uintptr_t)cookie; 320 struct seg *seg; 321 size_t len; 322 uint_t flags; 323 324 if (index < 0 || index >= SEGKP_MAX_CACHE) 325 return (NULL); 326 freelist = &segkp_cache[index]; 327 328 mutex_enter(&segkp_lock); 329 seg = freelist->kpf_seg; 330 flags = freelist->kpf_flags; 331 if (freelist->kpf_list != NULL) { 332 kpd = freelist->kpf_list; 333 freelist->kpf_list = kpd->kp_next; 334 freelist->kpf_count--; 335 mutex_exit(&segkp_lock); 336 kpd->kp_next = NULL; 337 segkp_insert(seg, kpd); 338 return (stom(kpd->kp_base, flags)); 339 } 340 len = freelist->kpf_len; 341 mutex_exit(&segkp_lock); 342 if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) { 343 kpd->kp_cookie = index; 344 return (stom(kpd->kp_base, flags)); 345 } 346 return (NULL); 347 } 348 349 caddr_t 350 segkp_get_withanonmap( 351 struct seg *seg, 352 size_t len, 353 uint_t flags, 354 struct anon_map *amp) 355 { 356 struct segkp_data *kpd = NULL; 357 358 ASSERT(amp != NULL); 359 flags |= KPD_HASAMP; 360 if (segkp_get_internal(seg, len, flags, &kpd, amp) != NULL) { 361 kpd->kp_cookie = -1; 362 return (stom(kpd->kp_base, flags)); 363 } 364 return (NULL); 365 } 366 367 /* 368 * This does the real work of segkp allocation. 369 * Return to client base addr. len must be page-aligned. A null value is 370 * returned if there are no more vm resources (e.g. pages, swap). The len 371 * and base recorded in the private data structure include the redzone 372 * and the redzone length (if applicable). If the user requests a redzone 373 * either the first or last page is left unmapped depending whether stacks 374 * grow to low or high memory. 375 * 376 * The client may also specify a no-wait flag. If that is set then the 377 * request will choose a non-blocking path when requesting resources. 378 * The default is make the client wait. 379 */ 380 static caddr_t 381 segkp_get_internal( 382 struct seg *seg, 383 size_t len, 384 uint_t flags, 385 struct segkp_data **tkpd, 386 struct anon_map *amp) 387 { 388 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 389 struct segkp_data *kpd; 390 caddr_t vbase = NULL; /* always first virtual, may not be mapped */ 391 pgcnt_t np = 0; /* number of pages in the resource */ 392 pgcnt_t segkpindex; 393 long i; 394 caddr_t va; 395 pgcnt_t pages = 0; 396 ulong_t anon_idx = 0; 397 int kmflag = (flags & KPD_NOWAIT) ? KM_NOSLEEP : KM_SLEEP; 398 caddr_t s_base = (segkp_fromheap) ? kvseg.s_base : seg->s_base; 399 400 if (len & PAGEOFFSET) { 401 panic("segkp_get: len is not page-aligned"); 402 /*NOTREACHED*/ 403 } 404 405 ASSERT(((flags & KPD_HASAMP) == 0) == (amp == NULL)); 406 407 /* Only allow KPD_NO_ANON if we are going to lock it down */ 408 if ((flags & (KPD_LOCKED|KPD_NO_ANON)) == KPD_NO_ANON) 409 return (NULL); 410 411 if ((kpd = kmem_zalloc(sizeof (struct segkp_data), kmflag)) == NULL) 412 return (NULL); 413 /* 414 * Fix up the len to reflect the REDZONE if applicable 415 */ 416 if (flags & KPD_HASREDZONE) 417 len += PAGESIZE; 418 np = btop(len); 419 420 vbase = vmem_alloc(SEGKP_VMEM(seg), len, kmflag | VM_BESTFIT); 421 if (vbase == NULL) { 422 kmem_free(kpd, sizeof (struct segkp_data)); 423 return (NULL); 424 } 425 426 /* If locking, reserve physical memory */ 427 if (flags & KPD_LOCKED) { 428 pages = btop(SEGKP_MAPLEN(len, flags)); 429 if (page_resv(pages, kmflag) == 0) { 430 vmem_free(SEGKP_VMEM(seg), vbase, len); 431 kmem_free(kpd, sizeof (struct segkp_data)); 432 return (NULL); 433 } 434 if ((flags & KPD_NO_ANON) == 0) 435 atomic_add_long(&anon_segkp_pages_locked, pages); 436 } 437 438 /* 439 * Reserve sufficient swap space for this vm resource. We'll 440 * actually allocate it in the loop below, but reserving it 441 * here allows us to back out more gracefully than if we 442 * had an allocation failure in the body of the loop. 443 * 444 * Note that we don't need swap space for the red zone page. 445 */ 446 if (amp != NULL) { 447 /* 448 * The swap reservation has been done, if required, and the 449 * anon_hdr is separate. 450 */ 451 anon_idx = 0; 452 kpd->kp_anon_idx = anon_idx; 453 kpd->kp_anon = amp->ahp; 454 455 TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u", 456 kpd, vbase, len, flags, 1); 457 458 } else if ((flags & KPD_NO_ANON) == 0) { 459 if (anon_resv_zone(SEGKP_MAPLEN(len, flags), NULL) == 0) { 460 if (flags & KPD_LOCKED) { 461 atomic_add_long(&anon_segkp_pages_locked, 462 -pages); 463 page_unresv(pages); 464 } 465 vmem_free(SEGKP_VMEM(seg), vbase, len); 466 kmem_free(kpd, sizeof (struct segkp_data)); 467 return (NULL); 468 } 469 atomic_add_long(&anon_segkp_pages_resv, 470 btop(SEGKP_MAPLEN(len, flags))); 471 anon_idx = ((uintptr_t)(vbase - s_base)) >> PAGESHIFT; 472 kpd->kp_anon_idx = anon_idx; 473 kpd->kp_anon = kpsd->kpsd_anon; 474 475 TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u", 476 kpd, vbase, len, flags, 1); 477 } else { 478 kpd->kp_anon = NULL; 479 kpd->kp_anon_idx = 0; 480 } 481 482 /* 483 * Allocate page and anon resources for the virtual address range 484 * except the redzone 485 */ 486 if (segkp_fromheap) 487 segkpindex = btop((uintptr_t)(vbase - kvseg.s_base)); 488 for (i = 0, va = vbase; i < np; i++, va += PAGESIZE) { 489 page_t *pl[2]; 490 struct vnode *vp; 491 anoff_t off; 492 int err; 493 page_t *pp = NULL; 494 495 /* 496 * Mark this page to be a segkp page in the bitmap. 497 */ 498 if (segkp_fromheap) { 499 BT_ATOMIC_SET(segkp_bitmap, segkpindex); 500 segkpindex++; 501 } 502 503 /* 504 * If this page is the red zone page, we don't need swap 505 * space for it. Note that we skip over the code that 506 * establishes MMU mappings, so that the page remains 507 * invalid. 508 */ 509 if ((flags & KPD_HASREDZONE) && KPD_REDZONE(kpd) == i) 510 continue; 511 512 if (kpd->kp_anon != NULL) { 513 struct anon *ap; 514 515 ASSERT(anon_get_ptr(kpd->kp_anon, anon_idx + i) 516 == NULL); 517 /* 518 * Determine the "vp" and "off" of the anon slot. 519 */ 520 ap = anon_alloc(NULL, 0); 521 if (amp != NULL) 522 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 523 (void) anon_set_ptr(kpd->kp_anon, anon_idx + i, 524 ap, ANON_SLEEP); 525 if (amp != NULL) 526 ANON_LOCK_EXIT(&->a_rwlock); 527 swap_xlate(ap, &vp, &off); 528 529 /* 530 * Create a page with the specified identity. The 531 * page is returned with the "shared" lock held. 532 */ 533 err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE, 534 NULL, pl, PAGESIZE, seg, va, S_CREATE, 535 kcred, NULL); 536 if (err) { 537 /* 538 * XXX - This should not fail. 539 */ 540 panic("segkp_get: no pages"); 541 /*NOTREACHED*/ 542 } 543 pp = pl[0]; 544 } else { 545 ASSERT(page_exists(&kvp, 546 (u_offset_t)(uintptr_t)va) == NULL); 547 548 if ((pp = page_create_va(&kvp, 549 (u_offset_t)(uintptr_t)va, PAGESIZE, 550 (flags & KPD_NOWAIT ? 0 : PG_WAIT) | PG_EXCL | 551 PG_NORELOC, seg, va)) == NULL) { 552 /* 553 * Legitimize resource; then destroy it. 554 * Easier than trying to unwind here. 555 */ 556 kpd->kp_flags = flags; 557 kpd->kp_base = vbase; 558 kpd->kp_len = len; 559 segkp_release_internal(seg, kpd, va - vbase); 560 return (NULL); 561 } 562 page_io_unlock(pp); 563 } 564 565 if (flags & KPD_ZERO) 566 pagezero(pp, 0, PAGESIZE); 567 568 /* 569 * Load and lock an MMU translation for the page. 570 */ 571 hat_memload(seg->s_as->a_hat, va, pp, (PROT_READ|PROT_WRITE), 572 ((flags & KPD_LOCKED) ? HAT_LOAD_LOCK : HAT_LOAD)); 573 574 /* 575 * Now, release lock on the page. 576 */ 577 if (flags & KPD_LOCKED) { 578 /* 579 * Indicate to page_retire framework that this 580 * page can only be retired when it is freed. 581 */ 582 PP_SETRAF(pp); 583 page_downgrade(pp); 584 } else 585 page_unlock(pp); 586 } 587 588 kpd->kp_flags = flags; 589 kpd->kp_base = vbase; 590 kpd->kp_len = len; 591 segkp_insert(seg, kpd); 592 *tkpd = kpd; 593 return (stom(kpd->kp_base, flags)); 594 } 595 596 /* 597 * Release the resource to cache if the pool(designate by the cookie) 598 * has less than the maximum allowable. If inserted in cache, 599 * segkp_delete insures element is taken off of active list. 600 */ 601 void 602 segkp_release(struct seg *seg, caddr_t vaddr) 603 { 604 struct segkp_cache *freelist; 605 struct segkp_data *kpd = NULL; 606 607 if ((kpd = segkp_find(seg, vaddr)) == NULL) { 608 panic("segkp_release: null kpd"); 609 /*NOTREACHED*/ 610 } 611 612 if (kpd->kp_cookie != -1) { 613 freelist = &segkp_cache[kpd->kp_cookie]; 614 mutex_enter(&segkp_lock); 615 if (!segkp_indel && freelist->kpf_count < freelist->kpf_max) { 616 segkp_delete(seg, kpd); 617 kpd->kp_next = freelist->kpf_list; 618 freelist->kpf_list = kpd; 619 freelist->kpf_count++; 620 mutex_exit(&segkp_lock); 621 return; 622 } else { 623 mutex_exit(&segkp_lock); 624 kpd->kp_cookie = -1; 625 } 626 } 627 segkp_release_internal(seg, kpd, kpd->kp_len); 628 } 629 630 /* 631 * Free the entire resource. segkp_unlock gets called with the start of the 632 * mapped portion of the resource. The length is the size of the mapped 633 * portion 634 */ 635 static void 636 segkp_release_internal(struct seg *seg, struct segkp_data *kpd, size_t len) 637 { 638 caddr_t va; 639 long i; 640 long redzone; 641 size_t np; 642 page_t *pp; 643 struct vnode *vp; 644 anoff_t off; 645 struct anon *ap; 646 pgcnt_t segkpindex; 647 648 ASSERT(kpd != NULL); 649 ASSERT((kpd->kp_flags & KPD_HASAMP) == 0 || kpd->kp_cookie == -1); 650 np = btop(len); 651 652 /* Remove from active hash list */ 653 if (kpd->kp_cookie == -1) { 654 mutex_enter(&segkp_lock); 655 segkp_delete(seg, kpd); 656 mutex_exit(&segkp_lock); 657 } 658 659 /* 660 * Precompute redzone page index. 661 */ 662 redzone = -1; 663 if (kpd->kp_flags & KPD_HASREDZONE) 664 redzone = KPD_REDZONE(kpd); 665 666 667 va = kpd->kp_base; 668 669 hat_unload(seg->s_as->a_hat, va, (np << PAGESHIFT), 670 ((kpd->kp_flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD)); 671 /* 672 * Free up those anon resources that are quiescent. 673 */ 674 if (segkp_fromheap) 675 segkpindex = btop((uintptr_t)(va - kvseg.s_base)); 676 for (i = 0; i < np; i++, va += PAGESIZE) { 677 678 /* 679 * Clear the bit for this page from the bitmap. 680 */ 681 if (segkp_fromheap) { 682 BT_ATOMIC_CLEAR(segkp_bitmap, segkpindex); 683 segkpindex++; 684 } 685 686 if (i == redzone) 687 continue; 688 if (kpd->kp_anon) { 689 /* 690 * Free up anon resources and destroy the 691 * associated pages. 692 * 693 * Release the lock if there is one. Have to get the 694 * page to do this, unfortunately. 695 */ 696 if (kpd->kp_flags & KPD_LOCKED) { 697 ap = anon_get_ptr(kpd->kp_anon, 698 kpd->kp_anon_idx + i); 699 swap_xlate(ap, &vp, &off); 700 /* Find the shared-locked page. */ 701 pp = page_find(vp, (u_offset_t)off); 702 if (pp == NULL) { 703 panic("segkp_release: " 704 "kp_anon: no page to unlock "); 705 /*NOTREACHED*/ 706 } 707 if (PP_ISRAF(pp)) 708 PP_CLRRAF(pp); 709 710 page_unlock(pp); 711 } 712 if ((kpd->kp_flags & KPD_HASAMP) == 0) { 713 anon_free(kpd->kp_anon, kpd->kp_anon_idx + i, 714 PAGESIZE); 715 anon_unresv_zone(PAGESIZE, NULL); 716 atomic_add_long(&anon_segkp_pages_resv, 717 -1); 718 } 719 TRACE_5(TR_FAC_VM, 720 TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u", 721 kpd, va, PAGESIZE, 0, 0); 722 } else { 723 if (kpd->kp_flags & KPD_LOCKED) { 724 pp = page_find(&kvp, (u_offset_t)(uintptr_t)va); 725 if (pp == NULL) { 726 panic("segkp_release: " 727 "no page to unlock"); 728 /*NOTREACHED*/ 729 } 730 if (PP_ISRAF(pp)) 731 PP_CLRRAF(pp); 732 /* 733 * We should just upgrade the lock here 734 * but there is no upgrade that waits. 735 */ 736 page_unlock(pp); 737 } 738 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)va, 739 SE_EXCL); 740 if (pp != NULL) 741 page_destroy(pp, 0); 742 } 743 } 744 745 /* If locked, release physical memory reservation */ 746 if (kpd->kp_flags & KPD_LOCKED) { 747 pgcnt_t pages = btop(SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)); 748 if ((kpd->kp_flags & KPD_NO_ANON) == 0) 749 atomic_add_long(&anon_segkp_pages_locked, -pages); 750 page_unresv(pages); 751 } 752 753 vmem_free(SEGKP_VMEM(seg), kpd->kp_base, kpd->kp_len); 754 kmem_free(kpd, sizeof (struct segkp_data)); 755 } 756 757 /* 758 * segkp_map_red() will check the current frame pointer against the 759 * stack base. If the amount of stack remaining is questionable 760 * (less than red_minavail), then segkp_map_red() will map in the redzone 761 * and return 1. Otherwise, it will return 0. segkp_map_red() can 762 * _only_ be called when it is safe to sleep on page_create_va(). 763 * 764 * It is up to the caller to remember whether segkp_map_red() successfully 765 * mapped the redzone, and, if so, to call segkp_unmap_red() at a later 766 * time. 767 * 768 * Currently, this routine is only called from pagefault() (which necessarily 769 * satisfies the above conditions). 770 */ 771 #if defined(STACK_GROWTH_DOWN) 772 int 773 segkp_map_red(void) 774 { 775 uintptr_t fp = STACK_BIAS + (uintptr_t)getfp(); 776 #ifndef _LP64 777 caddr_t stkbase; 778 #endif 779 780 /* 781 * Optimize for the common case where we simply return. 782 */ 783 if ((curthread->t_red_pp == NULL) && 784 (fp - (uintptr_t)curthread->t_stkbase >= red_minavail)) 785 return (0); 786 787 #if defined(_LP64) 788 /* 789 * XXX We probably need something better than this. 790 */ 791 panic("kernel stack overflow"); 792 /*NOTREACHED*/ 793 #else /* _LP64 */ 794 if (curthread->t_red_pp == NULL) { 795 page_t *red_pp; 796 struct seg kseg; 797 798 caddr_t red_va = (caddr_t) 799 (((uintptr_t)curthread->t_stkbase & (uintptr_t)PAGEMASK) - 800 PAGESIZE); 801 802 ASSERT(page_exists(&kvp, (u_offset_t)(uintptr_t)red_va) == 803 NULL); 804 805 /* 806 * Allocate the physical for the red page. 807 */ 808 /* 809 * No PG_NORELOC here to avoid waits. Unlikely to get 810 * a relocate happening in the short time the page exists 811 * and it will be OK anyway. 812 */ 813 814 kseg.s_as = &kas; 815 red_pp = page_create_va(&kvp, (u_offset_t)(uintptr_t)red_va, 816 PAGESIZE, PG_WAIT | PG_EXCL, &kseg, red_va); 817 ASSERT(red_pp != NULL); 818 819 /* 820 * So we now have a page to jam into the redzone... 821 */ 822 page_io_unlock(red_pp); 823 824 hat_memload(kas.a_hat, red_va, red_pp, 825 (PROT_READ|PROT_WRITE), HAT_LOAD_LOCK); 826 page_downgrade(red_pp); 827 828 /* 829 * The page is left SE_SHARED locked so we can hold on to 830 * the page_t pointer. 831 */ 832 curthread->t_red_pp = red_pp; 833 834 atomic_add_32(&red_nmapped, 1); 835 while (fp - (uintptr_t)curthread->t_stkbase < red_closest) { 836 (void) cas32(&red_closest, red_closest, 837 (uint32_t)(fp - (uintptr_t)curthread->t_stkbase)); 838 } 839 return (1); 840 } 841 842 stkbase = (caddr_t)(((uintptr_t)curthread->t_stkbase & 843 (uintptr_t)PAGEMASK) - PAGESIZE); 844 845 atomic_add_32(&red_ndoubles, 1); 846 847 if (fp - (uintptr_t)stkbase < RED_DEEP_THRESHOLD) { 848 /* 849 * Oh boy. We're already deep within the mapped-in 850 * redzone page, and the caller is trying to prepare 851 * for a deep stack run. We're running without a 852 * redzone right now: if the caller plows off the 853 * end of the stack, it'll plow another thread or 854 * LWP structure. That situation could result in 855 * a very hard-to-debug panic, so, in the spirit of 856 * recording the name of one's killer in one's own 857 * blood, we're going to record hrestime and the calling 858 * thread. 859 */ 860 red_deep_hires = hrestime.tv_nsec; 861 red_deep_thread = curthread; 862 } 863 864 /* 865 * If this is a DEBUG kernel, and we've run too deep for comfort, toss. 866 */ 867 ASSERT(fp - (uintptr_t)stkbase >= RED_DEEP_THRESHOLD); 868 return (0); 869 #endif /* _LP64 */ 870 } 871 872 void 873 segkp_unmap_red(void) 874 { 875 page_t *pp; 876 caddr_t red_va = (caddr_t)(((uintptr_t)curthread->t_stkbase & 877 (uintptr_t)PAGEMASK) - PAGESIZE); 878 879 ASSERT(curthread->t_red_pp != NULL); 880 881 /* 882 * Because we locked the mapping down, we can't simply rely 883 * on page_destroy() to clean everything up; we need to call 884 * hat_unload() to explicitly unlock the mapping resources. 885 */ 886 hat_unload(kas.a_hat, red_va, PAGESIZE, HAT_UNLOAD_UNLOCK); 887 888 pp = curthread->t_red_pp; 889 890 ASSERT(pp == page_find(&kvp, (u_offset_t)(uintptr_t)red_va)); 891 892 /* 893 * Need to upgrade the SE_SHARED lock to SE_EXCL. 894 */ 895 if (!page_tryupgrade(pp)) { 896 /* 897 * As there is now wait for upgrade, release the 898 * SE_SHARED lock and wait for SE_EXCL. 899 */ 900 page_unlock(pp); 901 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)red_va, SE_EXCL); 902 /* pp may be NULL here, hence the test below */ 903 } 904 905 /* 906 * Destroy the page, with dontfree set to zero (i.e. free it). 907 */ 908 if (pp != NULL) 909 page_destroy(pp, 0); 910 curthread->t_red_pp = NULL; 911 } 912 #else 913 #error Red stacks only supported with downwards stack growth. 914 #endif 915 916 /* 917 * Handle a fault on an address corresponding to one of the 918 * resources in the segkp segment. 919 */ 920 faultcode_t 921 segkp_fault( 922 struct hat *hat, 923 struct seg *seg, 924 caddr_t vaddr, 925 size_t len, 926 enum fault_type type, 927 enum seg_rw rw) 928 { 929 struct segkp_data *kpd = NULL; 930 int err; 931 932 ASSERT(seg->s_as == &kas && RW_READ_HELD(&seg->s_as->a_lock)); 933 934 /* 935 * Sanity checks. 936 */ 937 if (type == F_PROT) { 938 panic("segkp_fault: unexpected F_PROT fault"); 939 /*NOTREACHED*/ 940 } 941 942 if ((kpd = segkp_find(seg, vaddr)) == NULL) 943 return (FC_NOMAP); 944 945 mutex_enter(&kpd->kp_lock); 946 947 if (type == F_SOFTLOCK) { 948 ASSERT(!(kpd->kp_flags & KPD_LOCKED)); 949 /* 950 * The F_SOFTLOCK case has more stringent 951 * range requirements: the given range must exactly coincide 952 * with the resource's mapped portion. Note reference to 953 * redzone is handled since vaddr would not equal base 954 */ 955 if (vaddr != stom(kpd->kp_base, kpd->kp_flags) || 956 len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) { 957 mutex_exit(&kpd->kp_lock); 958 return (FC_MAKE_ERR(EFAULT)); 959 } 960 961 if ((err = segkp_load(hat, seg, vaddr, len, kpd, KPD_LOCKED))) { 962 mutex_exit(&kpd->kp_lock); 963 return (FC_MAKE_ERR(err)); 964 } 965 kpd->kp_flags |= KPD_LOCKED; 966 mutex_exit(&kpd->kp_lock); 967 return (0); 968 } 969 970 if (type == F_INVAL) { 971 ASSERT(!(kpd->kp_flags & KPD_NO_ANON)); 972 973 /* 974 * Check if we touched the redzone. Somewhat optimistic 975 * here if we are touching the redzone of our own stack 976 * since we wouldn't have a stack to get this far... 977 */ 978 if ((kpd->kp_flags & KPD_HASREDZONE) && 979 btop((uintptr_t)(vaddr - kpd->kp_base)) == KPD_REDZONE(kpd)) 980 panic("segkp_fault: accessing redzone"); 981 982 /* 983 * This fault may occur while the page is being F_SOFTLOCK'ed. 984 * Return since a 2nd segkp_load is unnecessary and also would 985 * result in the page being locked twice and eventually 986 * hang the thread_reaper thread. 987 */ 988 if (kpd->kp_flags & KPD_LOCKED) { 989 mutex_exit(&kpd->kp_lock); 990 return (0); 991 } 992 993 err = segkp_load(hat, seg, vaddr, len, kpd, kpd->kp_flags); 994 mutex_exit(&kpd->kp_lock); 995 return (err ? FC_MAKE_ERR(err) : 0); 996 } 997 998 if (type == F_SOFTUNLOCK) { 999 uint_t flags; 1000 1001 /* 1002 * Make sure the addr is LOCKED and it has anon backing 1003 * before unlocking 1004 */ 1005 if ((kpd->kp_flags & (KPD_LOCKED|KPD_NO_ANON)) != KPD_LOCKED) { 1006 panic("segkp_fault: bad unlock"); 1007 /*NOTREACHED*/ 1008 } 1009 1010 if (vaddr != stom(kpd->kp_base, kpd->kp_flags) || 1011 len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) { 1012 panic("segkp_fault: bad range"); 1013 /*NOTREACHED*/ 1014 } 1015 1016 if (rw == S_WRITE) 1017 flags = kpd->kp_flags | KPD_WRITEDIRTY; 1018 else 1019 flags = kpd->kp_flags; 1020 err = segkp_unlock(hat, seg, vaddr, len, kpd, flags); 1021 kpd->kp_flags &= ~KPD_LOCKED; 1022 mutex_exit(&kpd->kp_lock); 1023 return (err ? FC_MAKE_ERR(err) : 0); 1024 } 1025 mutex_exit(&kpd->kp_lock); 1026 panic("segkp_fault: bogus fault type: %d\n", type); 1027 /*NOTREACHED*/ 1028 } 1029 1030 /* 1031 * Check that the given protections suffice over the range specified by 1032 * vaddr and len. For this segment type, the only issue is whether or 1033 * not the range lies completely within the mapped part of an allocated 1034 * resource. 1035 */ 1036 /* ARGSUSED */ 1037 static int 1038 segkp_checkprot(struct seg *seg, caddr_t vaddr, size_t len, uint_t prot) 1039 { 1040 struct segkp_data *kpd = NULL; 1041 caddr_t mbase; 1042 size_t mlen; 1043 1044 if ((kpd = segkp_find(seg, vaddr)) == NULL) 1045 return (EACCES); 1046 1047 mutex_enter(&kpd->kp_lock); 1048 mbase = stom(kpd->kp_base, kpd->kp_flags); 1049 mlen = SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags); 1050 if (len > mlen || vaddr < mbase || 1051 ((vaddr + len) > (mbase + mlen))) { 1052 mutex_exit(&kpd->kp_lock); 1053 return (EACCES); 1054 } 1055 mutex_exit(&kpd->kp_lock); 1056 return (0); 1057 } 1058 1059 1060 /* 1061 * Check to see if it makes sense to do kluster/read ahead to 1062 * addr + delta relative to the mapping at addr. We assume here 1063 * that delta is a signed PAGESIZE'd multiple (which can be negative). 1064 * 1065 * For seg_u we always "approve" of this action from our standpoint. 1066 */ 1067 /*ARGSUSED*/ 1068 static int 1069 segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta) 1070 { 1071 return (0); 1072 } 1073 1074 /* 1075 * Load and possibly lock intra-slot resources in the range given by 1076 * vaddr and len. 1077 */ 1078 static int 1079 segkp_load( 1080 struct hat *hat, 1081 struct seg *seg, 1082 caddr_t vaddr, 1083 size_t len, 1084 struct segkp_data *kpd, 1085 uint_t flags) 1086 { 1087 caddr_t va; 1088 caddr_t vlim; 1089 ulong_t i; 1090 uint_t lock; 1091 1092 ASSERT(MUTEX_HELD(&kpd->kp_lock)); 1093 1094 len = P2ROUNDUP(len, PAGESIZE); 1095 1096 /* If locking, reserve physical memory */ 1097 if (flags & KPD_LOCKED) { 1098 pgcnt_t pages = btop(len); 1099 if ((kpd->kp_flags & KPD_NO_ANON) == 0) 1100 atomic_add_long(&anon_segkp_pages_locked, pages); 1101 (void) page_resv(pages, KM_SLEEP); 1102 } 1103 1104 /* 1105 * Loop through the pages in the given range. 1106 */ 1107 va = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK); 1108 vaddr = va; 1109 vlim = va + len; 1110 lock = flags & KPD_LOCKED; 1111 i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT; 1112 for (; va < vlim; va += PAGESIZE, i++) { 1113 page_t *pl[2]; /* second element NULL terminator */ 1114 struct vnode *vp; 1115 anoff_t off; 1116 int err; 1117 struct anon *ap; 1118 1119 /* 1120 * Summon the page. If it's not resident, arrange 1121 * for synchronous i/o to pull it in. 1122 */ 1123 ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i); 1124 swap_xlate(ap, &vp, &off); 1125 1126 /* 1127 * The returned page list will have exactly one entry, 1128 * which is returned to us already kept. 1129 */ 1130 err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE, NULL, 1131 pl, PAGESIZE, seg, va, S_READ, kcred, NULL); 1132 1133 if (err) { 1134 /* 1135 * Back out of what we've done so far. 1136 */ 1137 (void) segkp_unlock(hat, seg, vaddr, 1138 (va - vaddr), kpd, flags); 1139 return (err); 1140 } 1141 1142 /* 1143 * Load an MMU translation for the page. 1144 */ 1145 hat_memload(hat, va, pl[0], (PROT_READ|PROT_WRITE), 1146 lock ? HAT_LOAD_LOCK : HAT_LOAD); 1147 1148 if (!lock) { 1149 /* 1150 * Now, release "shared" lock on the page. 1151 */ 1152 page_unlock(pl[0]); 1153 } 1154 } 1155 return (0); 1156 } 1157 1158 /* 1159 * At the very least unload the mmu-translations and unlock the range if locked 1160 * Can be called with the following flag value KPD_WRITEDIRTY which specifies 1161 * any dirty pages should be written to disk. 1162 */ 1163 static int 1164 segkp_unlock( 1165 struct hat *hat, 1166 struct seg *seg, 1167 caddr_t vaddr, 1168 size_t len, 1169 struct segkp_data *kpd, 1170 uint_t flags) 1171 { 1172 caddr_t va; 1173 caddr_t vlim; 1174 ulong_t i; 1175 struct page *pp; 1176 struct vnode *vp; 1177 anoff_t off; 1178 struct anon *ap; 1179 1180 #ifdef lint 1181 seg = seg; 1182 #endif /* lint */ 1183 1184 ASSERT(MUTEX_HELD(&kpd->kp_lock)); 1185 1186 /* 1187 * Loop through the pages in the given range. It is assumed 1188 * segkp_unlock is called with page aligned base 1189 */ 1190 va = vaddr; 1191 vlim = va + len; 1192 i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT; 1193 hat_unload(hat, va, len, 1194 ((flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD)); 1195 for (; va < vlim; va += PAGESIZE, i++) { 1196 /* 1197 * Find the page associated with this part of the 1198 * slot, tracking it down through its associated swap 1199 * space. 1200 */ 1201 ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i); 1202 swap_xlate(ap, &vp, &off); 1203 1204 if (flags & KPD_LOCKED) { 1205 if ((pp = page_find(vp, off)) == NULL) { 1206 if (flags & KPD_LOCKED) { 1207 panic("segkp_softunlock: missing page"); 1208 /*NOTREACHED*/ 1209 } 1210 } 1211 } else { 1212 /* 1213 * Nothing to do if the slot is not locked and the 1214 * page doesn't exist. 1215 */ 1216 if ((pp = page_lookup(vp, off, SE_SHARED)) == NULL) 1217 continue; 1218 } 1219 1220 /* 1221 * If the page doesn't have any translations, is 1222 * dirty and not being shared, then push it out 1223 * asynchronously and avoid waiting for the 1224 * pageout daemon to do it for us. 1225 * 1226 * XXX - Do we really need to get the "exclusive" 1227 * lock via an upgrade? 1228 */ 1229 if ((flags & KPD_WRITEDIRTY) && !hat_page_is_mapped(pp) && 1230 hat_ismod(pp) && page_tryupgrade(pp)) { 1231 /* 1232 * Hold the vnode before releasing the page lock to 1233 * prevent it from being freed and re-used by some 1234 * other thread. 1235 */ 1236 VN_HOLD(vp); 1237 page_unlock(pp); 1238 1239 /* 1240 * Want most powerful credentials we can get so 1241 * use kcred. 1242 */ 1243 (void) VOP_PUTPAGE(vp, (offset_t)off, PAGESIZE, 1244 B_ASYNC | B_FREE, kcred, NULL); 1245 VN_RELE(vp); 1246 } else { 1247 page_unlock(pp); 1248 } 1249 } 1250 1251 /* If unlocking, release physical memory */ 1252 if (flags & KPD_LOCKED) { 1253 pgcnt_t pages = btopr(len); 1254 if ((kpd->kp_flags & KPD_NO_ANON) == 0) 1255 atomic_add_long(&anon_segkp_pages_locked, -pages); 1256 page_unresv(pages); 1257 } 1258 return (0); 1259 } 1260 1261 /* 1262 * Insert the kpd in the hash table. 1263 */ 1264 static void 1265 segkp_insert(struct seg *seg, struct segkp_data *kpd) 1266 { 1267 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 1268 int index; 1269 1270 /* 1271 * Insert the kpd based on the address that will be returned 1272 * via segkp_release. 1273 */ 1274 index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags)); 1275 mutex_enter(&segkp_lock); 1276 kpd->kp_next = kpsd->kpsd_hash[index]; 1277 kpsd->kpsd_hash[index] = kpd; 1278 mutex_exit(&segkp_lock); 1279 } 1280 1281 /* 1282 * Remove kpd from the hash table. 1283 */ 1284 static void 1285 segkp_delete(struct seg *seg, struct segkp_data *kpd) 1286 { 1287 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 1288 struct segkp_data **kpp; 1289 int index; 1290 1291 ASSERT(MUTEX_HELD(&segkp_lock)); 1292 1293 index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags)); 1294 for (kpp = &kpsd->kpsd_hash[index]; 1295 *kpp != NULL; kpp = &((*kpp)->kp_next)) { 1296 if (*kpp == kpd) { 1297 *kpp = kpd->kp_next; 1298 return; 1299 } 1300 } 1301 panic("segkp_delete: unable to find element to delete"); 1302 /*NOTREACHED*/ 1303 } 1304 1305 /* 1306 * Find the kpd associated with a vaddr. 1307 * 1308 * Most of the callers of segkp_find will pass the vaddr that 1309 * hashes to the desired index, but there are cases where 1310 * this is not true in which case we have to (potentially) scan 1311 * the whole table looking for it. This should be very rare 1312 * (e.g. a segkp_fault(F_INVAL) on an address somewhere in the 1313 * middle of the segkp_data region). 1314 */ 1315 static struct segkp_data * 1316 segkp_find(struct seg *seg, caddr_t vaddr) 1317 { 1318 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 1319 struct segkp_data *kpd; 1320 int i; 1321 int stop; 1322 1323 i = stop = SEGKP_HASH(vaddr); 1324 mutex_enter(&segkp_lock); 1325 do { 1326 for (kpd = kpsd->kpsd_hash[i]; kpd != NULL; 1327 kpd = kpd->kp_next) { 1328 if (vaddr >= kpd->kp_base && 1329 vaddr < kpd->kp_base + kpd->kp_len) { 1330 mutex_exit(&segkp_lock); 1331 return (kpd); 1332 } 1333 } 1334 if (--i < 0) 1335 i = SEGKP_HASHSZ - 1; /* Wrap */ 1336 } while (i != stop); 1337 mutex_exit(&segkp_lock); 1338 return (NULL); /* Not found */ 1339 } 1340 1341 /* 1342 * returns size of swappable area. 1343 */ 1344 size_t 1345 swapsize(caddr_t v) 1346 { 1347 struct segkp_data *kpd; 1348 1349 if ((kpd = segkp_find(segkp, v)) != NULL) 1350 return (SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)); 1351 else 1352 return (NULL); 1353 } 1354 1355 /* 1356 * Dump out all the active segkp pages 1357 */ 1358 static void 1359 segkp_dump(struct seg *seg) 1360 { 1361 int i; 1362 struct segkp_data *kpd; 1363 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 1364 1365 for (i = 0; i < SEGKP_HASHSZ; i++) { 1366 for (kpd = kpsd->kpsd_hash[i]; 1367 kpd != NULL; kpd = kpd->kp_next) { 1368 pfn_t pfn; 1369 caddr_t addr; 1370 caddr_t eaddr; 1371 1372 addr = kpd->kp_base; 1373 eaddr = addr + kpd->kp_len; 1374 while (addr < eaddr) { 1375 ASSERT(seg->s_as == &kas); 1376 pfn = hat_getpfnum(seg->s_as->a_hat, addr); 1377 if (pfn != PFN_INVALID) 1378 dump_addpage(seg->s_as, addr, pfn); 1379 addr += PAGESIZE; 1380 dump_timeleft = dump_timeout; 1381 } 1382 } 1383 } 1384 } 1385 1386 /*ARGSUSED*/ 1387 static int 1388 segkp_pagelock(struct seg *seg, caddr_t addr, size_t len, 1389 struct page ***ppp, enum lock_type type, enum seg_rw rw) 1390 { 1391 return (ENOTSUP); 1392 } 1393 1394 /*ARGSUSED*/ 1395 static int 1396 segkp_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp) 1397 { 1398 return (ENODEV); 1399 } 1400 1401 /*ARGSUSED*/ 1402 static lgrp_mem_policy_info_t * 1403 segkp_getpolicy(struct seg *seg, caddr_t addr) 1404 { 1405 return (NULL); 1406 } 1407 1408 /*ARGSUSED*/ 1409 static int 1410 segkp_capable(struct seg *seg, segcapability_t capability) 1411 { 1412 return (0); 1413 } 1414 1415 #include <sys/mem_config.h> 1416 1417 /*ARGSUSED*/ 1418 static void 1419 segkp_mem_config_post_add(void *arg, pgcnt_t delta_pages) 1420 {} 1421 1422 /* 1423 * During memory delete, turn off caches so that pages are not held. 1424 * A better solution may be to unlock the pages while they are 1425 * in the cache so that they may be collected naturally. 1426 */ 1427 1428 /*ARGSUSED*/ 1429 static int 1430 segkp_mem_config_pre_del(void *arg, pgcnt_t delta_pages) 1431 { 1432 atomic_add_32(&segkp_indel, 1); 1433 segkp_cache_free(); 1434 return (0); 1435 } 1436 1437 /*ARGSUSED*/ 1438 static void 1439 segkp_mem_config_post_del(void *arg, pgcnt_t delta_pages, int cancelled) 1440 { 1441 atomic_add_32(&segkp_indel, -1); 1442 } 1443 1444 static kphysm_setup_vector_t segkp_mem_config_vec = { 1445 KPHYSM_SETUP_VECTOR_VERSION, 1446 segkp_mem_config_post_add, 1447 segkp_mem_config_pre_del, 1448 segkp_mem_config_post_del, 1449 }; 1450 1451 static void 1452 segkpinit_mem_config(struct seg *seg) 1453 { 1454 int ret; 1455 1456 ret = kphysm_setup_func_register(&segkp_mem_config_vec, (void *)seg); 1457 ASSERT(ret == 0); 1458 }