Print this page
patch remove-dont-swap-flag
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/vm/seg_kp.c
+++ new/usr/src/uts/common/vm/seg_kp.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 */
24 24
25 25 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
26 26 /* All Rights Reserved */
27 27
28 28 /*
29 29 * Portions of this source code were derived from Berkeley 4.3 BSD
30 30 * under license from the Regents of the University of California.
31 31 */
32 32
33 33 /*
34 34 * segkp is a segment driver that administers the allocation and deallocation
35 35 * of pageable variable size chunks of kernel virtual address space. Each
36 36 * allocated resource is page-aligned.
37 37 *
38 38 * The user may specify whether the resource should be initialized to 0,
39 39 * include a redzone, or locked in memory.
40 40 */
41 41
42 42 #include <sys/types.h>
43 43 #include <sys/t_lock.h>
44 44 #include <sys/thread.h>
45 45 #include <sys/param.h>
46 46 #include <sys/errno.h>
47 47 #include <sys/sysmacros.h>
48 48 #include <sys/systm.h>
49 49 #include <sys/buf.h>
50 50 #include <sys/mman.h>
51 51 #include <sys/vnode.h>
52 52 #include <sys/cmn_err.h>
53 53 #include <sys/swap.h>
54 54 #include <sys/tuneable.h>
55 55 #include <sys/kmem.h>
56 56 #include <sys/vmem.h>
57 57 #include <sys/cred.h>
58 58 #include <sys/dumphdr.h>
59 59 #include <sys/debug.h>
60 60 #include <sys/vtrace.h>
61 61 #include <sys/stack.h>
62 62 #include <sys/atomic.h>
63 63 #include <sys/archsystm.h>
64 64 #include <sys/lgrp.h>
65 65
66 66 #include <vm/as.h>
67 67 #include <vm/seg.h>
68 68 #include <vm/seg_kp.h>
69 69 #include <vm/seg_kmem.h>
70 70 #include <vm/anon.h>
71 71 #include <vm/page.h>
72 72 #include <vm/hat.h>
73 73 #include <sys/bitmap.h>
74 74
75 75 /*
76 76 * Private seg op routines
77 77 */
78 78 static void segkp_badop(void);
79 79 static void segkp_dump(struct seg *seg);
80 80 static int segkp_checkprot(struct seg *seg, caddr_t addr, size_t len,
81 81 uint_t prot);
82 82 static int segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta);
83 83 static int segkp_pagelock(struct seg *seg, caddr_t addr, size_t len,
84 84 struct page ***page, enum lock_type type,
85 85 enum seg_rw rw);
86 86 static void segkp_insert(struct seg *seg, struct segkp_data *kpd);
87 87 static void segkp_delete(struct seg *seg, struct segkp_data *kpd);
88 88 static caddr_t segkp_get_internal(struct seg *seg, size_t len, uint_t flags,
89 89 struct segkp_data **tkpd, struct anon_map *amp);
90 90 static void segkp_release_internal(struct seg *seg,
91 91 struct segkp_data *kpd, size_t len);
92 92 static int segkp_unlock(struct hat *hat, struct seg *seg, caddr_t vaddr,
93 93 size_t len, struct segkp_data *kpd, uint_t flags);
94 94 static int segkp_load(struct hat *hat, struct seg *seg, caddr_t vaddr,
95 95 size_t len, struct segkp_data *kpd, uint_t flags);
96 96 static struct segkp_data *segkp_find(struct seg *seg, caddr_t vaddr);
97 97 static int segkp_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp);
98 98 static lgrp_mem_policy_info_t *segkp_getpolicy(struct seg *seg,
99 99 caddr_t addr);
100 100 static int segkp_capable(struct seg *seg, segcapability_t capability);
101 101
102 102 /*
103 103 * Lock used to protect the hash table(s) and caches.
104 104 */
105 105 static kmutex_t segkp_lock;
106 106
107 107 /*
108 108 * The segkp caches
109 109 */
110 110 static struct segkp_cache segkp_cache[SEGKP_MAX_CACHE];
111 111
112 112 #define SEGKP_BADOP(t) (t(*)())segkp_badop
113 113
114 114 /*
115 115 * When there are fewer than red_minavail bytes left on the stack,
116 116 * segkp_map_red() will map in the redzone (if called). 5000 seems
117 117 * to work reasonably well...
118 118 */
119 119 long red_minavail = 5000;
120 120
121 121 /*
122 122 * will be set to 1 for 32 bit x86 systems only, in startup.c
123 123 */
124 124 int segkp_fromheap = 0;
125 125 ulong_t *segkp_bitmap;
126 126
127 127 /*
128 128 * If segkp_map_red() is called with the redzone already mapped and
129 129 * with less than RED_DEEP_THRESHOLD bytes available on the stack,
130 130 * then the stack situation has become quite serious; if much more stack
131 131 * is consumed, we have the potential of scrogging the next thread/LWP
132 132 * structure. To help debug the "can't happen" panics which may
133 133 * result from this condition, we record hrestime and the calling thread
134 134 * in red_deep_hires and red_deep_thread respectively.
135 135 */
136 136 #define RED_DEEP_THRESHOLD 2000
137 137
138 138 hrtime_t red_deep_hires;
139 139 kthread_t *red_deep_thread;
140 140
141 141 uint32_t red_nmapped;
142 142 uint32_t red_closest = UINT_MAX;
143 143 uint32_t red_ndoubles;
144 144
145 145 pgcnt_t anon_segkp_pages_locked; /* See vm/anon.h */
146 146 pgcnt_t anon_segkp_pages_resv; /* anon reserved by seg_kp */
147 147
148 148 static struct seg_ops segkp_ops = {
149 149 SEGKP_BADOP(int), /* dup */
150 150 SEGKP_BADOP(int), /* unmap */
151 151 SEGKP_BADOP(void), /* free */
152 152 segkp_fault,
153 153 SEGKP_BADOP(faultcode_t), /* faulta */
154 154 SEGKP_BADOP(int), /* setprot */
155 155 segkp_checkprot,
156 156 segkp_kluster,
157 157 SEGKP_BADOP(size_t), /* swapout */
158 158 SEGKP_BADOP(int), /* sync */
159 159 SEGKP_BADOP(size_t), /* incore */
160 160 SEGKP_BADOP(int), /* lockop */
161 161 SEGKP_BADOP(int), /* getprot */
162 162 SEGKP_BADOP(u_offset_t), /* getoffset */
163 163 SEGKP_BADOP(int), /* gettype */
164 164 SEGKP_BADOP(int), /* getvp */
165 165 SEGKP_BADOP(int), /* advise */
166 166 segkp_dump, /* dump */
167 167 segkp_pagelock, /* pagelock */
168 168 SEGKP_BADOP(int), /* setpgsz */
169 169 segkp_getmemid, /* getmemid */
170 170 segkp_getpolicy, /* getpolicy */
171 171 segkp_capable, /* capable */
172 172 };
173 173
174 174
175 175 static void
176 176 segkp_badop(void)
177 177 {
178 178 panic("segkp_badop");
179 179 /*NOTREACHED*/
180 180 }
181 181
182 182 static void segkpinit_mem_config(struct seg *);
183 183
184 184 static uint32_t segkp_indel;
185 185
186 186 /*
187 187 * Allocate the segment specific private data struct and fill it in
188 188 * with the per kp segment mutex, anon ptr. array and hash table.
189 189 */
190 190 int
191 191 segkp_create(struct seg *seg)
192 192 {
193 193 struct segkp_segdata *kpsd;
194 194 size_t np;
195 195
196 196 ASSERT(seg != NULL && seg->s_as == &kas);
197 197 ASSERT(RW_WRITE_HELD(&seg->s_as->a_lock));
198 198
199 199 if (seg->s_size & PAGEOFFSET) {
200 200 panic("Bad segkp size");
201 201 /*NOTREACHED*/
202 202 }
203 203
204 204 kpsd = kmem_zalloc(sizeof (struct segkp_segdata), KM_SLEEP);
205 205
206 206 /*
207 207 * Allocate the virtual memory for segkp and initialize it
208 208 */
209 209 if (segkp_fromheap) {
210 210 np = btop(kvseg.s_size);
211 211 segkp_bitmap = kmem_zalloc(BT_SIZEOFMAP(np), KM_SLEEP);
212 212 kpsd->kpsd_arena = vmem_create("segkp", NULL, 0, PAGESIZE,
213 213 vmem_alloc, vmem_free, heap_arena, 5 * PAGESIZE, VM_SLEEP);
214 214 } else {
215 215 segkp_bitmap = NULL;
216 216 np = btop(seg->s_size);
217 217 kpsd->kpsd_arena = vmem_create("segkp", seg->s_base,
218 218 seg->s_size, PAGESIZE, NULL, NULL, NULL, 5 * PAGESIZE,
219 219 VM_SLEEP);
220 220 }
221 221
222 222 kpsd->kpsd_anon = anon_create(np, ANON_SLEEP | ANON_ALLOC_FORCE);
223 223
224 224 kpsd->kpsd_hash = kmem_zalloc(SEGKP_HASHSZ * sizeof (struct segkp *),
225 225 KM_SLEEP);
226 226 seg->s_data = (void *)kpsd;
227 227 seg->s_ops = &segkp_ops;
228 228 segkpinit_mem_config(seg);
229 229 return (0);
230 230 }
231 231
232 232
233 233 /*
234 234 * Find a free 'freelist' and initialize it with the appropriate attributes
235 235 */
236 236 void *
237 237 segkp_cache_init(struct seg *seg, int maxsize, size_t len, uint_t flags)
238 238 {
239 239 int i;
240 240
241 241 if ((flags & KPD_NO_ANON) && !(flags & KPD_LOCKED))
242 242 return ((void *)-1);
243 243
244 244 mutex_enter(&segkp_lock);
245 245 for (i = 0; i < SEGKP_MAX_CACHE; i++) {
246 246 if (segkp_cache[i].kpf_inuse)
247 247 continue;
248 248 segkp_cache[i].kpf_inuse = 1;
249 249 segkp_cache[i].kpf_max = maxsize;
250 250 segkp_cache[i].kpf_flags = flags;
251 251 segkp_cache[i].kpf_seg = seg;
252 252 segkp_cache[i].kpf_len = len;
253 253 mutex_exit(&segkp_lock);
254 254 return ((void *)(uintptr_t)i);
255 255 }
256 256 mutex_exit(&segkp_lock);
257 257 return ((void *)-1);
258 258 }
259 259
260 260 /*
261 261 * Free all the cache resources.
262 262 */
263 263 void
264 264 segkp_cache_free(void)
265 265 {
266 266 struct segkp_data *kpd;
267 267 struct seg *seg;
268 268 int i;
269 269
270 270 mutex_enter(&segkp_lock);
271 271 for (i = 0; i < SEGKP_MAX_CACHE; i++) {
272 272 if (!segkp_cache[i].kpf_inuse)
273 273 continue;
274 274 /*
275 275 * Disconnect the freelist and process each element
276 276 */
277 277 kpd = segkp_cache[i].kpf_list;
278 278 seg = segkp_cache[i].kpf_seg;
279 279 segkp_cache[i].kpf_list = NULL;
280 280 segkp_cache[i].kpf_count = 0;
281 281 mutex_exit(&segkp_lock);
282 282
283 283 while (kpd != NULL) {
284 284 struct segkp_data *next;
285 285
286 286 next = kpd->kp_next;
287 287 segkp_release_internal(seg, kpd, kpd->kp_len);
288 288 kpd = next;
289 289 }
290 290 mutex_enter(&segkp_lock);
291 291 }
292 292 mutex_exit(&segkp_lock);
293 293 }
294 294
295 295 /*
296 296 * There are 2 entries into segkp_get_internal. The first includes a cookie
297 297 * used to access a pool of cached segkp resources. The second does not
298 298 * use the cache.
299 299 */
300 300 caddr_t
301 301 segkp_get(struct seg *seg, size_t len, uint_t flags)
302 302 {
303 303 struct segkp_data *kpd = NULL;
304 304
305 305 if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) {
306 306 kpd->kp_cookie = -1;
307 307 return (stom(kpd->kp_base, flags));
308 308 }
309 309 return (NULL);
310 310 }
311 311
312 312 /*
313 313 * Return a 'cached' segkp address
314 314 */
315 315 caddr_t
316 316 segkp_cache_get(void *cookie)
317 317 {
318 318 struct segkp_cache *freelist = NULL;
319 319 struct segkp_data *kpd = NULL;
320 320 int index = (int)(uintptr_t)cookie;
321 321 struct seg *seg;
322 322 size_t len;
323 323 uint_t flags;
324 324
325 325 if (index < 0 || index >= SEGKP_MAX_CACHE)
326 326 return (NULL);
327 327 freelist = &segkp_cache[index];
328 328
329 329 mutex_enter(&segkp_lock);
330 330 seg = freelist->kpf_seg;
331 331 flags = freelist->kpf_flags;
332 332 if (freelist->kpf_list != NULL) {
333 333 kpd = freelist->kpf_list;
334 334 freelist->kpf_list = kpd->kp_next;
335 335 freelist->kpf_count--;
336 336 mutex_exit(&segkp_lock);
337 337 kpd->kp_next = NULL;
338 338 segkp_insert(seg, kpd);
339 339 return (stom(kpd->kp_base, flags));
340 340 }
341 341 len = freelist->kpf_len;
342 342 mutex_exit(&segkp_lock);
343 343 if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) {
344 344 kpd->kp_cookie = index;
345 345 return (stom(kpd->kp_base, flags));
346 346 }
347 347 return (NULL);
348 348 }
349 349
350 350 caddr_t
351 351 segkp_get_withanonmap(
352 352 struct seg *seg,
353 353 size_t len,
354 354 uint_t flags,
355 355 struct anon_map *amp)
356 356 {
357 357 struct segkp_data *kpd = NULL;
358 358
359 359 ASSERT(amp != NULL);
360 360 flags |= KPD_HASAMP;
361 361 if (segkp_get_internal(seg, len, flags, &kpd, amp) != NULL) {
362 362 kpd->kp_cookie = -1;
363 363 return (stom(kpd->kp_base, flags));
364 364 }
365 365 return (NULL);
366 366 }
367 367
368 368 /*
369 369 * This does the real work of segkp allocation.
370 370 * Return to client base addr. len must be page-aligned. A null value is
371 371 * returned if there are no more vm resources (e.g. pages, swap). The len
372 372 * and base recorded in the private data structure include the redzone
373 373 * and the redzone length (if applicable). If the user requests a redzone
374 374 * either the first or last page is left unmapped depending whether stacks
375 375 * grow to low or high memory.
376 376 *
377 377 * The client may also specify a no-wait flag. If that is set then the
378 378 * request will choose a non-blocking path when requesting resources.
379 379 * The default is make the client wait.
380 380 */
381 381 static caddr_t
382 382 segkp_get_internal(
383 383 struct seg *seg,
384 384 size_t len,
385 385 uint_t flags,
386 386 struct segkp_data **tkpd,
387 387 struct anon_map *amp)
388 388 {
389 389 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
390 390 struct segkp_data *kpd;
391 391 caddr_t vbase = NULL; /* always first virtual, may not be mapped */
392 392 pgcnt_t np = 0; /* number of pages in the resource */
393 393 pgcnt_t segkpindex;
394 394 long i;
395 395 caddr_t va;
396 396 pgcnt_t pages = 0;
397 397 ulong_t anon_idx = 0;
398 398 int kmflag = (flags & KPD_NOWAIT) ? KM_NOSLEEP : KM_SLEEP;
399 399 caddr_t s_base = (segkp_fromheap) ? kvseg.s_base : seg->s_base;
400 400
401 401 if (len & PAGEOFFSET) {
402 402 panic("segkp_get: len is not page-aligned");
403 403 /*NOTREACHED*/
404 404 }
405 405
406 406 ASSERT(((flags & KPD_HASAMP) == 0) == (amp == NULL));
407 407
408 408 /* Only allow KPD_NO_ANON if we are going to lock it down */
409 409 if ((flags & (KPD_LOCKED|KPD_NO_ANON)) == KPD_NO_ANON)
410 410 return (NULL);
411 411
412 412 if ((kpd = kmem_zalloc(sizeof (struct segkp_data), kmflag)) == NULL)
413 413 return (NULL);
414 414 /*
415 415 * Fix up the len to reflect the REDZONE if applicable
416 416 */
417 417 if (flags & KPD_HASREDZONE)
418 418 len += PAGESIZE;
419 419 np = btop(len);
420 420
421 421 vbase = vmem_alloc(SEGKP_VMEM(seg), len, kmflag | VM_BESTFIT);
422 422 if (vbase == NULL) {
423 423 kmem_free(kpd, sizeof (struct segkp_data));
424 424 return (NULL);
425 425 }
426 426
427 427 /* If locking, reserve physical memory */
428 428 if (flags & KPD_LOCKED) {
429 429 pages = btop(SEGKP_MAPLEN(len, flags));
430 430 if (page_resv(pages, kmflag) == 0) {
431 431 vmem_free(SEGKP_VMEM(seg), vbase, len);
432 432 kmem_free(kpd, sizeof (struct segkp_data));
433 433 return (NULL);
434 434 }
435 435 if ((flags & KPD_NO_ANON) == 0)
436 436 atomic_add_long(&anon_segkp_pages_locked, pages);
437 437 }
438 438
439 439 /*
440 440 * Reserve sufficient swap space for this vm resource. We'll
441 441 * actually allocate it in the loop below, but reserving it
442 442 * here allows us to back out more gracefully than if we
443 443 * had an allocation failure in the body of the loop.
444 444 *
445 445 * Note that we don't need swap space for the red zone page.
446 446 */
447 447 if (amp != NULL) {
448 448 /*
449 449 * The swap reservation has been done, if required, and the
450 450 * anon_hdr is separate.
451 451 */
452 452 anon_idx = 0;
453 453 kpd->kp_anon_idx = anon_idx;
454 454 kpd->kp_anon = amp->ahp;
455 455
456 456 TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u",
457 457 kpd, vbase, len, flags, 1);
458 458
459 459 } else if ((flags & KPD_NO_ANON) == 0) {
460 460 if (anon_resv_zone(SEGKP_MAPLEN(len, flags), NULL) == 0) {
461 461 if (flags & KPD_LOCKED) {
462 462 atomic_add_long(&anon_segkp_pages_locked,
463 463 -pages);
464 464 page_unresv(pages);
465 465 }
466 466 vmem_free(SEGKP_VMEM(seg), vbase, len);
467 467 kmem_free(kpd, sizeof (struct segkp_data));
468 468 return (NULL);
469 469 }
470 470 atomic_add_long(&anon_segkp_pages_resv,
471 471 btop(SEGKP_MAPLEN(len, flags)));
472 472 anon_idx = ((uintptr_t)(vbase - s_base)) >> PAGESHIFT;
473 473 kpd->kp_anon_idx = anon_idx;
474 474 kpd->kp_anon = kpsd->kpsd_anon;
475 475
476 476 TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u",
477 477 kpd, vbase, len, flags, 1);
478 478 } else {
479 479 kpd->kp_anon = NULL;
480 480 kpd->kp_anon_idx = 0;
481 481 }
482 482
483 483 /*
484 484 * Allocate page and anon resources for the virtual address range
485 485 * except the redzone
486 486 */
487 487 if (segkp_fromheap)
488 488 segkpindex = btop((uintptr_t)(vbase - kvseg.s_base));
489 489 for (i = 0, va = vbase; i < np; i++, va += PAGESIZE) {
490 490 page_t *pl[2];
491 491 struct vnode *vp;
492 492 anoff_t off;
493 493 int err;
494 494 page_t *pp = NULL;
495 495
496 496 /*
497 497 * Mark this page to be a segkp page in the bitmap.
498 498 */
499 499 if (segkp_fromheap) {
500 500 BT_ATOMIC_SET(segkp_bitmap, segkpindex);
501 501 segkpindex++;
502 502 }
503 503
504 504 /*
505 505 * If this page is the red zone page, we don't need swap
506 506 * space for it. Note that we skip over the code that
507 507 * establishes MMU mappings, so that the page remains
508 508 * invalid.
509 509 */
510 510 if ((flags & KPD_HASREDZONE) && KPD_REDZONE(kpd) == i)
511 511 continue;
512 512
513 513 if (kpd->kp_anon != NULL) {
514 514 struct anon *ap;
515 515
516 516 ASSERT(anon_get_ptr(kpd->kp_anon, anon_idx + i)
517 517 == NULL);
518 518 /*
519 519 * Determine the "vp" and "off" of the anon slot.
520 520 */
521 521 ap = anon_alloc(NULL, 0);
522 522 if (amp != NULL)
523 523 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER);
524 524 (void) anon_set_ptr(kpd->kp_anon, anon_idx + i,
525 525 ap, ANON_SLEEP);
526 526 if (amp != NULL)
527 527 ANON_LOCK_EXIT(&->a_rwlock);
528 528 swap_xlate(ap, &vp, &off);
529 529
530 530 /*
531 531 * Create a page with the specified identity. The
532 532 * page is returned with the "shared" lock held.
533 533 */
534 534 err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE,
535 535 NULL, pl, PAGESIZE, seg, va, S_CREATE,
536 536 kcred, NULL);
537 537 if (err) {
538 538 /*
539 539 * XXX - This should not fail.
540 540 */
541 541 panic("segkp_get: no pages");
542 542 /*NOTREACHED*/
543 543 }
544 544 pp = pl[0];
545 545 } else {
546 546 ASSERT(page_exists(&kvp,
547 547 (u_offset_t)(uintptr_t)va) == NULL);
548 548
549 549 if ((pp = page_create_va(&kvp,
550 550 (u_offset_t)(uintptr_t)va, PAGESIZE,
551 551 (flags & KPD_NOWAIT ? 0 : PG_WAIT) | PG_EXCL |
552 552 PG_NORELOC, seg, va)) == NULL) {
553 553 /*
554 554 * Legitimize resource; then destroy it.
555 555 * Easier than trying to unwind here.
556 556 */
557 557 kpd->kp_flags = flags;
558 558 kpd->kp_base = vbase;
559 559 kpd->kp_len = len;
560 560 segkp_release_internal(seg, kpd, va - vbase);
561 561 return (NULL);
562 562 }
563 563 page_io_unlock(pp);
564 564 }
565 565
566 566 if (flags & KPD_ZERO)
567 567 pagezero(pp, 0, PAGESIZE);
568 568
569 569 /*
570 570 * Load and lock an MMU translation for the page.
571 571 */
572 572 hat_memload(seg->s_as->a_hat, va, pp, (PROT_READ|PROT_WRITE),
573 573 ((flags & KPD_LOCKED) ? HAT_LOAD_LOCK : HAT_LOAD));
574 574
575 575 /*
576 576 * Now, release lock on the page.
577 577 */
578 578 if (flags & KPD_LOCKED) {
579 579 /*
580 580 * Indicate to page_retire framework that this
581 581 * page can only be retired when it is freed.
582 582 */
583 583 PP_SETRAF(pp);
584 584 page_downgrade(pp);
585 585 } else
586 586 page_unlock(pp);
587 587 }
588 588
589 589 kpd->kp_flags = flags;
590 590 kpd->kp_base = vbase;
591 591 kpd->kp_len = len;
592 592 segkp_insert(seg, kpd);
593 593 *tkpd = kpd;
594 594 return (stom(kpd->kp_base, flags));
595 595 }
596 596
597 597 /*
598 598 * Release the resource to cache if the pool(designate by the cookie)
599 599 * has less than the maximum allowable. If inserted in cache,
600 600 * segkp_delete insures element is taken off of active list.
601 601 */
602 602 void
603 603 segkp_release(struct seg *seg, caddr_t vaddr)
604 604 {
605 605 struct segkp_cache *freelist;
606 606 struct segkp_data *kpd = NULL;
607 607
608 608 if ((kpd = segkp_find(seg, vaddr)) == NULL) {
609 609 panic("segkp_release: null kpd");
610 610 /*NOTREACHED*/
611 611 }
612 612
613 613 if (kpd->kp_cookie != -1) {
614 614 freelist = &segkp_cache[kpd->kp_cookie];
615 615 mutex_enter(&segkp_lock);
616 616 if (!segkp_indel && freelist->kpf_count < freelist->kpf_max) {
617 617 segkp_delete(seg, kpd);
618 618 kpd->kp_next = freelist->kpf_list;
619 619 freelist->kpf_list = kpd;
620 620 freelist->kpf_count++;
621 621 mutex_exit(&segkp_lock);
622 622 return;
623 623 } else {
624 624 mutex_exit(&segkp_lock);
625 625 kpd->kp_cookie = -1;
626 626 }
627 627 }
628 628 segkp_release_internal(seg, kpd, kpd->kp_len);
629 629 }
630 630
631 631 /*
632 632 * Free the entire resource. segkp_unlock gets called with the start of the
633 633 * mapped portion of the resource. The length is the size of the mapped
634 634 * portion
635 635 */
636 636 static void
637 637 segkp_release_internal(struct seg *seg, struct segkp_data *kpd, size_t len)
638 638 {
639 639 caddr_t va;
640 640 long i;
641 641 long redzone;
642 642 size_t np;
643 643 page_t *pp;
644 644 struct vnode *vp;
645 645 anoff_t off;
646 646 struct anon *ap;
647 647 pgcnt_t segkpindex;
648 648
649 649 ASSERT(kpd != NULL);
650 650 ASSERT((kpd->kp_flags & KPD_HASAMP) == 0 || kpd->kp_cookie == -1);
651 651 np = btop(len);
652 652
653 653 /* Remove from active hash list */
654 654 if (kpd->kp_cookie == -1) {
655 655 mutex_enter(&segkp_lock);
656 656 segkp_delete(seg, kpd);
657 657 mutex_exit(&segkp_lock);
658 658 }
659 659
660 660 /*
661 661 * Precompute redzone page index.
662 662 */
663 663 redzone = -1;
664 664 if (kpd->kp_flags & KPD_HASREDZONE)
665 665 redzone = KPD_REDZONE(kpd);
666 666
667 667
668 668 va = kpd->kp_base;
669 669
670 670 hat_unload(seg->s_as->a_hat, va, (np << PAGESHIFT),
671 671 ((kpd->kp_flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD));
672 672 /*
673 673 * Free up those anon resources that are quiescent.
674 674 */
675 675 if (segkp_fromheap)
676 676 segkpindex = btop((uintptr_t)(va - kvseg.s_base));
677 677 for (i = 0; i < np; i++, va += PAGESIZE) {
678 678
679 679 /*
680 680 * Clear the bit for this page from the bitmap.
681 681 */
682 682 if (segkp_fromheap) {
683 683 BT_ATOMIC_CLEAR(segkp_bitmap, segkpindex);
684 684 segkpindex++;
685 685 }
686 686
687 687 if (i == redzone)
688 688 continue;
689 689 if (kpd->kp_anon) {
690 690 /*
691 691 * Free up anon resources and destroy the
692 692 * associated pages.
693 693 *
694 694 * Release the lock if there is one. Have to get the
695 695 * page to do this, unfortunately.
696 696 */
697 697 if (kpd->kp_flags & KPD_LOCKED) {
698 698 ap = anon_get_ptr(kpd->kp_anon,
699 699 kpd->kp_anon_idx + i);
700 700 swap_xlate(ap, &vp, &off);
701 701 /* Find the shared-locked page. */
702 702 pp = page_find(vp, (u_offset_t)off);
703 703 if (pp == NULL) {
704 704 panic("segkp_release: "
705 705 "kp_anon: no page to unlock ");
706 706 /*NOTREACHED*/
707 707 }
708 708 if (PP_ISRAF(pp))
709 709 PP_CLRRAF(pp);
710 710
711 711 page_unlock(pp);
712 712 }
713 713 if ((kpd->kp_flags & KPD_HASAMP) == 0) {
714 714 anon_free(kpd->kp_anon, kpd->kp_anon_idx + i,
715 715 PAGESIZE);
716 716 anon_unresv_zone(PAGESIZE, NULL);
717 717 atomic_add_long(&anon_segkp_pages_resv,
718 718 -1);
719 719 }
720 720 TRACE_5(TR_FAC_VM,
721 721 TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u",
722 722 kpd, va, PAGESIZE, 0, 0);
723 723 } else {
724 724 if (kpd->kp_flags & KPD_LOCKED) {
725 725 pp = page_find(&kvp, (u_offset_t)(uintptr_t)va);
726 726 if (pp == NULL) {
727 727 panic("segkp_release: "
728 728 "no page to unlock");
729 729 /*NOTREACHED*/
730 730 }
731 731 if (PP_ISRAF(pp))
732 732 PP_CLRRAF(pp);
733 733 /*
734 734 * We should just upgrade the lock here
735 735 * but there is no upgrade that waits.
736 736 */
737 737 page_unlock(pp);
738 738 }
739 739 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)va,
740 740 SE_EXCL);
741 741 if (pp != NULL)
742 742 page_destroy(pp, 0);
743 743 }
744 744 }
745 745
746 746 /* If locked, release physical memory reservation */
747 747 if (kpd->kp_flags & KPD_LOCKED) {
748 748 pgcnt_t pages = btop(SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags));
749 749 if ((kpd->kp_flags & KPD_NO_ANON) == 0)
750 750 atomic_add_long(&anon_segkp_pages_locked, -pages);
751 751 page_unresv(pages);
752 752 }
|
↓ open down ↓ |
752 lines elided |
↑ open up ↑ |
753 753
754 754 vmem_free(SEGKP_VMEM(seg), kpd->kp_base, kpd->kp_len);
755 755 kmem_free(kpd, sizeof (struct segkp_data));
756 756 }
757 757
758 758 /*
759 759 * segkp_map_red() will check the current frame pointer against the
760 760 * stack base. If the amount of stack remaining is questionable
761 761 * (less than red_minavail), then segkp_map_red() will map in the redzone
762 762 * and return 1. Otherwise, it will return 0. segkp_map_red() can
763 - * _only_ be called when:
764 - *
765 - * - it is safe to sleep on page_create_va().
766 - * - the caller is non-swappable.
763 + * _only_ be called when it is safe to sleep on page_create_va().
767 764 *
768 765 * It is up to the caller to remember whether segkp_map_red() successfully
769 766 * mapped the redzone, and, if so, to call segkp_unmap_red() at a later
770 - * time. Note that the caller must _remain_ non-swappable until after
771 - * calling segkp_unmap_red().
767 + * time.
772 768 *
773 769 * Currently, this routine is only called from pagefault() (which necessarily
774 770 * satisfies the above conditions).
775 771 */
776 772 #if defined(STACK_GROWTH_DOWN)
777 773 int
778 774 segkp_map_red(void)
779 775 {
780 776 uintptr_t fp = STACK_BIAS + (uintptr_t)getfp();
781 777 #ifndef _LP64
782 778 caddr_t stkbase;
783 779 #endif
784 780
785 - ASSERT(curthread->t_schedflag & TS_DONT_SWAP);
786 -
787 781 /*
788 782 * Optimize for the common case where we simply return.
789 783 */
790 784 if ((curthread->t_red_pp == NULL) &&
791 785 (fp - (uintptr_t)curthread->t_stkbase >= red_minavail))
792 786 return (0);
793 787
794 788 #if defined(_LP64)
795 789 /*
796 790 * XXX We probably need something better than this.
797 791 */
798 792 panic("kernel stack overflow");
799 793 /*NOTREACHED*/
800 794 #else /* _LP64 */
801 795 if (curthread->t_red_pp == NULL) {
802 796 page_t *red_pp;
803 797 struct seg kseg;
804 798
805 799 caddr_t red_va = (caddr_t)
806 800 (((uintptr_t)curthread->t_stkbase & (uintptr_t)PAGEMASK) -
807 801 PAGESIZE);
808 802
809 803 ASSERT(page_exists(&kvp, (u_offset_t)(uintptr_t)red_va) ==
810 804 NULL);
811 805
812 806 /*
813 807 * Allocate the physical for the red page.
814 808 */
815 809 /*
816 810 * No PG_NORELOC here to avoid waits. Unlikely to get
817 811 * a relocate happening in the short time the page exists
818 812 * and it will be OK anyway.
819 813 */
820 814
821 815 kseg.s_as = &kas;
822 816 red_pp = page_create_va(&kvp, (u_offset_t)(uintptr_t)red_va,
823 817 PAGESIZE, PG_WAIT | PG_EXCL, &kseg, red_va);
824 818 ASSERT(red_pp != NULL);
825 819
826 820 /*
827 821 * So we now have a page to jam into the redzone...
828 822 */
829 823 page_io_unlock(red_pp);
830 824
831 825 hat_memload(kas.a_hat, red_va, red_pp,
832 826 (PROT_READ|PROT_WRITE), HAT_LOAD_LOCK);
833 827 page_downgrade(red_pp);
834 828
835 829 /*
836 830 * The page is left SE_SHARED locked so we can hold on to
837 831 * the page_t pointer.
838 832 */
839 833 curthread->t_red_pp = red_pp;
840 834
841 835 atomic_add_32(&red_nmapped, 1);
842 836 while (fp - (uintptr_t)curthread->t_stkbase < red_closest) {
843 837 (void) cas32(&red_closest, red_closest,
844 838 (uint32_t)(fp - (uintptr_t)curthread->t_stkbase));
845 839 }
846 840 return (1);
847 841 }
848 842
849 843 stkbase = (caddr_t)(((uintptr_t)curthread->t_stkbase &
850 844 (uintptr_t)PAGEMASK) - PAGESIZE);
851 845
852 846 atomic_add_32(&red_ndoubles, 1);
853 847
854 848 if (fp - (uintptr_t)stkbase < RED_DEEP_THRESHOLD) {
855 849 /*
856 850 * Oh boy. We're already deep within the mapped-in
857 851 * redzone page, and the caller is trying to prepare
858 852 * for a deep stack run. We're running without a
859 853 * redzone right now: if the caller plows off the
860 854 * end of the stack, it'll plow another thread or
861 855 * LWP structure. That situation could result in
862 856 * a very hard-to-debug panic, so, in the spirit of
863 857 * recording the name of one's killer in one's own
864 858 * blood, we're going to record hrestime and the calling
865 859 * thread.
866 860 */
867 861 red_deep_hires = hrestime.tv_nsec;
868 862 red_deep_thread = curthread;
869 863 }
870 864
871 865 /*
872 866 * If this is a DEBUG kernel, and we've run too deep for comfort, toss.
873 867 */
874 868 ASSERT(fp - (uintptr_t)stkbase >= RED_DEEP_THRESHOLD);
875 869 return (0);
876 870 #endif /* _LP64 */
|
↓ open down ↓ |
80 lines elided |
↑ open up ↑ |
877 871 }
878 872
879 873 void
880 874 segkp_unmap_red(void)
881 875 {
882 876 page_t *pp;
883 877 caddr_t red_va = (caddr_t)(((uintptr_t)curthread->t_stkbase &
884 878 (uintptr_t)PAGEMASK) - PAGESIZE);
885 879
886 880 ASSERT(curthread->t_red_pp != NULL);
887 - ASSERT(curthread->t_schedflag & TS_DONT_SWAP);
888 881
889 882 /*
890 883 * Because we locked the mapping down, we can't simply rely
891 884 * on page_destroy() to clean everything up; we need to call
892 885 * hat_unload() to explicitly unlock the mapping resources.
893 886 */
894 887 hat_unload(kas.a_hat, red_va, PAGESIZE, HAT_UNLOAD_UNLOCK);
895 888
896 889 pp = curthread->t_red_pp;
897 890
898 891 ASSERT(pp == page_find(&kvp, (u_offset_t)(uintptr_t)red_va));
899 892
900 893 /*
901 894 * Need to upgrade the SE_SHARED lock to SE_EXCL.
902 895 */
903 896 if (!page_tryupgrade(pp)) {
904 897 /*
905 898 * As there is now wait for upgrade, release the
906 899 * SE_SHARED lock and wait for SE_EXCL.
907 900 */
908 901 page_unlock(pp);
909 902 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)red_va, SE_EXCL);
910 903 /* pp may be NULL here, hence the test below */
911 904 }
912 905
913 906 /*
914 907 * Destroy the page, with dontfree set to zero (i.e. free it).
915 908 */
916 909 if (pp != NULL)
917 910 page_destroy(pp, 0);
918 911 curthread->t_red_pp = NULL;
919 912 }
920 913 #else
921 914 #error Red stacks only supported with downwards stack growth.
922 915 #endif
923 916
924 917 /*
925 918 * Handle a fault on an address corresponding to one of the
926 919 * resources in the segkp segment.
927 920 */
928 921 faultcode_t
929 922 segkp_fault(
930 923 struct hat *hat,
931 924 struct seg *seg,
932 925 caddr_t vaddr,
933 926 size_t len,
934 927 enum fault_type type,
935 928 enum seg_rw rw)
936 929 {
937 930 struct segkp_data *kpd = NULL;
938 931 int err;
939 932
940 933 ASSERT(seg->s_as == &kas && RW_READ_HELD(&seg->s_as->a_lock));
941 934
942 935 /*
943 936 * Sanity checks.
944 937 */
945 938 if (type == F_PROT) {
946 939 panic("segkp_fault: unexpected F_PROT fault");
947 940 /*NOTREACHED*/
948 941 }
949 942
950 943 if ((kpd = segkp_find(seg, vaddr)) == NULL)
951 944 return (FC_NOMAP);
952 945
953 946 mutex_enter(&kpd->kp_lock);
954 947
955 948 if (type == F_SOFTLOCK) {
956 949 ASSERT(!(kpd->kp_flags & KPD_LOCKED));
957 950 /*
958 951 * The F_SOFTLOCK case has more stringent
959 952 * range requirements: the given range must exactly coincide
960 953 * with the resource's mapped portion. Note reference to
961 954 * redzone is handled since vaddr would not equal base
962 955 */
963 956 if (vaddr != stom(kpd->kp_base, kpd->kp_flags) ||
964 957 len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) {
965 958 mutex_exit(&kpd->kp_lock);
966 959 return (FC_MAKE_ERR(EFAULT));
967 960 }
968 961
969 962 if ((err = segkp_load(hat, seg, vaddr, len, kpd, KPD_LOCKED))) {
970 963 mutex_exit(&kpd->kp_lock);
971 964 return (FC_MAKE_ERR(err));
972 965 }
973 966 kpd->kp_flags |= KPD_LOCKED;
974 967 mutex_exit(&kpd->kp_lock);
975 968 return (0);
976 969 }
977 970
978 971 if (type == F_INVAL) {
979 972 ASSERT(!(kpd->kp_flags & KPD_NO_ANON));
980 973
981 974 /*
982 975 * Check if we touched the redzone. Somewhat optimistic
983 976 * here if we are touching the redzone of our own stack
984 977 * since we wouldn't have a stack to get this far...
985 978 */
986 979 if ((kpd->kp_flags & KPD_HASREDZONE) &&
987 980 btop((uintptr_t)(vaddr - kpd->kp_base)) == KPD_REDZONE(kpd))
988 981 panic("segkp_fault: accessing redzone");
989 982
990 983 /*
991 984 * This fault may occur while the page is being F_SOFTLOCK'ed.
992 985 * Return since a 2nd segkp_load is unnecessary and also would
993 986 * result in the page being locked twice and eventually
994 987 * hang the thread_reaper thread.
995 988 */
996 989 if (kpd->kp_flags & KPD_LOCKED) {
997 990 mutex_exit(&kpd->kp_lock);
998 991 return (0);
999 992 }
1000 993
1001 994 err = segkp_load(hat, seg, vaddr, len, kpd, kpd->kp_flags);
1002 995 mutex_exit(&kpd->kp_lock);
1003 996 return (err ? FC_MAKE_ERR(err) : 0);
1004 997 }
1005 998
1006 999 if (type == F_SOFTUNLOCK) {
1007 1000 uint_t flags;
1008 1001
1009 1002 /*
1010 1003 * Make sure the addr is LOCKED and it has anon backing
1011 1004 * before unlocking
1012 1005 */
1013 1006 if ((kpd->kp_flags & (KPD_LOCKED|KPD_NO_ANON)) != KPD_LOCKED) {
1014 1007 panic("segkp_fault: bad unlock");
1015 1008 /*NOTREACHED*/
1016 1009 }
1017 1010
1018 1011 if (vaddr != stom(kpd->kp_base, kpd->kp_flags) ||
1019 1012 len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) {
1020 1013 panic("segkp_fault: bad range");
1021 1014 /*NOTREACHED*/
1022 1015 }
1023 1016
1024 1017 if (rw == S_WRITE)
1025 1018 flags = kpd->kp_flags | KPD_WRITEDIRTY;
1026 1019 else
1027 1020 flags = kpd->kp_flags;
1028 1021 err = segkp_unlock(hat, seg, vaddr, len, kpd, flags);
1029 1022 kpd->kp_flags &= ~KPD_LOCKED;
1030 1023 mutex_exit(&kpd->kp_lock);
1031 1024 return (err ? FC_MAKE_ERR(err) : 0);
1032 1025 }
1033 1026 mutex_exit(&kpd->kp_lock);
1034 1027 panic("segkp_fault: bogus fault type: %d\n", type);
1035 1028 /*NOTREACHED*/
1036 1029 }
1037 1030
1038 1031 /*
1039 1032 * Check that the given protections suffice over the range specified by
1040 1033 * vaddr and len. For this segment type, the only issue is whether or
1041 1034 * not the range lies completely within the mapped part of an allocated
1042 1035 * resource.
1043 1036 */
1044 1037 /* ARGSUSED */
1045 1038 static int
1046 1039 segkp_checkprot(struct seg *seg, caddr_t vaddr, size_t len, uint_t prot)
1047 1040 {
1048 1041 struct segkp_data *kpd = NULL;
1049 1042 caddr_t mbase;
1050 1043 size_t mlen;
1051 1044
1052 1045 if ((kpd = segkp_find(seg, vaddr)) == NULL)
1053 1046 return (EACCES);
1054 1047
1055 1048 mutex_enter(&kpd->kp_lock);
1056 1049 mbase = stom(kpd->kp_base, kpd->kp_flags);
1057 1050 mlen = SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags);
1058 1051 if (len > mlen || vaddr < mbase ||
1059 1052 ((vaddr + len) > (mbase + mlen))) {
1060 1053 mutex_exit(&kpd->kp_lock);
1061 1054 return (EACCES);
1062 1055 }
1063 1056 mutex_exit(&kpd->kp_lock);
1064 1057 return (0);
1065 1058 }
1066 1059
1067 1060
1068 1061 /*
1069 1062 * Check to see if it makes sense to do kluster/read ahead to
1070 1063 * addr + delta relative to the mapping at addr. We assume here
1071 1064 * that delta is a signed PAGESIZE'd multiple (which can be negative).
1072 1065 *
1073 1066 * For seg_u we always "approve" of this action from our standpoint.
1074 1067 */
1075 1068 /*ARGSUSED*/
1076 1069 static int
1077 1070 segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
1078 1071 {
1079 1072 return (0);
1080 1073 }
1081 1074
1082 1075 /*
1083 1076 * Load and possibly lock intra-slot resources in the range given by
1084 1077 * vaddr and len.
1085 1078 */
1086 1079 static int
1087 1080 segkp_load(
1088 1081 struct hat *hat,
1089 1082 struct seg *seg,
1090 1083 caddr_t vaddr,
1091 1084 size_t len,
1092 1085 struct segkp_data *kpd,
1093 1086 uint_t flags)
1094 1087 {
1095 1088 caddr_t va;
1096 1089 caddr_t vlim;
1097 1090 ulong_t i;
1098 1091 uint_t lock;
1099 1092
1100 1093 ASSERT(MUTEX_HELD(&kpd->kp_lock));
1101 1094
1102 1095 len = P2ROUNDUP(len, PAGESIZE);
1103 1096
1104 1097 /* If locking, reserve physical memory */
1105 1098 if (flags & KPD_LOCKED) {
1106 1099 pgcnt_t pages = btop(len);
1107 1100 if ((kpd->kp_flags & KPD_NO_ANON) == 0)
1108 1101 atomic_add_long(&anon_segkp_pages_locked, pages);
1109 1102 (void) page_resv(pages, KM_SLEEP);
1110 1103 }
1111 1104
1112 1105 /*
1113 1106 * Loop through the pages in the given range.
1114 1107 */
1115 1108 va = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK);
1116 1109 vaddr = va;
1117 1110 vlim = va + len;
1118 1111 lock = flags & KPD_LOCKED;
1119 1112 i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT;
1120 1113 for (; va < vlim; va += PAGESIZE, i++) {
1121 1114 page_t *pl[2]; /* second element NULL terminator */
1122 1115 struct vnode *vp;
1123 1116 anoff_t off;
1124 1117 int err;
1125 1118 struct anon *ap;
1126 1119
1127 1120 /*
1128 1121 * Summon the page. If it's not resident, arrange
1129 1122 * for synchronous i/o to pull it in.
1130 1123 */
1131 1124 ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i);
1132 1125 swap_xlate(ap, &vp, &off);
1133 1126
1134 1127 /*
1135 1128 * The returned page list will have exactly one entry,
1136 1129 * which is returned to us already kept.
1137 1130 */
1138 1131 err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE, NULL,
1139 1132 pl, PAGESIZE, seg, va, S_READ, kcred, NULL);
1140 1133
1141 1134 if (err) {
1142 1135 /*
1143 1136 * Back out of what we've done so far.
1144 1137 */
1145 1138 (void) segkp_unlock(hat, seg, vaddr,
1146 1139 (va - vaddr), kpd, flags);
1147 1140 return (err);
1148 1141 }
1149 1142
1150 1143 /*
1151 1144 * Load an MMU translation for the page.
1152 1145 */
1153 1146 hat_memload(hat, va, pl[0], (PROT_READ|PROT_WRITE),
1154 1147 lock ? HAT_LOAD_LOCK : HAT_LOAD);
1155 1148
1156 1149 if (!lock) {
1157 1150 /*
1158 1151 * Now, release "shared" lock on the page.
1159 1152 */
1160 1153 page_unlock(pl[0]);
1161 1154 }
1162 1155 }
1163 1156 return (0);
1164 1157 }
1165 1158
1166 1159 /*
1167 1160 * At the very least unload the mmu-translations and unlock the range if locked
1168 1161 * Can be called with the following flag value KPD_WRITEDIRTY which specifies
1169 1162 * any dirty pages should be written to disk.
1170 1163 */
1171 1164 static int
1172 1165 segkp_unlock(
1173 1166 struct hat *hat,
1174 1167 struct seg *seg,
1175 1168 caddr_t vaddr,
1176 1169 size_t len,
1177 1170 struct segkp_data *kpd,
1178 1171 uint_t flags)
1179 1172 {
1180 1173 caddr_t va;
1181 1174 caddr_t vlim;
1182 1175 ulong_t i;
1183 1176 struct page *pp;
1184 1177 struct vnode *vp;
1185 1178 anoff_t off;
1186 1179 struct anon *ap;
1187 1180
1188 1181 #ifdef lint
1189 1182 seg = seg;
1190 1183 #endif /* lint */
1191 1184
1192 1185 ASSERT(MUTEX_HELD(&kpd->kp_lock));
1193 1186
1194 1187 /*
1195 1188 * Loop through the pages in the given range. It is assumed
1196 1189 * segkp_unlock is called with page aligned base
1197 1190 */
1198 1191 va = vaddr;
1199 1192 vlim = va + len;
1200 1193 i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT;
1201 1194 hat_unload(hat, va, len,
1202 1195 ((flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD));
1203 1196 for (; va < vlim; va += PAGESIZE, i++) {
1204 1197 /*
1205 1198 * Find the page associated with this part of the
1206 1199 * slot, tracking it down through its associated swap
1207 1200 * space.
1208 1201 */
1209 1202 ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i);
1210 1203 swap_xlate(ap, &vp, &off);
1211 1204
1212 1205 if (flags & KPD_LOCKED) {
1213 1206 if ((pp = page_find(vp, off)) == NULL) {
1214 1207 if (flags & KPD_LOCKED) {
1215 1208 panic("segkp_softunlock: missing page");
1216 1209 /*NOTREACHED*/
1217 1210 }
1218 1211 }
1219 1212 } else {
1220 1213 /*
1221 1214 * Nothing to do if the slot is not locked and the
1222 1215 * page doesn't exist.
1223 1216 */
1224 1217 if ((pp = page_lookup(vp, off, SE_SHARED)) == NULL)
1225 1218 continue;
1226 1219 }
1227 1220
1228 1221 /*
1229 1222 * If the page doesn't have any translations, is
1230 1223 * dirty and not being shared, then push it out
1231 1224 * asynchronously and avoid waiting for the
1232 1225 * pageout daemon to do it for us.
1233 1226 *
1234 1227 * XXX - Do we really need to get the "exclusive"
1235 1228 * lock via an upgrade?
1236 1229 */
1237 1230 if ((flags & KPD_WRITEDIRTY) && !hat_page_is_mapped(pp) &&
1238 1231 hat_ismod(pp) && page_tryupgrade(pp)) {
1239 1232 /*
1240 1233 * Hold the vnode before releasing the page lock to
1241 1234 * prevent it from being freed and re-used by some
1242 1235 * other thread.
1243 1236 */
1244 1237 VN_HOLD(vp);
1245 1238 page_unlock(pp);
1246 1239
1247 1240 /*
1248 1241 * Want most powerful credentials we can get so
1249 1242 * use kcred.
1250 1243 */
1251 1244 (void) VOP_PUTPAGE(vp, (offset_t)off, PAGESIZE,
1252 1245 B_ASYNC | B_FREE, kcred, NULL);
1253 1246 VN_RELE(vp);
1254 1247 } else {
1255 1248 page_unlock(pp);
1256 1249 }
1257 1250 }
1258 1251
1259 1252 /* If unlocking, release physical memory */
1260 1253 if (flags & KPD_LOCKED) {
1261 1254 pgcnt_t pages = btopr(len);
1262 1255 if ((kpd->kp_flags & KPD_NO_ANON) == 0)
1263 1256 atomic_add_long(&anon_segkp_pages_locked, -pages);
1264 1257 page_unresv(pages);
1265 1258 }
1266 1259 return (0);
1267 1260 }
1268 1261
1269 1262 /*
1270 1263 * Insert the kpd in the hash table.
1271 1264 */
1272 1265 static void
1273 1266 segkp_insert(struct seg *seg, struct segkp_data *kpd)
1274 1267 {
1275 1268 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1276 1269 int index;
1277 1270
1278 1271 /*
1279 1272 * Insert the kpd based on the address that will be returned
1280 1273 * via segkp_release.
1281 1274 */
1282 1275 index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags));
1283 1276 mutex_enter(&segkp_lock);
1284 1277 kpd->kp_next = kpsd->kpsd_hash[index];
1285 1278 kpsd->kpsd_hash[index] = kpd;
1286 1279 mutex_exit(&segkp_lock);
1287 1280 }
1288 1281
1289 1282 /*
1290 1283 * Remove kpd from the hash table.
1291 1284 */
1292 1285 static void
1293 1286 segkp_delete(struct seg *seg, struct segkp_data *kpd)
1294 1287 {
1295 1288 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1296 1289 struct segkp_data **kpp;
1297 1290 int index;
1298 1291
1299 1292 ASSERT(MUTEX_HELD(&segkp_lock));
1300 1293
1301 1294 index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags));
1302 1295 for (kpp = &kpsd->kpsd_hash[index];
1303 1296 *kpp != NULL; kpp = &((*kpp)->kp_next)) {
1304 1297 if (*kpp == kpd) {
1305 1298 *kpp = kpd->kp_next;
1306 1299 return;
1307 1300 }
1308 1301 }
1309 1302 panic("segkp_delete: unable to find element to delete");
1310 1303 /*NOTREACHED*/
1311 1304 }
1312 1305
1313 1306 /*
1314 1307 * Find the kpd associated with a vaddr.
1315 1308 *
1316 1309 * Most of the callers of segkp_find will pass the vaddr that
1317 1310 * hashes to the desired index, but there are cases where
1318 1311 * this is not true in which case we have to (potentially) scan
1319 1312 * the whole table looking for it. This should be very rare
1320 1313 * (e.g. a segkp_fault(F_INVAL) on an address somewhere in the
1321 1314 * middle of the segkp_data region).
1322 1315 */
1323 1316 static struct segkp_data *
1324 1317 segkp_find(struct seg *seg, caddr_t vaddr)
1325 1318 {
1326 1319 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1327 1320 struct segkp_data *kpd;
1328 1321 int i;
1329 1322 int stop;
1330 1323
1331 1324 i = stop = SEGKP_HASH(vaddr);
1332 1325 mutex_enter(&segkp_lock);
1333 1326 do {
1334 1327 for (kpd = kpsd->kpsd_hash[i]; kpd != NULL;
1335 1328 kpd = kpd->kp_next) {
1336 1329 if (vaddr >= kpd->kp_base &&
1337 1330 vaddr < kpd->kp_base + kpd->kp_len) {
1338 1331 mutex_exit(&segkp_lock);
1339 1332 return (kpd);
1340 1333 }
1341 1334 }
1342 1335 if (--i < 0)
1343 1336 i = SEGKP_HASHSZ - 1; /* Wrap */
1344 1337 } while (i != stop);
1345 1338 mutex_exit(&segkp_lock);
1346 1339 return (NULL); /* Not found */
1347 1340 }
1348 1341
1349 1342 /*
1350 1343 * returns size of swappable area.
1351 1344 */
1352 1345 size_t
1353 1346 swapsize(caddr_t v)
1354 1347 {
1355 1348 struct segkp_data *kpd;
1356 1349
1357 1350 if ((kpd = segkp_find(segkp, v)) != NULL)
1358 1351 return (SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags));
1359 1352 else
1360 1353 return (NULL);
1361 1354 }
1362 1355
1363 1356 /*
1364 1357 * Dump out all the active segkp pages
1365 1358 */
1366 1359 static void
1367 1360 segkp_dump(struct seg *seg)
1368 1361 {
1369 1362 int i;
1370 1363 struct segkp_data *kpd;
1371 1364 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1372 1365
1373 1366 for (i = 0; i < SEGKP_HASHSZ; i++) {
1374 1367 for (kpd = kpsd->kpsd_hash[i];
1375 1368 kpd != NULL; kpd = kpd->kp_next) {
1376 1369 pfn_t pfn;
1377 1370 caddr_t addr;
1378 1371 caddr_t eaddr;
1379 1372
1380 1373 addr = kpd->kp_base;
1381 1374 eaddr = addr + kpd->kp_len;
1382 1375 while (addr < eaddr) {
1383 1376 ASSERT(seg->s_as == &kas);
1384 1377 pfn = hat_getpfnum(seg->s_as->a_hat, addr);
1385 1378 if (pfn != PFN_INVALID)
1386 1379 dump_addpage(seg->s_as, addr, pfn);
1387 1380 addr += PAGESIZE;
1388 1381 dump_timeleft = dump_timeout;
1389 1382 }
1390 1383 }
1391 1384 }
1392 1385 }
1393 1386
1394 1387 /*ARGSUSED*/
1395 1388 static int
1396 1389 segkp_pagelock(struct seg *seg, caddr_t addr, size_t len,
1397 1390 struct page ***ppp, enum lock_type type, enum seg_rw rw)
1398 1391 {
1399 1392 return (ENOTSUP);
1400 1393 }
1401 1394
1402 1395 /*ARGSUSED*/
1403 1396 static int
1404 1397 segkp_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
1405 1398 {
1406 1399 return (ENODEV);
1407 1400 }
1408 1401
1409 1402 /*ARGSUSED*/
1410 1403 static lgrp_mem_policy_info_t *
1411 1404 segkp_getpolicy(struct seg *seg, caddr_t addr)
1412 1405 {
1413 1406 return (NULL);
1414 1407 }
1415 1408
1416 1409 /*ARGSUSED*/
1417 1410 static int
1418 1411 segkp_capable(struct seg *seg, segcapability_t capability)
1419 1412 {
1420 1413 return (0);
1421 1414 }
1422 1415
1423 1416 #include <sys/mem_config.h>
1424 1417
1425 1418 /*ARGSUSED*/
1426 1419 static void
1427 1420 segkp_mem_config_post_add(void *arg, pgcnt_t delta_pages)
1428 1421 {}
1429 1422
1430 1423 /*
1431 1424 * During memory delete, turn off caches so that pages are not held.
1432 1425 * A better solution may be to unlock the pages while they are
1433 1426 * in the cache so that they may be collected naturally.
1434 1427 */
1435 1428
1436 1429 /*ARGSUSED*/
1437 1430 static int
1438 1431 segkp_mem_config_pre_del(void *arg, pgcnt_t delta_pages)
1439 1432 {
1440 1433 atomic_add_32(&segkp_indel, 1);
1441 1434 segkp_cache_free();
1442 1435 return (0);
1443 1436 }
1444 1437
1445 1438 /*ARGSUSED*/
1446 1439 static void
1447 1440 segkp_mem_config_post_del(void *arg, pgcnt_t delta_pages, int cancelled)
1448 1441 {
1449 1442 atomic_add_32(&segkp_indel, -1);
1450 1443 }
1451 1444
1452 1445 static kphysm_setup_vector_t segkp_mem_config_vec = {
1453 1446 KPHYSM_SETUP_VECTOR_VERSION,
1454 1447 segkp_mem_config_post_add,
1455 1448 segkp_mem_config_pre_del,
1456 1449 segkp_mem_config_post_del,
1457 1450 };
1458 1451
1459 1452 static void
1460 1453 segkpinit_mem_config(struct seg *seg)
1461 1454 {
1462 1455 int ret;
1463 1456
1464 1457 ret = kphysm_setup_func_register(&segkp_mem_config_vec, (void *)seg);
1465 1458 ASSERT(ret == 0);
1466 1459 }
|
↓ open down ↓ |
569 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX