Print this page
6147 segop_getpolicy already checks for a NULL op
Reviewed by: Garrett D'Amore <garrett@damore.org>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/vm/seg_kmem.c
+++ new/usr/src/uts/common/vm/seg_kmem.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 */
24 24
25 25 #include <sys/types.h>
26 26 #include <sys/t_lock.h>
27 27 #include <sys/param.h>
28 28 #include <sys/sysmacros.h>
29 29 #include <sys/tuneable.h>
30 30 #include <sys/systm.h>
31 31 #include <sys/vm.h>
32 32 #include <sys/kmem.h>
33 33 #include <sys/vmem.h>
34 34 #include <sys/mman.h>
35 35 #include <sys/cmn_err.h>
36 36 #include <sys/debug.h>
37 37 #include <sys/dumphdr.h>
38 38 #include <sys/bootconf.h>
39 39 #include <sys/lgrp.h>
40 40 #include <vm/seg_kmem.h>
41 41 #include <vm/hat.h>
42 42 #include <vm/page.h>
43 43 #include <vm/vm_dep.h>
44 44 #include <vm/faultcode.h>
45 45 #include <sys/promif.h>
46 46 #include <vm/seg_kp.h>
47 47 #include <sys/bitmap.h>
48 48 #include <sys/mem_cage.h>
49 49
50 50 #ifdef __sparc
51 51 #include <sys/ivintr.h>
52 52 #include <sys/panic.h>
53 53 #endif
54 54
55 55 /*
56 56 * seg_kmem is the primary kernel memory segment driver. It
57 57 * maps the kernel heap [kernelheap, ekernelheap), module text,
58 58 * and all memory which was allocated before the VM was initialized
59 59 * into kas.
60 60 *
61 61 * Pages which belong to seg_kmem are hashed into &kvp vnode at
62 62 * an offset equal to (u_offset_t)virt_addr, and have p_lckcnt >= 1.
63 63 * They must never be paged out since segkmem_fault() is a no-op to
64 64 * prevent recursive faults.
65 65 *
66 66 * Currently, seg_kmem pages are sharelocked (p_sharelock == 1) on
67 67 * __x86 and are unlocked (p_sharelock == 0) on __sparc. Once __x86
68 68 * supports relocation the #ifdef kludges can be removed.
69 69 *
70 70 * seg_kmem pages may be subject to relocation by page_relocate(),
71 71 * provided that the HAT supports it; if this is so, segkmem_reloc
72 72 * will be set to a nonzero value. All boot time allocated memory as
73 73 * well as static memory is considered off limits to relocation.
74 74 * Pages are "relocatable" if p_state does not have P_NORELOC set, so
75 75 * we request P_NORELOC pages for memory that isn't safe to relocate.
76 76 *
77 77 * The kernel heap is logically divided up into four pieces:
78 78 *
79 79 * heap32_arena is for allocations that require 32-bit absolute
80 80 * virtual addresses (e.g. code that uses 32-bit pointers/offsets).
81 81 *
82 82 * heap_core is for allocations that require 2GB *relative*
83 83 * offsets; in other words all memory from heap_core is within
84 84 * 2GB of all other memory from the same arena. This is a requirement
85 85 * of the addressing modes of some processors in supervisor code.
86 86 *
87 87 * heap_arena is the general heap arena.
88 88 *
89 89 * static_arena is the static memory arena. Allocations from it
90 90 * are not subject to relocation so it is safe to use the memory
91 91 * physical address as well as the virtual address (e.g. the VA to
92 92 * PA translations are static). Caches may import from static_arena;
93 93 * all other static memory allocations should use static_alloc_arena.
94 94 *
95 95 * On some platforms which have limited virtual address space, seg_kmem
96 96 * may share [kernelheap, ekernelheap) with seg_kp; if this is so,
97 97 * segkp_bitmap is non-NULL, and each bit represents a page of virtual
98 98 * address space which is actually seg_kp mapped.
99 99 */
100 100
101 101 extern ulong_t *segkp_bitmap; /* Is set if segkp is from the kernel heap */
102 102
103 103 char *kernelheap; /* start of primary kernel heap */
104 104 char *ekernelheap; /* end of primary kernel heap */
105 105 struct seg kvseg; /* primary kernel heap segment */
106 106 struct seg kvseg_core; /* "core" kernel heap segment */
107 107 struct seg kzioseg; /* Segment for zio mappings */
108 108 vmem_t *heap_arena; /* primary kernel heap arena */
109 109 vmem_t *heap_core_arena; /* core kernel heap arena */
110 110 char *heap_core_base; /* start of core kernel heap arena */
111 111 char *heap_lp_base; /* start of kernel large page heap arena */
112 112 char *heap_lp_end; /* end of kernel large page heap arena */
113 113 vmem_t *hat_memload_arena; /* HAT translation data */
114 114 struct seg kvseg32; /* 32-bit kernel heap segment */
115 115 vmem_t *heap32_arena; /* 32-bit kernel heap arena */
116 116 vmem_t *heaptext_arena; /* heaptext arena */
117 117 struct as kas; /* kernel address space */
118 118 int segkmem_reloc; /* enable/disable relocatable segkmem pages */
119 119 vmem_t *static_arena; /* arena for caches to import static memory */
120 120 vmem_t *static_alloc_arena; /* arena for allocating static memory */
121 121 vmem_t *zio_arena = NULL; /* arena for allocating zio memory */
122 122 vmem_t *zio_alloc_arena = NULL; /* arena for allocating zio memory */
123 123
124 124 /*
125 125 * seg_kmem driver can map part of the kernel heap with large pages.
126 126 * Currently this functionality is implemented for sparc platforms only.
127 127 *
128 128 * The large page size "segkmem_lpsize" for kernel heap is selected in the
129 129 * platform specific code. It can also be modified via /etc/system file.
130 130 * Setting segkmem_lpsize to PAGESIZE in /etc/system disables usage of large
131 131 * pages for kernel heap. "segkmem_lpshift" is adjusted appropriately to
132 132 * match segkmem_lpsize.
133 133 *
134 134 * At boot time we carve from kernel heap arena a range of virtual addresses
135 135 * that will be used for large page mappings. This range [heap_lp_base,
136 136 * heap_lp_end) is set up as a separate vmem arena - "heap_lp_arena". We also
137 137 * create "kmem_lp_arena" that caches memory already backed up by large
138 138 * pages. kmem_lp_arena imports virtual segments from heap_lp_arena.
139 139 */
140 140
141 141 size_t segkmem_lpsize;
142 142 static uint_t segkmem_lpshift = PAGESHIFT;
143 143 int segkmem_lpszc = 0;
144 144
145 145 size_t segkmem_kmemlp_quantum = 0x400000; /* 4MB */
146 146 size_t segkmem_heaplp_quantum;
147 147 vmem_t *heap_lp_arena;
148 148 static vmem_t *kmem_lp_arena;
149 149 static vmem_t *segkmem_ppa_arena;
150 150 static segkmem_lpcb_t segkmem_lpcb;
151 151
152 152 /*
153 153 * We use "segkmem_kmemlp_max" to limit the total amount of physical memory
154 154 * consumed by the large page heap. By default this parameter is set to 1/8 of
155 155 * physmem but can be adjusted through /etc/system either directly or
156 156 * indirectly by setting "segkmem_kmemlp_pcnt" to the percent of physmem
157 157 * we allow for large page heap.
158 158 */
159 159 size_t segkmem_kmemlp_max;
160 160 static uint_t segkmem_kmemlp_pcnt;
161 161
162 162 /*
163 163 * Getting large pages for kernel heap could be problematic due to
164 164 * physical memory fragmentation. That's why we allow to preallocate
165 165 * "segkmem_kmemlp_min" bytes at boot time.
166 166 */
167 167 static size_t segkmem_kmemlp_min;
168 168
169 169 /*
170 170 * Throttling is used to avoid expensive tries to allocate large pages
171 171 * for kernel heap when a lot of succesive attempts to do so fail.
172 172 */
173 173 static ulong_t segkmem_lpthrottle_max = 0x400000;
174 174 static ulong_t segkmem_lpthrottle_start = 0x40;
175 175 static ulong_t segkmem_use_lpthrottle = 1;
176 176
177 177 /*
178 178 * Freed pages accumulate on a garbage list until segkmem is ready,
179 179 * at which point we call segkmem_gc() to free it all.
180 180 */
181 181 typedef struct segkmem_gc_list {
182 182 struct segkmem_gc_list *gc_next;
183 183 vmem_t *gc_arena;
184 184 size_t gc_size;
185 185 } segkmem_gc_list_t;
186 186
187 187 static segkmem_gc_list_t *segkmem_gc_list;
188 188
189 189 /*
190 190 * Allocations from the hat_memload arena add VM_MEMLOAD to their
191 191 * vmflags so that segkmem_xalloc() can inform the hat layer that it needs
192 192 * to take steps to prevent infinite recursion. HAT allocations also
193 193 * must be non-relocatable to prevent recursive page faults.
194 194 */
195 195 static void *
196 196 hat_memload_alloc(vmem_t *vmp, size_t size, int flags)
197 197 {
198 198 flags |= (VM_MEMLOAD | VM_NORELOC);
199 199 return (segkmem_alloc(vmp, size, flags));
200 200 }
201 201
202 202 /*
203 203 * Allocations from static_arena arena (or any other arena that uses
204 204 * segkmem_alloc_permanent()) require non-relocatable (permanently
205 205 * wired) memory pages, since these pages are referenced by physical
206 206 * as well as virtual address.
207 207 */
208 208 void *
209 209 segkmem_alloc_permanent(vmem_t *vmp, size_t size, int flags)
210 210 {
211 211 return (segkmem_alloc(vmp, size, flags | VM_NORELOC));
212 212 }
213 213
214 214 /*
215 215 * Initialize kernel heap boundaries.
216 216 */
217 217 void
218 218 kernelheap_init(
219 219 void *heap_start,
220 220 void *heap_end,
221 221 char *first_avail,
222 222 void *core_start,
223 223 void *core_end)
224 224 {
225 225 uintptr_t textbase;
226 226 size_t core_size;
227 227 size_t heap_size;
228 228 vmem_t *heaptext_parent;
229 229 size_t heap_lp_size = 0;
230 230 #ifdef __sparc
231 231 size_t kmem64_sz = kmem64_aligned_end - kmem64_base;
232 232 #endif /* __sparc */
233 233
234 234 kernelheap = heap_start;
235 235 ekernelheap = heap_end;
236 236
237 237 #ifdef __sparc
238 238 heap_lp_size = (((uintptr_t)heap_end - (uintptr_t)heap_start) / 4);
239 239 /*
240 240 * Bias heap_lp start address by kmem64_sz to reduce collisions
241 241 * in 4M kernel TSB between kmem64 area and heap_lp
242 242 */
243 243 kmem64_sz = P2ROUNDUP(kmem64_sz, MMU_PAGESIZE256M);
244 244 if (kmem64_sz <= heap_lp_size / 2)
245 245 heap_lp_size -= kmem64_sz;
246 246 heap_lp_base = ekernelheap - heap_lp_size;
247 247 heap_lp_end = heap_lp_base + heap_lp_size;
248 248 #endif /* __sparc */
249 249
250 250 /*
251 251 * If this platform has a 'core' heap area, then the space for
252 252 * overflow module text should be carved out of the end of that
253 253 * heap. Otherwise, it gets carved out of the general purpose
254 254 * heap.
255 255 */
256 256 core_size = (uintptr_t)core_end - (uintptr_t)core_start;
257 257 if (core_size > 0) {
258 258 ASSERT(core_size >= HEAPTEXT_SIZE);
259 259 textbase = (uintptr_t)core_end - HEAPTEXT_SIZE;
260 260 core_size -= HEAPTEXT_SIZE;
261 261 }
262 262 #ifndef __sparc
263 263 else {
264 264 ekernelheap -= HEAPTEXT_SIZE;
265 265 textbase = (uintptr_t)ekernelheap;
266 266 }
267 267 #endif
268 268
269 269 heap_size = (uintptr_t)ekernelheap - (uintptr_t)kernelheap;
270 270 heap_arena = vmem_init("heap", kernelheap, heap_size, PAGESIZE,
271 271 segkmem_alloc, segkmem_free);
272 272
273 273 if (core_size > 0) {
274 274 heap_core_arena = vmem_create("heap_core", core_start,
275 275 core_size, PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP);
276 276 heap_core_base = core_start;
277 277 } else {
278 278 heap_core_arena = heap_arena;
279 279 heap_core_base = kernelheap;
280 280 }
281 281
282 282 /*
283 283 * reserve space for the large page heap. If large pages for kernel
284 284 * heap is enabled large page heap arean will be created later in the
285 285 * boot sequence in segkmem_heap_lp_init(). Otherwise the allocated
286 286 * range will be returned back to the heap_arena.
287 287 */
288 288 if (heap_lp_size) {
289 289 (void) vmem_xalloc(heap_arena, heap_lp_size, PAGESIZE, 0, 0,
290 290 heap_lp_base, heap_lp_end,
291 291 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
292 292 }
293 293
294 294 /*
295 295 * Remove the already-spoken-for memory range [kernelheap, first_avail).
296 296 */
297 297 (void) vmem_xalloc(heap_arena, first_avail - kernelheap, PAGESIZE,
298 298 0, 0, kernelheap, first_avail, VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
299 299
300 300 #ifdef __sparc
301 301 heap32_arena = vmem_create("heap32", (void *)SYSBASE32,
302 302 SYSLIMIT32 - SYSBASE32 - HEAPTEXT_SIZE, PAGESIZE, NULL,
303 303 NULL, NULL, 0, VM_SLEEP);
304 304 /*
305 305 * Prom claims the physical and virtual resources used by panicbuf
306 306 * and inter_vec_table. So reserve space for panicbuf, intr_vec_table,
307 307 * reserved interrupt vector data structures from 32-bit heap.
308 308 */
309 309 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0,
310 310 panicbuf, panicbuf + PANICBUFSIZE,
311 311 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
312 312
313 313 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0,
314 314 intr_vec_table, (caddr_t)intr_vec_table + IVSIZE,
315 315 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
316 316
317 317 textbase = SYSLIMIT32 - HEAPTEXT_SIZE;
318 318 heaptext_parent = NULL;
319 319 #else /* __sparc */
320 320 heap32_arena = heap_core_arena;
321 321 heaptext_parent = heap_core_arena;
322 322 #endif /* __sparc */
323 323
324 324 heaptext_arena = vmem_create("heaptext", (void *)textbase,
325 325 HEAPTEXT_SIZE, PAGESIZE, NULL, NULL, heaptext_parent, 0, VM_SLEEP);
326 326
327 327 /*
328 328 * Create a set of arenas for memory with static translations
329 329 * (e.g. VA -> PA translations cannot change). Since using
330 330 * kernel pages by physical address implies it isn't safe to
331 331 * walk across page boundaries, the static_arena quantum must
332 332 * be PAGESIZE. Any kmem caches that require static memory
333 333 * should source from static_arena, while direct allocations
334 334 * should only use static_alloc_arena.
335 335 */
336 336 static_arena = vmem_create("static", NULL, 0, PAGESIZE,
337 337 segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP);
338 338 static_alloc_arena = vmem_create("static_alloc", NULL, 0,
339 339 sizeof (uint64_t), vmem_alloc, vmem_free, static_arena,
340 340 0, VM_SLEEP);
341 341
342 342 /*
343 343 * Create an arena for translation data (ptes, hmes, or hblks).
344 344 * We need an arena for this because hat_memload() is essential
345 345 * to vmem_populate() (see comments in common/os/vmem.c).
346 346 *
347 347 * Note: any kmem cache that allocates from hat_memload_arena
348 348 * must be created as a KMC_NOHASH cache (i.e. no external slab
349 349 * and bufctl structures to allocate) so that slab creation doesn't
350 350 * require anything more than a single vmem_alloc().
351 351 */
352 352 hat_memload_arena = vmem_create("hat_memload", NULL, 0, PAGESIZE,
353 353 hat_memload_alloc, segkmem_free, heap_arena, 0,
354 354 VM_SLEEP | VMC_POPULATOR | VMC_DUMPSAFE);
355 355 }
356 356
357 357 void
358 358 boot_mapin(caddr_t addr, size_t size)
359 359 {
360 360 caddr_t eaddr;
361 361 page_t *pp;
362 362 pfn_t pfnum;
363 363
364 364 if (page_resv(btop(size), KM_NOSLEEP) == 0)
365 365 panic("boot_mapin: page_resv failed");
366 366
367 367 for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
368 368 pfnum = va_to_pfn(addr);
369 369 if (pfnum == PFN_INVALID)
370 370 continue;
371 371 if ((pp = page_numtopp_nolock(pfnum)) == NULL)
372 372 panic("boot_mapin(): No pp for pfnum = %lx", pfnum);
373 373
374 374 /*
375 375 * must break up any large pages that may have constituent
376 376 * pages being utilized for BOP_ALLOC()'s before calling
377 377 * page_numtopp().The locking code (ie. page_reclaim())
378 378 * can't handle them
379 379 */
380 380 if (pp->p_szc != 0)
381 381 page_boot_demote(pp);
382 382
383 383 pp = page_numtopp(pfnum, SE_EXCL);
384 384 if (pp == NULL || PP_ISFREE(pp))
385 385 panic("boot_alloc: pp is NULL or free");
386 386
387 387 /*
388 388 * If the cage is on but doesn't yet contain this page,
389 389 * mark it as non-relocatable.
390 390 */
391 391 if (kcage_on && !PP_ISNORELOC(pp)) {
392 392 PP_SETNORELOC(pp);
393 393 PLCNT_XFER_NORELOC(pp);
394 394 }
395 395
396 396 (void) page_hashin(pp, &kvp, (u_offset_t)(uintptr_t)addr, NULL);
397 397 pp->p_lckcnt = 1;
398 398 #if defined(__x86)
399 399 page_downgrade(pp);
400 400 #else
401 401 page_unlock(pp);
402 402 #endif
403 403 }
404 404 }
405 405
406 406 /*
407 407 * Get pages from boot and hash them into the kernel's vp.
408 408 * Used after page structs have been allocated, but before segkmem is ready.
409 409 */
410 410 void *
411 411 boot_alloc(void *inaddr, size_t size, uint_t align)
412 412 {
413 413 caddr_t addr = inaddr;
414 414
415 415 if (bootops == NULL)
416 416 prom_panic("boot_alloc: attempt to allocate memory after "
417 417 "BOP_GONE");
418 418
419 419 size = ptob(btopr(size));
420 420 #ifdef __sparc
421 421 if (bop_alloc_chunk(addr, size, align) != (caddr_t)addr)
422 422 panic("boot_alloc: bop_alloc_chunk failed");
423 423 #else
424 424 if (BOP_ALLOC(bootops, addr, size, align) != addr)
425 425 panic("boot_alloc: BOP_ALLOC failed");
426 426 #endif
427 427 boot_mapin((caddr_t)addr, size);
428 428 return (addr);
429 429 }
430 430
431 431 static void
432 432 segkmem_badop()
433 433 {
434 434 panic("segkmem_badop");
435 435 }
436 436
437 437 #define SEGKMEM_BADOP(t) (t(*)())segkmem_badop
438 438
439 439 /*ARGSUSED*/
440 440 static faultcode_t
441 441 segkmem_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t size,
442 442 enum fault_type type, enum seg_rw rw)
443 443 {
444 444 pgcnt_t npages;
445 445 spgcnt_t pg;
446 446 page_t *pp;
447 447 struct vnode *vp = seg->s_data;
448 448
449 449 ASSERT(RW_READ_HELD(&seg->s_as->a_lock));
450 450
451 451 if (seg->s_as != &kas || size > seg->s_size ||
452 452 addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
453 453 panic("segkmem_fault: bad args");
454 454
455 455 /*
456 456 * If it is one of segkp pages, call segkp_fault.
457 457 */
458 458 if (segkp_bitmap && seg == &kvseg &&
459 459 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
460 460 return (segop_fault(hat, segkp, addr, size, type, rw));
461 461
462 462 if (rw != S_READ && rw != S_WRITE && rw != S_OTHER)
463 463 return (FC_NOSUPPORT);
464 464
465 465 npages = btopr(size);
466 466
467 467 switch (type) {
468 468 case F_SOFTLOCK: /* lock down already-loaded translations */
469 469 for (pg = 0; pg < npages; pg++) {
470 470 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr,
471 471 SE_SHARED);
472 472 if (pp == NULL) {
473 473 /*
474 474 * Hmm, no page. Does a kernel mapping
475 475 * exist for it?
476 476 */
477 477 if (!hat_probe(kas.a_hat, addr)) {
478 478 addr -= PAGESIZE;
479 479 while (--pg >= 0) {
480 480 pp = page_find(vp, (u_offset_t)
481 481 (uintptr_t)addr);
482 482 if (pp)
483 483 page_unlock(pp);
484 484 addr -= PAGESIZE;
485 485 }
486 486 return (FC_NOMAP);
487 487 }
488 488 }
489 489 addr += PAGESIZE;
490 490 }
491 491 if (rw == S_OTHER)
492 492 hat_reserve(seg->s_as, addr, size);
493 493 return (0);
494 494 case F_SOFTUNLOCK:
495 495 while (npages--) {
496 496 pp = page_find(vp, (u_offset_t)(uintptr_t)addr);
497 497 if (pp)
498 498 page_unlock(pp);
499 499 addr += PAGESIZE;
500 500 }
501 501 return (0);
502 502 default:
503 503 return (FC_NOSUPPORT);
504 504 }
505 505 /*NOTREACHED*/
506 506 }
507 507
508 508 static int
509 509 segkmem_setprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
510 510 {
511 511 ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
512 512
513 513 if (seg->s_as != &kas || size > seg->s_size ||
514 514 addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
515 515 panic("segkmem_setprot: bad args");
516 516
517 517 /*
518 518 * If it is one of segkp pages, call segkp.
519 519 */
520 520 if (segkp_bitmap && seg == &kvseg &&
521 521 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
522 522 return (segop_setprot(segkp, addr, size, prot));
523 523
524 524 if (prot == 0)
525 525 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD);
526 526 else
527 527 hat_chgprot(kas.a_hat, addr, size, prot);
528 528 return (0);
529 529 }
530 530
531 531 /*
532 532 * This is a dummy segkmem function overloaded to call segkp
533 533 * when segkp is under the heap.
534 534 */
535 535 /* ARGSUSED */
536 536 static int
537 537 segkmem_checkprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
538 538 {
539 539 ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
540 540
541 541 if (seg->s_as != &kas)
542 542 segkmem_badop();
543 543
544 544 /*
545 545 * If it is one of segkp pages, call into segkp.
546 546 */
547 547 if (segkp_bitmap && seg == &kvseg &&
548 548 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
549 549 return (segop_checkprot(segkp, addr, size, prot));
550 550
551 551 segkmem_badop();
552 552 return (0);
553 553 }
554 554
555 555 /*
556 556 * This is a dummy segkmem function overloaded to call segkp
557 557 * when segkp is under the heap.
558 558 */
559 559 /* ARGSUSED */
560 560 static int
561 561 segkmem_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
562 562 {
563 563 ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
564 564
565 565 if (seg->s_as != &kas)
566 566 segkmem_badop();
567 567
568 568 /*
569 569 * If it is one of segkp pages, call into segkp.
570 570 */
571 571 if (segkp_bitmap && seg == &kvseg &&
572 572 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
573 573 return (segop_kluster(segkp, addr, delta));
574 574
575 575 segkmem_badop();
576 576 return (0);
577 577 }
578 578
579 579 static void
580 580 segkmem_xdump_range(void *arg, void *start, size_t size)
581 581 {
582 582 struct as *as = arg;
583 583 caddr_t addr = start;
584 584 caddr_t addr_end = addr + size;
585 585
586 586 while (addr < addr_end) {
587 587 pfn_t pfn = hat_getpfnum(kas.a_hat, addr);
588 588 if (pfn != PFN_INVALID && pfn <= physmax && pf_is_memory(pfn))
589 589 dump_addpage(as, addr, pfn);
590 590 addr += PAGESIZE;
591 591 dump_timeleft = dump_timeout;
592 592 }
593 593 }
594 594
595 595 static void
596 596 segkmem_dump_range(void *arg, void *start, size_t size)
597 597 {
598 598 caddr_t addr = start;
599 599 caddr_t addr_end = addr + size;
600 600
601 601 /*
602 602 * If we are about to start dumping the range of addresses we
603 603 * carved out of the kernel heap for the large page heap walk
604 604 * heap_lp_arena to find what segments are actually populated
605 605 */
606 606 if (SEGKMEM_USE_LARGEPAGES &&
607 607 addr == heap_lp_base && addr_end == heap_lp_end &&
608 608 vmem_size(heap_lp_arena, VMEM_ALLOC) < size) {
609 609 vmem_walk(heap_lp_arena, VMEM_ALLOC | VMEM_REENTRANT,
610 610 segkmem_xdump_range, arg);
611 611 } else {
612 612 segkmem_xdump_range(arg, start, size);
613 613 }
614 614 }
615 615
616 616 static void
617 617 segkmem_dump(struct seg *seg)
618 618 {
619 619 /*
620 620 * The kernel's heap_arena (represented by kvseg) is a very large
621 621 * VA space, most of which is typically unused. To speed up dumping
622 622 * we use vmem_walk() to quickly find the pieces of heap_arena that
623 623 * are actually in use. We do the same for heap32_arena and
624 624 * heap_core.
625 625 *
626 626 * We specify VMEM_REENTRANT to vmem_walk() because dump_addpage()
627 627 * may ultimately need to allocate memory. Reentrant walks are
628 628 * necessarily imperfect snapshots. The kernel heap continues
629 629 * to change during a live crash dump, for example. For a normal
630 630 * crash dump, however, we know that there won't be any other threads
631 631 * messing with the heap. Therefore, at worst, we may fail to dump
632 632 * the pages that get allocated by the act of dumping; but we will
633 633 * always dump every page that was allocated when the walk began.
634 634 *
635 635 * The other segkmem segments are dense (fully populated), so there's
636 636 * no need to use this technique when dumping them.
637 637 *
638 638 * Note: when adding special dump handling for any new sparsely-
639 639 * populated segments, be sure to add similar handling to the ::kgrep
640 640 * code in mdb.
641 641 */
642 642 if (seg == &kvseg) {
643 643 vmem_walk(heap_arena, VMEM_ALLOC | VMEM_REENTRANT,
644 644 segkmem_dump_range, seg->s_as);
645 645 #ifndef __sparc
646 646 vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
647 647 segkmem_dump_range, seg->s_as);
648 648 #endif
649 649 } else if (seg == &kvseg_core) {
650 650 vmem_walk(heap_core_arena, VMEM_ALLOC | VMEM_REENTRANT,
651 651 segkmem_dump_range, seg->s_as);
652 652 } else if (seg == &kvseg32) {
653 653 vmem_walk(heap32_arena, VMEM_ALLOC | VMEM_REENTRANT,
654 654 segkmem_dump_range, seg->s_as);
655 655 vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
656 656 segkmem_dump_range, seg->s_as);
657 657 } else if (seg == &kzioseg) {
658 658 /*
659 659 * We don't want to dump pages attached to kzioseg since they
660 660 * contain file data from ZFS. If this page's segment is
661 661 * kzioseg return instead of writing it to the dump device.
662 662 */
663 663 return;
664 664 } else {
665 665 segkmem_dump_range(seg->s_as, seg->s_base, seg->s_size);
666 666 }
667 667 }
668 668
669 669 /*
670 670 * lock/unlock kmem pages over a given range [addr, addr+len).
671 671 * Returns a shadow list of pages in ppp. If there are holes
672 672 * in the range (e.g. some of the kernel mappings do not have
673 673 * underlying page_ts) returns ENOTSUP so that as_pagelock()
674 674 * will handle the range via as_fault(F_SOFTLOCK).
675 675 */
676 676 /*ARGSUSED*/
677 677 static int
678 678 segkmem_pagelock(struct seg *seg, caddr_t addr, size_t len,
679 679 page_t ***ppp, enum lock_type type, enum seg_rw rw)
680 680 {
681 681 page_t **pplist, *pp;
682 682 pgcnt_t npages;
683 683 spgcnt_t pg;
684 684 size_t nb;
685 685 struct vnode *vp = seg->s_data;
686 686
687 687 ASSERT(ppp != NULL);
688 688
689 689 /*
690 690 * If it is one of segkp pages, call into segkp.
691 691 */
692 692 if (segkp_bitmap && seg == &kvseg &&
693 693 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
694 694 return (segop_pagelock(segkp, addr, len, ppp, type, rw));
695 695
696 696 npages = btopr(len);
697 697 nb = sizeof (page_t *) * npages;
698 698
699 699 if (type == L_PAGEUNLOCK) {
700 700 pplist = *ppp;
701 701 ASSERT(pplist != NULL);
702 702
703 703 for (pg = 0; pg < npages; pg++) {
704 704 pp = pplist[pg];
705 705 page_unlock(pp);
706 706 }
707 707 kmem_free(pplist, nb);
708 708 return (0);
709 709 }
710 710
711 711 ASSERT(type == L_PAGELOCK);
712 712
713 713 pplist = kmem_alloc(nb, KM_NOSLEEP);
714 714 if (pplist == NULL) {
715 715 *ppp = NULL;
716 716 return (ENOTSUP); /* take the slow path */
717 717 }
718 718
719 719 for (pg = 0; pg < npages; pg++) {
720 720 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_SHARED);
721 721 if (pp == NULL) {
722 722 while (--pg >= 0)
723 723 page_unlock(pplist[pg]);
724 724 kmem_free(pplist, nb);
725 725 *ppp = NULL;
726 726 return (ENOTSUP);
727 727 }
728 728 pplist[pg] = pp;
729 729 addr += PAGESIZE;
730 730 }
731 731
732 732 *ppp = pplist;
733 733 return (0);
734 734 }
735 735
736 736 /*
737 737 * This is a dummy segkmem function overloaded to call segkp
738 738 * when segkp is under the heap.
739 739 */
740 740 /* ARGSUSED */
741 741 static int
742 742 segkmem_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
743 743 {
744 744 ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
745 745
746 746 if (seg->s_as != &kas)
747 747 segkmem_badop();
748 748
749 749 /*
750 750 * If it is one of segkp pages, call into segkp.
|
↓ open down ↓ |
750 lines elided |
↑ open up ↑ |
751 751 */
752 752 if (segkp_bitmap && seg == &kvseg &&
753 753 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
754 754 return (segop_getmemid(segkp, addr, memidp));
755 755
756 756 segkmem_badop();
757 757 return (0);
758 758 }
759 759
760 760 /*ARGSUSED*/
761 -static lgrp_mem_policy_info_t *
762 -segkmem_getpolicy(struct seg *seg, caddr_t addr)
763 -{
764 - return (NULL);
765 -}
766 -
767 -/*ARGSUSED*/
768 761 static int
769 762 segkmem_capable(struct seg *seg, segcapability_t capability)
770 763 {
771 764 if (capability == S_CAPABILITY_NOMINFLT)
772 765 return (1);
773 766 return (0);
774 767 }
775 768
776 769 static struct seg_ops segkmem_ops = {
777 770 .dup = SEGKMEM_BADOP(int),
778 771 .unmap = SEGKMEM_BADOP(int),
779 772 .free = SEGKMEM_BADOP(void),
780 773 .fault = segkmem_fault,
781 774 .faulta = SEGKMEM_BADOP(faultcode_t),
782 775 .setprot = segkmem_setprot,
783 776 .checkprot = segkmem_checkprot,
784 777 .kluster = segkmem_kluster,
785 778 .swapout = SEGKMEM_BADOP(size_t),
786 779 .sync = SEGKMEM_BADOP(int),
787 780 .incore = SEGKMEM_BADOP(size_t),
|
↓ open down ↓ |
10 lines elided |
↑ open up ↑ |
788 781 .lockop = SEGKMEM_BADOP(int),
789 782 .getprot = SEGKMEM_BADOP(int),
790 783 .getoffset = SEGKMEM_BADOP(u_offset_t),
791 784 .gettype = SEGKMEM_BADOP(int),
792 785 .getvp = SEGKMEM_BADOP(int),
793 786 .advise = SEGKMEM_BADOP(int),
794 787 .dump = segkmem_dump,
795 788 .pagelock = segkmem_pagelock,
796 789 .setpagesize = SEGKMEM_BADOP(int),
797 790 .getmemid = segkmem_getmemid,
798 - .getpolicy = segkmem_getpolicy,
799 791 .capable = segkmem_capable,
800 792 };
801 793
802 794 int
803 795 segkmem_zio_create(struct seg *seg)
804 796 {
805 797 ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock));
806 798 seg->s_ops = &segkmem_ops;
807 799 seg->s_data = &zvp;
808 800 kas.a_size += seg->s_size;
809 801 return (0);
810 802 }
811 803
812 804 int
813 805 segkmem_create(struct seg *seg)
814 806 {
815 807 ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock));
816 808 seg->s_ops = &segkmem_ops;
817 809 seg->s_data = &kvp;
818 810 kas.a_size += seg->s_size;
819 811 return (0);
820 812 }
821 813
822 814 /*ARGSUSED*/
823 815 page_t *
824 816 segkmem_page_create(void *addr, size_t size, int vmflag, void *arg)
825 817 {
826 818 struct seg kseg;
827 819 int pgflags;
828 820 struct vnode *vp = arg;
829 821
830 822 if (vp == NULL)
831 823 vp = &kvp;
832 824
833 825 kseg.s_as = &kas;
834 826 pgflags = PG_EXCL;
835 827
836 828 if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
837 829 pgflags |= PG_NORELOC;
838 830 if ((vmflag & VM_NOSLEEP) == 0)
839 831 pgflags |= PG_WAIT;
840 832 if (vmflag & VM_PANIC)
841 833 pgflags |= PG_PANIC;
842 834 if (vmflag & VM_PUSHPAGE)
843 835 pgflags |= PG_PUSHPAGE;
844 836 if (vmflag & VM_NORMALPRI) {
845 837 ASSERT(vmflag & VM_NOSLEEP);
846 838 pgflags |= PG_NORMALPRI;
847 839 }
848 840
849 841 return (page_create_va(vp, (u_offset_t)(uintptr_t)addr, size,
850 842 pgflags, &kseg, addr));
851 843 }
852 844
853 845 /*
854 846 * Allocate pages to back the virtual address range [addr, addr + size).
855 847 * If addr is NULL, allocate the virtual address space as well.
856 848 */
857 849 void *
858 850 segkmem_xalloc(vmem_t *vmp, void *inaddr, size_t size, int vmflag, uint_t attr,
859 851 page_t *(*page_create_func)(void *, size_t, int, void *), void *pcarg)
860 852 {
861 853 page_t *ppl;
862 854 caddr_t addr = inaddr;
863 855 pgcnt_t npages = btopr(size);
864 856 int allocflag;
865 857
866 858 if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
867 859 return (NULL);
868 860
869 861 ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
870 862
871 863 if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
872 864 if (inaddr == NULL)
873 865 vmem_free(vmp, addr, size);
874 866 return (NULL);
875 867 }
876 868
877 869 ppl = page_create_func(addr, size, vmflag, pcarg);
878 870 if (ppl == NULL) {
879 871 if (inaddr == NULL)
880 872 vmem_free(vmp, addr, size);
881 873 page_unresv(npages);
882 874 return (NULL);
883 875 }
884 876
885 877 /*
886 878 * Under certain conditions, we need to let the HAT layer know
887 879 * that it cannot safely allocate memory. Allocations from
888 880 * the hat_memload vmem arena always need this, to prevent
889 881 * infinite recursion.
890 882 *
891 883 * In addition, the x86 hat cannot safely do memory
892 884 * allocations while in vmem_populate(), because there
893 885 * is no simple bound on its usage.
894 886 */
895 887 if (vmflag & VM_MEMLOAD)
896 888 allocflag = HAT_NO_KALLOC;
897 889 #if defined(__x86)
898 890 else if (vmem_is_populator())
899 891 allocflag = HAT_NO_KALLOC;
900 892 #endif
901 893 else
902 894 allocflag = 0;
903 895
904 896 while (ppl != NULL) {
905 897 page_t *pp = ppl;
906 898 page_sub(&ppl, pp);
907 899 ASSERT(page_iolock_assert(pp));
908 900 ASSERT(PAGE_EXCL(pp));
909 901 page_io_unlock(pp);
910 902 hat_memload(kas.a_hat, (caddr_t)(uintptr_t)pp->p_offset, pp,
911 903 (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
912 904 HAT_LOAD_LOCK | allocflag);
913 905 pp->p_lckcnt = 1;
914 906 #if defined(__x86)
915 907 page_downgrade(pp);
916 908 #else
917 909 if (vmflag & SEGKMEM_SHARELOCKED)
918 910 page_downgrade(pp);
919 911 else
920 912 page_unlock(pp);
921 913 #endif
922 914 }
923 915
924 916 return (addr);
925 917 }
926 918
927 919 static void *
928 920 segkmem_alloc_vn(vmem_t *vmp, size_t size, int vmflag, struct vnode *vp)
929 921 {
930 922 void *addr;
931 923 segkmem_gc_list_t *gcp, **prev_gcpp;
932 924
933 925 ASSERT(vp != NULL);
934 926
935 927 if (kvseg.s_base == NULL) {
936 928 #ifndef __sparc
937 929 if (bootops->bsys_alloc == NULL)
938 930 halt("Memory allocation between bop_alloc() and "
939 931 "kmem_alloc().\n");
940 932 #endif
941 933
942 934 /*
943 935 * There's not a lot of memory to go around during boot,
944 936 * so recycle it if we can.
945 937 */
946 938 for (prev_gcpp = &segkmem_gc_list; (gcp = *prev_gcpp) != NULL;
947 939 prev_gcpp = &gcp->gc_next) {
948 940 if (gcp->gc_arena == vmp && gcp->gc_size == size) {
949 941 *prev_gcpp = gcp->gc_next;
950 942 return (gcp);
951 943 }
952 944 }
953 945
954 946 addr = vmem_alloc(vmp, size, vmflag | VM_PANIC);
955 947 if (boot_alloc(addr, size, BO_NO_ALIGN) != addr)
956 948 panic("segkmem_alloc: boot_alloc failed");
957 949 return (addr);
958 950 }
959 951 return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
960 952 segkmem_page_create, vp));
961 953 }
962 954
963 955 void *
964 956 segkmem_alloc(vmem_t *vmp, size_t size, int vmflag)
965 957 {
966 958 return (segkmem_alloc_vn(vmp, size, vmflag, &kvp));
967 959 }
968 960
969 961 void *
970 962 segkmem_zio_alloc(vmem_t *vmp, size_t size, int vmflag)
971 963 {
972 964 return (segkmem_alloc_vn(vmp, size, vmflag, &zvp));
973 965 }
974 966
975 967 /*
976 968 * Any changes to this routine must also be carried over to
977 969 * devmap_free_pages() in the seg_dev driver. This is because
978 970 * we currently don't have a special kernel segment for non-paged
979 971 * kernel memory that is exported by drivers to user space.
980 972 */
981 973 static void
982 974 segkmem_free_vn(vmem_t *vmp, void *inaddr, size_t size, struct vnode *vp,
983 975 void (*func)(page_t *))
984 976 {
985 977 page_t *pp;
986 978 caddr_t addr = inaddr;
987 979 caddr_t eaddr;
988 980 pgcnt_t npages = btopr(size);
989 981
990 982 ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
991 983 ASSERT(vp != NULL);
992 984
993 985 if (kvseg.s_base == NULL) {
994 986 segkmem_gc_list_t *gc = inaddr;
995 987 gc->gc_arena = vmp;
996 988 gc->gc_size = size;
997 989 gc->gc_next = segkmem_gc_list;
998 990 segkmem_gc_list = gc;
999 991 return;
1000 992 }
1001 993
1002 994 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1003 995
1004 996 for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
1005 997 #if defined(__x86)
1006 998 pp = page_find(vp, (u_offset_t)(uintptr_t)addr);
1007 999 if (pp == NULL)
1008 1000 panic("segkmem_free: page not found");
1009 1001 if (!page_tryupgrade(pp)) {
1010 1002 /*
1011 1003 * Some other thread has a sharelock. Wait for
1012 1004 * it to drop the lock so we can free this page.
1013 1005 */
1014 1006 page_unlock(pp);
1015 1007 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr,
1016 1008 SE_EXCL);
1017 1009 }
1018 1010 #else
1019 1011 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
1020 1012 #endif
1021 1013 if (pp == NULL)
1022 1014 panic("segkmem_free: page not found");
1023 1015 /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
1024 1016 pp->p_lckcnt = 0;
1025 1017 if (func)
1026 1018 func(pp);
1027 1019 else
1028 1020 page_destroy(pp, 0);
1029 1021 }
1030 1022 if (func == NULL)
1031 1023 page_unresv(npages);
1032 1024
1033 1025 if (vmp != NULL)
1034 1026 vmem_free(vmp, inaddr, size);
1035 1027
1036 1028 }
1037 1029
1038 1030 void
1039 1031 segkmem_xfree(vmem_t *vmp, void *inaddr, size_t size, void (*func)(page_t *))
1040 1032 {
1041 1033 segkmem_free_vn(vmp, inaddr, size, &kvp, func);
1042 1034 }
1043 1035
1044 1036 void
1045 1037 segkmem_free(vmem_t *vmp, void *inaddr, size_t size)
1046 1038 {
1047 1039 segkmem_free_vn(vmp, inaddr, size, &kvp, NULL);
1048 1040 }
1049 1041
1050 1042 void
1051 1043 segkmem_zio_free(vmem_t *vmp, void *inaddr, size_t size)
1052 1044 {
1053 1045 segkmem_free_vn(vmp, inaddr, size, &zvp, NULL);
1054 1046 }
1055 1047
1056 1048 void
1057 1049 segkmem_gc(void)
1058 1050 {
1059 1051 ASSERT(kvseg.s_base != NULL);
1060 1052 while (segkmem_gc_list != NULL) {
1061 1053 segkmem_gc_list_t *gc = segkmem_gc_list;
1062 1054 segkmem_gc_list = gc->gc_next;
1063 1055 segkmem_free(gc->gc_arena, gc, gc->gc_size);
1064 1056 }
1065 1057 }
1066 1058
1067 1059 /*
1068 1060 * Legacy entry points from here to end of file.
1069 1061 */
1070 1062 void
1071 1063 segkmem_mapin(struct seg *seg, void *addr, size_t size, uint_t vprot,
1072 1064 pfn_t pfn, uint_t flags)
1073 1065 {
1074 1066 hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1075 1067 hat_devload(seg->s_as->a_hat, addr, size, pfn, vprot,
1076 1068 flags | HAT_LOAD_LOCK);
1077 1069 }
1078 1070
1079 1071 void
1080 1072 segkmem_mapout(struct seg *seg, void *addr, size_t size)
1081 1073 {
1082 1074 hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1083 1075 }
1084 1076
1085 1077 void *
1086 1078 kmem_getpages(pgcnt_t npages, int kmflag)
1087 1079 {
1088 1080 return (kmem_alloc(ptob(npages), kmflag));
1089 1081 }
1090 1082
1091 1083 void
1092 1084 kmem_freepages(void *addr, pgcnt_t npages)
1093 1085 {
1094 1086 kmem_free(addr, ptob(npages));
1095 1087 }
1096 1088
1097 1089 /*
1098 1090 * segkmem_page_create_large() allocates a large page to be used for the kmem
1099 1091 * caches. If kpr is enabled we ask for a relocatable page unless requested
1100 1092 * otherwise. If kpr is disabled we have to ask for a non-reloc page
1101 1093 */
1102 1094 static page_t *
1103 1095 segkmem_page_create_large(void *addr, size_t size, int vmflag, void *arg)
1104 1096 {
1105 1097 int pgflags;
1106 1098
1107 1099 pgflags = PG_EXCL;
1108 1100
1109 1101 if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
1110 1102 pgflags |= PG_NORELOC;
1111 1103 if (!(vmflag & VM_NOSLEEP))
1112 1104 pgflags |= PG_WAIT;
1113 1105 if (vmflag & VM_PUSHPAGE)
1114 1106 pgflags |= PG_PUSHPAGE;
1115 1107 if (vmflag & VM_NORMALPRI)
1116 1108 pgflags |= PG_NORMALPRI;
1117 1109
1118 1110 return (page_create_va_large(&kvp, (u_offset_t)(uintptr_t)addr, size,
1119 1111 pgflags, &kvseg, addr, arg));
1120 1112 }
1121 1113
1122 1114 /*
1123 1115 * Allocate a large page to back the virtual address range
1124 1116 * [addr, addr + size). If addr is NULL, allocate the virtual address
1125 1117 * space as well.
1126 1118 */
1127 1119 static void *
1128 1120 segkmem_xalloc_lp(vmem_t *vmp, void *inaddr, size_t size, int vmflag,
1129 1121 uint_t attr, page_t *(*page_create_func)(void *, size_t, int, void *),
1130 1122 void *pcarg)
1131 1123 {
1132 1124 caddr_t addr = inaddr, pa;
1133 1125 size_t lpsize = segkmem_lpsize;
1134 1126 pgcnt_t npages = btopr(size);
1135 1127 pgcnt_t nbpages = btop(lpsize);
1136 1128 pgcnt_t nlpages = size >> segkmem_lpshift;
1137 1129 size_t ppasize = nbpages * sizeof (page_t *);
1138 1130 page_t *pp, *rootpp, **ppa, *pplist = NULL;
1139 1131 int i;
1140 1132
1141 1133 vmflag |= VM_NOSLEEP;
1142 1134
1143 1135 if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
1144 1136 return (NULL);
1145 1137 }
1146 1138
1147 1139 /*
1148 1140 * allocate an array we need for hat_memload_array.
1149 1141 * we use a separate arena to avoid recursion.
1150 1142 * we will not need this array when hat_memload_array learns pp++
1151 1143 */
1152 1144 if ((ppa = vmem_alloc(segkmem_ppa_arena, ppasize, vmflag)) == NULL) {
1153 1145 goto fail_array_alloc;
1154 1146 }
1155 1147
1156 1148 if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
1157 1149 goto fail_vmem_alloc;
1158 1150
1159 1151 ASSERT(((uintptr_t)addr & (lpsize - 1)) == 0);
1160 1152
1161 1153 /* create all the pages */
1162 1154 for (pa = addr, i = 0; i < nlpages; i++, pa += lpsize) {
1163 1155 if ((pp = page_create_func(pa, lpsize, vmflag, pcarg)) == NULL)
1164 1156 goto fail_page_create;
1165 1157 page_list_concat(&pplist, &pp);
1166 1158 }
1167 1159
1168 1160 /* at this point we have all the resource to complete the request */
1169 1161 while ((rootpp = pplist) != NULL) {
1170 1162 for (i = 0; i < nbpages; i++) {
1171 1163 ASSERT(pplist != NULL);
1172 1164 pp = pplist;
1173 1165 page_sub(&pplist, pp);
1174 1166 ASSERT(page_iolock_assert(pp));
1175 1167 page_io_unlock(pp);
1176 1168 ppa[i] = pp;
1177 1169 }
1178 1170 /*
1179 1171 * Load the locked entry. It's OK to preload the entry into the
1180 1172 * TSB since we now support large mappings in the kernel TSB.
1181 1173 */
1182 1174 hat_memload_array(kas.a_hat,
1183 1175 (caddr_t)(uintptr_t)rootpp->p_offset, lpsize,
1184 1176 ppa, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
1185 1177 HAT_LOAD_LOCK);
1186 1178
1187 1179 for (--i; i >= 0; --i) {
1188 1180 ppa[i]->p_lckcnt = 1;
1189 1181 page_unlock(ppa[i]);
1190 1182 }
1191 1183 }
1192 1184
1193 1185 vmem_free(segkmem_ppa_arena, ppa, ppasize);
1194 1186 return (addr);
1195 1187
1196 1188 fail_page_create:
1197 1189 while ((rootpp = pplist) != NULL) {
1198 1190 for (i = 0, pp = pplist; i < nbpages; i++, pp = pplist) {
1199 1191 ASSERT(pp != NULL);
1200 1192 page_sub(&pplist, pp);
1201 1193 ASSERT(page_iolock_assert(pp));
1202 1194 page_io_unlock(pp);
1203 1195 }
1204 1196 page_destroy_pages(rootpp);
1205 1197 }
1206 1198
1207 1199 if (inaddr == NULL)
1208 1200 vmem_free(vmp, addr, size);
1209 1201
1210 1202 fail_vmem_alloc:
1211 1203 vmem_free(segkmem_ppa_arena, ppa, ppasize);
1212 1204
1213 1205 fail_array_alloc:
1214 1206 page_unresv(npages);
1215 1207
1216 1208 return (NULL);
1217 1209 }
1218 1210
1219 1211 static void
1220 1212 segkmem_free_one_lp(caddr_t addr, size_t size)
1221 1213 {
1222 1214 page_t *pp, *rootpp = NULL;
1223 1215 pgcnt_t pgs_left = btopr(size);
1224 1216
1225 1217 ASSERT(size == segkmem_lpsize);
1226 1218
1227 1219 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1228 1220
1229 1221 for (; pgs_left > 0; addr += PAGESIZE, pgs_left--) {
1230 1222 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
1231 1223 if (pp == NULL)
1232 1224 panic("segkmem_free_one_lp: page not found");
1233 1225 ASSERT(PAGE_EXCL(pp));
1234 1226 pp->p_lckcnt = 0;
1235 1227 if (rootpp == NULL)
1236 1228 rootpp = pp;
1237 1229 }
1238 1230 ASSERT(rootpp != NULL);
1239 1231 page_destroy_pages(rootpp);
1240 1232
1241 1233 /* page_unresv() is done by the caller */
1242 1234 }
1243 1235
1244 1236 /*
1245 1237 * This function is called to import new spans into the vmem arenas like
1246 1238 * kmem_default_arena and kmem_oversize_arena. It first tries to import
1247 1239 * spans from large page arena - kmem_lp_arena. In order to do this it might
1248 1240 * have to "upgrade the requested size" to kmem_lp_arena quantum. If
1249 1241 * it was not able to satisfy the upgraded request it then calls regular
1250 1242 * segkmem_alloc() that satisfies the request by importing from "*vmp" arena
1251 1243 */
1252 1244 /*ARGSUSED*/
1253 1245 void *
1254 1246 segkmem_alloc_lp(vmem_t *vmp, size_t *sizep, size_t align, int vmflag)
1255 1247 {
1256 1248 size_t size;
1257 1249 kthread_t *t = curthread;
1258 1250 segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1259 1251
1260 1252 ASSERT(sizep != NULL);
1261 1253
1262 1254 size = *sizep;
1263 1255
1264 1256 if (lpcb->lp_uselp && !(t->t_flag & T_PANIC) &&
1265 1257 !(vmflag & SEGKMEM_SHARELOCKED)) {
1266 1258
1267 1259 size_t kmemlp_qnt = segkmem_kmemlp_quantum;
1268 1260 size_t asize = P2ROUNDUP(size, kmemlp_qnt);
1269 1261 void *addr = NULL;
1270 1262 ulong_t *lpthrtp = &lpcb->lp_throttle;
1271 1263 ulong_t lpthrt = *lpthrtp;
1272 1264 int dowakeup = 0;
1273 1265 int doalloc = 1;
1274 1266
1275 1267 ASSERT(kmem_lp_arena != NULL);
1276 1268 ASSERT(asize >= size);
1277 1269
1278 1270 if (lpthrt != 0) {
1279 1271 /* try to update the throttle value */
1280 1272 lpthrt = atomic_inc_ulong_nv(lpthrtp);
1281 1273 if (lpthrt >= segkmem_lpthrottle_max) {
1282 1274 lpthrt = atomic_cas_ulong(lpthrtp, lpthrt,
1283 1275 segkmem_lpthrottle_max / 4);
1284 1276 }
1285 1277
1286 1278 /*
1287 1279 * when we get above throttle start do an exponential
1288 1280 * backoff at trying large pages and reaping
1289 1281 */
1290 1282 if (lpthrt > segkmem_lpthrottle_start &&
1291 1283 !ISP2(lpthrt)) {
1292 1284 lpcb->allocs_throttled++;
1293 1285 lpthrt--;
1294 1286 if (ISP2(lpthrt))
1295 1287 kmem_reap();
1296 1288 return (segkmem_alloc(vmp, size, vmflag));
1297 1289 }
1298 1290 }
1299 1291
1300 1292 if (!(vmflag & VM_NOSLEEP) &&
1301 1293 segkmem_heaplp_quantum >= (8 * kmemlp_qnt) &&
1302 1294 vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt &&
1303 1295 asize < (segkmem_heaplp_quantum - kmemlp_qnt)) {
1304 1296
1305 1297 /*
1306 1298 * we are low on free memory in kmem_lp_arena
1307 1299 * we let only one guy to allocate heap_lp
1308 1300 * quantum size chunk that everybody is going to
1309 1301 * share
1310 1302 */
1311 1303 mutex_enter(&lpcb->lp_lock);
1312 1304
1313 1305 if (lpcb->lp_wait) {
1314 1306
1315 1307 /* we are not the first one - wait */
1316 1308 cv_wait(&lpcb->lp_cv, &lpcb->lp_lock);
1317 1309 if (vmem_size(kmem_lp_arena, VMEM_FREE) <
1318 1310 kmemlp_qnt) {
1319 1311 doalloc = 0;
1320 1312 }
1321 1313 } else if (vmem_size(kmem_lp_arena, VMEM_FREE) <=
1322 1314 kmemlp_qnt) {
1323 1315
1324 1316 /*
1325 1317 * we are the first one, make sure we import
1326 1318 * a large page
1327 1319 */
1328 1320 if (asize == kmemlp_qnt)
1329 1321 asize += kmemlp_qnt;
1330 1322 dowakeup = 1;
1331 1323 lpcb->lp_wait = 1;
1332 1324 }
1333 1325
1334 1326 mutex_exit(&lpcb->lp_lock);
1335 1327 }
1336 1328
1337 1329 /*
1338 1330 * VM_ABORT flag prevents sleeps in vmem_xalloc when
1339 1331 * large pages are not available. In that case this allocation
1340 1332 * attempt will fail and we will retry allocation with small
1341 1333 * pages. We also do not want to panic if this allocation fails
1342 1334 * because we are going to retry.
1343 1335 */
1344 1336 if (doalloc) {
1345 1337 addr = vmem_alloc(kmem_lp_arena, asize,
1346 1338 (vmflag | VM_ABORT) & ~VM_PANIC);
1347 1339
1348 1340 if (dowakeup) {
1349 1341 mutex_enter(&lpcb->lp_lock);
1350 1342 ASSERT(lpcb->lp_wait != 0);
1351 1343 lpcb->lp_wait = 0;
1352 1344 cv_broadcast(&lpcb->lp_cv);
1353 1345 mutex_exit(&lpcb->lp_lock);
1354 1346 }
1355 1347 }
1356 1348
1357 1349 if (addr != NULL) {
1358 1350 *sizep = asize;
1359 1351 *lpthrtp = 0;
1360 1352 return (addr);
1361 1353 }
1362 1354
1363 1355 if (vmflag & VM_NOSLEEP)
1364 1356 lpcb->nosleep_allocs_failed++;
1365 1357 else
1366 1358 lpcb->sleep_allocs_failed++;
1367 1359 lpcb->alloc_bytes_failed += size;
1368 1360
1369 1361 /* if large page throttling is not started yet do it */
1370 1362 if (segkmem_use_lpthrottle && lpthrt == 0) {
1371 1363 lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 1);
1372 1364 }
1373 1365 }
1374 1366 return (segkmem_alloc(vmp, size, vmflag));
1375 1367 }
1376 1368
1377 1369 void
1378 1370 segkmem_free_lp(vmem_t *vmp, void *inaddr, size_t size)
1379 1371 {
1380 1372 if (kmem_lp_arena == NULL || !IS_KMEM_VA_LARGEPAGE((caddr_t)inaddr)) {
1381 1373 segkmem_free(vmp, inaddr, size);
1382 1374 } else {
1383 1375 vmem_free(kmem_lp_arena, inaddr, size);
1384 1376 }
1385 1377 }
1386 1378
1387 1379 /*
1388 1380 * segkmem_alloc_lpi() imports virtual memory from large page heap arena
1389 1381 * into kmem_lp arena. In the process it maps the imported segment with
1390 1382 * large pages
1391 1383 */
1392 1384 static void *
1393 1385 segkmem_alloc_lpi(vmem_t *vmp, size_t size, int vmflag)
1394 1386 {
1395 1387 segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1396 1388 void *addr;
1397 1389
1398 1390 ASSERT(size != 0);
1399 1391 ASSERT(vmp == heap_lp_arena);
1400 1392
1401 1393 /* do not allow large page heap grow beyound limits */
1402 1394 if (vmem_size(vmp, VMEM_ALLOC) >= segkmem_kmemlp_max) {
1403 1395 lpcb->allocs_limited++;
1404 1396 return (NULL);
1405 1397 }
1406 1398
1407 1399 addr = segkmem_xalloc_lp(vmp, NULL, size, vmflag, 0,
1408 1400 segkmem_page_create_large, NULL);
1409 1401 return (addr);
1410 1402 }
1411 1403
1412 1404 /*
1413 1405 * segkmem_free_lpi() returns virtual memory back into large page heap arena
1414 1406 * from kmem_lp arena. Beore doing this it unmaps the segment and frees
1415 1407 * large pages used to map it.
1416 1408 */
1417 1409 static void
1418 1410 segkmem_free_lpi(vmem_t *vmp, void *inaddr, size_t size)
1419 1411 {
1420 1412 pgcnt_t nlpages = size >> segkmem_lpshift;
1421 1413 size_t lpsize = segkmem_lpsize;
1422 1414 caddr_t addr = inaddr;
1423 1415 pgcnt_t npages = btopr(size);
1424 1416 int i;
1425 1417
1426 1418 ASSERT(vmp == heap_lp_arena);
1427 1419 ASSERT(IS_KMEM_VA_LARGEPAGE(addr));
1428 1420 ASSERT(((uintptr_t)inaddr & (lpsize - 1)) == 0);
1429 1421
1430 1422 for (i = 0; i < nlpages; i++) {
1431 1423 segkmem_free_one_lp(addr, lpsize);
1432 1424 addr += lpsize;
1433 1425 }
1434 1426
1435 1427 page_unresv(npages);
1436 1428
1437 1429 vmem_free(vmp, inaddr, size);
1438 1430 }
1439 1431
1440 1432 /*
1441 1433 * This function is called at system boot time by kmem_init right after
1442 1434 * /etc/system file has been read. It checks based on hardware configuration
1443 1435 * and /etc/system settings if system is going to use large pages. The
1444 1436 * initialiazation necessary to actually start using large pages
1445 1437 * happens later in the process after segkmem_heap_lp_init() is called.
1446 1438 */
1447 1439 int
1448 1440 segkmem_lpsetup()
1449 1441 {
1450 1442 int use_large_pages = 0;
1451 1443
1452 1444 #ifdef __sparc
1453 1445
1454 1446 size_t memtotal = physmem * PAGESIZE;
1455 1447
1456 1448 if (heap_lp_base == NULL) {
1457 1449 segkmem_lpsize = PAGESIZE;
1458 1450 return (0);
1459 1451 }
1460 1452
1461 1453 /* get a platform dependent value of large page size for kernel heap */
1462 1454 segkmem_lpsize = get_segkmem_lpsize(segkmem_lpsize);
1463 1455
1464 1456 if (segkmem_lpsize <= PAGESIZE) {
1465 1457 /*
1466 1458 * put virtual space reserved for the large page kernel
1467 1459 * back to the regular heap
1468 1460 */
1469 1461 vmem_xfree(heap_arena, heap_lp_base,
1470 1462 heap_lp_end - heap_lp_base);
1471 1463 heap_lp_base = NULL;
1472 1464 heap_lp_end = NULL;
1473 1465 segkmem_lpsize = PAGESIZE;
1474 1466 return (0);
1475 1467 }
1476 1468
1477 1469 /* set heap_lp quantum if necessary */
1478 1470 if (segkmem_heaplp_quantum == 0 || !ISP2(segkmem_heaplp_quantum) ||
1479 1471 P2PHASE(segkmem_heaplp_quantum, segkmem_lpsize)) {
1480 1472 segkmem_heaplp_quantum = segkmem_lpsize;
1481 1473 }
1482 1474
1483 1475 /* set kmem_lp quantum if necessary */
1484 1476 if (segkmem_kmemlp_quantum == 0 || !ISP2(segkmem_kmemlp_quantum) ||
1485 1477 segkmem_kmemlp_quantum > segkmem_heaplp_quantum) {
1486 1478 segkmem_kmemlp_quantum = segkmem_heaplp_quantum;
1487 1479 }
1488 1480
1489 1481 /* set total amount of memory allowed for large page kernel heap */
1490 1482 if (segkmem_kmemlp_max == 0) {
1491 1483 if (segkmem_kmemlp_pcnt == 0 || segkmem_kmemlp_pcnt > 100)
1492 1484 segkmem_kmemlp_pcnt = 12;
1493 1485 segkmem_kmemlp_max = (memtotal * segkmem_kmemlp_pcnt) / 100;
1494 1486 }
1495 1487 segkmem_kmemlp_max = P2ROUNDUP(segkmem_kmemlp_max,
1496 1488 segkmem_heaplp_quantum);
1497 1489
1498 1490 /* fix lp kmem preallocation request if necesssary */
1499 1491 if (segkmem_kmemlp_min) {
1500 1492 segkmem_kmemlp_min = P2ROUNDUP(segkmem_kmemlp_min,
1501 1493 segkmem_heaplp_quantum);
1502 1494 if (segkmem_kmemlp_min > segkmem_kmemlp_max)
1503 1495 segkmem_kmemlp_min = segkmem_kmemlp_max;
1504 1496 }
1505 1497
1506 1498 use_large_pages = 1;
1507 1499 segkmem_lpszc = page_szc(segkmem_lpsize);
1508 1500 segkmem_lpshift = page_get_shift(segkmem_lpszc);
1509 1501
1510 1502 #endif
1511 1503 return (use_large_pages);
1512 1504 }
1513 1505
1514 1506 void
1515 1507 segkmem_zio_init(void *zio_mem_base, size_t zio_mem_size)
1516 1508 {
1517 1509 ASSERT(zio_mem_base != NULL);
1518 1510 ASSERT(zio_mem_size != 0);
1519 1511
1520 1512 /*
1521 1513 * To reduce VA space fragmentation, we set up quantum caches for the
1522 1514 * smaller sizes; we chose 32k because that translates to 128k VA
1523 1515 * slabs, which matches nicely with the common 128k zio_data bufs.
1524 1516 */
1525 1517 zio_arena = vmem_create("zfs_file_data", zio_mem_base, zio_mem_size,
1526 1518 PAGESIZE, NULL, NULL, NULL, 32 * 1024, VM_SLEEP);
1527 1519
1528 1520 zio_alloc_arena = vmem_create("zfs_file_data_buf", NULL, 0, PAGESIZE,
1529 1521 segkmem_zio_alloc, segkmem_zio_free, zio_arena, 0, VM_SLEEP);
1530 1522
1531 1523 ASSERT(zio_arena != NULL);
1532 1524 ASSERT(zio_alloc_arena != NULL);
1533 1525 }
1534 1526
1535 1527 #ifdef __sparc
1536 1528
1537 1529
1538 1530 static void *
1539 1531 segkmem_alloc_ppa(vmem_t *vmp, size_t size, int vmflag)
1540 1532 {
1541 1533 size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
1542 1534 void *addr;
1543 1535
1544 1536 if (ppaquantum <= PAGESIZE)
1545 1537 return (segkmem_alloc(vmp, size, vmflag));
1546 1538
1547 1539 ASSERT((size & (ppaquantum - 1)) == 0);
1548 1540
1549 1541 addr = vmem_xalloc(vmp, size, ppaquantum, 0, 0, NULL, NULL, vmflag);
1550 1542 if (addr != NULL && segkmem_xalloc(vmp, addr, size, vmflag, 0,
1551 1543 segkmem_page_create, NULL) == NULL) {
1552 1544 vmem_xfree(vmp, addr, size);
1553 1545 addr = NULL;
1554 1546 }
1555 1547
1556 1548 return (addr);
1557 1549 }
1558 1550
1559 1551 static void
1560 1552 segkmem_free_ppa(vmem_t *vmp, void *addr, size_t size)
1561 1553 {
1562 1554 size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
1563 1555
1564 1556 ASSERT(addr != NULL);
1565 1557
1566 1558 if (ppaquantum <= PAGESIZE) {
1567 1559 segkmem_free(vmp, addr, size);
1568 1560 } else {
1569 1561 segkmem_free(NULL, addr, size);
1570 1562 vmem_xfree(vmp, addr, size);
1571 1563 }
1572 1564 }
1573 1565
1574 1566 void
1575 1567 segkmem_heap_lp_init()
1576 1568 {
1577 1569 segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1578 1570 size_t heap_lp_size = heap_lp_end - heap_lp_base;
1579 1571 size_t lpsize = segkmem_lpsize;
1580 1572 size_t ppaquantum;
1581 1573 void *addr;
1582 1574
1583 1575 if (segkmem_lpsize <= PAGESIZE) {
1584 1576 ASSERT(heap_lp_base == NULL);
1585 1577 ASSERT(heap_lp_end == NULL);
1586 1578 return;
1587 1579 }
1588 1580
1589 1581 ASSERT(segkmem_heaplp_quantum >= lpsize);
1590 1582 ASSERT((segkmem_heaplp_quantum & (lpsize - 1)) == 0);
1591 1583 ASSERT(lpcb->lp_uselp == 0);
1592 1584 ASSERT(heap_lp_base != NULL);
1593 1585 ASSERT(heap_lp_end != NULL);
1594 1586 ASSERT(heap_lp_base < heap_lp_end);
1595 1587 ASSERT(heap_lp_arena == NULL);
1596 1588 ASSERT(((uintptr_t)heap_lp_base & (lpsize - 1)) == 0);
1597 1589 ASSERT(((uintptr_t)heap_lp_end & (lpsize - 1)) == 0);
1598 1590
1599 1591 /* create large page heap arena */
1600 1592 heap_lp_arena = vmem_create("heap_lp", heap_lp_base, heap_lp_size,
1601 1593 segkmem_heaplp_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1602 1594
1603 1595 ASSERT(heap_lp_arena != NULL);
1604 1596
1605 1597 /* This arena caches memory already mapped by large pages */
1606 1598 kmem_lp_arena = vmem_create("kmem_lp", NULL, 0, segkmem_kmemlp_quantum,
1607 1599 segkmem_alloc_lpi, segkmem_free_lpi, heap_lp_arena, 0, VM_SLEEP);
1608 1600
1609 1601 ASSERT(kmem_lp_arena != NULL);
1610 1602
1611 1603 mutex_init(&lpcb->lp_lock, NULL, MUTEX_DEFAULT, NULL);
1612 1604 cv_init(&lpcb->lp_cv, NULL, CV_DEFAULT, NULL);
1613 1605
1614 1606 /*
1615 1607 * this arena is used for the array of page_t pointers necessary
1616 1608 * to call hat_mem_load_array
1617 1609 */
1618 1610 ppaquantum = btopr(lpsize) * sizeof (page_t *);
1619 1611 segkmem_ppa_arena = vmem_create("segkmem_ppa", NULL, 0, ppaquantum,
1620 1612 segkmem_alloc_ppa, segkmem_free_ppa, heap_arena, ppaquantum,
1621 1613 VM_SLEEP);
1622 1614
1623 1615 ASSERT(segkmem_ppa_arena != NULL);
1624 1616
1625 1617 /* prealloacate some memory for the lp kernel heap */
1626 1618 if (segkmem_kmemlp_min) {
1627 1619
1628 1620 ASSERT(P2PHASE(segkmem_kmemlp_min,
1629 1621 segkmem_heaplp_quantum) == 0);
1630 1622
1631 1623 if ((addr = segkmem_alloc_lpi(heap_lp_arena,
1632 1624 segkmem_kmemlp_min, VM_SLEEP)) != NULL) {
1633 1625
1634 1626 addr = vmem_add(kmem_lp_arena, addr,
1635 1627 segkmem_kmemlp_min, VM_SLEEP);
1636 1628 ASSERT(addr != NULL);
1637 1629 }
1638 1630 }
1639 1631
1640 1632 lpcb->lp_uselp = 1;
1641 1633 }
1642 1634
1643 1635 #endif
|
↓ open down ↓ |
835 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX