1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ 27 /* All Rights Reserved */ 28 29 /* 30 * Portions of this source code were derived from Berkeley 4.3 BSD 31 * under license from the Regents of the University of California. 32 */ 33 34 /* 35 * VM - segment for non-faulting loads. 36 */ 37 38 #include <sys/types.h> 39 #include <sys/t_lock.h> 40 #include <sys/param.h> 41 #include <sys/mman.h> 42 #include <sys/errno.h> 43 #include <sys/kmem.h> 44 #include <sys/cmn_err.h> 45 #include <sys/vnode.h> 46 #include <sys/proc.h> 47 #include <sys/conf.h> 48 #include <sys/debug.h> 49 #include <sys/archsystm.h> 50 #include <sys/lgrp.h> 51 52 #include <vm/page.h> 53 #include <vm/hat.h> 54 #include <vm/as.h> 55 #include <vm/seg.h> 56 #include <vm/vpage.h> 57 58 /* 59 * Private seg op routines. 60 */ 61 static int segnf_dup(struct seg *seg, struct seg *newseg); 62 static int segnf_unmap(struct seg *seg, caddr_t addr, size_t len); 63 static void segnf_free(struct seg *seg); 64 static faultcode_t segnf_nomap(void); 65 static int segnf_setprot(struct seg *seg, caddr_t addr, 66 size_t len, uint_t prot); 67 static int segnf_checkprot(struct seg *seg, caddr_t addr, 68 size_t len, uint_t prot); 69 static void segnf_badop(void); 70 static int segnf_nop(void); 71 static int segnf_getprot(struct seg *seg, caddr_t addr, 72 size_t len, uint_t *protv); 73 static u_offset_t segnf_getoffset(struct seg *seg, caddr_t addr); 74 static int segnf_gettype(struct seg *seg, caddr_t addr); 75 static int segnf_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp); 76 static void segnf_dump(struct seg *seg); 77 static int segnf_pagelock(struct seg *seg, caddr_t addr, size_t len, 78 struct page ***ppp, enum lock_type type, enum seg_rw rw); 79 static int segnf_setpagesize(struct seg *seg, caddr_t addr, size_t len, 80 uint_t szc); 81 static int segnf_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp); 82 static lgrp_mem_policy_info_t *segnf_getpolicy(struct seg *seg, 83 caddr_t addr); 84 85 86 struct seg_ops segnf_ops = { 87 .dup = segnf_dup, 88 .unmap = segnf_unmap, 89 .free = segnf_free, 90 .fault = (faultcode_t (*)(struct hat *, struct seg *, caddr_t, 91 size_t, enum fault_type, enum seg_rw))segnf_nomap, 92 .faulta = (faultcode_t (*)(struct seg *, caddr_t)) segnf_nomap, 93 .setprot = segnf_setprot, 94 .checkprot = segnf_checkprot, 95 .kluster = (int (*)())segnf_badop, 96 .sync = (int (*)(struct seg *, caddr_t, size_t, int, uint_t)) 97 segnf_nop, 98 .incore = (size_t (*)(struct seg *, caddr_t, size_t, char *)) 99 segnf_nop, 100 .lockop = (int (*)(struct seg *, caddr_t, size_t, int, int, 101 ulong_t *, size_t))segnf_nop, 102 .getprot = segnf_getprot, 103 .getoffset = segnf_getoffset, 104 .gettype = segnf_gettype, 105 .getvp = segnf_getvp, 106 .advise = (int (*)(struct seg *, caddr_t, size_t, uint_t)) 107 segnf_nop, 108 .dump = segnf_dump, 109 .pagelock = segnf_pagelock, 110 .setpagesize = segnf_setpagesize, 111 .getmemid = segnf_getmemid, 112 .getpolicy = segnf_getpolicy, 113 }; 114 115 /* 116 * vnode and page for the page of zeros we use for the nf mappings. 117 */ 118 static kmutex_t segnf_lock; 119 static struct vnode nfvp; 120 static struct page **nfpp; 121 122 #define addr_to_vcolor(addr) \ 123 (shm_alignment) ? \ 124 ((int)(((uintptr_t)(addr) & (shm_alignment - 1)) >> PAGESHIFT)) : 0 125 126 /* 127 * We try to limit the number of Non-fault segments created. 128 * Non fault segments are created to optimize sparc V9 code which uses 129 * the sparc nonfaulting load ASI (ASI_PRIMARY_NOFAULT). 130 * 131 * There are several reasons why creating too many non-fault segments 132 * could cause problems. 133 * 134 * First, excessive allocation of kernel resources for the seg 135 * structures and the HAT data to map the zero pages. 136 * 137 * Secondly, creating nofault segments actually uses up user virtual 138 * address space. This makes it unavailable for subsequent mmap(0, ...) 139 * calls which use as_gap() to find empty va regions. Creation of too 140 * many nofault segments could thus interfere with the ability of the 141 * runtime linker to load a shared object. 142 */ 143 #define MAXSEGFORNF (10000) 144 #define MAXNFSEARCH (5) 145 146 147 /* 148 * Must be called from startup() 149 */ 150 void 151 segnf_init() 152 { 153 mutex_init(&segnf_lock, NULL, MUTEX_DEFAULT, NULL); 154 } 155 156 157 /* 158 * Create a no-fault segment. 159 * 160 * The no-fault segment is not technically necessary, as the code in 161 * nfload() in trap.c will emulate the SPARC instruction and load 162 * a value of zero in the destination register. 163 * 164 * However, this code tries to put a page of zero's at the nofault address 165 * so that subsequent non-faulting loads to the same page will not 166 * trap with a tlb miss. 167 * 168 * In order to help limit the number of segments we merge adjacent nofault 169 * segments into a single segment. If we get a large number of segments 170 * we'll also try to delete a random other nf segment. 171 */ 172 /* ARGSUSED */ 173 int 174 segnf_create(struct seg *seg, void *argsp) 175 { 176 uint_t prot; 177 pgcnt_t vacpgs; 178 u_offset_t off = 0; 179 caddr_t vaddr = NULL; 180 int i, color; 181 struct seg *s1; 182 struct seg *s2; 183 size_t size; 184 struct as *as = seg->s_as; 185 186 ASSERT(as && AS_WRITE_HELD(as, &as->a_lock)); 187 188 /* 189 * Need a page per virtual color or just 1 if no vac. 190 */ 191 mutex_enter(&segnf_lock); 192 if (nfpp == NULL) { 193 struct seg kseg; 194 195 vacpgs = 1; 196 if (shm_alignment > PAGESIZE) { 197 vacpgs = shm_alignment >> PAGESHIFT; 198 } 199 200 nfpp = kmem_alloc(sizeof (*nfpp) * vacpgs, KM_SLEEP); 201 202 kseg.s_as = &kas; 203 for (i = 0; i < vacpgs; i++, off += PAGESIZE, 204 vaddr += PAGESIZE) { 205 nfpp[i] = page_create_va(&nfvp, off, PAGESIZE, 206 PG_WAIT | PG_NORELOC, &kseg, vaddr); 207 page_io_unlock(nfpp[i]); 208 page_downgrade(nfpp[i]); 209 pagezero(nfpp[i], 0, PAGESIZE); 210 } 211 } 212 mutex_exit(&segnf_lock); 213 214 hat_map(as->a_hat, seg->s_base, seg->s_size, HAT_MAP); 215 216 /* 217 * s_data can't be NULL because of ASSERTS in the common vm code. 218 */ 219 seg->s_ops = &segnf_ops; 220 seg->s_data = seg; 221 seg->s_flags |= S_PURGE; 222 223 mutex_enter(&as->a_contents); 224 as->a_flags |= AS_NEEDSPURGE; 225 mutex_exit(&as->a_contents); 226 227 prot = PROT_READ; 228 color = addr_to_vcolor(seg->s_base); 229 if (as != &kas) 230 prot |= PROT_USER; 231 hat_memload(as->a_hat, seg->s_base, nfpp[color], 232 prot | HAT_NOFAULT, HAT_LOAD); 233 234 /* 235 * At this point see if we can concatenate a segment to 236 * a non-fault segment immediately before and/or after it. 237 */ 238 if ((s1 = AS_SEGPREV(as, seg)) != NULL && 239 s1->s_ops == &segnf_ops && 240 s1->s_base + s1->s_size == seg->s_base) { 241 size = s1->s_size; 242 seg_free(s1); 243 seg->s_base -= size; 244 seg->s_size += size; 245 } 246 247 if ((s2 = AS_SEGNEXT(as, seg)) != NULL && 248 s2->s_ops == &segnf_ops && 249 seg->s_base + seg->s_size == s2->s_base) { 250 size = s2->s_size; 251 seg_free(s2); 252 seg->s_size += size; 253 } 254 255 /* 256 * if we already have a lot of segments, try to delete some other 257 * nofault segment to reduce the probability of uncontrolled segment 258 * creation. 259 * 260 * the code looks around quickly (no more than MAXNFSEARCH segments 261 * each way) for another NF segment and then deletes it. 262 */ 263 if (avl_numnodes(&as->a_segtree) > MAXSEGFORNF) { 264 size = 0; 265 s2 = NULL; 266 s1 = AS_SEGPREV(as, seg); 267 while (size++ < MAXNFSEARCH && s1 != NULL) { 268 if (s1->s_ops == &segnf_ops) 269 s2 = s1; 270 s1 = AS_SEGPREV(s1->s_as, seg); 271 } 272 if (s2 == NULL) { 273 s1 = AS_SEGNEXT(as, seg); 274 while (size-- > 0 && s1 != NULL) { 275 if (s1->s_ops == &segnf_ops) 276 s2 = s1; 277 s1 = AS_SEGNEXT(as, seg); 278 } 279 } 280 if (s2 != NULL) 281 seg_unmap(s2); 282 } 283 284 return (0); 285 } 286 287 /* 288 * Never really need "No fault" segments, so they aren't dup'd. 289 */ 290 /* ARGSUSED */ 291 static int 292 segnf_dup(struct seg *seg, struct seg *newseg) 293 { 294 panic("segnf_dup"); 295 return (0); 296 } 297 298 /* 299 * Split a segment at addr for length len. 300 */ 301 static int 302 segnf_unmap(struct seg *seg, caddr_t addr, size_t len) 303 { 304 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 305 306 /* 307 * Check for bad sizes. 308 */ 309 if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size || 310 (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET)) { 311 cmn_err(CE_PANIC, "segnf_unmap: bad unmap size"); 312 } 313 314 /* 315 * Unload any hardware translations in the range to be taken out. 316 */ 317 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP); 318 319 if (addr == seg->s_base && len == seg->s_size) { 320 /* 321 * Freeing entire segment. 322 */ 323 seg_free(seg); 324 } else if (addr == seg->s_base) { 325 /* 326 * Freeing the beginning of the segment. 327 */ 328 seg->s_base += len; 329 seg->s_size -= len; 330 } else if (addr + len == seg->s_base + seg->s_size) { 331 /* 332 * Freeing the end of the segment. 333 */ 334 seg->s_size -= len; 335 } else { 336 /* 337 * The section to go is in the middle of the segment, so we 338 * have to cut it into two segments. We shrink the existing 339 * "seg" at the low end, and create "nseg" for the high end. 340 */ 341 caddr_t nbase = addr + len; 342 size_t nsize = (seg->s_base + seg->s_size) - nbase; 343 struct seg *nseg; 344 345 /* 346 * Trim down "seg" before trying to stick "nseg" into the as. 347 */ 348 seg->s_size = addr - seg->s_base; 349 nseg = seg_alloc(seg->s_as, nbase, nsize); 350 if (nseg == NULL) 351 cmn_err(CE_PANIC, "segnf_unmap: seg_alloc failed"); 352 353 /* 354 * s_data can't be NULL because of ASSERTs in common VM code. 355 */ 356 nseg->s_ops = seg->s_ops; 357 nseg->s_data = nseg; 358 nseg->s_flags |= S_PURGE; 359 mutex_enter(&seg->s_as->a_contents); 360 seg->s_as->a_flags |= AS_NEEDSPURGE; 361 mutex_exit(&seg->s_as->a_contents); 362 } 363 364 return (0); 365 } 366 367 /* 368 * Free a segment. 369 */ 370 static void 371 segnf_free(struct seg *seg) 372 { 373 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 374 } 375 376 /* 377 * No faults allowed on segnf. 378 */ 379 static faultcode_t 380 segnf_nomap(void) 381 { 382 return (FC_NOMAP); 383 } 384 385 /* ARGSUSED */ 386 static int 387 segnf_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot) 388 { 389 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 390 return (EACCES); 391 } 392 393 /* ARGSUSED */ 394 static int 395 segnf_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot) 396 { 397 uint_t sprot; 398 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 399 400 sprot = seg->s_as == &kas ? PROT_READ : PROT_READ|PROT_USER; 401 return ((prot & sprot) == prot ? 0 : EACCES); 402 } 403 404 static void 405 segnf_badop(void) 406 { 407 panic("segnf_badop"); 408 /*NOTREACHED*/ 409 } 410 411 static int 412 segnf_nop(void) 413 { 414 return (0); 415 } 416 417 static int 418 segnf_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv) 419 { 420 size_t pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1; 421 size_t p; 422 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 423 424 for (p = 0; p < pgno; ++p) 425 protv[p] = PROT_READ; 426 return (0); 427 } 428 429 /* ARGSUSED */ 430 static u_offset_t 431 segnf_getoffset(struct seg *seg, caddr_t addr) 432 { 433 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 434 435 return ((u_offset_t)0); 436 } 437 438 /* ARGSUSED */ 439 static int 440 segnf_gettype(struct seg *seg, caddr_t addr) 441 { 442 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 443 444 return (MAP_SHARED); 445 } 446 447 /* ARGSUSED */ 448 static int 449 segnf_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp) 450 { 451 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 452 453 *vpp = &nfvp; 454 return (0); 455 } 456 457 /* 458 * segnf pages are not dumped, so we just return 459 */ 460 /* ARGSUSED */ 461 static void 462 segnf_dump(struct seg *seg) 463 {} 464 465 /*ARGSUSED*/ 466 static int 467 segnf_pagelock(struct seg *seg, caddr_t addr, size_t len, 468 struct page ***ppp, enum lock_type type, enum seg_rw rw) 469 { 470 return (ENOTSUP); 471 } 472 473 /*ARGSUSED*/ 474 static int 475 segnf_setpagesize(struct seg *seg, caddr_t addr, size_t len, 476 uint_t szc) 477 { 478 return (ENOTSUP); 479 } 480 481 /*ARGSUSED*/ 482 static int 483 segnf_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp) 484 { 485 return (ENODEV); 486 } 487 488 /*ARGSUSED*/ 489 static lgrp_mem_policy_info_t * 490 segnf_getpolicy(struct seg *seg, caddr_t addr) 491 { 492 return (NULL); 493 }