Print this page
6144 use C99 initializers in segment ops structures
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/vm/seg_map.c
+++ new/usr/src/uts/common/vm/seg_map.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
27 27 /* All Rights Reserved */
28 28
29 29 /*
30 30 * Portions of this source code were derived from Berkeley 4.3 BSD
31 31 * under license from the Regents of the University of California.
32 32 */
33 33
34 34 /*
35 35 * VM - generic vnode mapping segment.
36 36 *
37 37 * The segmap driver is used only by the kernel to get faster (than seg_vn)
38 38 * mappings [lower routine overhead; more persistent cache] to random
39 39 * vnode/offsets. Note than the kernel may (and does) use seg_vn as well.
40 40 */
41 41
42 42 #include <sys/types.h>
43 43 #include <sys/t_lock.h>
44 44 #include <sys/param.h>
45 45 #include <sys/sysmacros.h>
46 46 #include <sys/buf.h>
47 47 #include <sys/systm.h>
48 48 #include <sys/vnode.h>
49 49 #include <sys/mman.h>
50 50 #include <sys/errno.h>
51 51 #include <sys/cred.h>
52 52 #include <sys/kmem.h>
53 53 #include <sys/vtrace.h>
54 54 #include <sys/cmn_err.h>
55 55 #include <sys/debug.h>
56 56 #include <sys/thread.h>
57 57 #include <sys/dumphdr.h>
58 58 #include <sys/bitmap.h>
59 59 #include <sys/lgrp.h>
60 60
61 61 #include <vm/seg_kmem.h>
62 62 #include <vm/hat.h>
63 63 #include <vm/as.h>
64 64 #include <vm/seg.h>
65 65 #include <vm/seg_kpm.h>
66 66 #include <vm/seg_map.h>
67 67 #include <vm/page.h>
68 68 #include <vm/pvn.h>
69 69 #include <vm/rm.h>
70 70
71 71 /*
72 72 * Private seg op routines.
73 73 */
74 74 static void segmap_free(struct seg *seg);
75 75 faultcode_t segmap_fault(struct hat *hat, struct seg *seg, caddr_t addr,
76 76 size_t len, enum fault_type type, enum seg_rw rw);
77 77 static faultcode_t segmap_faulta(struct seg *seg, caddr_t addr);
78 78 static int segmap_checkprot(struct seg *seg, caddr_t addr, size_t len,
79 79 uint_t prot);
80 80 static int segmap_kluster(struct seg *seg, caddr_t addr, ssize_t);
81 81 static int segmap_getprot(struct seg *seg, caddr_t addr, size_t len,
82 82 uint_t *protv);
83 83 static u_offset_t segmap_getoffset(struct seg *seg, caddr_t addr);
84 84 static int segmap_gettype(struct seg *seg, caddr_t addr);
85 85 static int segmap_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp);
86 86 static void segmap_dump(struct seg *seg);
87 87 static int segmap_pagelock(struct seg *seg, caddr_t addr, size_t len,
88 88 struct page ***ppp, enum lock_type type,
89 89 enum seg_rw rw);
90 90 static void segmap_badop(void);
91 91 static int segmap_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp);
92 92 static lgrp_mem_policy_info_t *segmap_getpolicy(struct seg *seg,
93 93 caddr_t addr);
|
↓ open down ↓ |
93 lines elided |
↑ open up ↑ |
94 94 static int segmap_capable(struct seg *seg, segcapability_t capability);
95 95
96 96 /* segkpm support */
97 97 static caddr_t segmap_pagecreate_kpm(struct seg *, vnode_t *, u_offset_t,
98 98 struct smap *, enum seg_rw);
99 99 struct smap *get_smap_kpm(caddr_t, page_t **);
100 100
101 101 #define SEGMAP_BADOP(t) (t(*)())segmap_badop
102 102
103 103 static struct seg_ops segmap_ops = {
104 - SEGMAP_BADOP(int), /* dup */
105 - SEGMAP_BADOP(int), /* unmap */
106 - segmap_free,
107 - segmap_fault,
108 - segmap_faulta,
109 - SEGMAP_BADOP(int), /* setprot */
110 - segmap_checkprot,
111 - segmap_kluster,
112 - SEGMAP_BADOP(size_t), /* swapout */
113 - SEGMAP_BADOP(int), /* sync */
114 - SEGMAP_BADOP(size_t), /* incore */
115 - SEGMAP_BADOP(int), /* lockop */
116 - segmap_getprot,
117 - segmap_getoffset,
118 - segmap_gettype,
119 - segmap_getvp,
120 - SEGMAP_BADOP(int), /* advise */
121 - segmap_dump,
122 - segmap_pagelock, /* pagelock */
123 - SEGMAP_BADOP(int), /* setpgsz */
124 - segmap_getmemid, /* getmemid */
125 - segmap_getpolicy, /* getpolicy */
126 - segmap_capable, /* capable */
127 - seg_inherit_notsup /* inherit */
104 + .dup = SEGMAP_BADOP(int),
105 + .unmap = SEGMAP_BADOP(int),
106 + .free = segmap_free,
107 + .fault = segmap_fault,
108 + .faulta = segmap_faulta,
109 + .setprot = SEGMAP_BADOP(int),
110 + .checkprot = segmap_checkprot,
111 + .kluster = segmap_kluster,
112 + .swapout = SEGMAP_BADOP(size_t),
113 + .sync = SEGMAP_BADOP(int),
114 + .incore = SEGMAP_BADOP(size_t),
115 + .lockop = SEGMAP_BADOP(int),
116 + .getprot = segmap_getprot,
117 + .getoffset = segmap_getoffset,
118 + .gettype = segmap_gettype,
119 + .getvp = segmap_getvp,
120 + .advise = SEGMAP_BADOP(int),
121 + .dump = segmap_dump,
122 + .pagelock = segmap_pagelock,
123 + .setpagesize = SEGMAP_BADOP(int),
124 + .getmemid = segmap_getmemid,
125 + .getpolicy = segmap_getpolicy,
126 + .capable = segmap_capable,
127 + .inherit = seg_inherit_notsup,
128 128 };
129 129
130 130 /*
131 131 * Private segmap routines.
132 132 */
133 133 static void segmap_unlock(struct hat *hat, struct seg *seg, caddr_t addr,
134 134 size_t len, enum seg_rw rw, struct smap *smp);
135 135 static void segmap_smapadd(struct smap *smp);
136 136 static struct smap *segmap_hashin(struct smap *smp, struct vnode *vp,
137 137 u_offset_t off, int hashid);
138 138 static void segmap_hashout(struct smap *smp);
139 139
140 140
141 141 /*
142 142 * Statistics for segmap operations.
143 143 *
144 144 * No explicit locking to protect these stats.
145 145 */
146 146 struct segmapcnt segmapcnt = {
147 147 { "fault", KSTAT_DATA_ULONG },
148 148 { "faulta", KSTAT_DATA_ULONG },
149 149 { "getmap", KSTAT_DATA_ULONG },
150 150 { "get_use", KSTAT_DATA_ULONG },
151 151 { "get_reclaim", KSTAT_DATA_ULONG },
152 152 { "get_reuse", KSTAT_DATA_ULONG },
153 153 { "get_unused", KSTAT_DATA_ULONG },
154 154 { "get_nofree", KSTAT_DATA_ULONG },
155 155 { "rel_async", KSTAT_DATA_ULONG },
156 156 { "rel_write", KSTAT_DATA_ULONG },
157 157 { "rel_free", KSTAT_DATA_ULONG },
158 158 { "rel_abort", KSTAT_DATA_ULONG },
159 159 { "rel_dontneed", KSTAT_DATA_ULONG },
160 160 { "release", KSTAT_DATA_ULONG },
161 161 { "pagecreate", KSTAT_DATA_ULONG },
162 162 { "free_notfree", KSTAT_DATA_ULONG },
163 163 { "free_dirty", KSTAT_DATA_ULONG },
164 164 { "free", KSTAT_DATA_ULONG },
165 165 { "stolen", KSTAT_DATA_ULONG },
166 166 { "get_nomtx", KSTAT_DATA_ULONG }
167 167 };
168 168
169 169 kstat_named_t *segmapcnt_ptr = (kstat_named_t *)&segmapcnt;
170 170 uint_t segmapcnt_ndata = sizeof (segmapcnt) / sizeof (kstat_named_t);
171 171
172 172 /*
173 173 * Return number of map pages in segment.
174 174 */
175 175 #define MAP_PAGES(seg) ((seg)->s_size >> MAXBSHIFT)
176 176
177 177 /*
178 178 * Translate addr into smap number within segment.
179 179 */
180 180 #define MAP_PAGE(seg, addr) (((addr) - (seg)->s_base) >> MAXBSHIFT)
181 181
182 182 /*
183 183 * Translate addr in seg into struct smap pointer.
184 184 */
185 185 #define GET_SMAP(seg, addr) \
186 186 &(((struct segmap_data *)((seg)->s_data))->smd_sm[MAP_PAGE(seg, addr)])
187 187
188 188 /*
189 189 * Bit in map (16 bit bitmap).
190 190 */
191 191 #define SMAP_BIT_MASK(bitindex) (1 << ((bitindex) & 0xf))
192 192
193 193 static int smd_colormsk = 0;
194 194 static int smd_ncolor = 0;
195 195 static int smd_nfree = 0;
196 196 static int smd_freemsk = 0;
197 197 #ifdef DEBUG
198 198 static int *colors_used;
199 199 #endif
200 200 static struct smap *smd_smap;
201 201 static struct smaphash *smd_hash;
202 202 #ifdef SEGMAP_HASHSTATS
203 203 static unsigned int *smd_hash_len;
204 204 #endif
205 205 static struct smfree *smd_free;
206 206 static ulong_t smd_hashmsk = 0;
207 207
208 208 #define SEGMAP_MAXCOLOR 2
209 209 #define SEGMAP_CACHE_PAD 64
210 210
211 211 union segmap_cpu {
212 212 struct {
213 213 uint32_t scpu_free_ndx[SEGMAP_MAXCOLOR];
214 214 struct smap *scpu_last_smap;
215 215 ulong_t scpu_getmap;
216 216 ulong_t scpu_release;
217 217 ulong_t scpu_get_reclaim;
218 218 ulong_t scpu_fault;
219 219 ulong_t scpu_pagecreate;
220 220 ulong_t scpu_get_reuse;
221 221 } scpu;
222 222 char scpu_pad[SEGMAP_CACHE_PAD];
223 223 };
224 224 static union segmap_cpu *smd_cpu;
225 225
226 226 /*
227 227 * There are three locks in seg_map:
228 228 * - per freelist mutexes
229 229 * - per hashchain mutexes
230 230 * - per smap mutexes
231 231 *
232 232 * The lock ordering is to get the smap mutex to lock down the slot
233 233 * first then the hash lock (for hash in/out (vp, off) list) or the
234 234 * freelist lock to put the slot back on the free list.
235 235 *
236 236 * The hash search is done by only holding the hashchain lock, when a wanted
237 237 * slot is found, we drop the hashchain lock then lock the slot so there
238 238 * is no overlapping of hashchain and smap locks. After the slot is
239 239 * locked, we verify again if the slot is still what we are looking
240 240 * for.
241 241 *
242 242 * Allocation of a free slot is done by holding the freelist lock,
243 243 * then locking the smap slot at the head of the freelist. This is
244 244 * in reversed lock order so mutex_tryenter() is used.
245 245 *
246 246 * The smap lock protects all fields in smap structure except for
247 247 * the link fields for hash/free lists which are protected by
248 248 * hashchain and freelist locks.
249 249 */
250 250
251 251 #define SHASHMTX(hashid) (&smd_hash[hashid].sh_mtx)
252 252
253 253 #define SMP2SMF(smp) (&smd_free[(smp - smd_smap) & smd_freemsk])
254 254 #define SMP2SMF_NDX(smp) (ushort_t)((smp - smd_smap) & smd_freemsk)
255 255
256 256 #define SMAPMTX(smp) (&smp->sm_mtx)
257 257
258 258 #define SMAP_HASHFUNC(vp, off, hashid) \
259 259 { \
260 260 hashid = ((((uintptr_t)(vp) >> 6) + ((uintptr_t)(vp) >> 3) + \
261 261 ((off) >> MAXBSHIFT)) & smd_hashmsk); \
262 262 }
263 263
264 264 /*
265 265 * The most frequently updated kstat counters are kept in the
266 266 * per cpu array to avoid hot cache blocks. The update function
267 267 * sums the cpu local counters to update the global counters.
268 268 */
269 269
270 270 /* ARGSUSED */
271 271 int
272 272 segmap_kstat_update(kstat_t *ksp, int rw)
273 273 {
274 274 int i;
275 275 ulong_t getmap, release, get_reclaim;
276 276 ulong_t fault, pagecreate, get_reuse;
277 277
278 278 if (rw == KSTAT_WRITE)
279 279 return (EACCES);
280 280 getmap = release = get_reclaim = (ulong_t)0;
281 281 fault = pagecreate = get_reuse = (ulong_t)0;
282 282 for (i = 0; i < max_ncpus; i++) {
283 283 getmap += smd_cpu[i].scpu.scpu_getmap;
284 284 release += smd_cpu[i].scpu.scpu_release;
285 285 get_reclaim += smd_cpu[i].scpu.scpu_get_reclaim;
286 286 fault += smd_cpu[i].scpu.scpu_fault;
287 287 pagecreate += smd_cpu[i].scpu.scpu_pagecreate;
288 288 get_reuse += smd_cpu[i].scpu.scpu_get_reuse;
289 289 }
290 290 segmapcnt.smp_getmap.value.ul = getmap;
291 291 segmapcnt.smp_release.value.ul = release;
292 292 segmapcnt.smp_get_reclaim.value.ul = get_reclaim;
293 293 segmapcnt.smp_fault.value.ul = fault;
294 294 segmapcnt.smp_pagecreate.value.ul = pagecreate;
295 295 segmapcnt.smp_get_reuse.value.ul = get_reuse;
296 296 return (0);
297 297 }
298 298
299 299 int
300 300 segmap_create(struct seg *seg, void *argsp)
301 301 {
302 302 struct segmap_data *smd;
303 303 struct smap *smp;
304 304 struct smfree *sm;
305 305 struct segmap_crargs *a = (struct segmap_crargs *)argsp;
306 306 struct smaphash *shashp;
307 307 union segmap_cpu *scpu;
308 308 long i, npages;
309 309 size_t hashsz;
310 310 uint_t nfreelist;
311 311 extern void prefetch_smap_w(void *);
312 312 extern int max_ncpus;
313 313
314 314 ASSERT(seg->s_as && RW_WRITE_HELD(&seg->s_as->a_lock));
315 315
316 316 if (((uintptr_t)seg->s_base | seg->s_size) & MAXBOFFSET) {
317 317 panic("segkmap not MAXBSIZE aligned");
318 318 /*NOTREACHED*/
319 319 }
320 320
321 321 smd = kmem_zalloc(sizeof (struct segmap_data), KM_SLEEP);
322 322
323 323 seg->s_data = (void *)smd;
324 324 seg->s_ops = &segmap_ops;
325 325 smd->smd_prot = a->prot;
326 326
327 327 /*
328 328 * Scale the number of smap freelists to be
329 329 * proportional to max_ncpus * number of virtual colors.
330 330 * The caller can over-ride this scaling by providing
331 331 * a non-zero a->nfreelist argument.
332 332 */
333 333 nfreelist = a->nfreelist;
334 334 if (nfreelist == 0)
335 335 nfreelist = max_ncpus;
336 336 else if (nfreelist < 0 || nfreelist > 4 * max_ncpus) {
337 337 cmn_err(CE_WARN, "segmap_create: nfreelist out of range "
338 338 "%d, using %d", nfreelist, max_ncpus);
339 339 nfreelist = max_ncpus;
340 340 }
341 341 if (!ISP2(nfreelist)) {
342 342 /* round up nfreelist to the next power of two. */
343 343 nfreelist = 1 << (highbit(nfreelist));
344 344 }
345 345
346 346 /*
347 347 * Get the number of virtual colors - must be a power of 2.
348 348 */
349 349 if (a->shmsize)
350 350 smd_ncolor = a->shmsize >> MAXBSHIFT;
351 351 else
352 352 smd_ncolor = 1;
353 353 ASSERT((smd_ncolor & (smd_ncolor - 1)) == 0);
354 354 ASSERT(smd_ncolor <= SEGMAP_MAXCOLOR);
355 355 smd_colormsk = smd_ncolor - 1;
356 356 smd->smd_nfree = smd_nfree = smd_ncolor * nfreelist;
357 357 smd_freemsk = smd_nfree - 1;
358 358
359 359 /*
360 360 * Allocate and initialize the freelist headers.
361 361 * Note that sm_freeq[1] starts out as the release queue. This
362 362 * is known when the smap structures are initialized below.
363 363 */
364 364 smd_free = smd->smd_free =
365 365 kmem_zalloc(smd_nfree * sizeof (struct smfree), KM_SLEEP);
366 366 for (i = 0; i < smd_nfree; i++) {
367 367 sm = &smd->smd_free[i];
368 368 mutex_init(&sm->sm_freeq[0].smq_mtx, NULL, MUTEX_DEFAULT, NULL);
369 369 mutex_init(&sm->sm_freeq[1].smq_mtx, NULL, MUTEX_DEFAULT, NULL);
370 370 sm->sm_allocq = &sm->sm_freeq[0];
371 371 sm->sm_releq = &sm->sm_freeq[1];
372 372 }
373 373
374 374 /*
375 375 * Allocate and initialize the smap hash chain headers.
376 376 * Compute hash size rounding down to the next power of two.
377 377 */
378 378 npages = MAP_PAGES(seg);
379 379 smd->smd_npages = npages;
380 380 hashsz = npages / SMAP_HASHAVELEN;
381 381 hashsz = 1 << (highbit(hashsz)-1);
382 382 smd_hashmsk = hashsz - 1;
383 383 smd_hash = smd->smd_hash =
384 384 kmem_alloc(hashsz * sizeof (struct smaphash), KM_SLEEP);
385 385 #ifdef SEGMAP_HASHSTATS
386 386 smd_hash_len =
387 387 kmem_zalloc(hashsz * sizeof (unsigned int), KM_SLEEP);
388 388 #endif
389 389 for (i = 0, shashp = smd_hash; i < hashsz; i++, shashp++) {
390 390 shashp->sh_hash_list = NULL;
391 391 mutex_init(&shashp->sh_mtx, NULL, MUTEX_DEFAULT, NULL);
392 392 }
393 393
394 394 /*
395 395 * Allocate and initialize the smap structures.
396 396 * Link all slots onto the appropriate freelist.
397 397 * The smap array is large enough to affect boot time
398 398 * on large systems, so use memory prefetching and only
399 399 * go through the array 1 time. Inline a optimized version
400 400 * of segmap_smapadd to add structures to freelists with
401 401 * knowledge that no locks are needed here.
402 402 */
403 403 smd_smap = smd->smd_sm =
404 404 kmem_alloc(sizeof (struct smap) * npages, KM_SLEEP);
405 405
406 406 for (smp = &smd->smd_sm[MAP_PAGES(seg) - 1];
407 407 smp >= smd->smd_sm; smp--) {
408 408 struct smap *smpfreelist;
409 409 struct sm_freeq *releq;
410 410
411 411 prefetch_smap_w((char *)smp);
412 412
413 413 smp->sm_vp = NULL;
414 414 smp->sm_hash = NULL;
415 415 smp->sm_off = 0;
416 416 smp->sm_bitmap = 0;
417 417 smp->sm_refcnt = 0;
418 418 mutex_init(&smp->sm_mtx, NULL, MUTEX_DEFAULT, NULL);
419 419 smp->sm_free_ndx = SMP2SMF_NDX(smp);
420 420
421 421 sm = SMP2SMF(smp);
422 422 releq = sm->sm_releq;
423 423
424 424 smpfreelist = releq->smq_free;
425 425 if (smpfreelist == 0) {
426 426 releq->smq_free = smp->sm_next = smp->sm_prev = smp;
427 427 } else {
428 428 smp->sm_next = smpfreelist;
429 429 smp->sm_prev = smpfreelist->sm_prev;
430 430 smpfreelist->sm_prev = smp;
431 431 smp->sm_prev->sm_next = smp;
432 432 releq->smq_free = smp->sm_next;
433 433 }
434 434
435 435 /*
436 436 * sm_flag = 0 (no SM_QNDX_ZERO) implies smap on sm_freeq[1]
437 437 */
438 438 smp->sm_flags = 0;
439 439
440 440 #ifdef SEGKPM_SUPPORT
441 441 /*
442 442 * Due to the fragile prefetch loop no
443 443 * separate function is used here.
444 444 */
445 445 smp->sm_kpme_next = NULL;
446 446 smp->sm_kpme_prev = NULL;
447 447 smp->sm_kpme_page = NULL;
448 448 #endif
449 449 }
450 450
451 451 /*
452 452 * Allocate the per color indices that distribute allocation
453 453 * requests over the free lists. Each cpu will have a private
454 454 * rotor index to spread the allocations even across the available
455 455 * smap freelists. Init the scpu_last_smap field to the first
456 456 * smap element so there is no need to check for NULL.
457 457 */
458 458 smd_cpu =
459 459 kmem_zalloc(sizeof (union segmap_cpu) * max_ncpus, KM_SLEEP);
460 460 for (i = 0, scpu = smd_cpu; i < max_ncpus; i++, scpu++) {
461 461 int j;
462 462 for (j = 0; j < smd_ncolor; j++)
463 463 scpu->scpu.scpu_free_ndx[j] = j;
464 464 scpu->scpu.scpu_last_smap = smd_smap;
465 465 }
466 466
467 467 vpm_init();
468 468
469 469 #ifdef DEBUG
470 470 /*
471 471 * Keep track of which colors are used more often.
472 472 */
473 473 colors_used = kmem_zalloc(smd_nfree * sizeof (int), KM_SLEEP);
474 474 #endif /* DEBUG */
475 475
476 476 return (0);
477 477 }
478 478
479 479 static void
480 480 segmap_free(seg)
481 481 struct seg *seg;
482 482 {
483 483 ASSERT(seg->s_as && RW_WRITE_HELD(&seg->s_as->a_lock));
484 484 }
485 485
486 486 /*
487 487 * Do a F_SOFTUNLOCK call over the range requested.
488 488 * The range must have already been F_SOFTLOCK'ed.
489 489 */
490 490 static void
491 491 segmap_unlock(
492 492 struct hat *hat,
493 493 struct seg *seg,
494 494 caddr_t addr,
495 495 size_t len,
496 496 enum seg_rw rw,
497 497 struct smap *smp)
498 498 {
499 499 page_t *pp;
500 500 caddr_t adr;
501 501 u_offset_t off;
502 502 struct vnode *vp;
503 503 kmutex_t *smtx;
504 504
505 505 ASSERT(smp->sm_refcnt > 0);
506 506
507 507 #ifdef lint
508 508 seg = seg;
509 509 #endif
510 510
511 511 if (segmap_kpm && IS_KPM_ADDR(addr)) {
512 512
513 513 /*
514 514 * We're called only from segmap_fault and this was a
515 515 * NOP in case of a kpm based smap, so dangerous things
516 516 * must have happened in the meantime. Pages are prefaulted
517 517 * and locked in segmap_getmapflt and they will not be
518 518 * unlocked until segmap_release.
519 519 */
520 520 panic("segmap_unlock: called with kpm addr %p", (void *)addr);
521 521 /*NOTREACHED*/
522 522 }
523 523
524 524 vp = smp->sm_vp;
525 525 off = smp->sm_off + (u_offset_t)((uintptr_t)addr & MAXBOFFSET);
526 526
527 527 hat_unlock(hat, addr, P2ROUNDUP(len, PAGESIZE));
528 528 for (adr = addr; adr < addr + len; adr += PAGESIZE, off += PAGESIZE) {
529 529 ushort_t bitmask;
530 530
531 531 /*
532 532 * Use page_find() instead of page_lookup() to
533 533 * find the page since we know that it has
534 534 * "shared" lock.
535 535 */
536 536 pp = page_find(vp, off);
537 537 if (pp == NULL) {
538 538 panic("segmap_unlock: page not found");
539 539 /*NOTREACHED*/
540 540 }
541 541
542 542 if (rw == S_WRITE) {
543 543 hat_setrefmod(pp);
544 544 } else if (rw != S_OTHER) {
545 545 TRACE_3(TR_FAC_VM, TR_SEGMAP_FAULT,
546 546 "segmap_fault:pp %p vp %p offset %llx", pp, vp, off);
547 547 hat_setref(pp);
548 548 }
549 549
550 550 /*
551 551 * Clear bitmap, if the bit corresponding to "off" is set,
552 552 * since the page and translation are being unlocked.
553 553 */
554 554 bitmask = SMAP_BIT_MASK((off - smp->sm_off) >> PAGESHIFT);
555 555
556 556 /*
557 557 * Large Files: Following assertion is to verify
558 558 * the correctness of the cast to (int) above.
559 559 */
560 560 ASSERT((u_offset_t)(off - smp->sm_off) <= INT_MAX);
561 561 smtx = SMAPMTX(smp);
562 562 mutex_enter(smtx);
563 563 if (smp->sm_bitmap & bitmask) {
564 564 smp->sm_bitmap &= ~bitmask;
565 565 }
566 566 mutex_exit(smtx);
567 567
568 568 page_unlock(pp);
569 569 }
570 570 }
571 571
572 572 #define MAXPPB (MAXBSIZE/4096) /* assumes minimum page size of 4k */
573 573
574 574 /*
575 575 * This routine is called via a machine specific fault handling
576 576 * routine. It is also called by software routines wishing to
577 577 * lock or unlock a range of addresses.
578 578 *
579 579 * Note that this routine expects a page-aligned "addr".
580 580 */
581 581 faultcode_t
582 582 segmap_fault(
583 583 struct hat *hat,
584 584 struct seg *seg,
585 585 caddr_t addr,
586 586 size_t len,
587 587 enum fault_type type,
588 588 enum seg_rw rw)
589 589 {
590 590 struct segmap_data *smd = (struct segmap_data *)seg->s_data;
591 591 struct smap *smp;
592 592 page_t *pp, **ppp;
593 593 struct vnode *vp;
594 594 u_offset_t off;
595 595 page_t *pl[MAXPPB + 1];
596 596 uint_t prot;
597 597 u_offset_t addroff;
598 598 caddr_t adr;
599 599 int err;
600 600 u_offset_t sm_off;
601 601 int hat_flag;
602 602
603 603 if (segmap_kpm && IS_KPM_ADDR(addr)) {
604 604 int newpage;
605 605 kmutex_t *smtx;
606 606
607 607 /*
608 608 * Pages are successfully prefaulted and locked in
609 609 * segmap_getmapflt and can't be unlocked until
610 610 * segmap_release. No hat mappings have to be locked
611 611 * and they also can't be unlocked as long as the
612 612 * caller owns an active kpm addr.
613 613 */
614 614 #ifndef DEBUG
615 615 if (type != F_SOFTUNLOCK)
616 616 return (0);
617 617 #endif
618 618
619 619 if ((smp = get_smap_kpm(addr, NULL)) == NULL) {
620 620 panic("segmap_fault: smap not found "
621 621 "for addr %p", (void *)addr);
622 622 /*NOTREACHED*/
623 623 }
624 624
625 625 smtx = SMAPMTX(smp);
626 626 #ifdef DEBUG
627 627 newpage = smp->sm_flags & SM_KPM_NEWPAGE;
628 628 if (newpage) {
629 629 cmn_err(CE_WARN, "segmap_fault: newpage? smp %p",
630 630 (void *)smp);
631 631 }
632 632
633 633 if (type != F_SOFTUNLOCK) {
634 634 mutex_exit(smtx);
635 635 return (0);
636 636 }
637 637 #endif
638 638 mutex_exit(smtx);
639 639 vp = smp->sm_vp;
640 640 sm_off = smp->sm_off;
641 641
642 642 if (vp == NULL)
643 643 return (FC_MAKE_ERR(EIO));
644 644
645 645 ASSERT(smp->sm_refcnt > 0);
646 646
647 647 addroff = (u_offset_t)((uintptr_t)addr & MAXBOFFSET);
648 648 if (addroff + len > MAXBSIZE)
649 649 panic("segmap_fault: endaddr %p exceeds MAXBSIZE chunk",
650 650 (void *)(addr + len));
651 651
652 652 off = sm_off + addroff;
653 653
654 654 pp = page_find(vp, off);
655 655
656 656 if (pp == NULL)
657 657 panic("segmap_fault: softunlock page not found");
658 658
659 659 /*
660 660 * Set ref bit also here in case of S_OTHER to avoid the
661 661 * overhead of supporting other cases than F_SOFTUNLOCK
662 662 * with segkpm. We can do this because the underlying
663 663 * pages are locked anyway.
664 664 */
665 665 if (rw == S_WRITE) {
666 666 hat_setrefmod(pp);
667 667 } else {
668 668 TRACE_3(TR_FAC_VM, TR_SEGMAP_FAULT,
669 669 "segmap_fault:pp %p vp %p offset %llx",
670 670 pp, vp, off);
671 671 hat_setref(pp);
672 672 }
673 673
674 674 return (0);
675 675 }
676 676
677 677 smd_cpu[CPU->cpu_seqid].scpu.scpu_fault++;
678 678 smp = GET_SMAP(seg, addr);
679 679 vp = smp->sm_vp;
680 680 sm_off = smp->sm_off;
681 681
682 682 if (vp == NULL)
683 683 return (FC_MAKE_ERR(EIO));
684 684
685 685 ASSERT(smp->sm_refcnt > 0);
686 686
687 687 addroff = (u_offset_t)((uintptr_t)addr & MAXBOFFSET);
688 688 if (addroff + len > MAXBSIZE) {
689 689 panic("segmap_fault: endaddr %p "
690 690 "exceeds MAXBSIZE chunk", (void *)(addr + len));
691 691 /*NOTREACHED*/
692 692 }
693 693 off = sm_off + addroff;
694 694
695 695 /*
696 696 * First handle the easy stuff
697 697 */
698 698 if (type == F_SOFTUNLOCK) {
699 699 segmap_unlock(hat, seg, addr, len, rw, smp);
700 700 return (0);
701 701 }
702 702
703 703 TRACE_3(TR_FAC_VM, TR_SEGMAP_GETPAGE,
704 704 "segmap_getpage:seg %p addr %p vp %p", seg, addr, vp);
705 705 err = VOP_GETPAGE(vp, (offset_t)off, len, &prot, pl, MAXBSIZE,
706 706 seg, addr, rw, CRED(), NULL);
707 707
708 708 if (err)
709 709 return (FC_MAKE_ERR(err));
710 710
711 711 prot &= smd->smd_prot;
712 712
713 713 /*
714 714 * Handle all pages returned in the pl[] array.
715 715 * This loop is coded on the assumption that if
716 716 * there was no error from the VOP_GETPAGE routine,
717 717 * that the page list returned will contain all the
718 718 * needed pages for the vp from [off..off + len].
719 719 */
720 720 ppp = pl;
721 721 while ((pp = *ppp++) != NULL) {
722 722 u_offset_t poff;
723 723 ASSERT(pp->p_vnode == vp);
724 724 hat_flag = HAT_LOAD;
725 725
726 726 /*
727 727 * Verify that the pages returned are within the range
728 728 * of this segmap region. Note that it is theoretically
729 729 * possible for pages outside this range to be returned,
730 730 * but it is not very likely. If we cannot use the
731 731 * page here, just release it and go on to the next one.
732 732 */
733 733 if (pp->p_offset < sm_off ||
734 734 pp->p_offset >= sm_off + MAXBSIZE) {
735 735 (void) page_release(pp, 1);
736 736 continue;
737 737 }
738 738
739 739 ASSERT(hat == kas.a_hat);
740 740 poff = pp->p_offset;
741 741 adr = addr + (poff - off);
742 742 if (adr >= addr && adr < addr + len) {
743 743 hat_setref(pp);
744 744 TRACE_3(TR_FAC_VM, TR_SEGMAP_FAULT,
745 745 "segmap_fault:pp %p vp %p offset %llx",
746 746 pp, vp, poff);
747 747 if (type == F_SOFTLOCK)
748 748 hat_flag = HAT_LOAD_LOCK;
749 749 }
750 750
751 751 /*
752 752 * Deal with VMODSORT pages here. If we know this is a write
753 753 * do the setmod now and allow write protection.
754 754 * As long as it's modified or not S_OTHER, remove write
755 755 * protection. With S_OTHER it's up to the FS to deal with this.
756 756 */
757 757 if (IS_VMODSORT(vp)) {
758 758 if (rw == S_WRITE)
759 759 hat_setmod(pp);
760 760 else if (rw != S_OTHER && !hat_ismod(pp))
761 761 prot &= ~PROT_WRITE;
762 762 }
763 763
764 764 hat_memload(hat, adr, pp, prot, hat_flag);
765 765 if (hat_flag != HAT_LOAD_LOCK)
766 766 page_unlock(pp);
767 767 }
768 768 return (0);
769 769 }
770 770
771 771 /*
772 772 * This routine is used to start I/O on pages asynchronously.
773 773 */
774 774 static faultcode_t
775 775 segmap_faulta(struct seg *seg, caddr_t addr)
776 776 {
777 777 struct smap *smp;
778 778 struct vnode *vp;
779 779 u_offset_t off;
780 780 int err;
781 781
782 782 if (segmap_kpm && IS_KPM_ADDR(addr)) {
783 783 int newpage;
784 784 kmutex_t *smtx;
785 785
786 786 /*
787 787 * Pages are successfully prefaulted and locked in
788 788 * segmap_getmapflt and can't be unlocked until
789 789 * segmap_release. No hat mappings have to be locked
790 790 * and they also can't be unlocked as long as the
791 791 * caller owns an active kpm addr.
792 792 */
793 793 #ifdef DEBUG
794 794 if ((smp = get_smap_kpm(addr, NULL)) == NULL) {
795 795 panic("segmap_faulta: smap not found "
796 796 "for addr %p", (void *)addr);
797 797 /*NOTREACHED*/
798 798 }
799 799
800 800 smtx = SMAPMTX(smp);
801 801 newpage = smp->sm_flags & SM_KPM_NEWPAGE;
802 802 mutex_exit(smtx);
803 803 if (newpage)
804 804 cmn_err(CE_WARN, "segmap_faulta: newpage? smp %p",
805 805 (void *)smp);
806 806 #endif
807 807 return (0);
808 808 }
809 809
810 810 segmapcnt.smp_faulta.value.ul++;
811 811 smp = GET_SMAP(seg, addr);
812 812
813 813 ASSERT(smp->sm_refcnt > 0);
814 814
815 815 vp = smp->sm_vp;
816 816 off = smp->sm_off;
817 817
818 818 if (vp == NULL) {
819 819 cmn_err(CE_WARN, "segmap_faulta - no vp");
820 820 return (FC_MAKE_ERR(EIO));
821 821 }
822 822
823 823 TRACE_3(TR_FAC_VM, TR_SEGMAP_GETPAGE,
824 824 "segmap_getpage:seg %p addr %p vp %p", seg, addr, vp);
825 825
826 826 err = VOP_GETPAGE(vp, (offset_t)(off + ((offset_t)((uintptr_t)addr
827 827 & MAXBOFFSET))), PAGESIZE, (uint_t *)NULL, (page_t **)NULL, 0,
828 828 seg, addr, S_READ, CRED(), NULL);
829 829
830 830 if (err)
831 831 return (FC_MAKE_ERR(err));
832 832 return (0);
833 833 }
834 834
835 835 /*ARGSUSED*/
836 836 static int
837 837 segmap_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
838 838 {
839 839 struct segmap_data *smd = (struct segmap_data *)seg->s_data;
840 840
841 841 ASSERT(seg->s_as && RW_LOCK_HELD(&seg->s_as->a_lock));
842 842
843 843 /*
844 844 * Need not acquire the segment lock since
845 845 * "smd_prot" is a read-only field.
846 846 */
847 847 return (((smd->smd_prot & prot) != prot) ? EACCES : 0);
848 848 }
849 849
850 850 static int
851 851 segmap_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
852 852 {
853 853 struct segmap_data *smd = (struct segmap_data *)seg->s_data;
854 854 size_t pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
855 855
856 856 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
857 857
858 858 if (pgno != 0) {
859 859 do {
860 860 protv[--pgno] = smd->smd_prot;
861 861 } while (pgno != 0);
862 862 }
863 863 return (0);
864 864 }
865 865
866 866 static u_offset_t
867 867 segmap_getoffset(struct seg *seg, caddr_t addr)
868 868 {
869 869 struct segmap_data *smd = (struct segmap_data *)seg->s_data;
870 870
871 871 ASSERT(seg->s_as && RW_READ_HELD(&seg->s_as->a_lock));
872 872
873 873 return ((u_offset_t)smd->smd_sm->sm_off + (addr - seg->s_base));
874 874 }
875 875
876 876 /*ARGSUSED*/
877 877 static int
878 878 segmap_gettype(struct seg *seg, caddr_t addr)
879 879 {
880 880 ASSERT(seg->s_as && RW_READ_HELD(&seg->s_as->a_lock));
881 881
882 882 return (MAP_SHARED);
883 883 }
884 884
885 885 /*ARGSUSED*/
886 886 static int
887 887 segmap_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp)
888 888 {
889 889 struct segmap_data *smd = (struct segmap_data *)seg->s_data;
890 890
891 891 ASSERT(seg->s_as && RW_READ_HELD(&seg->s_as->a_lock));
892 892
893 893 /* XXX - This doesn't make any sense */
894 894 *vpp = smd->smd_sm->sm_vp;
895 895 return (0);
896 896 }
897 897
898 898 /*
899 899 * Check to see if it makes sense to do kluster/read ahead to
900 900 * addr + delta relative to the mapping at addr. We assume here
901 901 * that delta is a signed PAGESIZE'd multiple (which can be negative).
902 902 *
903 903 * For segmap we always "approve" of this action from our standpoint.
904 904 */
905 905 /*ARGSUSED*/
906 906 static int
907 907 segmap_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
908 908 {
909 909 return (0);
910 910 }
911 911
912 912 static void
913 913 segmap_badop()
914 914 {
915 915 panic("segmap_badop");
916 916 /*NOTREACHED*/
917 917 }
918 918
919 919 /*
920 920 * Special private segmap operations
921 921 */
922 922
923 923 /*
924 924 * Add smap to the appropriate free list.
925 925 */
926 926 static void
927 927 segmap_smapadd(struct smap *smp)
928 928 {
929 929 struct smfree *sm;
930 930 struct smap *smpfreelist;
931 931 struct sm_freeq *releq;
932 932
933 933 ASSERT(MUTEX_HELD(SMAPMTX(smp)));
934 934
935 935 if (smp->sm_refcnt != 0) {
936 936 panic("segmap_smapadd");
937 937 /*NOTREACHED*/
938 938 }
939 939
940 940 sm = &smd_free[smp->sm_free_ndx];
941 941 /*
942 942 * Add to the tail of the release queue
943 943 * Note that sm_releq and sm_allocq could toggle
944 944 * before we get the lock. This does not affect
945 945 * correctness as the 2 queues are only maintained
946 946 * to reduce lock pressure.
947 947 */
948 948 releq = sm->sm_releq;
949 949 if (releq == &sm->sm_freeq[0])
950 950 smp->sm_flags |= SM_QNDX_ZERO;
951 951 else
952 952 smp->sm_flags &= ~SM_QNDX_ZERO;
953 953 mutex_enter(&releq->smq_mtx);
954 954 smpfreelist = releq->smq_free;
955 955 if (smpfreelist == 0) {
956 956 int want;
957 957
958 958 releq->smq_free = smp->sm_next = smp->sm_prev = smp;
959 959 /*
960 960 * Both queue mutexes held to set sm_want;
961 961 * snapshot the value before dropping releq mutex.
962 962 * If sm_want appears after the releq mutex is dropped,
963 963 * then the smap just freed is already gone.
964 964 */
965 965 want = sm->sm_want;
966 966 mutex_exit(&releq->smq_mtx);
967 967 /*
968 968 * See if there was a waiter before dropping the releq mutex
969 969 * then recheck after obtaining sm_freeq[0] mutex as
970 970 * the another thread may have already signaled.
971 971 */
972 972 if (want) {
973 973 mutex_enter(&sm->sm_freeq[0].smq_mtx);
974 974 if (sm->sm_want)
975 975 cv_signal(&sm->sm_free_cv);
976 976 mutex_exit(&sm->sm_freeq[0].smq_mtx);
977 977 }
978 978 } else {
979 979 smp->sm_next = smpfreelist;
980 980 smp->sm_prev = smpfreelist->sm_prev;
981 981 smpfreelist->sm_prev = smp;
982 982 smp->sm_prev->sm_next = smp;
983 983 mutex_exit(&releq->smq_mtx);
984 984 }
985 985 }
986 986
987 987
988 988 static struct smap *
989 989 segmap_hashin(struct smap *smp, struct vnode *vp, u_offset_t off, int hashid)
990 990 {
991 991 struct smap **hpp;
992 992 struct smap *tmp;
993 993 kmutex_t *hmtx;
994 994
995 995 ASSERT(MUTEX_HELD(SMAPMTX(smp)));
996 996 ASSERT(smp->sm_vp == NULL);
997 997 ASSERT(smp->sm_hash == NULL);
998 998 ASSERT(smp->sm_prev == NULL);
999 999 ASSERT(smp->sm_next == NULL);
1000 1000 ASSERT(hashid >= 0 && hashid <= smd_hashmsk);
1001 1001
1002 1002 hmtx = SHASHMTX(hashid);
1003 1003
1004 1004 mutex_enter(hmtx);
1005 1005 /*
1006 1006 * First we need to verify that no one has created a smp
1007 1007 * with (vp,off) as its tag before we us.
1008 1008 */
1009 1009 for (tmp = smd_hash[hashid].sh_hash_list;
1010 1010 tmp != NULL; tmp = tmp->sm_hash)
1011 1011 if (tmp->sm_vp == vp && tmp->sm_off == off)
1012 1012 break;
1013 1013
1014 1014 if (tmp == NULL) {
1015 1015 /*
1016 1016 * No one created one yet.
1017 1017 *
1018 1018 * Funniness here - we don't increment the ref count on the
1019 1019 * vnode * even though we have another pointer to it here.
1020 1020 * The reason for this is that we don't want the fact that
1021 1021 * a seg_map entry somewhere refers to a vnode to prevent the
1022 1022 * vnode * itself from going away. This is because this
1023 1023 * reference to the vnode is a "soft one". In the case where
1024 1024 * a mapping is being used by a rdwr [or directory routine?]
1025 1025 * there already has to be a non-zero ref count on the vnode.
1026 1026 * In the case where the vp has been freed and the the smap
1027 1027 * structure is on the free list, there are no pages in memory
1028 1028 * that can refer to the vnode. Thus even if we reuse the same
1029 1029 * vnode/smap structure for a vnode which has the same
1030 1030 * address but represents a different object, we are ok.
1031 1031 */
1032 1032 smp->sm_vp = vp;
1033 1033 smp->sm_off = off;
1034 1034
1035 1035 hpp = &smd_hash[hashid].sh_hash_list;
1036 1036 smp->sm_hash = *hpp;
1037 1037 *hpp = smp;
1038 1038 #ifdef SEGMAP_HASHSTATS
1039 1039 smd_hash_len[hashid]++;
1040 1040 #endif
1041 1041 }
1042 1042 mutex_exit(hmtx);
1043 1043
1044 1044 return (tmp);
1045 1045 }
1046 1046
1047 1047 static void
1048 1048 segmap_hashout(struct smap *smp)
1049 1049 {
1050 1050 struct smap **hpp, *hp;
1051 1051 struct vnode *vp;
1052 1052 kmutex_t *mtx;
1053 1053 int hashid;
1054 1054 u_offset_t off;
1055 1055
1056 1056 ASSERT(MUTEX_HELD(SMAPMTX(smp)));
1057 1057
1058 1058 vp = smp->sm_vp;
1059 1059 off = smp->sm_off;
1060 1060
1061 1061 SMAP_HASHFUNC(vp, off, hashid); /* macro assigns hashid */
1062 1062 mtx = SHASHMTX(hashid);
1063 1063 mutex_enter(mtx);
1064 1064
1065 1065 hpp = &smd_hash[hashid].sh_hash_list;
1066 1066 for (;;) {
1067 1067 hp = *hpp;
1068 1068 if (hp == NULL) {
1069 1069 panic("segmap_hashout");
1070 1070 /*NOTREACHED*/
1071 1071 }
1072 1072 if (hp == smp)
1073 1073 break;
1074 1074 hpp = &hp->sm_hash;
1075 1075 }
1076 1076
1077 1077 *hpp = smp->sm_hash;
1078 1078 smp->sm_hash = NULL;
1079 1079 #ifdef SEGMAP_HASHSTATS
1080 1080 smd_hash_len[hashid]--;
1081 1081 #endif
1082 1082 mutex_exit(mtx);
1083 1083
1084 1084 smp->sm_vp = NULL;
1085 1085 smp->sm_off = (u_offset_t)0;
1086 1086
1087 1087 }
1088 1088
1089 1089 /*
1090 1090 * Attempt to free unmodified, unmapped, and non locked segmap
1091 1091 * pages.
1092 1092 */
1093 1093 void
1094 1094 segmap_pagefree(struct vnode *vp, u_offset_t off)
1095 1095 {
1096 1096 u_offset_t pgoff;
1097 1097 page_t *pp;
1098 1098
1099 1099 for (pgoff = off; pgoff < off + MAXBSIZE; pgoff += PAGESIZE) {
1100 1100
1101 1101 if ((pp = page_lookup_nowait(vp, pgoff, SE_EXCL)) == NULL)
1102 1102 continue;
1103 1103
1104 1104 switch (page_release(pp, 1)) {
1105 1105 case PGREL_NOTREL:
1106 1106 segmapcnt.smp_free_notfree.value.ul++;
1107 1107 break;
1108 1108 case PGREL_MOD:
1109 1109 segmapcnt.smp_free_dirty.value.ul++;
1110 1110 break;
1111 1111 case PGREL_CLEAN:
1112 1112 segmapcnt.smp_free.value.ul++;
1113 1113 break;
1114 1114 }
1115 1115 }
1116 1116 }
1117 1117
1118 1118 /*
1119 1119 * Locks held on entry: smap lock
1120 1120 * Locks held on exit : smap lock.
1121 1121 */
1122 1122
1123 1123 static void
1124 1124 grab_smp(struct smap *smp, page_t *pp)
1125 1125 {
1126 1126 ASSERT(MUTEX_HELD(SMAPMTX(smp)));
1127 1127 ASSERT(smp->sm_refcnt == 0);
1128 1128
1129 1129 if (smp->sm_vp != (struct vnode *)NULL) {
1130 1130 struct vnode *vp = smp->sm_vp;
1131 1131 u_offset_t off = smp->sm_off;
1132 1132 /*
1133 1133 * Destroy old vnode association and
1134 1134 * unload any hardware translations to
1135 1135 * the old object.
1136 1136 */
1137 1137 smd_cpu[CPU->cpu_seqid].scpu.scpu_get_reuse++;
1138 1138 segmap_hashout(smp);
1139 1139
1140 1140 /*
1141 1141 * This node is off freelist and hashlist,
1142 1142 * so there is no reason to drop/reacquire sm_mtx
1143 1143 * across calls to hat_unload.
1144 1144 */
1145 1145 if (segmap_kpm) {
1146 1146 caddr_t vaddr;
1147 1147 int hat_unload_needed = 0;
1148 1148
1149 1149 /*
1150 1150 * unload kpm mapping
1151 1151 */
1152 1152 if (pp != NULL) {
1153 1153 vaddr = hat_kpm_page2va(pp, 1);
1154 1154 hat_kpm_mapout(pp, GET_KPME(smp), vaddr);
1155 1155 page_unlock(pp);
1156 1156 }
1157 1157
1158 1158 /*
1159 1159 * Check if we have (also) the rare case of a
1160 1160 * non kpm mapping.
1161 1161 */
1162 1162 if (smp->sm_flags & SM_NOTKPM_RELEASED) {
1163 1163 hat_unload_needed = 1;
1164 1164 smp->sm_flags &= ~SM_NOTKPM_RELEASED;
1165 1165 }
1166 1166
1167 1167 if (hat_unload_needed) {
1168 1168 hat_unload(kas.a_hat, segkmap->s_base +
1169 1169 ((smp - smd_smap) * MAXBSIZE),
1170 1170 MAXBSIZE, HAT_UNLOAD);
1171 1171 }
1172 1172
1173 1173 } else {
1174 1174 ASSERT(smp->sm_flags & SM_NOTKPM_RELEASED);
1175 1175 smp->sm_flags &= ~SM_NOTKPM_RELEASED;
1176 1176 hat_unload(kas.a_hat, segkmap->s_base +
1177 1177 ((smp - smd_smap) * MAXBSIZE),
1178 1178 MAXBSIZE, HAT_UNLOAD);
1179 1179 }
1180 1180 segmap_pagefree(vp, off);
1181 1181 }
1182 1182 }
1183 1183
1184 1184 static struct smap *
1185 1185 get_free_smp(int free_ndx)
1186 1186 {
1187 1187 struct smfree *sm;
1188 1188 kmutex_t *smtx;
1189 1189 struct smap *smp, *first;
1190 1190 struct sm_freeq *allocq, *releq;
1191 1191 struct kpme *kpme;
1192 1192 page_t *pp = NULL;
1193 1193 int end_ndx, page_locked = 0;
1194 1194
1195 1195 end_ndx = free_ndx;
1196 1196 sm = &smd_free[free_ndx];
1197 1197
1198 1198 retry_queue:
1199 1199 allocq = sm->sm_allocq;
1200 1200 mutex_enter(&allocq->smq_mtx);
1201 1201
1202 1202 if ((smp = allocq->smq_free) == NULL) {
1203 1203
1204 1204 skip_queue:
1205 1205 /*
1206 1206 * The alloc list is empty or this queue is being skipped;
1207 1207 * first see if the allocq toggled.
1208 1208 */
1209 1209 if (sm->sm_allocq != allocq) {
1210 1210 /* queue changed */
1211 1211 mutex_exit(&allocq->smq_mtx);
1212 1212 goto retry_queue;
1213 1213 }
1214 1214 releq = sm->sm_releq;
1215 1215 if (!mutex_tryenter(&releq->smq_mtx)) {
1216 1216 /* cannot get releq; a free smp may be there now */
1217 1217 mutex_exit(&allocq->smq_mtx);
1218 1218
1219 1219 /*
1220 1220 * This loop could spin forever if this thread has
1221 1221 * higher priority than the thread that is holding
1222 1222 * releq->smq_mtx. In order to force the other thread
1223 1223 * to run, we'll lock/unlock the mutex which is safe
1224 1224 * since we just unlocked the allocq mutex.
1225 1225 */
1226 1226 mutex_enter(&releq->smq_mtx);
1227 1227 mutex_exit(&releq->smq_mtx);
1228 1228 goto retry_queue;
1229 1229 }
1230 1230 if (releq->smq_free == NULL) {
1231 1231 /*
1232 1232 * This freelist is empty.
1233 1233 * This should not happen unless clients
1234 1234 * are failing to release the segmap
1235 1235 * window after accessing the data.
1236 1236 * Before resorting to sleeping, try
1237 1237 * the next list of the same color.
1238 1238 */
1239 1239 free_ndx = (free_ndx + smd_ncolor) & smd_freemsk;
1240 1240 if (free_ndx != end_ndx) {
1241 1241 mutex_exit(&releq->smq_mtx);
1242 1242 mutex_exit(&allocq->smq_mtx);
1243 1243 sm = &smd_free[free_ndx];
1244 1244 goto retry_queue;
1245 1245 }
1246 1246 /*
1247 1247 * Tried all freelists of the same color once,
1248 1248 * wait on this list and hope something gets freed.
1249 1249 */
1250 1250 segmapcnt.smp_get_nofree.value.ul++;
1251 1251 sm->sm_want++;
1252 1252 mutex_exit(&sm->sm_freeq[1].smq_mtx);
1253 1253 cv_wait(&sm->sm_free_cv,
1254 1254 &sm->sm_freeq[0].smq_mtx);
1255 1255 sm->sm_want--;
1256 1256 mutex_exit(&sm->sm_freeq[0].smq_mtx);
1257 1257 sm = &smd_free[free_ndx];
1258 1258 goto retry_queue;
1259 1259 } else {
1260 1260 /*
1261 1261 * Something on the rele queue; flip the alloc
1262 1262 * and rele queues and retry.
1263 1263 */
1264 1264 sm->sm_allocq = releq;
1265 1265 sm->sm_releq = allocq;
1266 1266 mutex_exit(&allocq->smq_mtx);
1267 1267 mutex_exit(&releq->smq_mtx);
1268 1268 if (page_locked) {
1269 1269 delay(hz >> 2);
1270 1270 page_locked = 0;
1271 1271 }
1272 1272 goto retry_queue;
1273 1273 }
1274 1274 } else {
1275 1275 /*
1276 1276 * Fastpath the case we get the smap mutex
1277 1277 * on the first try.
1278 1278 */
1279 1279 first = smp;
1280 1280 next_smap:
1281 1281 smtx = SMAPMTX(smp);
1282 1282 if (!mutex_tryenter(smtx)) {
1283 1283 /*
1284 1284 * Another thread is trying to reclaim this slot.
1285 1285 * Skip to the next queue or smap.
1286 1286 */
1287 1287 if ((smp = smp->sm_next) == first) {
1288 1288 goto skip_queue;
1289 1289 } else {
1290 1290 goto next_smap;
1291 1291 }
1292 1292 } else {
1293 1293 /*
1294 1294 * if kpme exists, get shared lock on the page
1295 1295 */
1296 1296 if (segmap_kpm && smp->sm_vp != NULL) {
1297 1297
1298 1298 kpme = GET_KPME(smp);
1299 1299 pp = kpme->kpe_page;
1300 1300
1301 1301 if (pp != NULL) {
1302 1302 if (!page_trylock(pp, SE_SHARED)) {
1303 1303 smp = smp->sm_next;
1304 1304 mutex_exit(smtx);
1305 1305 page_locked = 1;
1306 1306
1307 1307 pp = NULL;
1308 1308
1309 1309 if (smp == first) {
1310 1310 goto skip_queue;
1311 1311 } else {
1312 1312 goto next_smap;
1313 1313 }
1314 1314 } else {
1315 1315 if (kpme->kpe_page == NULL) {
1316 1316 page_unlock(pp);
1317 1317 pp = NULL;
1318 1318 }
1319 1319 }
1320 1320 }
1321 1321 }
1322 1322
1323 1323 /*
1324 1324 * At this point, we've selected smp. Remove smp
1325 1325 * from its freelist. If smp is the first one in
1326 1326 * the freelist, update the head of the freelist.
1327 1327 */
1328 1328 if (first == smp) {
1329 1329 ASSERT(first == allocq->smq_free);
1330 1330 allocq->smq_free = smp->sm_next;
1331 1331 }
1332 1332
1333 1333 /*
1334 1334 * if the head of the freelist still points to smp,
1335 1335 * then there are no more free smaps in that list.
1336 1336 */
1337 1337 if (allocq->smq_free == smp)
1338 1338 /*
1339 1339 * Took the last one
1340 1340 */
1341 1341 allocq->smq_free = NULL;
1342 1342 else {
1343 1343 smp->sm_prev->sm_next = smp->sm_next;
1344 1344 smp->sm_next->sm_prev = smp->sm_prev;
1345 1345 }
1346 1346 mutex_exit(&allocq->smq_mtx);
1347 1347 smp->sm_prev = smp->sm_next = NULL;
1348 1348
1349 1349 /*
1350 1350 * if pp != NULL, pp must have been locked;
1351 1351 * grab_smp() unlocks pp.
1352 1352 */
1353 1353 ASSERT((pp == NULL) || PAGE_LOCKED(pp));
1354 1354 grab_smp(smp, pp);
1355 1355 /* return smp locked. */
1356 1356 ASSERT(SMAPMTX(smp) == smtx);
1357 1357 ASSERT(MUTEX_HELD(smtx));
1358 1358 return (smp);
1359 1359 }
1360 1360 }
1361 1361 }
1362 1362
1363 1363 /*
1364 1364 * Special public segmap operations
1365 1365 */
1366 1366
1367 1367 /*
1368 1368 * Create pages (without using VOP_GETPAGE) and load up translations to them.
1369 1369 * If softlock is TRUE, then set things up so that it looks like a call
1370 1370 * to segmap_fault with F_SOFTLOCK.
1371 1371 *
1372 1372 * Returns 1, if a page is created by calling page_create_va(), or 0 otherwise.
1373 1373 *
1374 1374 * All fields in the generic segment (struct seg) are considered to be
1375 1375 * read-only for "segmap" even though the kernel address space (kas) may
1376 1376 * not be locked, hence no lock is needed to access them.
1377 1377 */
1378 1378 int
1379 1379 segmap_pagecreate(struct seg *seg, caddr_t addr, size_t len, int softlock)
1380 1380 {
1381 1381 struct segmap_data *smd = (struct segmap_data *)seg->s_data;
1382 1382 page_t *pp;
1383 1383 u_offset_t off;
1384 1384 struct smap *smp;
1385 1385 struct vnode *vp;
1386 1386 caddr_t eaddr;
1387 1387 int newpage = 0;
1388 1388 uint_t prot;
1389 1389 kmutex_t *smtx;
1390 1390 int hat_flag;
1391 1391
1392 1392 ASSERT(seg->s_as == &kas);
1393 1393
1394 1394 if (segmap_kpm && IS_KPM_ADDR(addr)) {
1395 1395 /*
1396 1396 * Pages are successfully prefaulted and locked in
1397 1397 * segmap_getmapflt and can't be unlocked until
1398 1398 * segmap_release. The SM_KPM_NEWPAGE flag is set
1399 1399 * in segmap_pagecreate_kpm when new pages are created.
1400 1400 * and it is returned as "newpage" indication here.
1401 1401 */
1402 1402 if ((smp = get_smap_kpm(addr, NULL)) == NULL) {
1403 1403 panic("segmap_pagecreate: smap not found "
1404 1404 "for addr %p", (void *)addr);
1405 1405 /*NOTREACHED*/
1406 1406 }
1407 1407
1408 1408 smtx = SMAPMTX(smp);
1409 1409 newpage = smp->sm_flags & SM_KPM_NEWPAGE;
1410 1410 smp->sm_flags &= ~SM_KPM_NEWPAGE;
1411 1411 mutex_exit(smtx);
1412 1412
1413 1413 return (newpage);
1414 1414 }
1415 1415
1416 1416 smd_cpu[CPU->cpu_seqid].scpu.scpu_pagecreate++;
1417 1417
1418 1418 eaddr = addr + len;
1419 1419 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1420 1420
1421 1421 smp = GET_SMAP(seg, addr);
1422 1422
1423 1423 /*
1424 1424 * We don't grab smp mutex here since we assume the smp
1425 1425 * has a refcnt set already which prevents the slot from
1426 1426 * changing its id.
1427 1427 */
1428 1428 ASSERT(smp->sm_refcnt > 0);
1429 1429
1430 1430 vp = smp->sm_vp;
1431 1431 off = smp->sm_off + ((u_offset_t)((uintptr_t)addr & MAXBOFFSET));
1432 1432 prot = smd->smd_prot;
1433 1433
1434 1434 for (; addr < eaddr; addr += PAGESIZE, off += PAGESIZE) {
1435 1435 hat_flag = HAT_LOAD;
1436 1436 pp = page_lookup(vp, off, SE_SHARED);
1437 1437 if (pp == NULL) {
1438 1438 ushort_t bitindex;
1439 1439
1440 1440 if ((pp = page_create_va(vp, off,
1441 1441 PAGESIZE, PG_WAIT, seg, addr)) == NULL) {
1442 1442 panic("segmap_pagecreate: page_create failed");
1443 1443 /*NOTREACHED*/
1444 1444 }
1445 1445 newpage = 1;
1446 1446 page_io_unlock(pp);
1447 1447
1448 1448 /*
1449 1449 * Since pages created here do not contain valid
1450 1450 * data until the caller writes into them, the
1451 1451 * "exclusive" lock will not be dropped to prevent
1452 1452 * other users from accessing the page. We also
1453 1453 * have to lock the translation to prevent a fault
1454 1454 * from occurring when the virtual address mapped by
1455 1455 * this page is written into. This is necessary to
1456 1456 * avoid a deadlock since we haven't dropped the
1457 1457 * "exclusive" lock.
1458 1458 */
1459 1459 bitindex = (ushort_t)((off - smp->sm_off) >> PAGESHIFT);
1460 1460
1461 1461 /*
1462 1462 * Large Files: The following assertion is to
1463 1463 * verify the cast above.
1464 1464 */
1465 1465 ASSERT((u_offset_t)(off - smp->sm_off) <= INT_MAX);
1466 1466 smtx = SMAPMTX(smp);
1467 1467 mutex_enter(smtx);
1468 1468 smp->sm_bitmap |= SMAP_BIT_MASK(bitindex);
1469 1469 mutex_exit(smtx);
1470 1470
1471 1471 hat_flag = HAT_LOAD_LOCK;
1472 1472 } else if (softlock) {
1473 1473 hat_flag = HAT_LOAD_LOCK;
1474 1474 }
1475 1475
1476 1476 if (IS_VMODSORT(pp->p_vnode) && (prot & PROT_WRITE))
1477 1477 hat_setmod(pp);
1478 1478
1479 1479 hat_memload(kas.a_hat, addr, pp, prot, hat_flag);
1480 1480
1481 1481 if (hat_flag != HAT_LOAD_LOCK)
1482 1482 page_unlock(pp);
1483 1483
1484 1484 TRACE_5(TR_FAC_VM, TR_SEGMAP_PAGECREATE,
1485 1485 "segmap_pagecreate:seg %p addr %p pp %p vp %p offset %llx",
1486 1486 seg, addr, pp, vp, off);
1487 1487 }
1488 1488
1489 1489 return (newpage);
1490 1490 }
1491 1491
1492 1492 void
1493 1493 segmap_pageunlock(struct seg *seg, caddr_t addr, size_t len, enum seg_rw rw)
1494 1494 {
1495 1495 struct smap *smp;
1496 1496 ushort_t bitmask;
1497 1497 page_t *pp;
1498 1498 struct vnode *vp;
1499 1499 u_offset_t off;
1500 1500 caddr_t eaddr;
1501 1501 kmutex_t *smtx;
1502 1502
1503 1503 ASSERT(seg->s_as == &kas);
1504 1504
1505 1505 eaddr = addr + len;
1506 1506 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1507 1507
1508 1508 if (segmap_kpm && IS_KPM_ADDR(addr)) {
1509 1509 /*
1510 1510 * Pages are successfully prefaulted and locked in
1511 1511 * segmap_getmapflt and can't be unlocked until
1512 1512 * segmap_release, so no pages or hat mappings have
1513 1513 * to be unlocked at this point.
1514 1514 */
1515 1515 #ifdef DEBUG
1516 1516 if ((smp = get_smap_kpm(addr, NULL)) == NULL) {
1517 1517 panic("segmap_pageunlock: smap not found "
1518 1518 "for addr %p", (void *)addr);
1519 1519 /*NOTREACHED*/
1520 1520 }
1521 1521
1522 1522 ASSERT(smp->sm_refcnt > 0);
1523 1523 mutex_exit(SMAPMTX(smp));
1524 1524 #endif
1525 1525 return;
1526 1526 }
1527 1527
1528 1528 smp = GET_SMAP(seg, addr);
1529 1529 smtx = SMAPMTX(smp);
1530 1530
1531 1531 ASSERT(smp->sm_refcnt > 0);
1532 1532
1533 1533 vp = smp->sm_vp;
1534 1534 off = smp->sm_off + ((u_offset_t)((uintptr_t)addr & MAXBOFFSET));
1535 1535
1536 1536 for (; addr < eaddr; addr += PAGESIZE, off += PAGESIZE) {
1537 1537 bitmask = SMAP_BIT_MASK((int)(off - smp->sm_off) >> PAGESHIFT);
1538 1538
1539 1539 /*
1540 1540 * Large Files: Following assertion is to verify
1541 1541 * the correctness of the cast to (int) above.
1542 1542 */
1543 1543 ASSERT((u_offset_t)(off - smp->sm_off) <= INT_MAX);
1544 1544
1545 1545 /*
1546 1546 * If the bit corresponding to "off" is set,
1547 1547 * clear this bit in the bitmap, unlock translations,
1548 1548 * and release the "exclusive" lock on the page.
1549 1549 */
1550 1550 if (smp->sm_bitmap & bitmask) {
1551 1551 mutex_enter(smtx);
1552 1552 smp->sm_bitmap &= ~bitmask;
1553 1553 mutex_exit(smtx);
1554 1554
1555 1555 hat_unlock(kas.a_hat, addr, PAGESIZE);
1556 1556
1557 1557 /*
1558 1558 * Use page_find() instead of page_lookup() to
1559 1559 * find the page since we know that it has
1560 1560 * "exclusive" lock.
1561 1561 */
1562 1562 pp = page_find(vp, off);
1563 1563 if (pp == NULL) {
1564 1564 panic("segmap_pageunlock: page not found");
1565 1565 /*NOTREACHED*/
1566 1566 }
1567 1567 if (rw == S_WRITE) {
1568 1568 hat_setrefmod(pp);
1569 1569 } else if (rw != S_OTHER) {
1570 1570 hat_setref(pp);
1571 1571 }
1572 1572
1573 1573 page_unlock(pp);
1574 1574 }
1575 1575 }
1576 1576 }
1577 1577
1578 1578 caddr_t
1579 1579 segmap_getmap(struct seg *seg, struct vnode *vp, u_offset_t off)
1580 1580 {
1581 1581 return (segmap_getmapflt(seg, vp, off, MAXBSIZE, 0, S_OTHER));
1582 1582 }
1583 1583
1584 1584 /*
1585 1585 * This is the magic virtual address that offset 0 of an ELF
1586 1586 * file gets mapped to in user space. This is used to pick
1587 1587 * the vac color on the freelist.
1588 1588 */
1589 1589 #define ELF_OFFZERO_VA (0x10000)
1590 1590 /*
1591 1591 * segmap_getmap allocates a MAXBSIZE big slot to map the vnode vp
1592 1592 * in the range <off, off + len). off doesn't need to be MAXBSIZE aligned.
1593 1593 * The return address is always MAXBSIZE aligned.
1594 1594 *
1595 1595 * If forcefault is nonzero and the MMU translations haven't yet been created,
1596 1596 * segmap_getmap will call segmap_fault(..., F_INVAL, rw) to create them.
1597 1597 */
1598 1598 caddr_t
1599 1599 segmap_getmapflt(
1600 1600 struct seg *seg,
1601 1601 struct vnode *vp,
1602 1602 u_offset_t off,
1603 1603 size_t len,
1604 1604 int forcefault,
1605 1605 enum seg_rw rw)
1606 1606 {
1607 1607 struct smap *smp, *nsmp;
1608 1608 extern struct vnode *common_specvp();
1609 1609 caddr_t baseaddr; /* MAXBSIZE aligned */
1610 1610 u_offset_t baseoff;
1611 1611 int newslot;
1612 1612 caddr_t vaddr;
1613 1613 int color, hashid;
1614 1614 kmutex_t *hashmtx, *smapmtx;
1615 1615 struct smfree *sm;
1616 1616 page_t *pp;
1617 1617 struct kpme *kpme;
1618 1618 uint_t prot;
1619 1619 caddr_t base;
1620 1620 page_t *pl[MAXPPB + 1];
1621 1621 int error;
1622 1622 int is_kpm = 1;
1623 1623
1624 1624 ASSERT(seg->s_as == &kas);
1625 1625 ASSERT(seg == segkmap);
1626 1626
1627 1627 baseoff = off & (offset_t)MAXBMASK;
1628 1628 if (off + len > baseoff + MAXBSIZE) {
1629 1629 panic("segmap_getmap bad len");
1630 1630 /*NOTREACHED*/
1631 1631 }
1632 1632
1633 1633 /*
1634 1634 * If this is a block device we have to be sure to use the
1635 1635 * "common" block device vnode for the mapping.
1636 1636 */
1637 1637 if (vp->v_type == VBLK)
1638 1638 vp = common_specvp(vp);
1639 1639
1640 1640 smd_cpu[CPU->cpu_seqid].scpu.scpu_getmap++;
1641 1641
1642 1642 if (segmap_kpm == 0 ||
1643 1643 (forcefault == SM_PAGECREATE && rw != S_WRITE)) {
1644 1644 is_kpm = 0;
1645 1645 }
1646 1646
1647 1647 SMAP_HASHFUNC(vp, off, hashid); /* macro assigns hashid */
1648 1648 hashmtx = SHASHMTX(hashid);
1649 1649
1650 1650 retry_hash:
1651 1651 mutex_enter(hashmtx);
1652 1652 for (smp = smd_hash[hashid].sh_hash_list;
1653 1653 smp != NULL; smp = smp->sm_hash)
1654 1654 if (smp->sm_vp == vp && smp->sm_off == baseoff)
1655 1655 break;
1656 1656 mutex_exit(hashmtx);
1657 1657
1658 1658 vrfy_smp:
1659 1659 if (smp != NULL) {
1660 1660
1661 1661 ASSERT(vp->v_count != 0);
1662 1662
1663 1663 /*
1664 1664 * Get smap lock and recheck its tag. The hash lock
1665 1665 * is dropped since the hash is based on (vp, off)
1666 1666 * and (vp, off) won't change when we have smap mtx.
1667 1667 */
1668 1668 smapmtx = SMAPMTX(smp);
1669 1669 mutex_enter(smapmtx);
1670 1670 if (smp->sm_vp != vp || smp->sm_off != baseoff) {
1671 1671 mutex_exit(smapmtx);
1672 1672 goto retry_hash;
1673 1673 }
1674 1674
1675 1675 if (smp->sm_refcnt == 0) {
1676 1676
1677 1677 smd_cpu[CPU->cpu_seqid].scpu.scpu_get_reclaim++;
1678 1678
1679 1679 /*
1680 1680 * Could still be on the free list. However, this
1681 1681 * could also be an smp that is transitioning from
1682 1682 * the free list when we have too much contention
1683 1683 * for the smapmtx's. In this case, we have an
1684 1684 * unlocked smp that is not on the free list any
1685 1685 * longer, but still has a 0 refcnt. The only way
1686 1686 * to be sure is to check the freelist pointers.
1687 1687 * Since we now have the smapmtx, we are guaranteed
1688 1688 * that the (vp, off) won't change, so we are safe
1689 1689 * to reclaim it. get_free_smp() knows that this
1690 1690 * can happen, and it will check the refcnt.
1691 1691 */
1692 1692
1693 1693 if ((smp->sm_next != NULL)) {
1694 1694 struct sm_freeq *freeq;
1695 1695
1696 1696 ASSERT(smp->sm_prev != NULL);
1697 1697 sm = &smd_free[smp->sm_free_ndx];
1698 1698
1699 1699 if (smp->sm_flags & SM_QNDX_ZERO)
1700 1700 freeq = &sm->sm_freeq[0];
1701 1701 else
1702 1702 freeq = &sm->sm_freeq[1];
1703 1703
1704 1704 mutex_enter(&freeq->smq_mtx);
1705 1705 if (freeq->smq_free != smp) {
1706 1706 /*
1707 1707 * fastpath normal case
1708 1708 */
1709 1709 smp->sm_prev->sm_next = smp->sm_next;
1710 1710 smp->sm_next->sm_prev = smp->sm_prev;
1711 1711 } else if (smp == smp->sm_next) {
1712 1712 /*
1713 1713 * Taking the last smap on freelist
1714 1714 */
1715 1715 freeq->smq_free = NULL;
1716 1716 } else {
1717 1717 /*
1718 1718 * Reclaiming 1st smap on list
1719 1719 */
1720 1720 freeq->smq_free = smp->sm_next;
1721 1721 smp->sm_prev->sm_next = smp->sm_next;
1722 1722 smp->sm_next->sm_prev = smp->sm_prev;
1723 1723 }
1724 1724 mutex_exit(&freeq->smq_mtx);
1725 1725 smp->sm_prev = smp->sm_next = NULL;
1726 1726 } else {
1727 1727 ASSERT(smp->sm_prev == NULL);
1728 1728 segmapcnt.smp_stolen.value.ul++;
1729 1729 }
1730 1730
1731 1731 } else {
1732 1732 segmapcnt.smp_get_use.value.ul++;
1733 1733 }
1734 1734 smp->sm_refcnt++; /* another user */
1735 1735
1736 1736 /*
1737 1737 * We don't invoke segmap_fault via TLB miss, so we set ref
1738 1738 * and mod bits in advance. For S_OTHER we set them in
1739 1739 * segmap_fault F_SOFTUNLOCK.
1740 1740 */
1741 1741 if (is_kpm) {
1742 1742 if (rw == S_WRITE) {
1743 1743 smp->sm_flags |= SM_WRITE_DATA;
1744 1744 } else if (rw == S_READ) {
1745 1745 smp->sm_flags |= SM_READ_DATA;
1746 1746 }
1747 1747 }
1748 1748 mutex_exit(smapmtx);
1749 1749
1750 1750 newslot = 0;
1751 1751 } else {
1752 1752
1753 1753 uint32_t free_ndx, *free_ndxp;
1754 1754 union segmap_cpu *scpu;
1755 1755
1756 1756 /*
1757 1757 * On a PAC machine or a machine with anti-alias
1758 1758 * hardware, smd_colormsk will be zero.
1759 1759 *
1760 1760 * On a VAC machine- pick color by offset in the file
1761 1761 * so we won't get VAC conflicts on elf files.
1762 1762 * On data files, color does not matter but we
1763 1763 * don't know what kind of file it is so we always
1764 1764 * pick color by offset. This causes color
1765 1765 * corresponding to file offset zero to be used more
1766 1766 * heavily.
1767 1767 */
1768 1768 color = (baseoff >> MAXBSHIFT) & smd_colormsk;
1769 1769 scpu = smd_cpu+CPU->cpu_seqid;
1770 1770 free_ndxp = &scpu->scpu.scpu_free_ndx[color];
1771 1771 free_ndx = (*free_ndxp += smd_ncolor) & smd_freemsk;
1772 1772 #ifdef DEBUG
1773 1773 colors_used[free_ndx]++;
1774 1774 #endif /* DEBUG */
1775 1775
1776 1776 /*
1777 1777 * Get a locked smp slot from the free list.
1778 1778 */
1779 1779 smp = get_free_smp(free_ndx);
1780 1780 smapmtx = SMAPMTX(smp);
1781 1781
1782 1782 ASSERT(smp->sm_vp == NULL);
1783 1783
1784 1784 if ((nsmp = segmap_hashin(smp, vp, baseoff, hashid)) != NULL) {
1785 1785 /*
1786 1786 * Failed to hashin, there exists one now.
1787 1787 * Return the smp we just allocated.
1788 1788 */
1789 1789 segmap_smapadd(smp);
1790 1790 mutex_exit(smapmtx);
1791 1791
1792 1792 smp = nsmp;
1793 1793 goto vrfy_smp;
1794 1794 }
1795 1795 smp->sm_refcnt++; /* another user */
1796 1796
1797 1797 /*
1798 1798 * We don't invoke segmap_fault via TLB miss, so we set ref
1799 1799 * and mod bits in advance. For S_OTHER we set them in
1800 1800 * segmap_fault F_SOFTUNLOCK.
1801 1801 */
1802 1802 if (is_kpm) {
1803 1803 if (rw == S_WRITE) {
1804 1804 smp->sm_flags |= SM_WRITE_DATA;
1805 1805 } else if (rw == S_READ) {
1806 1806 smp->sm_flags |= SM_READ_DATA;
1807 1807 }
1808 1808 }
1809 1809 mutex_exit(smapmtx);
1810 1810
1811 1811 newslot = 1;
1812 1812 }
1813 1813
1814 1814 if (!is_kpm)
1815 1815 goto use_segmap_range;
1816 1816
1817 1817 /*
1818 1818 * Use segkpm
1819 1819 */
1820 1820 /* Lint directive required until 6746211 is fixed */
1821 1821 /*CONSTCOND*/
1822 1822 ASSERT(PAGESIZE == MAXBSIZE);
1823 1823
1824 1824 /*
1825 1825 * remember the last smp faulted on this cpu.
1826 1826 */
1827 1827 (smd_cpu+CPU->cpu_seqid)->scpu.scpu_last_smap = smp;
1828 1828
1829 1829 if (forcefault == SM_PAGECREATE) {
1830 1830 baseaddr = segmap_pagecreate_kpm(seg, vp, baseoff, smp, rw);
1831 1831 return (baseaddr);
1832 1832 }
1833 1833
1834 1834 if (newslot == 0 &&
1835 1835 (pp = GET_KPME(smp)->kpe_page) != NULL) {
1836 1836
1837 1837 /* fastpath */
1838 1838 switch (rw) {
1839 1839 case S_READ:
1840 1840 case S_WRITE:
1841 1841 if (page_trylock(pp, SE_SHARED)) {
1842 1842 if (PP_ISFREE(pp) ||
1843 1843 !(pp->p_vnode == vp &&
1844 1844 pp->p_offset == baseoff)) {
1845 1845 page_unlock(pp);
1846 1846 pp = page_lookup(vp, baseoff,
1847 1847 SE_SHARED);
1848 1848 }
1849 1849 } else {
1850 1850 pp = page_lookup(vp, baseoff, SE_SHARED);
1851 1851 }
1852 1852
1853 1853 if (pp == NULL) {
1854 1854 ASSERT(GET_KPME(smp)->kpe_page == NULL);
1855 1855 break;
1856 1856 }
1857 1857
1858 1858 if (rw == S_WRITE &&
1859 1859 hat_page_getattr(pp, P_MOD | P_REF) !=
1860 1860 (P_MOD | P_REF)) {
1861 1861 page_unlock(pp);
1862 1862 break;
1863 1863 }
1864 1864
1865 1865 /*
1866 1866 * We have the p_selock as reader, grab_smp
1867 1867 * can't hit us, we have bumped the smap
1868 1868 * refcnt and hat_pageunload needs the
1869 1869 * p_selock exclusive.
1870 1870 */
1871 1871 kpme = GET_KPME(smp);
1872 1872 if (kpme->kpe_page == pp) {
1873 1873 baseaddr = hat_kpm_page2va(pp, 0);
1874 1874 } else if (kpme->kpe_page == NULL) {
1875 1875 baseaddr = hat_kpm_mapin(pp, kpme);
1876 1876 } else {
1877 1877 panic("segmap_getmapflt: stale "
1878 1878 "kpme page, kpme %p", (void *)kpme);
1879 1879 /*NOTREACHED*/
1880 1880 }
1881 1881
1882 1882 /*
1883 1883 * We don't invoke segmap_fault via TLB miss,
1884 1884 * so we set ref and mod bits in advance.
1885 1885 * For S_OTHER and we set them in segmap_fault
1886 1886 * F_SOFTUNLOCK.
1887 1887 */
1888 1888 if (rw == S_READ && !hat_isref(pp))
1889 1889 hat_setref(pp);
1890 1890
1891 1891 return (baseaddr);
1892 1892 default:
1893 1893 break;
1894 1894 }
1895 1895 }
1896 1896
1897 1897 base = segkpm_create_va(baseoff);
1898 1898 error = VOP_GETPAGE(vp, (offset_t)baseoff, len, &prot, pl, MAXBSIZE,
1899 1899 seg, base, rw, CRED(), NULL);
1900 1900
1901 1901 pp = pl[0];
1902 1902 if (error || pp == NULL) {
1903 1903 /*
1904 1904 * Use segmap address slot and let segmap_fault deal
1905 1905 * with the error cases. There is no error return
1906 1906 * possible here.
1907 1907 */
1908 1908 goto use_segmap_range;
1909 1909 }
1910 1910
1911 1911 ASSERT(pl[1] == NULL);
1912 1912
1913 1913 /*
1914 1914 * When prot is not returned w/ PROT_ALL the returned pages
1915 1915 * are not backed by fs blocks. For most of the segmap users
1916 1916 * this is no problem, they don't write to the pages in the
1917 1917 * same request and therefore don't rely on a following
1918 1918 * trap driven segmap_fault. With SM_LOCKPROTO users it
1919 1919 * is more secure to use segkmap adresses to allow
1920 1920 * protection segmap_fault's.
1921 1921 */
1922 1922 if (prot != PROT_ALL && forcefault == SM_LOCKPROTO) {
1923 1923 /*
1924 1924 * Use segmap address slot and let segmap_fault
1925 1925 * do the error return.
1926 1926 */
1927 1927 ASSERT(rw != S_WRITE);
1928 1928 ASSERT(PAGE_LOCKED(pp));
1929 1929 page_unlock(pp);
1930 1930 forcefault = 0;
1931 1931 goto use_segmap_range;
1932 1932 }
1933 1933
1934 1934 /*
1935 1935 * We have the p_selock as reader, grab_smp can't hit us, we
1936 1936 * have bumped the smap refcnt and hat_pageunload needs the
1937 1937 * p_selock exclusive.
1938 1938 */
1939 1939 kpme = GET_KPME(smp);
1940 1940 if (kpme->kpe_page == pp) {
1941 1941 baseaddr = hat_kpm_page2va(pp, 0);
1942 1942 } else if (kpme->kpe_page == NULL) {
1943 1943 baseaddr = hat_kpm_mapin(pp, kpme);
1944 1944 } else {
1945 1945 panic("segmap_getmapflt: stale kpme page after "
1946 1946 "VOP_GETPAGE, kpme %p", (void *)kpme);
1947 1947 /*NOTREACHED*/
1948 1948 }
1949 1949
1950 1950 smd_cpu[CPU->cpu_seqid].scpu.scpu_fault++;
1951 1951
1952 1952 return (baseaddr);
1953 1953
1954 1954
1955 1955 use_segmap_range:
1956 1956 baseaddr = seg->s_base + ((smp - smd_smap) * MAXBSIZE);
1957 1957 TRACE_4(TR_FAC_VM, TR_SEGMAP_GETMAP,
1958 1958 "segmap_getmap:seg %p addr %p vp %p offset %llx",
1959 1959 seg, baseaddr, vp, baseoff);
1960 1960
1961 1961 /*
1962 1962 * Prefault the translations
1963 1963 */
1964 1964 vaddr = baseaddr + (off - baseoff);
1965 1965 if (forcefault && (newslot || !hat_probe(kas.a_hat, vaddr))) {
1966 1966
1967 1967 caddr_t pgaddr = (caddr_t)((uintptr_t)vaddr &
1968 1968 (uintptr_t)PAGEMASK);
1969 1969
1970 1970 (void) segmap_fault(kas.a_hat, seg, pgaddr,
1971 1971 (vaddr + len - pgaddr + PAGESIZE - 1) & (uintptr_t)PAGEMASK,
1972 1972 F_INVAL, rw);
1973 1973 }
1974 1974
1975 1975 return (baseaddr);
1976 1976 }
1977 1977
1978 1978 int
1979 1979 segmap_release(struct seg *seg, caddr_t addr, uint_t flags)
1980 1980 {
1981 1981 struct smap *smp;
1982 1982 int error;
1983 1983 int bflags = 0;
1984 1984 struct vnode *vp;
1985 1985 u_offset_t offset;
1986 1986 kmutex_t *smtx;
1987 1987 int is_kpm = 0;
1988 1988 page_t *pp;
1989 1989
1990 1990 if (segmap_kpm && IS_KPM_ADDR(addr)) {
1991 1991
1992 1992 if (((uintptr_t)addr & MAXBOFFSET) != 0) {
1993 1993 panic("segmap_release: addr %p not "
1994 1994 "MAXBSIZE aligned", (void *)addr);
1995 1995 /*NOTREACHED*/
1996 1996 }
1997 1997
1998 1998 if ((smp = get_smap_kpm(addr, &pp)) == NULL) {
1999 1999 panic("segmap_release: smap not found "
2000 2000 "for addr %p", (void *)addr);
2001 2001 /*NOTREACHED*/
2002 2002 }
2003 2003
2004 2004 TRACE_3(TR_FAC_VM, TR_SEGMAP_RELMAP,
2005 2005 "segmap_relmap:seg %p addr %p smp %p",
2006 2006 seg, addr, smp);
2007 2007
2008 2008 smtx = SMAPMTX(smp);
2009 2009
2010 2010 /*
2011 2011 * For compatibility reasons segmap_pagecreate_kpm sets this
2012 2012 * flag to allow a following segmap_pagecreate to return
2013 2013 * this as "newpage" flag. When segmap_pagecreate is not
2014 2014 * called at all we clear it now.
2015 2015 */
2016 2016 smp->sm_flags &= ~SM_KPM_NEWPAGE;
2017 2017 is_kpm = 1;
2018 2018 if (smp->sm_flags & SM_WRITE_DATA) {
2019 2019 hat_setrefmod(pp);
2020 2020 } else if (smp->sm_flags & SM_READ_DATA) {
2021 2021 hat_setref(pp);
2022 2022 }
2023 2023 } else {
2024 2024 if (addr < seg->s_base || addr >= seg->s_base + seg->s_size ||
2025 2025 ((uintptr_t)addr & MAXBOFFSET) != 0) {
2026 2026 panic("segmap_release: bad addr %p", (void *)addr);
2027 2027 /*NOTREACHED*/
2028 2028 }
2029 2029 smp = GET_SMAP(seg, addr);
2030 2030
2031 2031 TRACE_3(TR_FAC_VM, TR_SEGMAP_RELMAP,
2032 2032 "segmap_relmap:seg %p addr %p smp %p",
2033 2033 seg, addr, smp);
2034 2034
2035 2035 smtx = SMAPMTX(smp);
2036 2036 mutex_enter(smtx);
2037 2037 smp->sm_flags |= SM_NOTKPM_RELEASED;
2038 2038 }
2039 2039
2040 2040 ASSERT(smp->sm_refcnt > 0);
2041 2041
2042 2042 /*
2043 2043 * Need to call VOP_PUTPAGE() if any flags (except SM_DONTNEED)
2044 2044 * are set.
2045 2045 */
2046 2046 if ((flags & ~SM_DONTNEED) != 0) {
2047 2047 if (flags & SM_WRITE)
2048 2048 segmapcnt.smp_rel_write.value.ul++;
2049 2049 if (flags & SM_ASYNC) {
2050 2050 bflags |= B_ASYNC;
2051 2051 segmapcnt.smp_rel_async.value.ul++;
2052 2052 }
2053 2053 if (flags & SM_INVAL) {
2054 2054 bflags |= B_INVAL;
2055 2055 segmapcnt.smp_rel_abort.value.ul++;
2056 2056 }
2057 2057 if (flags & SM_DESTROY) {
2058 2058 bflags |= (B_INVAL|B_TRUNC);
2059 2059 segmapcnt.smp_rel_abort.value.ul++;
2060 2060 }
2061 2061 if (smp->sm_refcnt == 1) {
2062 2062 /*
2063 2063 * We only bother doing the FREE and DONTNEED flags
2064 2064 * if no one else is still referencing this mapping.
2065 2065 */
2066 2066 if (flags & SM_FREE) {
2067 2067 bflags |= B_FREE;
2068 2068 segmapcnt.smp_rel_free.value.ul++;
2069 2069 }
2070 2070 if (flags & SM_DONTNEED) {
2071 2071 bflags |= B_DONTNEED;
2072 2072 segmapcnt.smp_rel_dontneed.value.ul++;
2073 2073 }
2074 2074 }
2075 2075 } else {
2076 2076 smd_cpu[CPU->cpu_seqid].scpu.scpu_release++;
2077 2077 }
2078 2078
2079 2079 vp = smp->sm_vp;
2080 2080 offset = smp->sm_off;
2081 2081
2082 2082 if (--smp->sm_refcnt == 0) {
2083 2083
2084 2084 smp->sm_flags &= ~(SM_WRITE_DATA | SM_READ_DATA);
2085 2085
2086 2086 if (flags & (SM_INVAL|SM_DESTROY)) {
2087 2087 segmap_hashout(smp); /* remove map info */
2088 2088 if (is_kpm) {
2089 2089 hat_kpm_mapout(pp, GET_KPME(smp), addr);
2090 2090 if (smp->sm_flags & SM_NOTKPM_RELEASED) {
2091 2091 smp->sm_flags &= ~SM_NOTKPM_RELEASED;
2092 2092 hat_unload(kas.a_hat, segkmap->s_base +
2093 2093 ((smp - smd_smap) * MAXBSIZE),
2094 2094 MAXBSIZE, HAT_UNLOAD);
2095 2095 }
2096 2096
2097 2097 } else {
2098 2098 if (segmap_kpm)
2099 2099 segkpm_mapout_validkpme(GET_KPME(smp));
2100 2100
2101 2101 smp->sm_flags &= ~SM_NOTKPM_RELEASED;
2102 2102 hat_unload(kas.a_hat, addr, MAXBSIZE,
2103 2103 HAT_UNLOAD);
2104 2104 }
2105 2105 }
2106 2106 segmap_smapadd(smp); /* add to free list */
2107 2107 }
2108 2108
2109 2109 mutex_exit(smtx);
2110 2110
2111 2111 if (is_kpm)
2112 2112 page_unlock(pp);
2113 2113 /*
2114 2114 * Now invoke VOP_PUTPAGE() if any flags (except SM_DONTNEED)
2115 2115 * are set.
2116 2116 */
2117 2117 if ((flags & ~SM_DONTNEED) != 0) {
2118 2118 error = VOP_PUTPAGE(vp, offset, MAXBSIZE,
2119 2119 bflags, CRED(), NULL);
2120 2120 } else {
2121 2121 error = 0;
2122 2122 }
2123 2123
2124 2124 return (error);
2125 2125 }
2126 2126
2127 2127 /*
2128 2128 * Dump the pages belonging to this segmap segment.
2129 2129 */
2130 2130 static void
2131 2131 segmap_dump(struct seg *seg)
2132 2132 {
2133 2133 struct segmap_data *smd;
2134 2134 struct smap *smp, *smp_end;
2135 2135 page_t *pp;
2136 2136 pfn_t pfn;
2137 2137 u_offset_t off;
2138 2138 caddr_t addr;
2139 2139
2140 2140 smd = (struct segmap_data *)seg->s_data;
2141 2141 addr = seg->s_base;
2142 2142 for (smp = smd->smd_sm, smp_end = smp + smd->smd_npages;
2143 2143 smp < smp_end; smp++) {
2144 2144
2145 2145 if (smp->sm_refcnt) {
2146 2146 for (off = 0; off < MAXBSIZE; off += PAGESIZE) {
2147 2147 int we_own_it = 0;
2148 2148
2149 2149 /*
2150 2150 * If pp == NULL, the page either does
2151 2151 * not exist or is exclusively locked.
2152 2152 * So determine if it exists before
2153 2153 * searching for it.
2154 2154 */
2155 2155 if ((pp = page_lookup_nowait(smp->sm_vp,
2156 2156 smp->sm_off + off, SE_SHARED)))
2157 2157 we_own_it = 1;
2158 2158 else
2159 2159 pp = page_exists(smp->sm_vp,
2160 2160 smp->sm_off + off);
2161 2161
2162 2162 if (pp) {
2163 2163 pfn = page_pptonum(pp);
2164 2164 dump_addpage(seg->s_as,
2165 2165 addr + off, pfn);
2166 2166 if (we_own_it)
2167 2167 page_unlock(pp);
2168 2168 }
2169 2169 dump_timeleft = dump_timeout;
2170 2170 }
2171 2171 }
2172 2172 addr += MAXBSIZE;
2173 2173 }
2174 2174 }
2175 2175
2176 2176 /*ARGSUSED*/
2177 2177 static int
2178 2178 segmap_pagelock(struct seg *seg, caddr_t addr, size_t len,
2179 2179 struct page ***ppp, enum lock_type type, enum seg_rw rw)
2180 2180 {
2181 2181 return (ENOTSUP);
2182 2182 }
2183 2183
2184 2184 static int
2185 2185 segmap_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
2186 2186 {
2187 2187 struct segmap_data *smd = (struct segmap_data *)seg->s_data;
2188 2188
2189 2189 memidp->val[0] = (uintptr_t)smd->smd_sm->sm_vp;
2190 2190 memidp->val[1] = smd->smd_sm->sm_off + (uintptr_t)(addr - seg->s_base);
2191 2191 return (0);
2192 2192 }
2193 2193
2194 2194 /*ARGSUSED*/
2195 2195 static lgrp_mem_policy_info_t *
2196 2196 segmap_getpolicy(struct seg *seg, caddr_t addr)
2197 2197 {
2198 2198 return (NULL);
2199 2199 }
2200 2200
2201 2201 /*ARGSUSED*/
2202 2202 static int
2203 2203 segmap_capable(struct seg *seg, segcapability_t capability)
2204 2204 {
2205 2205 return (0);
2206 2206 }
2207 2207
2208 2208
2209 2209 #ifdef SEGKPM_SUPPORT
2210 2210
2211 2211 /*
2212 2212 * segkpm support routines
2213 2213 */
2214 2214
2215 2215 static caddr_t
2216 2216 segmap_pagecreate_kpm(struct seg *seg, vnode_t *vp, u_offset_t off,
2217 2217 struct smap *smp, enum seg_rw rw)
2218 2218 {
2219 2219 caddr_t base;
2220 2220 page_t *pp;
2221 2221 int newpage = 0;
2222 2222 struct kpme *kpme;
2223 2223
2224 2224 ASSERT(smp->sm_refcnt > 0);
2225 2225
2226 2226 if ((pp = page_lookup(vp, off, SE_SHARED)) == NULL) {
2227 2227 kmutex_t *smtx;
2228 2228
2229 2229 base = segkpm_create_va(off);
2230 2230
2231 2231 if ((pp = page_create_va(vp, off, PAGESIZE, PG_WAIT,
2232 2232 seg, base)) == NULL) {
2233 2233 panic("segmap_pagecreate_kpm: "
2234 2234 "page_create failed");
2235 2235 /*NOTREACHED*/
2236 2236 }
2237 2237
2238 2238 newpage = 1;
2239 2239 page_io_unlock(pp);
2240 2240 ASSERT((u_offset_t)(off - smp->sm_off) <= INT_MAX);
2241 2241
2242 2242 /*
2243 2243 * Mark this here until the following segmap_pagecreate
2244 2244 * or segmap_release.
2245 2245 */
2246 2246 smtx = SMAPMTX(smp);
2247 2247 mutex_enter(smtx);
2248 2248 smp->sm_flags |= SM_KPM_NEWPAGE;
2249 2249 mutex_exit(smtx);
2250 2250 }
2251 2251
2252 2252 kpme = GET_KPME(smp);
2253 2253 if (!newpage && kpme->kpe_page == pp)
2254 2254 base = hat_kpm_page2va(pp, 0);
2255 2255 else
2256 2256 base = hat_kpm_mapin(pp, kpme);
2257 2257
2258 2258 /*
2259 2259 * FS code may decide not to call segmap_pagecreate and we
2260 2260 * don't invoke segmap_fault via TLB miss, so we have to set
2261 2261 * ref and mod bits in advance.
2262 2262 */
2263 2263 if (rw == S_WRITE) {
2264 2264 hat_setrefmod(pp);
2265 2265 } else {
2266 2266 ASSERT(rw == S_READ);
2267 2267 hat_setref(pp);
2268 2268 }
2269 2269
2270 2270 smd_cpu[CPU->cpu_seqid].scpu.scpu_pagecreate++;
2271 2271
2272 2272 return (base);
2273 2273 }
2274 2274
2275 2275 /*
2276 2276 * Find the smap structure corresponding to the
2277 2277 * KPM addr and return it locked.
2278 2278 */
2279 2279 struct smap *
2280 2280 get_smap_kpm(caddr_t addr, page_t **ppp)
2281 2281 {
2282 2282 struct smap *smp;
2283 2283 struct vnode *vp;
2284 2284 u_offset_t offset;
2285 2285 caddr_t baseaddr = (caddr_t)((uintptr_t)addr & MAXBMASK);
2286 2286 int hashid;
2287 2287 kmutex_t *hashmtx;
2288 2288 page_t *pp;
2289 2289 union segmap_cpu *scpu;
2290 2290
2291 2291 pp = hat_kpm_vaddr2page(baseaddr);
2292 2292
2293 2293 ASSERT(pp && !PP_ISFREE(pp));
2294 2294 ASSERT(PAGE_LOCKED(pp));
2295 2295 ASSERT(((uintptr_t)pp->p_offset & MAXBOFFSET) == 0);
2296 2296
2297 2297 vp = pp->p_vnode;
2298 2298 offset = pp->p_offset;
2299 2299 ASSERT(vp != NULL);
2300 2300
2301 2301 /*
2302 2302 * Assume the last smap used on this cpu is the one needed.
2303 2303 */
2304 2304 scpu = smd_cpu+CPU->cpu_seqid;
2305 2305 smp = scpu->scpu.scpu_last_smap;
2306 2306 mutex_enter(&smp->sm_mtx);
2307 2307 if (smp->sm_vp == vp && smp->sm_off == offset) {
2308 2308 ASSERT(smp->sm_refcnt > 0);
2309 2309 } else {
2310 2310 /*
2311 2311 * Assumption wrong, find the smap on the hash chain.
2312 2312 */
2313 2313 mutex_exit(&smp->sm_mtx);
2314 2314 SMAP_HASHFUNC(vp, offset, hashid); /* macro assigns hashid */
2315 2315 hashmtx = SHASHMTX(hashid);
2316 2316
2317 2317 mutex_enter(hashmtx);
2318 2318 smp = smd_hash[hashid].sh_hash_list;
2319 2319 for (; smp != NULL; smp = smp->sm_hash) {
2320 2320 if (smp->sm_vp == vp && smp->sm_off == offset)
2321 2321 break;
2322 2322 }
2323 2323 mutex_exit(hashmtx);
2324 2324 if (smp) {
2325 2325 mutex_enter(&smp->sm_mtx);
2326 2326 ASSERT(smp->sm_vp == vp && smp->sm_off == offset);
2327 2327 }
2328 2328 }
2329 2329
2330 2330 if (ppp)
2331 2331 *ppp = smp ? pp : NULL;
2332 2332
2333 2333 return (smp);
2334 2334 }
2335 2335
2336 2336 #else /* SEGKPM_SUPPORT */
2337 2337
2338 2338 /* segkpm stubs */
2339 2339
2340 2340 /*ARGSUSED*/
2341 2341 static caddr_t
2342 2342 segmap_pagecreate_kpm(struct seg *seg, vnode_t *vp, u_offset_t off,
2343 2343 struct smap *smp, enum seg_rw rw)
2344 2344 {
2345 2345 return (NULL);
2346 2346 }
2347 2347
2348 2348 /*ARGSUSED*/
2349 2349 struct smap *
2350 2350 get_smap_kpm(caddr_t addr, page_t **ppp)
2351 2351 {
2352 2352 return (NULL);
2353 2353 }
2354 2354
2355 2355 #endif /* SEGKPM_SUPPORT */
|
↓ open down ↓ |
2218 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX