Print this page
6147 segop_getpolicy already checks for a NULL op
Reviewed by: Garrett D'Amore <garrett@damore.org>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/vm/seg_dev.c
+++ new/usr/src/uts/common/vm/seg_dev.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 */
26 26
27 27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
28 28 /* All Rights Reserved */
29 29
30 30 /*
31 31 * University Copyright- Copyright (c) 1982, 1986, 1988
32 32 * The Regents of the University of California
33 33 * All Rights Reserved
34 34 *
35 35 * University Acknowledgment- Portions of this document are derived from
36 36 * software developed by the University of California, Berkeley, and its
37 37 * contributors.
38 38 */
39 39
40 40 /*
41 41 * VM - segment of a mapped device.
42 42 *
43 43 * This segment driver is used when mapping character special devices.
44 44 */
45 45
46 46 #include <sys/types.h>
47 47 #include <sys/t_lock.h>
48 48 #include <sys/sysmacros.h>
49 49 #include <sys/vtrace.h>
50 50 #include <sys/systm.h>
51 51 #include <sys/vmsystm.h>
52 52 #include <sys/mman.h>
53 53 #include <sys/errno.h>
54 54 #include <sys/kmem.h>
55 55 #include <sys/cmn_err.h>
56 56 #include <sys/vnode.h>
57 57 #include <sys/proc.h>
58 58 #include <sys/conf.h>
59 59 #include <sys/debug.h>
60 60 #include <sys/ddidevmap.h>
61 61 #include <sys/ddi_implfuncs.h>
62 62 #include <sys/lgrp.h>
63 63
64 64 #include <vm/page.h>
65 65 #include <vm/hat.h>
66 66 #include <vm/as.h>
67 67 #include <vm/seg.h>
68 68 #include <vm/seg_dev.h>
69 69 #include <vm/seg_kp.h>
70 70 #include <vm/seg_kmem.h>
71 71 #include <vm/vpage.h>
72 72
73 73 #include <sys/sunddi.h>
74 74 #include <sys/esunddi.h>
75 75 #include <sys/fs/snode.h>
76 76
77 77
78 78 #if DEBUG
79 79 int segdev_debug;
80 80 #define DEBUGF(level, args) { if (segdev_debug >= (level)) cmn_err args; }
81 81 #else
82 82 #define DEBUGF(level, args)
83 83 #endif
84 84
85 85 /* Default timeout for devmap context management */
86 86 #define CTX_TIMEOUT_VALUE 0
87 87
88 88 #define HOLD_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
89 89 { mutex_enter(&dhp->dh_lock); }
90 90
91 91 #define RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
92 92 { mutex_exit(&dhp->dh_lock); }
93 93
94 94 #define round_down_p2(a, s) ((a) & ~((s) - 1))
95 95 #define round_up_p2(a, s) (((a) + (s) - 1) & ~((s) - 1))
96 96
97 97 /*
98 98 * VA_PA_ALIGNED checks to see if both VA and PA are on pgsize boundary
99 99 * VA_PA_PGSIZE_ALIGNED check to see if VA is aligned with PA w.r.t. pgsize
100 100 */
101 101 #define VA_PA_ALIGNED(uvaddr, paddr, pgsize) \
102 102 (((uvaddr | paddr) & (pgsize - 1)) == 0)
103 103 #define VA_PA_PGSIZE_ALIGNED(uvaddr, paddr, pgsize) \
104 104 (((uvaddr ^ paddr) & (pgsize - 1)) == 0)
105 105
106 106 #define vpgtob(n) ((n) * sizeof (struct vpage)) /* For brevity */
107 107
108 108 #define VTOCVP(vp) (VTOS(vp)->s_commonvp) /* we "know" it's an snode */
109 109
110 110 static struct devmap_ctx *devmapctx_list = NULL;
111 111 static struct devmap_softlock *devmap_slist = NULL;
112 112
113 113 /*
114 114 * mutex, vnode and page for the page of zeros we use for the trash mappings.
115 115 * One trash page is allocated on the first ddi_umem_setup call that uses it
116 116 * XXX Eventually, we may want to combine this with what segnf does when all
117 117 * hat layers implement HAT_NOFAULT.
118 118 *
119 119 * The trash page is used when the backing store for a userland mapping is
120 120 * removed but the application semantics do not take kindly to a SIGBUS.
121 121 * In that scenario, the applications pages are mapped to some dummy page
122 122 * which returns garbage on read and writes go into a common place.
123 123 * (Perfect for NO_FAULT semantics)
124 124 * The device driver is responsible to communicating to the app with some
125 125 * other mechanism that such remapping has happened and the app should take
126 126 * corrective action.
127 127 * We can also use an anonymous memory page as there is no requirement to
128 128 * keep the page locked, however this complicates the fault code. RFE.
129 129 */
130 130 static struct vnode trashvp;
131 131 static struct page *trashpp;
132 132
133 133 /* Non-pageable kernel memory is allocated from the umem_np_arena. */
134 134 static vmem_t *umem_np_arena;
135 135
136 136 /* Set the cookie to a value we know will never be a valid umem_cookie */
137 137 #define DEVMAP_DEVMEM_COOKIE ((ddi_umem_cookie_t)0x1)
138 138
139 139 /*
140 140 * Macros to check if type of devmap handle
141 141 */
142 142 #define cookie_is_devmem(c) \
143 143 ((c) == (struct ddi_umem_cookie *)DEVMAP_DEVMEM_COOKIE)
144 144
145 145 #define cookie_is_pmem(c) \
146 146 ((c) == (struct ddi_umem_cookie *)DEVMAP_PMEM_COOKIE)
147 147
148 148 #define cookie_is_kpmem(c) (!cookie_is_devmem(c) && !cookie_is_pmem(c) &&\
149 149 ((c)->type == KMEM_PAGEABLE))
150 150
151 151 #define dhp_is_devmem(dhp) \
152 152 (cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
153 153
154 154 #define dhp_is_pmem(dhp) \
155 155 (cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
156 156
157 157 #define dhp_is_kpmem(dhp) \
158 158 (cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
159 159
160 160 /*
161 161 * Private seg op routines.
162 162 */
163 163 static int segdev_dup(struct seg *, struct seg *);
164 164 static int segdev_unmap(struct seg *, caddr_t, size_t);
165 165 static void segdev_free(struct seg *);
166 166 static faultcode_t segdev_fault(struct hat *, struct seg *, caddr_t, size_t,
167 167 enum fault_type, enum seg_rw);
168 168 static faultcode_t segdev_faulta(struct seg *, caddr_t);
169 169 static int segdev_setprot(struct seg *, caddr_t, size_t, uint_t);
170 170 static int segdev_checkprot(struct seg *, caddr_t, size_t, uint_t);
171 171 static void segdev_badop(void);
172 172 static int segdev_sync(struct seg *, caddr_t, size_t, int, uint_t);
173 173 static size_t segdev_incore(struct seg *, caddr_t, size_t, char *);
174 174 static int segdev_lockop(struct seg *, caddr_t, size_t, int, int,
175 175 ulong_t *, size_t);
|
↓ open down ↓ |
175 lines elided |
↑ open up ↑ |
176 176 static int segdev_getprot(struct seg *, caddr_t, size_t, uint_t *);
177 177 static u_offset_t segdev_getoffset(struct seg *, caddr_t);
178 178 static int segdev_gettype(struct seg *, caddr_t);
179 179 static int segdev_getvp(struct seg *, caddr_t, struct vnode **);
180 180 static int segdev_advise(struct seg *, caddr_t, size_t, uint_t);
181 181 static void segdev_dump(struct seg *);
182 182 static int segdev_pagelock(struct seg *, caddr_t, size_t,
183 183 struct page ***, enum lock_type, enum seg_rw);
184 184 static int segdev_setpagesize(struct seg *, caddr_t, size_t, uint_t);
185 185 static int segdev_getmemid(struct seg *, caddr_t, memid_t *);
186 -static lgrp_mem_policy_info_t *segdev_getpolicy(struct seg *, caddr_t);
187 186 static int segdev_capable(struct seg *, segcapability_t);
188 187
189 188 /*
190 189 * XXX this struct is used by rootnex_map_fault to identify
191 190 * the segment it has been passed. So if you make it
192 191 * "static" you'll need to fix rootnex_map_fault.
193 192 */
194 193 struct seg_ops segdev_ops = {
195 194 .dup = segdev_dup,
196 195 .unmap = segdev_unmap,
197 196 .free = segdev_free,
198 197 .fault = segdev_fault,
199 198 .faulta = segdev_faulta,
200 199 .setprot = segdev_setprot,
201 200 .checkprot = segdev_checkprot,
202 201 .kluster = (int (*)())segdev_badop,
203 202 .sync = segdev_sync,
204 203 .incore = segdev_incore,
|
↓ open down ↓ |
8 lines elided |
↑ open up ↑ |
205 204 .lockop = segdev_lockop,
206 205 .getprot = segdev_getprot,
207 206 .getoffset = segdev_getoffset,
208 207 .gettype = segdev_gettype,
209 208 .getvp = segdev_getvp,
210 209 .advise = segdev_advise,
211 210 .dump = segdev_dump,
212 211 .pagelock = segdev_pagelock,
213 212 .setpagesize = segdev_setpagesize,
214 213 .getmemid = segdev_getmemid,
215 - .getpolicy = segdev_getpolicy,
216 214 .capable = segdev_capable,
217 215 };
218 216
219 217 /*
220 218 * Private segdev support routines
221 219 */
222 220 static struct segdev_data *sdp_alloc(void);
223 221
224 222 static void segdev_softunlock(struct hat *, struct seg *, caddr_t,
225 223 size_t, enum seg_rw);
226 224
227 225 static faultcode_t segdev_faultpage(struct hat *, struct seg *, caddr_t,
228 226 struct vpage *, enum fault_type, enum seg_rw, devmap_handle_t *);
229 227
230 228 static faultcode_t segdev_faultpages(struct hat *, struct seg *, caddr_t,
231 229 size_t, enum fault_type, enum seg_rw, devmap_handle_t *);
232 230
233 231 static struct devmap_ctx *devmap_ctxinit(dev_t, ulong_t);
234 232 static struct devmap_softlock *devmap_softlock_init(dev_t, ulong_t);
235 233 static void devmap_softlock_rele(devmap_handle_t *);
236 234 static void devmap_ctx_rele(devmap_handle_t *);
237 235
238 236 static void devmap_ctxto(void *);
239 237
240 238 static devmap_handle_t *devmap_find_handle(devmap_handle_t *dhp_head,
241 239 caddr_t addr);
242 240
243 241 static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
244 242 ulong_t *opfn, ulong_t *pagesize);
245 243
246 244 static void free_devmap_handle(devmap_handle_t *dhp);
247 245
248 246 static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
249 247 struct seg *newseg);
250 248
251 249 static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp);
252 250
253 251 static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len);
254 252
255 253 static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr);
256 254
257 255 static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
258 256 offset_t off, size_t len, uint_t flags);
259 257
260 258 static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len,
261 259 caddr_t addr, size_t *llen, caddr_t *laddr);
262 260
263 261 static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len);
264 262
265 263 static void *devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag);
266 264 static void devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size);
267 265
268 266 static void *devmap_umem_alloc_np(size_t size, size_t flags);
269 267 static void devmap_umem_free_np(void *addr, size_t size);
270 268
271 269 /*
272 270 * routines to lock and unlock underlying segkp segment for
273 271 * KMEM_PAGEABLE type cookies.
274 272 */
275 273 static faultcode_t acquire_kpmem_lock(struct ddi_umem_cookie *, size_t);
276 274 static void release_kpmem_lock(struct ddi_umem_cookie *, size_t);
277 275
278 276 /*
279 277 * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
280 278 * drivers with devmap_access callbacks
281 279 */
282 280 static int devmap_softlock_enter(struct devmap_softlock *, size_t,
283 281 enum fault_type);
284 282 static void devmap_softlock_exit(struct devmap_softlock *, size_t,
285 283 enum fault_type);
286 284
287 285 static kmutex_t devmapctx_lock;
288 286
289 287 static kmutex_t devmap_slock;
290 288
291 289 /*
292 290 * Initialize the thread callbacks and thread private data.
293 291 */
294 292 static struct devmap_ctx *
295 293 devmap_ctxinit(dev_t dev, ulong_t id)
296 294 {
297 295 struct devmap_ctx *devctx;
298 296 struct devmap_ctx *tmp;
299 297 dev_info_t *dip;
300 298
301 299 tmp = kmem_zalloc(sizeof (struct devmap_ctx), KM_SLEEP);
302 300
303 301 mutex_enter(&devmapctx_lock);
304 302
305 303 dip = e_ddi_hold_devi_by_dev(dev, 0);
306 304 ASSERT(dip != NULL);
307 305 ddi_release_devi(dip);
308 306
309 307 for (devctx = devmapctx_list; devctx != NULL; devctx = devctx->next)
310 308 if ((devctx->dip == dip) && (devctx->id == id))
311 309 break;
312 310
313 311 if (devctx == NULL) {
314 312 devctx = tmp;
315 313 devctx->dip = dip;
316 314 devctx->id = id;
317 315 mutex_init(&devctx->lock, NULL, MUTEX_DEFAULT, NULL);
318 316 cv_init(&devctx->cv, NULL, CV_DEFAULT, NULL);
319 317 devctx->next = devmapctx_list;
320 318 devmapctx_list = devctx;
321 319 } else
322 320 kmem_free(tmp, sizeof (struct devmap_ctx));
323 321
324 322 mutex_enter(&devctx->lock);
325 323 devctx->refcnt++;
326 324 mutex_exit(&devctx->lock);
327 325 mutex_exit(&devmapctx_lock);
328 326
329 327 return (devctx);
330 328 }
331 329
332 330 /*
333 331 * Timeout callback called if a CPU has not given up the device context
334 332 * within dhp->dh_timeout_length ticks
335 333 */
336 334 static void
337 335 devmap_ctxto(void *data)
338 336 {
339 337 struct devmap_ctx *devctx = data;
340 338
341 339 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_CTXTO,
342 340 "devmap_ctxto:timeout expired, devctx=%p", (void *)devctx);
343 341 mutex_enter(&devctx->lock);
344 342 /*
345 343 * Set oncpu = 0 so the next mapping trying to get the device context
346 344 * can.
347 345 */
348 346 devctx->oncpu = 0;
349 347 devctx->timeout = 0;
350 348 cv_signal(&devctx->cv);
351 349 mutex_exit(&devctx->lock);
352 350 }
353 351
354 352 /*
355 353 * Create a device segment.
356 354 */
357 355 int
358 356 segdev_create(struct seg *seg, void *argsp)
359 357 {
360 358 struct segdev_data *sdp;
361 359 struct segdev_crargs *a = (struct segdev_crargs *)argsp;
362 360 devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data;
363 361 int error;
364 362
365 363 /*
366 364 * Since the address space is "write" locked, we
367 365 * don't need the segment lock to protect "segdev" data.
368 366 */
369 367 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
370 368
371 369 hat_map(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_MAP);
372 370
373 371 sdp = sdp_alloc();
374 372
375 373 sdp->mapfunc = a->mapfunc;
376 374 sdp->offset = a->offset;
377 375 sdp->prot = a->prot;
378 376 sdp->maxprot = a->maxprot;
379 377 sdp->type = a->type;
380 378 sdp->pageprot = 0;
381 379 sdp->softlockcnt = 0;
382 380 sdp->vpage = NULL;
383 381
384 382 if (sdp->mapfunc == NULL)
385 383 sdp->devmap_data = dhp;
386 384 else
387 385 sdp->devmap_data = dhp = NULL;
388 386
389 387 sdp->hat_flags = a->hat_flags;
390 388 sdp->hat_attr = a->hat_attr;
391 389
392 390 /*
393 391 * Currently, hat_flags supports only HAT_LOAD_NOCONSIST
394 392 */
395 393 ASSERT(!(sdp->hat_flags & ~HAT_LOAD_NOCONSIST));
396 394
397 395 /*
398 396 * Hold shadow vnode -- segdev only deals with
399 397 * character (VCHR) devices. We use the common
400 398 * vp to hang pages on.
401 399 */
402 400 sdp->vp = specfind(a->dev, VCHR);
403 401 ASSERT(sdp->vp != NULL);
404 402
405 403 seg->s_ops = &segdev_ops;
406 404 seg->s_data = sdp;
407 405
408 406 while (dhp != NULL) {
409 407 dhp->dh_seg = seg;
410 408 dhp = dhp->dh_next;
411 409 }
412 410
413 411 /*
414 412 * Inform the vnode of the new mapping.
415 413 */
416 414 /*
417 415 * It is ok to use pass sdp->maxprot to ADDMAP rather than to use
418 416 * dhp specific maxprot because spec_addmap does not use maxprot.
419 417 */
420 418 error = VOP_ADDMAP(VTOCVP(sdp->vp), sdp->offset,
421 419 seg->s_as, seg->s_base, seg->s_size,
422 420 sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
423 421
424 422 if (error != 0) {
425 423 sdp->devmap_data = NULL;
426 424 hat_unload(seg->s_as->a_hat, seg->s_base, seg->s_size,
427 425 HAT_UNLOAD_UNMAP);
428 426 } else {
429 427 /*
430 428 * Mappings of /dev/null don't count towards the VSZ of a
431 429 * process. Mappings of /dev/null have no mapping type.
432 430 */
433 431 if ((segop_gettype(seg, seg->s_base) & (MAP_SHARED |
434 432 MAP_PRIVATE)) == 0) {
435 433 seg->s_as->a_resvsize -= seg->s_size;
436 434 }
437 435 }
438 436
439 437 return (error);
440 438 }
441 439
442 440 static struct segdev_data *
443 441 sdp_alloc(void)
444 442 {
445 443 struct segdev_data *sdp;
446 444
447 445 sdp = kmem_zalloc(sizeof (struct segdev_data), KM_SLEEP);
448 446 rw_init(&sdp->lock, NULL, RW_DEFAULT, NULL);
449 447
450 448 return (sdp);
451 449 }
452 450
453 451 /*
454 452 * Duplicate seg and return new segment in newseg.
455 453 */
456 454 static int
457 455 segdev_dup(struct seg *seg, struct seg *newseg)
458 456 {
459 457 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
460 458 struct segdev_data *newsdp;
461 459 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
462 460 size_t npages;
463 461 int ret;
464 462
465 463 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DUP,
466 464 "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg);
467 465
468 466 DEBUGF(3, (CE_CONT, "segdev_dup: dhp %p seg %p\n",
469 467 (void *)dhp, (void *)seg));
470 468
471 469 /*
472 470 * Since the address space is "write" locked, we
473 471 * don't need the segment lock to protect "segdev" data.
474 472 */
475 473 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
476 474
477 475 newsdp = sdp_alloc();
478 476
479 477 newseg->s_ops = seg->s_ops;
480 478 newseg->s_data = (void *)newsdp;
481 479
482 480 VN_HOLD(sdp->vp);
483 481 newsdp->vp = sdp->vp;
484 482 newsdp->mapfunc = sdp->mapfunc;
485 483 newsdp->offset = sdp->offset;
486 484 newsdp->pageprot = sdp->pageprot;
487 485 newsdp->prot = sdp->prot;
488 486 newsdp->maxprot = sdp->maxprot;
489 487 newsdp->type = sdp->type;
490 488 newsdp->hat_attr = sdp->hat_attr;
491 489 newsdp->hat_flags = sdp->hat_flags;
492 490 newsdp->softlockcnt = 0;
493 491
494 492 /*
495 493 * Initialize per page data if the segment we are
496 494 * dup'ing has per page information.
497 495 */
498 496 npages = seg_pages(newseg);
499 497
500 498 if (sdp->vpage != NULL) {
501 499 size_t nbytes = vpgtob(npages);
502 500
503 501 newsdp->vpage = kmem_zalloc(nbytes, KM_SLEEP);
504 502 bcopy(sdp->vpage, newsdp->vpage, nbytes);
505 503 } else
506 504 newsdp->vpage = NULL;
507 505
508 506 /*
509 507 * duplicate devmap handles
510 508 */
511 509 if (dhp != NULL) {
512 510 ret = devmap_handle_dup(dhp,
513 511 (devmap_handle_t **)&newsdp->devmap_data, newseg);
514 512 if (ret != 0) {
515 513 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DUP_CK1,
516 514 "segdev_dup:ret1 ret=%x, dhp=%p seg=%p",
517 515 ret, (void *)dhp, (void *)seg);
518 516 DEBUGF(1, (CE_CONT,
519 517 "segdev_dup: ret %x dhp %p seg %p\n",
520 518 ret, (void *)dhp, (void *)seg));
521 519 return (ret);
522 520 }
523 521 }
524 522
525 523 /*
526 524 * Inform the common vnode of the new mapping.
527 525 */
528 526 return (VOP_ADDMAP(VTOCVP(newsdp->vp),
529 527 newsdp->offset, newseg->s_as,
530 528 newseg->s_base, newseg->s_size, newsdp->prot,
531 529 newsdp->maxprot, sdp->type, CRED(), NULL));
532 530 }
533 531
534 532 /*
535 533 * duplicate devmap handles
536 534 */
537 535 static int
538 536 devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
539 537 struct seg *newseg)
540 538 {
541 539 devmap_handle_t *newdhp_save = NULL;
542 540 devmap_handle_t *newdhp = NULL;
543 541 struct devmap_callback_ctl *callbackops;
544 542
545 543 while (dhp != NULL) {
546 544 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
547 545
548 546 /* Need to lock the original dhp while copying if REMAP */
549 547 HOLD_DHP_LOCK(dhp);
550 548 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
551 549 RELE_DHP_LOCK(dhp);
552 550 newdhp->dh_seg = newseg;
553 551 newdhp->dh_next = NULL;
554 552 if (newdhp_save != NULL)
555 553 newdhp_save->dh_next = newdhp;
556 554 else
557 555 *new_dhp = newdhp;
558 556 newdhp_save = newdhp;
559 557
560 558 callbackops = &newdhp->dh_callbackops;
561 559
562 560 if (dhp->dh_softlock != NULL)
563 561 newdhp->dh_softlock = devmap_softlock_init(
564 562 newdhp->dh_dev,
565 563 (ulong_t)callbackops->devmap_access);
566 564 if (dhp->dh_ctx != NULL)
567 565 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
568 566 (ulong_t)callbackops->devmap_access);
569 567
570 568 /*
571 569 * Initialize dh_lock if we want to do remap.
572 570 */
573 571 if (newdhp->dh_flags & DEVMAP_ALLOW_REMAP) {
574 572 mutex_init(&newdhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
575 573 newdhp->dh_flags |= DEVMAP_LOCK_INITED;
576 574 }
577 575
578 576 if (callbackops->devmap_dup != NULL) {
579 577 int ret;
580 578
581 579 /*
582 580 * Call the dup callback so that the driver can
583 581 * duplicate its private data.
584 582 */
585 583 ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp,
586 584 (devmap_cookie_t *)newdhp, &newdhp->dh_pvtp);
587 585
588 586 if (ret != 0) {
589 587 /*
590 588 * We want to free up this segment as the driver
591 589 * has indicated that we can't dup it. But we
592 590 * don't want to call the drivers, devmap_unmap,
593 591 * callback function as the driver does not
594 592 * think this segment exists. The caller of
595 593 * devmap_dup will call seg_free on newseg
596 594 * as it was the caller that allocated the
597 595 * segment.
598 596 */
599 597 DEBUGF(1, (CE_CONT, "devmap_handle_dup ERROR: "
600 598 "newdhp %p dhp %p\n", (void *)newdhp,
601 599 (void *)dhp));
602 600 callbackops->devmap_unmap = NULL;
603 601 return (ret);
604 602 }
605 603 }
606 604
607 605 dhp = dhp->dh_next;
608 606 }
609 607
610 608 return (0);
611 609 }
612 610
613 611 /*
614 612 * Split a segment at addr for length len.
615 613 */
616 614 /*ARGSUSED*/
617 615 static int
618 616 segdev_unmap(struct seg *seg, caddr_t addr, size_t len)
619 617 {
620 618 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
621 619 register struct segdev_data *nsdp;
622 620 register struct seg *nseg;
623 621 register size_t opages; /* old segment size in pages */
624 622 register size_t npages; /* new segment size in pages */
625 623 register size_t dpages; /* pages being deleted (unmapped) */
626 624 register size_t nbytes;
627 625 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
628 626 devmap_handle_t *dhpp;
629 627 devmap_handle_t *newdhp;
630 628 struct devmap_callback_ctl *callbackops;
631 629 caddr_t nbase;
632 630 offset_t off;
633 631 ulong_t nsize;
634 632 size_t mlen, sz;
635 633
636 634 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP,
637 635 "segdev_unmap:start dhp=%p, seg=%p addr=%p len=%lx",
638 636 (void *)dhp, (void *)seg, (void *)addr, len);
639 637
640 638 DEBUGF(3, (CE_CONT, "segdev_unmap: dhp %p seg %p addr %p len %lx\n",
641 639 (void *)dhp, (void *)seg, (void *)addr, len));
642 640
643 641 /*
644 642 * Since the address space is "write" locked, we
645 643 * don't need the segment lock to protect "segdev" data.
646 644 */
647 645 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
648 646
649 647 if ((sz = sdp->softlockcnt) > 0) {
650 648 /*
651 649 * Fail the unmap if pages are SOFTLOCKed through this mapping.
652 650 * softlockcnt is protected from change by the as write lock.
653 651 */
654 652 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK1,
655 653 "segdev_unmap:error softlockcnt = %ld", sz);
656 654 DEBUGF(1, (CE_CONT, "segdev_unmap: softlockcnt %ld\n", sz));
657 655 return (EAGAIN);
658 656 }
659 657
660 658 /*
661 659 * Check for bad sizes
662 660 */
663 661 if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size ||
664 662 (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET))
665 663 panic("segdev_unmap");
666 664
667 665 if (dhp != NULL) {
668 666 devmap_handle_t *tdhp;
669 667 /*
670 668 * If large page size was used in hat_devload(),
671 669 * the same page size must be used in hat_unload().
672 670 */
673 671 dhpp = tdhp = devmap_find_handle(dhp, addr);
674 672 while (tdhp != NULL) {
675 673 if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
676 674 break;
677 675 }
678 676 tdhp = tdhp->dh_next;
679 677 }
680 678 if (tdhp != NULL) { /* found a dhp using large pages */
681 679 size_t slen = len;
682 680 size_t mlen;
683 681 size_t soff;
684 682
685 683 soff = (ulong_t)(addr - dhpp->dh_uvaddr);
686 684 while (slen != 0) {
687 685 mlen = MIN(slen, (dhpp->dh_len - soff));
688 686 hat_unload(seg->s_as->a_hat, dhpp->dh_uvaddr,
689 687 dhpp->dh_len, HAT_UNLOAD_UNMAP);
690 688 dhpp = dhpp->dh_next;
691 689 ASSERT(slen >= mlen);
692 690 slen -= mlen;
693 691 soff = 0;
694 692 }
695 693 } else
696 694 hat_unload(seg->s_as->a_hat, addr, len,
697 695 HAT_UNLOAD_UNMAP);
698 696 } else {
699 697 /*
700 698 * Unload any hardware translations in the range
701 699 * to be taken out.
702 700 */
703 701 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP);
704 702 }
705 703
706 704 /*
707 705 * get the user offset which will used in the driver callbacks
708 706 */
709 707 off = sdp->offset + (offset_t)(addr - seg->s_base);
710 708
711 709 /*
712 710 * Inform the vnode of the unmapping.
713 711 */
714 712 ASSERT(sdp->vp != NULL);
715 713 (void) VOP_DELMAP(VTOCVP(sdp->vp), off, seg->s_as, addr, len,
716 714 sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
717 715
718 716 /*
719 717 * Check for entire segment
720 718 */
721 719 if (addr == seg->s_base && len == seg->s_size) {
722 720 seg_free(seg);
723 721 return (0);
724 722 }
725 723
726 724 opages = seg_pages(seg);
727 725 dpages = btop(len);
728 726 npages = opages - dpages;
729 727
730 728 /*
731 729 * Check for beginning of segment
732 730 */
733 731 if (addr == seg->s_base) {
734 732 if (sdp->vpage != NULL) {
735 733 register struct vpage *ovpage;
736 734
737 735 ovpage = sdp->vpage; /* keep pointer to vpage */
738 736
739 737 nbytes = vpgtob(npages);
740 738 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
741 739 bcopy(&ovpage[dpages], sdp->vpage, nbytes);
742 740
743 741 /* free up old vpage */
744 742 kmem_free(ovpage, vpgtob(opages));
745 743 }
746 744
747 745 /*
748 746 * free devmap handles from the beginning of the mapping.
749 747 */
750 748 if (dhp != NULL)
751 749 devmap_handle_unmap_head(dhp, len);
752 750
753 751 sdp->offset += (offset_t)len;
754 752
755 753 seg->s_base += len;
756 754 seg->s_size -= len;
757 755
758 756 return (0);
759 757 }
760 758
761 759 /*
762 760 * Check for end of segment
763 761 */
764 762 if (addr + len == seg->s_base + seg->s_size) {
765 763 if (sdp->vpage != NULL) {
766 764 register struct vpage *ovpage;
767 765
768 766 ovpage = sdp->vpage; /* keep pointer to vpage */
769 767
770 768 nbytes = vpgtob(npages);
771 769 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
772 770 bcopy(ovpage, sdp->vpage, nbytes);
773 771
774 772 /* free up old vpage */
775 773 kmem_free(ovpage, vpgtob(opages));
776 774 }
777 775 seg->s_size -= len;
778 776
779 777 /*
780 778 * free devmap handles from addr to the end of the mapping.
781 779 */
782 780 if (dhp != NULL)
783 781 devmap_handle_unmap_tail(dhp, addr);
784 782
785 783 return (0);
786 784 }
787 785
788 786 /*
789 787 * The section to go is in the middle of the segment,
790 788 * have to make it into two segments. nseg is made for
791 789 * the high end while seg is cut down at the low end.
792 790 */
793 791 nbase = addr + len; /* new seg base */
794 792 nsize = (seg->s_base + seg->s_size) - nbase; /* new seg size */
795 793 seg->s_size = addr - seg->s_base; /* shrink old seg */
796 794 nseg = seg_alloc(seg->s_as, nbase, nsize);
797 795 if (nseg == NULL)
798 796 panic("segdev_unmap seg_alloc");
799 797
800 798 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK2,
801 799 "segdev_unmap: seg=%p nseg=%p", (void *)seg, (void *)nseg);
802 800 DEBUGF(3, (CE_CONT, "segdev_unmap: segdev_dup seg %p nseg %p\n",
803 801 (void *)seg, (void *)nseg));
804 802 nsdp = sdp_alloc();
805 803
806 804 nseg->s_ops = seg->s_ops;
807 805 nseg->s_data = (void *)nsdp;
808 806
809 807 VN_HOLD(sdp->vp);
810 808 nsdp->mapfunc = sdp->mapfunc;
811 809 nsdp->offset = sdp->offset + (offset_t)(nseg->s_base - seg->s_base);
812 810 nsdp->vp = sdp->vp;
813 811 nsdp->pageprot = sdp->pageprot;
814 812 nsdp->prot = sdp->prot;
815 813 nsdp->maxprot = sdp->maxprot;
816 814 nsdp->type = sdp->type;
817 815 nsdp->hat_attr = sdp->hat_attr;
818 816 nsdp->hat_flags = sdp->hat_flags;
819 817 nsdp->softlockcnt = 0;
820 818
821 819 /*
822 820 * Initialize per page data if the segment we are
823 821 * dup'ing has per page information.
824 822 */
825 823 if (sdp->vpage != NULL) {
826 824 /* need to split vpage into two arrays */
827 825 register size_t nnbytes;
828 826 register size_t nnpages;
829 827 register struct vpage *ovpage;
830 828
831 829 ovpage = sdp->vpage; /* keep pointer to vpage */
832 830
833 831 npages = seg_pages(seg); /* seg has shrunk */
834 832 nbytes = vpgtob(npages);
835 833 nnpages = seg_pages(nseg);
836 834 nnbytes = vpgtob(nnpages);
837 835
838 836 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
839 837 bcopy(ovpage, sdp->vpage, nbytes);
840 838
841 839 nsdp->vpage = kmem_alloc(nnbytes, KM_SLEEP);
842 840 bcopy(&ovpage[npages + dpages], nsdp->vpage, nnbytes);
843 841
844 842 /* free up old vpage */
845 843 kmem_free(ovpage, vpgtob(opages));
846 844 } else
847 845 nsdp->vpage = NULL;
848 846
849 847 /*
850 848 * unmap dhps.
851 849 */
852 850 if (dhp == NULL) {
853 851 nsdp->devmap_data = NULL;
854 852 return (0);
855 853 }
856 854 while (dhp != NULL) {
857 855 callbackops = &dhp->dh_callbackops;
858 856 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK3,
859 857 "segdev_unmap: dhp=%p addr=%p", dhp, addr);
860 858 DEBUGF(3, (CE_CONT, "unmap: dhp %p addr %p uvaddr %p len %lx\n",
861 859 (void *)dhp, (void *)addr,
862 860 (void *)dhp->dh_uvaddr, dhp->dh_len));
863 861
864 862 if (addr == (dhp->dh_uvaddr + dhp->dh_len)) {
865 863 dhpp = dhp->dh_next;
866 864 dhp->dh_next = NULL;
867 865 dhp = dhpp;
868 866 } else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) {
869 867 dhp = dhp->dh_next;
870 868 } else if (addr > dhp->dh_uvaddr &&
871 869 (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) {
872 870 /*
873 871 * <addr, addr+len> is enclosed by dhp.
874 872 * create a newdhp that begins at addr+len and
875 873 * ends at dhp->dh_uvaddr+dhp->dh_len.
876 874 */
877 875 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
878 876 HOLD_DHP_LOCK(dhp);
879 877 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
880 878 RELE_DHP_LOCK(dhp);
881 879 newdhp->dh_seg = nseg;
882 880 newdhp->dh_next = dhp->dh_next;
883 881 if (dhp->dh_softlock != NULL)
884 882 newdhp->dh_softlock = devmap_softlock_init(
885 883 newdhp->dh_dev,
886 884 (ulong_t)callbackops->devmap_access);
887 885 if (dhp->dh_ctx != NULL)
888 886 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
889 887 (ulong_t)callbackops->devmap_access);
890 888 if (newdhp->dh_flags & DEVMAP_LOCK_INITED) {
891 889 mutex_init(&newdhp->dh_lock,
892 890 NULL, MUTEX_DEFAULT, NULL);
893 891 }
894 892 if (callbackops->devmap_unmap != NULL)
895 893 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
896 894 off, len, dhp, &dhp->dh_pvtp,
897 895 newdhp, &newdhp->dh_pvtp);
898 896 mlen = len + (addr - dhp->dh_uvaddr);
899 897 devmap_handle_reduce_len(newdhp, mlen);
900 898 nsdp->devmap_data = newdhp;
901 899 /* XX Changing len should recalculate LARGE flag */
902 900 dhp->dh_len = addr - dhp->dh_uvaddr;
903 901 dhpp = dhp->dh_next;
904 902 dhp->dh_next = NULL;
905 903 dhp = dhpp;
906 904 } else if ((addr > dhp->dh_uvaddr) &&
907 905 ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) {
908 906 mlen = dhp->dh_len + dhp->dh_uvaddr - addr;
909 907 /*
910 908 * <addr, addr+len> spans over dhps.
911 909 */
912 910 if (callbackops->devmap_unmap != NULL)
913 911 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
914 912 off, mlen, (devmap_cookie_t *)dhp,
915 913 &dhp->dh_pvtp, NULL, NULL);
916 914 /* XX Changing len should recalculate LARGE flag */
917 915 dhp->dh_len = addr - dhp->dh_uvaddr;
918 916 dhpp = dhp->dh_next;
919 917 dhp->dh_next = NULL;
920 918 dhp = dhpp;
921 919 nsdp->devmap_data = dhp;
922 920 } else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) {
923 921 /*
924 922 * dhp is enclosed by <addr, addr+len>.
925 923 */
926 924 dhp->dh_seg = nseg;
927 925 nsdp->devmap_data = dhp;
928 926 dhp = devmap_handle_unmap(dhp);
929 927 nsdp->devmap_data = dhp; /* XX redundant? */
930 928 } else if (((addr + len) > dhp->dh_uvaddr) &&
931 929 ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) {
932 930 mlen = addr + len - dhp->dh_uvaddr;
933 931 if (callbackops->devmap_unmap != NULL)
934 932 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
935 933 dhp->dh_uoff, mlen, NULL,
936 934 NULL, dhp, &dhp->dh_pvtp);
937 935 devmap_handle_reduce_len(dhp, mlen);
938 936 nsdp->devmap_data = dhp;
939 937 dhp->dh_seg = nseg;
940 938 dhp = dhp->dh_next;
941 939 } else {
942 940 dhp->dh_seg = nseg;
943 941 dhp = dhp->dh_next;
944 942 }
945 943 }
946 944 return (0);
947 945 }
948 946
949 947 /*
950 948 * Utility function handles reducing the length of a devmap handle during unmap
951 949 * Note that is only used for unmapping the front portion of the handler,
952 950 * i.e., we are bumping up the offset/pfn etc up by len
953 951 * Do not use if reducing length at the tail.
954 952 */
955 953 static void
956 954 devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len)
957 955 {
958 956 struct ddi_umem_cookie *cp;
959 957 struct devmap_pmem_cookie *pcp;
960 958 /*
961 959 * adjust devmap handle fields
962 960 */
963 961 ASSERT(len < dhp->dh_len);
964 962
965 963 /* Make sure only page-aligned changes are done */
966 964 ASSERT((len & PAGEOFFSET) == 0);
967 965
968 966 dhp->dh_len -= len;
969 967 dhp->dh_uoff += (offset_t)len;
970 968 dhp->dh_roff += (offset_t)len;
971 969 dhp->dh_uvaddr += len;
972 970 /* Need to grab dhp lock if REMAP */
973 971 HOLD_DHP_LOCK(dhp);
974 972 cp = dhp->dh_cookie;
975 973 if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) {
976 974 if (cookie_is_devmem(cp)) {
977 975 dhp->dh_pfn += btop(len);
978 976 } else if (cookie_is_pmem(cp)) {
979 977 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
980 978 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
981 979 dhp->dh_roff < ptob(pcp->dp_npages));
982 980 } else {
983 981 ASSERT(dhp->dh_roff < cp->size);
984 982 ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
985 983 dhp->dh_cvaddr < (cp->cvaddr + cp->size));
986 984 ASSERT((dhp->dh_cvaddr + len) <=
987 985 (cp->cvaddr + cp->size));
988 986
989 987 dhp->dh_cvaddr += len;
990 988 }
991 989 }
992 990 /* XXX - Should recalculate the DEVMAP_FLAG_LARGE after changes */
993 991 RELE_DHP_LOCK(dhp);
994 992 }
995 993
996 994 /*
997 995 * Free devmap handle, dhp.
998 996 * Return the next devmap handle on the linked list.
999 997 */
1000 998 static devmap_handle_t *
1001 999 devmap_handle_unmap(devmap_handle_t *dhp)
1002 1000 {
1003 1001 struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops;
1004 1002 struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data;
1005 1003 devmap_handle_t *dhpp = (devmap_handle_t *)sdp->devmap_data;
1006 1004
1007 1005 ASSERT(dhp != NULL);
1008 1006
1009 1007 /*
1010 1008 * before we free up dhp, call the driver's devmap_unmap entry point
1011 1009 * to free resources allocated for this dhp.
1012 1010 */
1013 1011 if (callbackops->devmap_unmap != NULL) {
1014 1012 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff,
1015 1013 dhp->dh_len, NULL, NULL, NULL, NULL);
1016 1014 }
1017 1015
1018 1016 if (dhpp == dhp) { /* releasing first dhp, change sdp data */
1019 1017 sdp->devmap_data = dhp->dh_next;
1020 1018 } else {
1021 1019 while (dhpp->dh_next != dhp) {
1022 1020 dhpp = dhpp->dh_next;
1023 1021 }
1024 1022 dhpp->dh_next = dhp->dh_next;
1025 1023 }
1026 1024 dhpp = dhp->dh_next; /* return value is next dhp in chain */
1027 1025
1028 1026 if (dhp->dh_softlock != NULL)
1029 1027 devmap_softlock_rele(dhp);
1030 1028
1031 1029 if (dhp->dh_ctx != NULL)
1032 1030 devmap_ctx_rele(dhp);
1033 1031
1034 1032 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1035 1033 mutex_destroy(&dhp->dh_lock);
1036 1034 }
1037 1035 kmem_free(dhp, sizeof (devmap_handle_t));
1038 1036
1039 1037 return (dhpp);
1040 1038 }
1041 1039
1042 1040 /*
1043 1041 * Free complete devmap handles from dhp for len bytes
1044 1042 * dhp can be either the first handle or a subsequent handle
1045 1043 */
1046 1044 static void
1047 1045 devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len)
1048 1046 {
1049 1047 struct devmap_callback_ctl *callbackops;
1050 1048
1051 1049 /*
1052 1050 * free the devmap handles covered by len.
1053 1051 */
1054 1052 while (len >= dhp->dh_len) {
1055 1053 len -= dhp->dh_len;
1056 1054 dhp = devmap_handle_unmap(dhp);
1057 1055 }
1058 1056 if (len != 0) { /* partial unmap at head of first remaining dhp */
1059 1057 callbackops = &dhp->dh_callbackops;
1060 1058
1061 1059 /*
1062 1060 * Call the unmap callback so the drivers can make
1063 1061 * adjustment on its private data.
1064 1062 */
1065 1063 if (callbackops->devmap_unmap != NULL)
1066 1064 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
1067 1065 dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp);
1068 1066 devmap_handle_reduce_len(dhp, len);
1069 1067 }
1070 1068 }
1071 1069
1072 1070 /*
1073 1071 * Free devmap handles to truncate the mapping after addr
1074 1072 * RFE: Simpler to pass in dhp pointing at correct dhp (avoid find again)
1075 1073 * Also could then use the routine in middle unmap case too
1076 1074 */
1077 1075 static void
1078 1076 devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr)
1079 1077 {
1080 1078 register struct seg *seg = dhp->dh_seg;
1081 1079 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1082 1080 register devmap_handle_t *dhph = (devmap_handle_t *)sdp->devmap_data;
1083 1081 struct devmap_callback_ctl *callbackops;
1084 1082 register devmap_handle_t *dhpp;
1085 1083 size_t maplen;
1086 1084 ulong_t off;
1087 1085 size_t len;
1088 1086
1089 1087 maplen = (size_t)(addr - dhp->dh_uvaddr);
1090 1088 dhph = devmap_find_handle(dhph, addr);
1091 1089
1092 1090 while (dhph != NULL) {
1093 1091 if (maplen == 0) {
1094 1092 dhph = devmap_handle_unmap(dhph);
1095 1093 } else {
1096 1094 callbackops = &dhph->dh_callbackops;
1097 1095 len = dhph->dh_len - maplen;
1098 1096 off = (ulong_t)sdp->offset + (addr - seg->s_base);
1099 1097 /*
1100 1098 * Call the unmap callback so the driver
1101 1099 * can make adjustments on its private data.
1102 1100 */
1103 1101 if (callbackops->devmap_unmap != NULL)
1104 1102 (*callbackops->devmap_unmap)(dhph,
1105 1103 dhph->dh_pvtp, off, len,
1106 1104 (devmap_cookie_t *)dhph,
1107 1105 &dhph->dh_pvtp, NULL, NULL);
1108 1106 /* XXX Reducing len needs to recalculate LARGE flag */
1109 1107 dhph->dh_len = maplen;
1110 1108 maplen = 0;
1111 1109 dhpp = dhph->dh_next;
1112 1110 dhph->dh_next = NULL;
1113 1111 dhph = dhpp;
1114 1112 }
1115 1113 } /* end while */
1116 1114 }
1117 1115
1118 1116 /*
1119 1117 * Free a segment.
1120 1118 */
1121 1119 static void
1122 1120 segdev_free(struct seg *seg)
1123 1121 {
1124 1122 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1125 1123 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
1126 1124
1127 1125 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FREE,
1128 1126 "segdev_free: dhp=%p seg=%p", (void *)dhp, (void *)seg);
1129 1127 DEBUGF(3, (CE_CONT, "segdev_free: dhp %p seg %p\n",
1130 1128 (void *)dhp, (void *)seg));
1131 1129
1132 1130 /*
1133 1131 * Since the address space is "write" locked, we
1134 1132 * don't need the segment lock to protect "segdev" data.
1135 1133 */
1136 1134 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
1137 1135
1138 1136 while (dhp != NULL)
1139 1137 dhp = devmap_handle_unmap(dhp);
1140 1138
1141 1139 VN_RELE(sdp->vp);
1142 1140 if (sdp->vpage != NULL)
1143 1141 kmem_free(sdp->vpage, vpgtob(seg_pages(seg)));
1144 1142
1145 1143 rw_destroy(&sdp->lock);
1146 1144 kmem_free(sdp, sizeof (*sdp));
1147 1145 }
1148 1146
1149 1147 static void
1150 1148 free_devmap_handle(devmap_handle_t *dhp)
1151 1149 {
1152 1150 register devmap_handle_t *dhpp;
1153 1151
1154 1152 /*
1155 1153 * free up devmap handle
1156 1154 */
1157 1155 while (dhp != NULL) {
1158 1156 dhpp = dhp->dh_next;
1159 1157 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1160 1158 mutex_destroy(&dhp->dh_lock);
1161 1159 }
1162 1160
1163 1161 if (dhp->dh_softlock != NULL)
1164 1162 devmap_softlock_rele(dhp);
1165 1163
1166 1164 if (dhp->dh_ctx != NULL)
1167 1165 devmap_ctx_rele(dhp);
1168 1166
1169 1167 kmem_free(dhp, sizeof (devmap_handle_t));
1170 1168 dhp = dhpp;
1171 1169 }
1172 1170 }
1173 1171
1174 1172 /*
1175 1173 * routines to lock and unlock underlying segkp segment for
1176 1174 * KMEM_PAGEABLE type cookies.
1177 1175 * segkp only allows a single pending F_SOFTLOCK
1178 1176 * we keep track of number of locks in the cookie so we can
1179 1177 * have multiple pending faults and manage the calls to segkp.
1180 1178 * RFE: if segkp supports either pagelock or can support multiple
1181 1179 * calls to F_SOFTLOCK, then these routines can go away.
1182 1180 * If pagelock, segdev_faultpage can fault on a page by page basis
1183 1181 * and simplifies the code quite a bit.
1184 1182 * if multiple calls allowed but not partial ranges, then need for
1185 1183 * cookie->lock and locked count goes away, code can call as_fault directly
1186 1184 */
1187 1185 static faultcode_t
1188 1186 acquire_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1189 1187 {
1190 1188 int err = 0;
1191 1189 ASSERT(cookie_is_kpmem(cookie));
1192 1190 /*
1193 1191 * Fault in pages in segkp with F_SOFTLOCK.
1194 1192 * We want to hold the lock until all pages have been loaded.
1195 1193 * segkp only allows single caller to hold SOFTLOCK, so cookie
1196 1194 * holds a count so we dont call into segkp multiple times
1197 1195 */
1198 1196 mutex_enter(&cookie->lock);
1199 1197
1200 1198 /*
1201 1199 * Check for overflow in locked field
1202 1200 */
1203 1201 if ((UINT32_MAX - cookie->locked) < npages) {
1204 1202 err = FC_MAKE_ERR(ENOMEM);
1205 1203 } else if (cookie->locked == 0) {
1206 1204 /* First time locking */
1207 1205 err = as_fault(kas.a_hat, &kas, cookie->cvaddr,
1208 1206 cookie->size, F_SOFTLOCK, PROT_READ|PROT_WRITE);
1209 1207 }
1210 1208 if (!err) {
1211 1209 cookie->locked += npages;
1212 1210 }
1213 1211 mutex_exit(&cookie->lock);
1214 1212 return (err);
1215 1213 }
1216 1214
1217 1215 static void
1218 1216 release_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1219 1217 {
1220 1218 mutex_enter(&cookie->lock);
1221 1219 ASSERT(cookie_is_kpmem(cookie));
1222 1220 ASSERT(cookie->locked >= npages);
1223 1221 cookie->locked -= (uint_t)npages;
1224 1222 if (cookie->locked == 0) {
1225 1223 /* Last unlock */
1226 1224 if (as_fault(kas.a_hat, &kas, cookie->cvaddr,
1227 1225 cookie->size, F_SOFTUNLOCK, PROT_READ|PROT_WRITE))
1228 1226 panic("segdev releasing kpmem lock %p", (void *)cookie);
1229 1227 }
1230 1228 mutex_exit(&cookie->lock);
1231 1229 }
1232 1230
1233 1231 /*
1234 1232 * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
1235 1233 * drivers with devmap_access callbacks
1236 1234 * slock->softlocked basically works like a rw lock
1237 1235 * -ve counts => F_SOFTLOCK in progress
1238 1236 * +ve counts => F_INVAL/F_PROT in progress
1239 1237 * We allow only one F_SOFTLOCK at a time
1240 1238 * but can have multiple pending F_INVAL/F_PROT calls
1241 1239 *
1242 1240 * This routine waits using cv_wait_sig so killing processes is more graceful
1243 1241 * Returns EINTR if coming out of this routine due to a signal, 0 otherwise
1244 1242 */
1245 1243 static int devmap_softlock_enter(
1246 1244 struct devmap_softlock *slock,
1247 1245 size_t npages,
1248 1246 enum fault_type type)
1249 1247 {
1250 1248 if (npages == 0)
1251 1249 return (0);
1252 1250 mutex_enter(&(slock->lock));
1253 1251 switch (type) {
1254 1252 case F_SOFTLOCK :
1255 1253 while (slock->softlocked) {
1256 1254 if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1257 1255 /* signalled */
1258 1256 mutex_exit(&(slock->lock));
1259 1257 return (EINTR);
1260 1258 }
1261 1259 }
1262 1260 slock->softlocked -= npages; /* -ve count => locked */
1263 1261 break;
1264 1262 case F_INVAL :
1265 1263 case F_PROT :
1266 1264 while (slock->softlocked < 0)
1267 1265 if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1268 1266 /* signalled */
1269 1267 mutex_exit(&(slock->lock));
1270 1268 return (EINTR);
1271 1269 }
1272 1270 slock->softlocked += npages; /* +ve count => f_invals */
1273 1271 break;
1274 1272 default:
1275 1273 ASSERT(0);
1276 1274 }
1277 1275 mutex_exit(&(slock->lock));
1278 1276 return (0);
1279 1277 }
1280 1278
1281 1279 static void devmap_softlock_exit(
1282 1280 struct devmap_softlock *slock,
1283 1281 size_t npages,
1284 1282 enum fault_type type)
1285 1283 {
1286 1284 if (slock == NULL)
1287 1285 return;
1288 1286 mutex_enter(&(slock->lock));
1289 1287 switch (type) {
1290 1288 case F_SOFTLOCK :
1291 1289 ASSERT(-slock->softlocked >= npages);
1292 1290 slock->softlocked += npages; /* -ve count is softlocked */
1293 1291 if (slock->softlocked == 0)
1294 1292 cv_signal(&slock->cv);
1295 1293 break;
1296 1294 case F_INVAL :
1297 1295 case F_PROT:
1298 1296 ASSERT(slock->softlocked >= npages);
1299 1297 slock->softlocked -= npages;
1300 1298 if (slock->softlocked == 0)
1301 1299 cv_signal(&slock->cv);
1302 1300 break;
1303 1301 default:
1304 1302 ASSERT(0);
1305 1303 }
1306 1304 mutex_exit(&(slock->lock));
1307 1305 }
1308 1306
1309 1307 /*
1310 1308 * Do a F_SOFTUNLOCK call over the range requested.
1311 1309 * The range must have already been F_SOFTLOCK'ed.
1312 1310 * The segment lock should be held, (but not the segment private lock?)
1313 1311 * The softunlock code below does not adjust for large page sizes
1314 1312 * assumes the caller already did any addr/len adjustments for
1315 1313 * pagesize mappings before calling.
1316 1314 */
1317 1315 /*ARGSUSED*/
1318 1316 static void
1319 1317 segdev_softunlock(
1320 1318 struct hat *hat, /* the hat */
1321 1319 struct seg *seg, /* seg_dev of interest */
1322 1320 caddr_t addr, /* base address of range */
1323 1321 size_t len, /* number of bytes */
1324 1322 enum seg_rw rw) /* type of access at fault */
1325 1323 {
1326 1324 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1327 1325 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1328 1326
1329 1327 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SOFTUNLOCK,
1330 1328 "segdev_softunlock:dhp_head=%p sdp=%p addr=%p len=%lx",
1331 1329 dhp_head, sdp, addr, len);
1332 1330 DEBUGF(3, (CE_CONT, "segdev_softunlock: dhp %p lockcnt %lx "
1333 1331 "addr %p len %lx\n",
1334 1332 (void *)dhp_head, sdp->softlockcnt, (void *)addr, len));
1335 1333
1336 1334 hat_unlock(hat, addr, len);
1337 1335
1338 1336 if (dhp_head != NULL) {
1339 1337 devmap_handle_t *dhp;
1340 1338 size_t mlen;
1341 1339 size_t tlen = len;
1342 1340 ulong_t off;
1343 1341
1344 1342 dhp = devmap_find_handle(dhp_head, addr);
1345 1343 ASSERT(dhp != NULL);
1346 1344
1347 1345 off = (ulong_t)(addr - dhp->dh_uvaddr);
1348 1346 while (tlen != 0) {
1349 1347 mlen = MIN(tlen, (dhp->dh_len - off));
1350 1348
1351 1349 /*
1352 1350 * unlock segkp memory, locked during F_SOFTLOCK
1353 1351 */
1354 1352 if (dhp_is_kpmem(dhp)) {
1355 1353 release_kpmem_lock(
1356 1354 (struct ddi_umem_cookie *)dhp->dh_cookie,
1357 1355 btopr(mlen));
1358 1356 }
1359 1357
1360 1358 /*
1361 1359 * Do the softlock accounting for devmap_access
1362 1360 */
1363 1361 if (dhp->dh_callbackops.devmap_access != NULL) {
1364 1362 devmap_softlock_exit(dhp->dh_softlock,
1365 1363 btopr(mlen), F_SOFTLOCK);
1366 1364 }
1367 1365
1368 1366 tlen -= mlen;
1369 1367 dhp = dhp->dh_next;
1370 1368 off = 0;
1371 1369 }
1372 1370 }
1373 1371
1374 1372 mutex_enter(&freemem_lock);
1375 1373 ASSERT(sdp->softlockcnt >= btopr(len));
1376 1374 sdp->softlockcnt -= btopr(len);
1377 1375 mutex_exit(&freemem_lock);
1378 1376 if (sdp->softlockcnt == 0) {
1379 1377 /*
1380 1378 * All SOFTLOCKS are gone. Wakeup any waiting
1381 1379 * unmappers so they can try again to unmap.
1382 1380 * Check for waiters first without the mutex
1383 1381 * held so we don't always grab the mutex on
1384 1382 * softunlocks.
1385 1383 */
1386 1384 if (AS_ISUNMAPWAIT(seg->s_as)) {
1387 1385 mutex_enter(&seg->s_as->a_contents);
1388 1386 if (AS_ISUNMAPWAIT(seg->s_as)) {
1389 1387 AS_CLRUNMAPWAIT(seg->s_as);
1390 1388 cv_broadcast(&seg->s_as->a_cv);
1391 1389 }
1392 1390 mutex_exit(&seg->s_as->a_contents);
1393 1391 }
1394 1392 }
1395 1393
1396 1394 }
1397 1395
1398 1396 /*
1399 1397 * Handle fault for a single page.
1400 1398 * Done in a separate routine so we can handle errors more easily.
1401 1399 * This routine is called only from segdev_faultpages()
1402 1400 * when looping over the range of addresses requested. The segment lock is held.
1403 1401 */
1404 1402 static faultcode_t
1405 1403 segdev_faultpage(
1406 1404 struct hat *hat, /* the hat */
1407 1405 struct seg *seg, /* seg_dev of interest */
1408 1406 caddr_t addr, /* address in as */
1409 1407 struct vpage *vpage, /* pointer to vpage for seg, addr */
1410 1408 enum fault_type type, /* type of fault */
1411 1409 enum seg_rw rw, /* type of access at fault */
1412 1410 devmap_handle_t *dhp) /* devmap handle if any for this page */
1413 1411 {
1414 1412 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1415 1413 uint_t prot;
1416 1414 pfn_t pfnum = PFN_INVALID;
1417 1415 u_offset_t offset;
1418 1416 uint_t hat_flags;
1419 1417 dev_info_t *dip;
1420 1418
1421 1419 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE,
1422 1420 "segdev_faultpage: dhp=%p seg=%p addr=%p", dhp, seg, addr);
1423 1421 DEBUGF(8, (CE_CONT, "segdev_faultpage: dhp %p seg %p addr %p \n",
1424 1422 (void *)dhp, (void *)seg, (void *)addr));
1425 1423
1426 1424 /*
1427 1425 * Initialize protection value for this page.
1428 1426 * If we have per page protection values check it now.
1429 1427 */
1430 1428 if (sdp->pageprot) {
1431 1429 uint_t protchk;
1432 1430
1433 1431 switch (rw) {
1434 1432 case S_READ:
1435 1433 protchk = PROT_READ;
1436 1434 break;
1437 1435 case S_WRITE:
1438 1436 protchk = PROT_WRITE;
1439 1437 break;
1440 1438 case S_EXEC:
1441 1439 protchk = PROT_EXEC;
1442 1440 break;
1443 1441 case S_OTHER:
1444 1442 default:
1445 1443 protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1446 1444 break;
1447 1445 }
1448 1446
1449 1447 prot = VPP_PROT(vpage);
1450 1448 if ((prot & protchk) == 0)
1451 1449 return (FC_PROT); /* illegal access type */
1452 1450 } else {
1453 1451 prot = sdp->prot;
1454 1452 /* caller has already done segment level protection check */
1455 1453 }
1456 1454
1457 1455 if (type == F_SOFTLOCK) {
1458 1456 mutex_enter(&freemem_lock);
1459 1457 sdp->softlockcnt++;
1460 1458 mutex_exit(&freemem_lock);
1461 1459 }
1462 1460
1463 1461 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
1464 1462 offset = sdp->offset + (u_offset_t)(addr - seg->s_base);
1465 1463 /*
1466 1464 * In the devmap framework, sdp->mapfunc is set to NULL. we can get
1467 1465 * pfnum from dhp->dh_pfn (at beginning of segment) and offset from
1468 1466 * seg->s_base.
1469 1467 */
1470 1468 if (dhp == NULL) {
1471 1469 /* If segment has devmap_data, then dhp should be non-NULL */
1472 1470 ASSERT(sdp->devmap_data == NULL);
1473 1471 pfnum = (pfn_t)cdev_mmap(sdp->mapfunc, sdp->vp->v_rdev,
1474 1472 (off_t)offset, prot);
1475 1473 prot |= sdp->hat_attr;
1476 1474 } else {
1477 1475 ulong_t off;
1478 1476 struct ddi_umem_cookie *cp;
1479 1477 struct devmap_pmem_cookie *pcp;
1480 1478
1481 1479 /* ensure the dhp passed in contains addr. */
1482 1480 ASSERT(dhp == devmap_find_handle(
1483 1481 (devmap_handle_t *)sdp->devmap_data, addr));
1484 1482
1485 1483 off = addr - dhp->dh_uvaddr;
1486 1484
1487 1485 /*
1488 1486 * This routine assumes that the caller makes sure that the
1489 1487 * fields in dhp used below are unchanged due to remap during
1490 1488 * this call. Caller does HOLD_DHP_LOCK if neeed
1491 1489 */
1492 1490 cp = dhp->dh_cookie;
1493 1491 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1494 1492 pfnum = PFN_INVALID;
1495 1493 } else if (cookie_is_devmem(cp)) {
1496 1494 pfnum = dhp->dh_pfn + btop(off);
1497 1495 } else if (cookie_is_pmem(cp)) {
1498 1496 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
1499 1497 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
1500 1498 dhp->dh_roff < ptob(pcp->dp_npages));
1501 1499 pfnum = page_pptonum(
1502 1500 pcp->dp_pparray[btop(off + dhp->dh_roff)]);
1503 1501 } else {
1504 1502 ASSERT(dhp->dh_roff < cp->size);
1505 1503 ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
1506 1504 dhp->dh_cvaddr < (cp->cvaddr + cp->size));
1507 1505 ASSERT((dhp->dh_cvaddr + off) <=
1508 1506 (cp->cvaddr + cp->size));
1509 1507 ASSERT((dhp->dh_cvaddr + off + PAGESIZE) <=
1510 1508 (cp->cvaddr + cp->size));
1511 1509
1512 1510 switch (cp->type) {
1513 1511 case UMEM_LOCKED :
1514 1512 if (cp->pparray != NULL) {
1515 1513 ASSERT((dhp->dh_roff &
1516 1514 PAGEOFFSET) == 0);
1517 1515 pfnum = page_pptonum(
1518 1516 cp->pparray[btop(off +
1519 1517 dhp->dh_roff)]);
1520 1518 } else {
1521 1519 pfnum = hat_getpfnum(
1522 1520 ((proc_t *)cp->procp)->p_as->a_hat,
1523 1521 cp->cvaddr + off);
1524 1522 }
1525 1523 break;
1526 1524 case UMEM_TRASH :
1527 1525 pfnum = page_pptonum(trashpp);
1528 1526 /*
1529 1527 * We should set hat_flags to HAT_NOFAULT also
1530 1528 * However, not all hat layers implement this
1531 1529 */
1532 1530 break;
1533 1531 case KMEM_PAGEABLE:
1534 1532 case KMEM_NON_PAGEABLE:
1535 1533 pfnum = hat_getpfnum(kas.a_hat,
1536 1534 dhp->dh_cvaddr + off);
1537 1535 break;
1538 1536 default :
1539 1537 pfnum = PFN_INVALID;
1540 1538 break;
1541 1539 }
1542 1540 }
1543 1541 prot |= dhp->dh_hat_attr;
1544 1542 }
1545 1543 if (pfnum == PFN_INVALID) {
1546 1544 return (FC_MAKE_ERR(EFAULT));
1547 1545 }
1548 1546 /* prot should already be OR'ed in with hat_attributes if needed */
1549 1547
1550 1548 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE_CK1,
1551 1549 "segdev_faultpage: pfnum=%lx memory=%x prot=%x flags=%x",
1552 1550 pfnum, pf_is_memory(pfnum), prot, hat_flags);
1553 1551 DEBUGF(9, (CE_CONT, "segdev_faultpage: pfnum %lx memory %x "
1554 1552 "prot %x flags %x\n", pfnum, pf_is_memory(pfnum), prot, hat_flags));
1555 1553
1556 1554 if (pf_is_memory(pfnum) || (dhp != NULL)) {
1557 1555 /*
1558 1556 * It's not _really_ required here to pass sdp->hat_flags
1559 1557 * to hat_devload even though we do it.
1560 1558 * This is because hat figures it out DEVMEM mappings
1561 1559 * are non-consistent, anyway.
1562 1560 */
1563 1561 hat_devload(hat, addr, PAGESIZE, pfnum,
1564 1562 prot, hat_flags | sdp->hat_flags);
1565 1563 return (0);
1566 1564 }
1567 1565
1568 1566 /*
1569 1567 * Fall through to the case where devmap is not used and need to call
1570 1568 * up the device tree to set up the mapping
1571 1569 */
1572 1570
1573 1571 dip = VTOS(VTOCVP(sdp->vp))->s_dip;
1574 1572 ASSERT(dip);
1575 1573
1576 1574 /*
1577 1575 * When calling ddi_map_fault, we do not OR in sdp->hat_attr
1578 1576 * This is because this calls drivers which may not expect
1579 1577 * prot to have any other values than PROT_ALL
1580 1578 * The root nexus driver has a hack to peek into the segment
1581 1579 * structure and then OR in sdp->hat_attr.
1582 1580 * XX In case the bus_ops interfaces are ever revisited
1583 1581 * we need to fix this. prot should include other hat attributes
1584 1582 */
1585 1583 if (ddi_map_fault(dip, hat, seg, addr, NULL, pfnum, prot & PROT_ALL,
1586 1584 (uint_t)(type == F_SOFTLOCK)) != DDI_SUCCESS) {
1587 1585 return (FC_MAKE_ERR(EFAULT));
1588 1586 }
1589 1587 return (0);
1590 1588 }
1591 1589
1592 1590 static faultcode_t
1593 1591 segdev_fault(
1594 1592 struct hat *hat, /* the hat */
1595 1593 struct seg *seg, /* the seg_dev of interest */
1596 1594 caddr_t addr, /* the address of the fault */
1597 1595 size_t len, /* the length of the range */
1598 1596 enum fault_type type, /* type of fault */
1599 1597 enum seg_rw rw) /* type of access at fault */
1600 1598 {
1601 1599 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1602 1600 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1603 1601 devmap_handle_t *dhp;
1604 1602 struct devmap_softlock *slock = NULL;
1605 1603 ulong_t slpage = 0;
1606 1604 ulong_t off;
1607 1605 caddr_t maddr = addr;
1608 1606 int err;
1609 1607 int err_is_faultcode = 0;
1610 1608
1611 1609 TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_FAULT,
1612 1610 "segdev_fault: dhp_head=%p seg=%p addr=%p len=%lx type=%x",
1613 1611 (void *)dhp_head, (void *)seg, (void *)addr, len, type);
1614 1612 DEBUGF(7, (CE_CONT, "segdev_fault: dhp_head %p seg %p "
1615 1613 "addr %p len %lx type %x\n",
1616 1614 (void *)dhp_head, (void *)seg, (void *)addr, len, type));
1617 1615
1618 1616 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
1619 1617
1620 1618 /* Handle non-devmap case */
1621 1619 if (dhp_head == NULL)
1622 1620 return (segdev_faultpages(hat, seg, addr, len, type, rw, NULL));
1623 1621
1624 1622 /* Find devmap handle */
1625 1623 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
1626 1624 return (FC_NOMAP);
1627 1625
1628 1626 /*
1629 1627 * The seg_dev driver does not implement copy-on-write,
1630 1628 * and always loads translations with maximal allowed permissions
1631 1629 * but we got an fault trying to access the device.
1632 1630 * Servicing the fault is not going to result in any better result
1633 1631 * RFE: If we want devmap_access callbacks to be involved in F_PROT
1634 1632 * faults, then the code below is written for that
1635 1633 * Pending resolution of the following:
1636 1634 * - determine if the F_INVAL/F_SOFTLOCK syncing
1637 1635 * is needed for F_PROT also or not. The code below assumes it does
1638 1636 * - If driver sees F_PROT and calls devmap_load with same type,
1639 1637 * then segdev_faultpages will fail with FC_PROT anyway, need to
1640 1638 * change that so calls from devmap_load to segdev_faultpages for
1641 1639 * F_PROT type are retagged to F_INVAL.
1642 1640 * RFE: Today we dont have drivers that use devmap and want to handle
1643 1641 * F_PROT calls. The code in segdev_fault* is written to allow
1644 1642 * this case but is not tested. A driver that needs this capability
1645 1643 * should be able to remove the short-circuit case; resolve the
1646 1644 * above issues and "should" work.
1647 1645 */
1648 1646 if (type == F_PROT) {
1649 1647 return (FC_PROT);
1650 1648 }
1651 1649
1652 1650 /*
1653 1651 * Loop through dhp list calling devmap_access or segdev_faultpages for
1654 1652 * each devmap handle.
1655 1653 * drivers which implement devmap_access can interpose on faults and do
1656 1654 * device-appropriate special actions before calling devmap_load.
1657 1655 */
1658 1656
1659 1657 /*
1660 1658 * Unfortunately, this simple loop has turned out to expose a variety
1661 1659 * of complex problems which results in the following convoluted code.
1662 1660 *
1663 1661 * First, a desire to handle a serialization of F_SOFTLOCK calls
1664 1662 * to the driver within the framework.
1665 1663 * This results in a dh_softlock structure that is on a per device
1666 1664 * (or device instance) basis and serializes devmap_access calls.
1667 1665 * Ideally we would need to do this for underlying
1668 1666 * memory/device regions that are being faulted on
1669 1667 * but that is hard to identify and with REMAP, harder
1670 1668 * Second, a desire to serialize F_INVAL(and F_PROT) calls w.r.t.
1671 1669 * to F_SOFTLOCK calls to the driver.
1672 1670 * These serializations are to simplify the driver programmer model.
1673 1671 * To support these two features, the code first goes through the
1674 1672 * devmap handles and counts the pages (slpage) that are covered
1675 1673 * by devmap_access callbacks.
1676 1674 * This part ends with a devmap_softlock_enter call
1677 1675 * which allows only one F_SOFTLOCK active on a device instance,
1678 1676 * but multiple F_INVAL/F_PROTs can be active except when a
1679 1677 * F_SOFTLOCK is active
1680 1678 *
1681 1679 * Next, we dont short-circuit the fault code upfront to call
1682 1680 * segdev_softunlock for F_SOFTUNLOCK, because we must use
1683 1681 * the same length when we softlock and softunlock.
1684 1682 *
1685 1683 * -Hat layers may not support softunlocking lengths less than the
1686 1684 * original length when there is large page support.
1687 1685 * -kpmem locking is dependent on keeping the lengths same.
1688 1686 * -if drivers handled F_SOFTLOCK, they probably also expect to
1689 1687 * see an F_SOFTUNLOCK of the same length
1690 1688 * Hence, if extending lengths during softlock,
1691 1689 * softunlock has to make the same adjustments and goes through
1692 1690 * the same loop calling segdev_faultpages/segdev_softunlock
1693 1691 * But some of the synchronization and error handling is different
1694 1692 */
1695 1693
1696 1694 if (type != F_SOFTUNLOCK) {
1697 1695 devmap_handle_t *dhpp = dhp;
1698 1696 size_t slen = len;
1699 1697
1700 1698 /*
1701 1699 * Calculate count of pages that are :
1702 1700 * a) within the (potentially extended) fault region
1703 1701 * b) AND covered by devmap handle with devmap_access
1704 1702 */
1705 1703 off = (ulong_t)(addr - dhpp->dh_uvaddr);
1706 1704 while (slen != 0) {
1707 1705 size_t mlen;
1708 1706
1709 1707 /*
1710 1708 * Softlocking on a region that allows remap is
1711 1709 * unsupported due to unresolved locking issues
1712 1710 * XXX: unclear what these are?
1713 1711 * One potential is that if there is a pending
1714 1712 * softlock, then a remap should not be allowed
1715 1713 * until the unlock is done. This is easily
1716 1714 * fixed by returning error in devmap*remap on
1717 1715 * checking the dh->dh_softlock->softlocked value
1718 1716 */
1719 1717 if ((type == F_SOFTLOCK) &&
1720 1718 (dhpp->dh_flags & DEVMAP_ALLOW_REMAP)) {
1721 1719 return (FC_NOSUPPORT);
1722 1720 }
1723 1721
1724 1722 mlen = MIN(slen, (dhpp->dh_len - off));
1725 1723 if (dhpp->dh_callbackops.devmap_access) {
1726 1724 size_t llen;
1727 1725 caddr_t laddr;
1728 1726 /*
1729 1727 * use extended length for large page mappings
1730 1728 */
1731 1729 HOLD_DHP_LOCK(dhpp);
1732 1730 if ((sdp->pageprot == 0) &&
1733 1731 (dhpp->dh_flags & DEVMAP_FLAG_LARGE)) {
1734 1732 devmap_get_large_pgsize(dhpp,
1735 1733 mlen, maddr, &llen, &laddr);
1736 1734 } else {
1737 1735 llen = mlen;
1738 1736 }
1739 1737 RELE_DHP_LOCK(dhpp);
1740 1738
1741 1739 slpage += btopr(llen);
1742 1740 slock = dhpp->dh_softlock;
1743 1741 }
1744 1742 maddr += mlen;
1745 1743 ASSERT(slen >= mlen);
1746 1744 slen -= mlen;
1747 1745 dhpp = dhpp->dh_next;
1748 1746 off = 0;
1749 1747 }
1750 1748 /*
1751 1749 * synchonize with other faulting threads and wait till safe
1752 1750 * devmap_softlock_enter might return due to signal in cv_wait
1753 1751 *
1754 1752 * devmap_softlock_enter has to be called outside of while loop
1755 1753 * to prevent a deadlock if len spans over multiple dhps.
1756 1754 * dh_softlock is based on device instance and if multiple dhps
1757 1755 * use the same device instance, the second dhp's LOCK call
1758 1756 * will hang waiting on the first to complete.
1759 1757 * devmap_setup verifies that slocks in a dhp_chain are same.
1760 1758 * RFE: this deadlock only hold true for F_SOFTLOCK. For
1761 1759 * F_INVAL/F_PROT, since we now allow multiple in parallel,
1762 1760 * we could have done the softlock_enter inside the loop
1763 1761 * and supported multi-dhp mappings with dissimilar devices
1764 1762 */
1765 1763 if (err = devmap_softlock_enter(slock, slpage, type))
1766 1764 return (FC_MAKE_ERR(err));
1767 1765 }
1768 1766
1769 1767 /* reset 'maddr' to the start addr of the range of fault. */
1770 1768 maddr = addr;
1771 1769
1772 1770 /* calculate the offset corresponds to 'addr' in the first dhp. */
1773 1771 off = (ulong_t)(addr - dhp->dh_uvaddr);
1774 1772
1775 1773 /*
1776 1774 * The fault length may span over multiple dhps.
1777 1775 * Loop until the total length is satisfied.
1778 1776 */
1779 1777 while (len != 0) {
1780 1778 size_t llen;
1781 1779 size_t mlen;
1782 1780 caddr_t laddr;
1783 1781
1784 1782 /*
1785 1783 * mlen is the smaller of 'len' and the length
1786 1784 * from addr to the end of mapping defined by dhp.
1787 1785 */
1788 1786 mlen = MIN(len, (dhp->dh_len - off));
1789 1787
1790 1788 HOLD_DHP_LOCK(dhp);
1791 1789 /*
1792 1790 * Pass the extended length and address to devmap_access
1793 1791 * if large pagesize is used for loading address translations.
1794 1792 */
1795 1793 if ((sdp->pageprot == 0) &&
1796 1794 (dhp->dh_flags & DEVMAP_FLAG_LARGE)) {
1797 1795 devmap_get_large_pgsize(dhp, mlen, maddr,
1798 1796 &llen, &laddr);
1799 1797 ASSERT(maddr == addr || laddr == maddr);
1800 1798 } else {
1801 1799 llen = mlen;
1802 1800 laddr = maddr;
1803 1801 }
1804 1802
1805 1803 if (dhp->dh_callbackops.devmap_access != NULL) {
1806 1804 offset_t aoff;
1807 1805
1808 1806 aoff = sdp->offset + (offset_t)(laddr - seg->s_base);
1809 1807
1810 1808 /*
1811 1809 * call driver's devmap_access entry point which will
1812 1810 * call devmap_load/contextmgmt to load the translations
1813 1811 *
1814 1812 * We drop the dhp_lock before calling access so
1815 1813 * drivers can call devmap_*_remap within access
1816 1814 */
1817 1815 RELE_DHP_LOCK(dhp);
1818 1816
1819 1817 err = (*dhp->dh_callbackops.devmap_access)(
1820 1818 dhp, (void *)dhp->dh_pvtp, aoff, llen, type, rw);
1821 1819 } else {
1822 1820 /*
1823 1821 * If no devmap_access entry point, then load mappings
1824 1822 * hold dhp_lock across faultpages if REMAP
1825 1823 */
1826 1824 err = segdev_faultpages(hat, seg, laddr, llen,
1827 1825 type, rw, dhp);
1828 1826 err_is_faultcode = 1;
1829 1827 RELE_DHP_LOCK(dhp);
1830 1828 }
1831 1829
1832 1830 if (err) {
1833 1831 if ((type == F_SOFTLOCK) && (maddr > addr)) {
1834 1832 /*
1835 1833 * If not first dhp, use
1836 1834 * segdev_fault(F_SOFTUNLOCK) for prior dhps
1837 1835 * While this is recursion, it is incorrect to
1838 1836 * call just segdev_softunlock
1839 1837 * if we are using either large pages
1840 1838 * or devmap_access. It will be more right
1841 1839 * to go through the same loop as above
1842 1840 * rather than call segdev_softunlock directly
1843 1841 * It will use the right lenghths as well as
1844 1842 * call into the driver devmap_access routines.
1845 1843 */
1846 1844 size_t done = (size_t)(maddr - addr);
1847 1845 (void) segdev_fault(hat, seg, addr, done,
1848 1846 F_SOFTUNLOCK, S_OTHER);
1849 1847 /*
1850 1848 * reduce slpage by number of pages
1851 1849 * released by segdev_softunlock
1852 1850 */
1853 1851 ASSERT(slpage >= btopr(done));
1854 1852 devmap_softlock_exit(slock,
1855 1853 slpage - btopr(done), type);
1856 1854 } else {
1857 1855 devmap_softlock_exit(slock, slpage, type);
1858 1856 }
1859 1857
1860 1858
1861 1859 /*
1862 1860 * Segdev_faultpages() already returns a faultcode,
1863 1861 * hence, result from segdev_faultpages() should be
1864 1862 * returned directly.
1865 1863 */
1866 1864 if (err_is_faultcode)
1867 1865 return (err);
1868 1866 return (FC_MAKE_ERR(err));
1869 1867 }
1870 1868
1871 1869 maddr += mlen;
1872 1870 ASSERT(len >= mlen);
1873 1871 len -= mlen;
1874 1872 dhp = dhp->dh_next;
1875 1873 off = 0;
1876 1874
1877 1875 ASSERT(!dhp || len == 0 || maddr == dhp->dh_uvaddr);
1878 1876 }
1879 1877 /*
1880 1878 * release the softlock count at end of fault
1881 1879 * For F_SOFTLOCk this is done in the later F_SOFTUNLOCK
1882 1880 */
1883 1881 if ((type == F_INVAL) || (type == F_PROT))
1884 1882 devmap_softlock_exit(slock, slpage, type);
1885 1883 return (0);
1886 1884 }
1887 1885
1888 1886 /*
1889 1887 * segdev_faultpages
1890 1888 *
1891 1889 * Used to fault in seg_dev segment pages. Called by segdev_fault or devmap_load
1892 1890 * This routine assumes that the callers makes sure that the fields
1893 1891 * in dhp used below are not changed due to remap during this call.
1894 1892 * Caller does HOLD_DHP_LOCK if neeed
1895 1893 * This routine returns a faultcode_t as a return value for segdev_fault.
1896 1894 */
1897 1895 static faultcode_t
1898 1896 segdev_faultpages(
1899 1897 struct hat *hat, /* the hat */
1900 1898 struct seg *seg, /* the seg_dev of interest */
1901 1899 caddr_t addr, /* the address of the fault */
1902 1900 size_t len, /* the length of the range */
1903 1901 enum fault_type type, /* type of fault */
1904 1902 enum seg_rw rw, /* type of access at fault */
1905 1903 devmap_handle_t *dhp) /* devmap handle */
1906 1904 {
1907 1905 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1908 1906 register caddr_t a;
1909 1907 struct vpage *vpage;
1910 1908 struct ddi_umem_cookie *kpmem_cookie = NULL;
1911 1909 int err;
1912 1910
1913 1911 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGES,
1914 1912 "segdev_faultpages: dhp=%p seg=%p addr=%p len=%lx",
1915 1913 (void *)dhp, (void *)seg, (void *)addr, len);
1916 1914 DEBUGF(5, (CE_CONT, "segdev_faultpages: "
1917 1915 "dhp %p seg %p addr %p len %lx\n",
1918 1916 (void *)dhp, (void *)seg, (void *)addr, len));
1919 1917
1920 1918 /*
1921 1919 * The seg_dev driver does not implement copy-on-write,
1922 1920 * and always loads translations with maximal allowed permissions
1923 1921 * but we got an fault trying to access the device.
1924 1922 * Servicing the fault is not going to result in any better result
1925 1923 * XXX: If we want to allow devmap_access to handle F_PROT calls,
1926 1924 * This code should be removed and let the normal fault handling
1927 1925 * take care of finding the error
1928 1926 */
1929 1927 if (type == F_PROT) {
1930 1928 return (FC_PROT);
1931 1929 }
1932 1930
1933 1931 if (type == F_SOFTUNLOCK) {
1934 1932 segdev_softunlock(hat, seg, addr, len, rw);
1935 1933 return (0);
1936 1934 }
1937 1935
1938 1936 /*
1939 1937 * For kernel pageable memory, fault/lock segkp pages
1940 1938 * We hold this until the completion of this
1941 1939 * fault (INVAL/PROT) or till unlock (SOFTLOCK).
1942 1940 */
1943 1941 if ((dhp != NULL) && dhp_is_kpmem(dhp)) {
1944 1942 kpmem_cookie = (struct ddi_umem_cookie *)dhp->dh_cookie;
1945 1943 if (err = acquire_kpmem_lock(kpmem_cookie, btopr(len)))
1946 1944 return (err);
1947 1945 }
1948 1946
1949 1947 /*
1950 1948 * If we have the same protections for the entire segment,
1951 1949 * insure that the access being attempted is legitimate.
1952 1950 */
1953 1951 rw_enter(&sdp->lock, RW_READER);
1954 1952 if (sdp->pageprot == 0) {
1955 1953 uint_t protchk;
1956 1954
1957 1955 switch (rw) {
1958 1956 case S_READ:
1959 1957 protchk = PROT_READ;
1960 1958 break;
1961 1959 case S_WRITE:
1962 1960 protchk = PROT_WRITE;
1963 1961 break;
1964 1962 case S_EXEC:
1965 1963 protchk = PROT_EXEC;
1966 1964 break;
1967 1965 case S_OTHER:
1968 1966 default:
1969 1967 protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1970 1968 break;
1971 1969 }
1972 1970
1973 1971 if ((sdp->prot & protchk) == 0) {
1974 1972 rw_exit(&sdp->lock);
1975 1973 /* undo kpmem locking */
1976 1974 if (kpmem_cookie != NULL) {
1977 1975 release_kpmem_lock(kpmem_cookie, btopr(len));
1978 1976 }
1979 1977 return (FC_PROT); /* illegal access type */
1980 1978 }
1981 1979 }
1982 1980
1983 1981 /*
1984 1982 * we do a single hat_devload for the range if
1985 1983 * - devmap framework (dhp is not NULL),
1986 1984 * - pageprot == 0, i.e., no per-page protection set and
1987 1985 * - is device pages, irrespective of whether we are using large pages
1988 1986 */
1989 1987 if ((sdp->pageprot == 0) && (dhp != NULL) && dhp_is_devmem(dhp)) {
1990 1988 pfn_t pfnum;
1991 1989 uint_t hat_flags;
1992 1990
1993 1991 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1994 1992 rw_exit(&sdp->lock);
1995 1993 return (FC_NOMAP);
1996 1994 }
1997 1995
1998 1996 if (type == F_SOFTLOCK) {
1999 1997 mutex_enter(&freemem_lock);
2000 1998 sdp->softlockcnt += btopr(len);
2001 1999 mutex_exit(&freemem_lock);
2002 2000 }
2003 2001
2004 2002 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
2005 2003 pfnum = dhp->dh_pfn + btop((uintptr_t)(addr - dhp->dh_uvaddr));
2006 2004 ASSERT(!pf_is_memory(pfnum));
2007 2005
2008 2006 hat_devload(hat, addr, len, pfnum, sdp->prot | dhp->dh_hat_attr,
2009 2007 hat_flags | sdp->hat_flags);
2010 2008 rw_exit(&sdp->lock);
2011 2009 return (0);
2012 2010 }
2013 2011
2014 2012 /* Handle cases where we have to loop through fault handling per-page */
2015 2013
2016 2014 if (sdp->vpage == NULL)
2017 2015 vpage = NULL;
2018 2016 else
2019 2017 vpage = &sdp->vpage[seg_page(seg, addr)];
2020 2018
2021 2019 /* loop over the address range handling each fault */
2022 2020 for (a = addr; a < addr + len; a += PAGESIZE) {
2023 2021 if (err = segdev_faultpage(hat, seg, a, vpage, type, rw, dhp)) {
2024 2022 break;
2025 2023 }
2026 2024 if (vpage != NULL)
2027 2025 vpage++;
2028 2026 }
2029 2027 rw_exit(&sdp->lock);
2030 2028 if (err && (type == F_SOFTLOCK)) { /* error handling for F_SOFTLOCK */
2031 2029 size_t done = (size_t)(a - addr); /* pages fault successfully */
2032 2030 if (done > 0) {
2033 2031 /* use softunlock for those pages */
2034 2032 segdev_softunlock(hat, seg, addr, done, S_OTHER);
2035 2033 }
2036 2034 if (kpmem_cookie != NULL) {
2037 2035 /* release kpmem lock for rest of pages */
2038 2036 ASSERT(len >= done);
2039 2037 release_kpmem_lock(kpmem_cookie, btopr(len - done));
2040 2038 }
2041 2039 } else if ((kpmem_cookie != NULL) && (type != F_SOFTLOCK)) {
2042 2040 /* for non-SOFTLOCK cases, release kpmem */
2043 2041 release_kpmem_lock(kpmem_cookie, btopr(len));
2044 2042 }
2045 2043 return (err);
2046 2044 }
2047 2045
2048 2046 /*
2049 2047 * Asynchronous page fault. We simply do nothing since this
2050 2048 * entry point is not supposed to load up the translation.
2051 2049 */
2052 2050 /*ARGSUSED*/
2053 2051 static faultcode_t
2054 2052 segdev_faulta(struct seg *seg, caddr_t addr)
2055 2053 {
2056 2054 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FAULTA,
2057 2055 "segdev_faulta: seg=%p addr=%p", (void *)seg, (void *)addr);
2058 2056 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2059 2057
2060 2058 return (0);
2061 2059 }
2062 2060
2063 2061 static int
2064 2062 segdev_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2065 2063 {
2066 2064 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2067 2065 register devmap_handle_t *dhp;
2068 2066 register struct vpage *vp, *evp;
2069 2067 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
2070 2068 ulong_t off;
2071 2069 size_t mlen, sz;
2072 2070
2073 2071 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT,
2074 2072 "segdev_setprot:start seg=%p addr=%p len=%lx prot=%x",
2075 2073 (void *)seg, (void *)addr, len, prot);
2076 2074 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2077 2075
2078 2076 if ((sz = sdp->softlockcnt) > 0 && dhp_head != NULL) {
2079 2077 /*
2080 2078 * Fail the setprot if pages are SOFTLOCKed through this
2081 2079 * mapping.
2082 2080 * Softlockcnt is protected from change by the as read lock.
2083 2081 */
2084 2082 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT_CK1,
2085 2083 "segdev_setprot:error softlockcnt=%lx", sz);
2086 2084 DEBUGF(1, (CE_CONT, "segdev_setprot: softlockcnt %ld\n", sz));
2087 2085 return (EAGAIN);
2088 2086 }
2089 2087
2090 2088 if (dhp_head != NULL) {
2091 2089 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
2092 2090 return (EINVAL);
2093 2091
2094 2092 /*
2095 2093 * check if violate maxprot.
2096 2094 */
2097 2095 off = (ulong_t)(addr - dhp->dh_uvaddr);
2098 2096 mlen = len;
2099 2097 while (dhp) {
2100 2098 if ((dhp->dh_maxprot & prot) != prot)
2101 2099 return (EACCES); /* violated maxprot */
2102 2100
2103 2101 if (mlen > (dhp->dh_len - off)) {
2104 2102 mlen -= dhp->dh_len - off;
2105 2103 dhp = dhp->dh_next;
2106 2104 off = 0;
2107 2105 } else
2108 2106 break;
2109 2107 }
2110 2108 } else {
2111 2109 if ((sdp->maxprot & prot) != prot)
2112 2110 return (EACCES);
2113 2111 }
2114 2112
2115 2113 rw_enter(&sdp->lock, RW_WRITER);
2116 2114 if (addr == seg->s_base && len == seg->s_size && sdp->pageprot == 0) {
2117 2115 if (sdp->prot == prot) {
2118 2116 rw_exit(&sdp->lock);
2119 2117 return (0); /* all done */
2120 2118 }
2121 2119 sdp->prot = (uchar_t)prot;
2122 2120 } else {
2123 2121 sdp->pageprot = 1;
2124 2122 if (sdp->vpage == NULL) {
2125 2123 /*
2126 2124 * First time through setting per page permissions,
2127 2125 * initialize all the vpage structures to prot
2128 2126 */
2129 2127 sdp->vpage = kmem_zalloc(vpgtob(seg_pages(seg)),
2130 2128 KM_SLEEP);
2131 2129 evp = &sdp->vpage[seg_pages(seg)];
2132 2130 for (vp = sdp->vpage; vp < evp; vp++)
2133 2131 VPP_SETPROT(vp, sdp->prot);
2134 2132 }
2135 2133 /*
2136 2134 * Now go change the needed vpages protections.
2137 2135 */
2138 2136 evp = &sdp->vpage[seg_page(seg, addr + len)];
2139 2137 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++)
2140 2138 VPP_SETPROT(vp, prot);
2141 2139 }
2142 2140 rw_exit(&sdp->lock);
2143 2141
2144 2142 if (dhp_head != NULL) {
2145 2143 devmap_handle_t *tdhp;
2146 2144 /*
2147 2145 * If large page size was used in hat_devload(),
2148 2146 * the same page size must be used in hat_unload().
2149 2147 */
2150 2148 dhp = tdhp = devmap_find_handle(dhp_head, addr);
2151 2149 while (tdhp != NULL) {
2152 2150 if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
2153 2151 break;
2154 2152 }
2155 2153 tdhp = tdhp->dh_next;
2156 2154 }
2157 2155 if (tdhp) {
2158 2156 size_t slen = len;
2159 2157 size_t mlen;
2160 2158 size_t soff;
2161 2159
2162 2160 soff = (ulong_t)(addr - dhp->dh_uvaddr);
2163 2161 while (slen != 0) {
2164 2162 mlen = MIN(slen, (dhp->dh_len - soff));
2165 2163 hat_unload(seg->s_as->a_hat, dhp->dh_uvaddr,
2166 2164 dhp->dh_len, HAT_UNLOAD);
2167 2165 dhp = dhp->dh_next;
2168 2166 ASSERT(slen >= mlen);
2169 2167 slen -= mlen;
2170 2168 soff = 0;
2171 2169 }
2172 2170 return (0);
2173 2171 }
2174 2172 }
2175 2173
2176 2174 if ((prot & ~PROT_USER) == PROT_NONE) {
2177 2175 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD);
2178 2176 } else {
2179 2177 /*
2180 2178 * RFE: the segment should keep track of all attributes
2181 2179 * allowing us to remove the deprecated hat_chgprot
2182 2180 * and use hat_chgattr.
2183 2181 */
2184 2182 hat_chgprot(seg->s_as->a_hat, addr, len, prot);
2185 2183 }
2186 2184
2187 2185 return (0);
2188 2186 }
2189 2187
2190 2188 static int
2191 2189 segdev_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2192 2190 {
2193 2191 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2194 2192 struct vpage *vp, *evp;
2195 2193
2196 2194 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_CHECKPROT,
2197 2195 "segdev_checkprot:start seg=%p addr=%p len=%lx prot=%x",
2198 2196 (void *)seg, (void *)addr, len, prot);
2199 2197 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2200 2198
2201 2199 /*
2202 2200 * If segment protection can be used, simply check against them
2203 2201 */
2204 2202 rw_enter(&sdp->lock, RW_READER);
2205 2203 if (sdp->pageprot == 0) {
2206 2204 register int err;
2207 2205
2208 2206 err = ((sdp->prot & prot) != prot) ? EACCES : 0;
2209 2207 rw_exit(&sdp->lock);
2210 2208 return (err);
2211 2209 }
2212 2210
2213 2211 /*
2214 2212 * Have to check down to the vpage level
2215 2213 */
2216 2214 evp = &sdp->vpage[seg_page(seg, addr + len)];
2217 2215 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++) {
2218 2216 if ((VPP_PROT(vp) & prot) != prot) {
2219 2217 rw_exit(&sdp->lock);
2220 2218 return (EACCES);
2221 2219 }
2222 2220 }
2223 2221 rw_exit(&sdp->lock);
2224 2222 return (0);
2225 2223 }
2226 2224
2227 2225 static int
2228 2226 segdev_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
2229 2227 {
2230 2228 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2231 2229 size_t pgno;
2232 2230
2233 2231 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_GETPROT,
2234 2232 "segdev_getprot:start seg=%p addr=%p len=%lx protv=%p",
2235 2233 (void *)seg, (void *)addr, len, (void *)protv);
2236 2234 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2237 2235
2238 2236 pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
2239 2237 if (pgno != 0) {
2240 2238 rw_enter(&sdp->lock, RW_READER);
2241 2239 if (sdp->pageprot == 0) {
2242 2240 do {
2243 2241 protv[--pgno] = sdp->prot;
2244 2242 } while (pgno != 0);
2245 2243 } else {
2246 2244 size_t pgoff = seg_page(seg, addr);
2247 2245
2248 2246 do {
2249 2247 pgno--;
2250 2248 protv[pgno] =
2251 2249 VPP_PROT(&sdp->vpage[pgno + pgoff]);
2252 2250 } while (pgno != 0);
2253 2251 }
2254 2252 rw_exit(&sdp->lock);
2255 2253 }
2256 2254 return (0);
2257 2255 }
2258 2256
2259 2257 static u_offset_t
2260 2258 segdev_getoffset(register struct seg *seg, caddr_t addr)
2261 2259 {
2262 2260 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2263 2261
2264 2262 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETOFFSET,
2265 2263 "segdev_getoffset:start seg=%p addr=%p", (void *)seg, (void *)addr);
2266 2264
2267 2265 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2268 2266
2269 2267 return ((u_offset_t)sdp->offset + (addr - seg->s_base));
2270 2268 }
2271 2269
2272 2270 /*ARGSUSED*/
2273 2271 static int
2274 2272 segdev_gettype(register struct seg *seg, caddr_t addr)
2275 2273 {
2276 2274 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2277 2275
2278 2276 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETTYPE,
2279 2277 "segdev_gettype:start seg=%p addr=%p", (void *)seg, (void *)addr);
2280 2278
2281 2279 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2282 2280
2283 2281 return (sdp->type);
2284 2282 }
2285 2283
2286 2284
2287 2285 /*ARGSUSED*/
2288 2286 static int
2289 2287 segdev_getvp(register struct seg *seg, caddr_t addr, struct vnode **vpp)
2290 2288 {
2291 2289 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2292 2290
2293 2291 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETVP,
2294 2292 "segdev_getvp:start seg=%p addr=%p", (void *)seg, (void *)addr);
2295 2293
2296 2294 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2297 2295
2298 2296 /*
2299 2297 * Note that this vp is the common_vp of the device, where the
2300 2298 * pages are hung ..
2301 2299 */
2302 2300 *vpp = VTOCVP(sdp->vp);
2303 2301
2304 2302 return (0);
2305 2303 }
2306 2304
2307 2305 static void
2308 2306 segdev_badop(void)
2309 2307 {
2310 2308 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGDEV_BADOP,
2311 2309 "segdev_badop:start");
2312 2310 panic("segdev_badop");
2313 2311 /*NOTREACHED*/
2314 2312 }
2315 2313
2316 2314 /*
2317 2315 * segdev pages are not in the cache, and thus can't really be controlled.
2318 2316 * Hence, syncs are simply always successful.
2319 2317 */
2320 2318 /*ARGSUSED*/
2321 2319 static int
2322 2320 segdev_sync(struct seg *seg, caddr_t addr, size_t len, int attr, uint_t flags)
2323 2321 {
2324 2322 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SYNC, "segdev_sync:start");
2325 2323
2326 2324 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2327 2325
2328 2326 return (0);
2329 2327 }
2330 2328
2331 2329 /*
2332 2330 * segdev pages are always "in core".
2333 2331 */
2334 2332 /*ARGSUSED*/
2335 2333 static size_t
2336 2334 segdev_incore(struct seg *seg, caddr_t addr, size_t len, char *vec)
2337 2335 {
2338 2336 size_t v = 0;
2339 2337
2340 2338 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_INCORE, "segdev_incore:start");
2341 2339
2342 2340 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2343 2341
2344 2342 for (len = (len + PAGEOFFSET) & PAGEMASK; len; len -= PAGESIZE,
2345 2343 v += PAGESIZE)
2346 2344 *vec++ = 1;
2347 2345 return (v);
2348 2346 }
2349 2347
2350 2348 /*
2351 2349 * segdev pages are not in the cache, and thus can't really be controlled.
2352 2350 * Hence, locks are simply always successful.
2353 2351 */
2354 2352 /*ARGSUSED*/
2355 2353 static int
2356 2354 segdev_lockop(struct seg *seg, caddr_t addr,
2357 2355 size_t len, int attr, int op, ulong_t *lockmap, size_t pos)
2358 2356 {
2359 2357 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_LOCKOP, "segdev_lockop:start");
2360 2358
2361 2359 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2362 2360
2363 2361 return (0);
2364 2362 }
2365 2363
2366 2364 /*
2367 2365 * segdev pages are not in the cache, and thus can't really be controlled.
2368 2366 * Hence, advise is simply always successful.
2369 2367 */
2370 2368 /*ARGSUSED*/
2371 2369 static int
2372 2370 segdev_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
2373 2371 {
2374 2372 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_ADVISE, "segdev_advise:start");
2375 2373
2376 2374 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2377 2375
2378 2376 return (0);
2379 2377 }
2380 2378
2381 2379 /*
2382 2380 * segdev pages are not dumped, so we just return
2383 2381 */
2384 2382 /*ARGSUSED*/
2385 2383 static void
2386 2384 segdev_dump(struct seg *seg)
2387 2385 {}
2388 2386
2389 2387 /*
2390 2388 * ddi_segmap_setup: Used by drivers who wish specify mapping attributes
2391 2389 * for a segment. Called from a drivers segmap(9E)
2392 2390 * routine.
2393 2391 */
2394 2392 /*ARGSUSED*/
2395 2393 int
2396 2394 ddi_segmap_setup(dev_t dev, off_t offset, struct as *as, caddr_t *addrp,
2397 2395 off_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cred,
2398 2396 ddi_device_acc_attr_t *accattrp, uint_t rnumber)
2399 2397 {
2400 2398 struct segdev_crargs dev_a;
2401 2399 int (*mapfunc)(dev_t dev, off_t off, int prot);
2402 2400 uint_t hat_attr;
2403 2401 pfn_t pfn;
2404 2402 int error, i;
2405 2403
2406 2404 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP_SETUP,
2407 2405 "ddi_segmap_setup:start");
2408 2406
2409 2407 if ((mapfunc = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap) == nodev)
2410 2408 return (ENODEV);
2411 2409
2412 2410 /*
2413 2411 * Character devices that support the d_mmap
2414 2412 * interface can only be mmap'ed shared.
2415 2413 */
2416 2414 if ((flags & MAP_TYPE) != MAP_SHARED)
2417 2415 return (EINVAL);
2418 2416
2419 2417 /*
2420 2418 * Check that this region is indeed mappable on this platform.
2421 2419 * Use the mapping function.
2422 2420 */
2423 2421 if (ddi_device_mapping_check(dev, accattrp, rnumber, &hat_attr) == -1)
2424 2422 return (ENXIO);
2425 2423
2426 2424 /*
2427 2425 * Check to ensure that the entire range is
2428 2426 * legal and we are not trying to map in
2429 2427 * more than the device will let us.
2430 2428 */
2431 2429 for (i = 0; i < len; i += PAGESIZE) {
2432 2430 if (i == 0) {
2433 2431 /*
2434 2432 * Save the pfn at offset here. This pfn will be
2435 2433 * used later to get user address.
2436 2434 */
2437 2435 if ((pfn = (pfn_t)cdev_mmap(mapfunc, dev, offset,
2438 2436 maxprot)) == PFN_INVALID)
2439 2437 return (ENXIO);
2440 2438 } else {
2441 2439 if (cdev_mmap(mapfunc, dev, offset + i, maxprot) ==
2442 2440 PFN_INVALID)
2443 2441 return (ENXIO);
2444 2442 }
2445 2443 }
2446 2444
2447 2445 as_rangelock(as);
2448 2446 /* Pick an address w/o worrying about any vac alignment constraints. */
2449 2447 error = choose_addr(as, addrp, len, ptob(pfn), ADDR_NOVACALIGN, flags);
2450 2448 if (error != 0) {
2451 2449 as_rangeunlock(as);
2452 2450 return (error);
2453 2451 }
2454 2452
2455 2453 dev_a.mapfunc = mapfunc;
2456 2454 dev_a.dev = dev;
2457 2455 dev_a.offset = (offset_t)offset;
2458 2456 dev_a.type = flags & MAP_TYPE;
2459 2457 dev_a.prot = (uchar_t)prot;
2460 2458 dev_a.maxprot = (uchar_t)maxprot;
2461 2459 dev_a.hat_attr = hat_attr;
2462 2460 dev_a.hat_flags = 0;
2463 2461 dev_a.devmap_data = NULL;
2464 2462
2465 2463 error = as_map(as, *addrp, len, segdev_create, &dev_a);
2466 2464 as_rangeunlock(as);
2467 2465 return (error);
2468 2466
2469 2467 }
2470 2468
2471 2469 /*ARGSUSED*/
2472 2470 static int
2473 2471 segdev_pagelock(struct seg *seg, caddr_t addr, size_t len,
2474 2472 struct page ***ppp, enum lock_type type, enum seg_rw rw)
2475 2473 {
2476 2474 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_PAGELOCK,
2477 2475 "segdev_pagelock:start");
2478 2476 return (ENOTSUP);
2479 2477 }
2480 2478
2481 2479 /*ARGSUSED*/
2482 2480 static int
2483 2481 segdev_setpagesize(struct seg *seg, caddr_t addr, size_t len,
2484 2482 uint_t szc)
2485 2483 {
2486 2484 return (ENOTSUP);
2487 2485 }
2488 2486
2489 2487 /*
2490 2488 * devmap_device: Used by devmap framework to establish mapping
2491 2489 * called by devmap_seup(9F) during map setup time.
2492 2490 */
2493 2491 /*ARGSUSED*/
2494 2492 static int
2495 2493 devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
2496 2494 offset_t off, size_t len, uint_t flags)
2497 2495 {
2498 2496 devmap_handle_t *rdhp, *maxdhp;
2499 2497 struct segdev_crargs dev_a;
2500 2498 int err;
2501 2499 uint_t maxprot = PROT_ALL;
2502 2500 offset_t offset = 0;
2503 2501 pfn_t pfn;
2504 2502 struct devmap_pmem_cookie *pcp;
2505 2503
2506 2504 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVICE,
2507 2505 "devmap_device:start dhp=%p addr=%p off=%llx, len=%lx",
2508 2506 (void *)dhp, (void *)addr, off, len);
2509 2507
2510 2508 DEBUGF(2, (CE_CONT, "devmap_device: dhp %p addr %p off %llx len %lx\n",
2511 2509 (void *)dhp, (void *)addr, off, len));
2512 2510
2513 2511 as_rangelock(as);
2514 2512 if ((flags & MAP_FIXED) == 0) {
2515 2513 offset_t aligned_off;
2516 2514
2517 2515 rdhp = maxdhp = dhp;
2518 2516 while (rdhp != NULL) {
2519 2517 maxdhp = (maxdhp->dh_len > rdhp->dh_len) ?
2520 2518 maxdhp : rdhp;
2521 2519 rdhp = rdhp->dh_next;
2522 2520 maxprot |= dhp->dh_maxprot;
2523 2521 }
2524 2522 offset = maxdhp->dh_uoff - dhp->dh_uoff;
2525 2523
2526 2524 /*
2527 2525 * Use the dhp that has the
2528 2526 * largest len to get user address.
2529 2527 */
2530 2528 /*
2531 2529 * If MAPPING_INVALID, cannot use dh_pfn/dh_cvaddr,
2532 2530 * use 0 which is as good as any other.
2533 2531 */
2534 2532 if (maxdhp->dh_flags & DEVMAP_MAPPING_INVALID) {
2535 2533 aligned_off = (offset_t)0;
2536 2534 } else if (dhp_is_devmem(maxdhp)) {
2537 2535 aligned_off = (offset_t)ptob(maxdhp->dh_pfn) - offset;
2538 2536 } else if (dhp_is_pmem(maxdhp)) {
2539 2537 pcp = (struct devmap_pmem_cookie *)maxdhp->dh_pcookie;
2540 2538 pfn = page_pptonum(
2541 2539 pcp->dp_pparray[btop(maxdhp->dh_roff)]);
2542 2540 aligned_off = (offset_t)ptob(pfn) - offset;
2543 2541 } else {
2544 2542 aligned_off = (offset_t)(uintptr_t)maxdhp->dh_cvaddr -
2545 2543 offset;
2546 2544 }
2547 2545
2548 2546 /*
2549 2547 * Pick an address aligned to dh_cookie.
2550 2548 * for kernel memory/user memory, cookie is cvaddr.
2551 2549 * for device memory, cookie is physical address.
2552 2550 */
2553 2551 map_addr(addr, len, aligned_off, 1, flags);
2554 2552 if (*addr == NULL) {
2555 2553 as_rangeunlock(as);
2556 2554 return (ENOMEM);
2557 2555 }
2558 2556 } else {
2559 2557 /*
2560 2558 * User-specified address; blow away any previous mappings.
2561 2559 */
2562 2560 (void) as_unmap(as, *addr, len);
2563 2561 }
2564 2562
2565 2563 dev_a.mapfunc = NULL;
2566 2564 dev_a.dev = dhp->dh_dev;
2567 2565 dev_a.type = flags & MAP_TYPE;
2568 2566 dev_a.offset = off;
2569 2567 /*
2570 2568 * sdp->maxprot has the least restrict protection of all dhps.
2571 2569 */
2572 2570 dev_a.maxprot = maxprot;
2573 2571 dev_a.prot = dhp->dh_prot;
2574 2572 /*
2575 2573 * devmap uses dhp->dh_hat_attr for hat.
2576 2574 */
2577 2575 dev_a.hat_flags = 0;
2578 2576 dev_a.hat_attr = 0;
2579 2577 dev_a.devmap_data = (void *)dhp;
2580 2578
2581 2579 err = as_map(as, *addr, len, segdev_create, &dev_a);
2582 2580 as_rangeunlock(as);
2583 2581 return (err);
2584 2582 }
2585 2583
2586 2584 int
2587 2585 devmap_do_ctxmgt(devmap_cookie_t dhc, void *pvtp, offset_t off, size_t len,
2588 2586 uint_t type, uint_t rw, int (*ctxmgt)(devmap_cookie_t, void *, offset_t,
2589 2587 size_t, uint_t, uint_t))
2590 2588 {
2591 2589 register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2592 2590 struct devmap_ctx *devctx;
2593 2591 int do_timeout = 0;
2594 2592 int ret;
2595 2593
2596 2594 #ifdef lint
2597 2595 pvtp = pvtp;
2598 2596 #endif
2599 2597
2600 2598 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT,
2601 2599 "devmap_do_ctxmgt:start dhp=%p off=%llx, len=%lx",
2602 2600 (void *)dhp, off, len);
2603 2601 DEBUGF(7, (CE_CONT, "devmap_do_ctxmgt: dhp %p off %llx len %lx\n",
2604 2602 (void *)dhp, off, len));
2605 2603
2606 2604 if (ctxmgt == NULL)
2607 2605 return (FC_HWERR);
2608 2606
2609 2607 devctx = dhp->dh_ctx;
2610 2608
2611 2609 /*
2612 2610 * If we are on an MP system with more than one cpu running
2613 2611 * and if a thread on some CPU already has the context, wait
2614 2612 * for it to finish if there is a hysteresis timeout.
2615 2613 *
2616 2614 * We call cv_wait() instead of cv_wait_sig() because
2617 2615 * it does not matter much if it returned due to a signal
2618 2616 * or due to a cv_signal() or cv_broadcast(). In either event
2619 2617 * we need to complete the mapping otherwise the processes
2620 2618 * will die with a SEGV.
2621 2619 */
2622 2620 if ((dhp->dh_timeout_length > 0) && (ncpus > 1)) {
2623 2621 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK1,
2624 2622 "devmap_do_ctxmgt:doing hysteresis, devctl %p dhp %p",
2625 2623 devctx, dhp);
2626 2624 do_timeout = 1;
2627 2625 mutex_enter(&devctx->lock);
2628 2626 while (devctx->oncpu)
2629 2627 cv_wait(&devctx->cv, &devctx->lock);
2630 2628 devctx->oncpu = 1;
2631 2629 mutex_exit(&devctx->lock);
2632 2630 }
2633 2631
2634 2632 /*
2635 2633 * Call the contextmgt callback so that the driver can handle
2636 2634 * the fault.
2637 2635 */
2638 2636 ret = (*ctxmgt)(dhp, dhp->dh_pvtp, off, len, type, rw);
2639 2637
2640 2638 /*
2641 2639 * If devmap_access() returned -1, then there was a hardware
2642 2640 * error so we need to convert the return value to something
2643 2641 * that trap() will understand. Otherwise, the return value
2644 2642 * is already a fault code generated by devmap_unload()
2645 2643 * or devmap_load().
2646 2644 */
2647 2645 if (ret) {
2648 2646 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK2,
2649 2647 "devmap_do_ctxmgt: ret=%x dhp=%p devctx=%p",
2650 2648 ret, dhp, devctx);
2651 2649 DEBUGF(1, (CE_CONT, "devmap_do_ctxmgt: ret %x dhp %p\n",
2652 2650 ret, (void *)dhp));
2653 2651 if (devctx->oncpu) {
2654 2652 mutex_enter(&devctx->lock);
2655 2653 devctx->oncpu = 0;
2656 2654 cv_signal(&devctx->cv);
2657 2655 mutex_exit(&devctx->lock);
2658 2656 }
2659 2657 return (FC_HWERR);
2660 2658 }
2661 2659
2662 2660 /*
2663 2661 * Setup the timeout if we need to
2664 2662 */
2665 2663 if (do_timeout) {
2666 2664 mutex_enter(&devctx->lock);
2667 2665 if (dhp->dh_timeout_length > 0) {
2668 2666 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK3,
2669 2667 "devmap_do_ctxmgt:timeout set");
2670 2668 devctx->timeout = timeout(devmap_ctxto,
2671 2669 devctx, dhp->dh_timeout_length);
2672 2670 } else {
2673 2671 /*
2674 2672 * We don't want to wait so set oncpu to
2675 2673 * 0 and wake up anyone waiting.
2676 2674 */
2677 2675 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK4,
2678 2676 "devmap_do_ctxmgt:timeout not set");
2679 2677 devctx->oncpu = 0;
2680 2678 cv_signal(&devctx->cv);
2681 2679 }
2682 2680 mutex_exit(&devctx->lock);
2683 2681 }
2684 2682
2685 2683 return (DDI_SUCCESS);
2686 2684 }
2687 2685
2688 2686 /*
2689 2687 * end of mapping
2690 2688 * poff fault_offset |
2691 2689 * base | | |
2692 2690 * | | | |
2693 2691 * V V V V
2694 2692 * +-----------+---------------+-------+---------+-------+
2695 2693 * ^ ^ ^ ^
2696 2694 * |<--- offset--->|<-len->| |
2697 2695 * |<--- dh_len(size of mapping) --->|
2698 2696 * |<-- pg -->|
2699 2697 * -->|rlen|<--
2700 2698 */
2701 2699 static ulong_t
2702 2700 devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
2703 2701 ulong_t *opfn, ulong_t *pagesize)
2704 2702 {
2705 2703 register int level;
2706 2704 ulong_t pg;
2707 2705 ulong_t poff;
2708 2706 ulong_t base;
2709 2707 caddr_t uvaddr;
2710 2708 long rlen;
2711 2709
2712 2710 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP,
2713 2711 "devmap_roundup:start dhp=%p off=%lx len=%lx",
2714 2712 (void *)dhp, offset, len);
2715 2713 DEBUGF(2, (CE_CONT, "devmap_roundup: dhp %p off %lx len %lx\n",
2716 2714 (void *)dhp, offset, len));
2717 2715
2718 2716 /*
2719 2717 * get the max. pagesize that is aligned within the range
2720 2718 * <dh_pfn, dh_pfn+offset>.
2721 2719 *
2722 2720 * The calculations below use physical address to ddetermine
2723 2721 * the page size to use. The same calculations can use the
2724 2722 * virtual address to determine the page size.
2725 2723 */
2726 2724 base = (ulong_t)ptob(dhp->dh_pfn);
2727 2725 for (level = dhp->dh_mmulevel; level >= 0; level--) {
2728 2726 pg = page_get_pagesize(level);
2729 2727 poff = ((base + offset) & ~(pg - 1));
2730 2728 uvaddr = dhp->dh_uvaddr + (poff - base);
2731 2729 if ((poff >= base) &&
2732 2730 ((poff + pg) <= (base + dhp->dh_len)) &&
2733 2731 VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg))
2734 2732 break;
2735 2733 }
2736 2734
2737 2735 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK1,
2738 2736 "devmap_roundup: base=%lx poff=%lx dhp=%p",
2739 2737 base, poff, dhp);
2740 2738 DEBUGF(2, (CE_CONT, "devmap_roundup: base %lx poff %lx pfn %lx\n",
2741 2739 base, poff, dhp->dh_pfn));
2742 2740
2743 2741 ASSERT(VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg));
2744 2742 ASSERT(level >= 0);
2745 2743
2746 2744 *pagesize = pg;
2747 2745 *opfn = dhp->dh_pfn + btop(poff - base);
2748 2746
2749 2747 rlen = len + offset - (poff - base + pg);
2750 2748
2751 2749 ASSERT(rlen < (long)len);
2752 2750
2753 2751 TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK2,
2754 2752 "devmap_roundup:ret dhp=%p level=%x rlen=%lx psiz=%p opfn=%p",
2755 2753 (void *)dhp, level, rlen, pagesize, opfn);
2756 2754 DEBUGF(1, (CE_CONT, "devmap_roundup: dhp %p "
2757 2755 "level %x rlen %lx psize %lx opfn %lx\n",
2758 2756 (void *)dhp, level, rlen, *pagesize, *opfn));
2759 2757
2760 2758 return ((ulong_t)((rlen > 0) ? rlen : 0));
2761 2759 }
2762 2760
2763 2761 /*
2764 2762 * find the dhp that contains addr.
2765 2763 */
2766 2764 static devmap_handle_t *
2767 2765 devmap_find_handle(devmap_handle_t *dhp_head, caddr_t addr)
2768 2766 {
2769 2767 devmap_handle_t *dhp;
2770 2768
2771 2769 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_FIND_HANDLE,
2772 2770 "devmap_find_handle:start");
2773 2771
2774 2772 dhp = dhp_head;
2775 2773 while (dhp) {
2776 2774 if (addr >= dhp->dh_uvaddr &&
2777 2775 addr < (dhp->dh_uvaddr + dhp->dh_len))
2778 2776 return (dhp);
2779 2777 dhp = dhp->dh_next;
2780 2778 }
2781 2779
2782 2780 return ((devmap_handle_t *)NULL);
2783 2781 }
2784 2782
2785 2783 /*
2786 2784 * devmap_unload:
2787 2785 * Marks a segdev segment or pages if offset->offset+len
2788 2786 * is not the entire segment as intercept and unloads the
2789 2787 * pages in the range offset -> offset+len.
2790 2788 */
2791 2789 int
2792 2790 devmap_unload(devmap_cookie_t dhc, offset_t offset, size_t len)
2793 2791 {
2794 2792 register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2795 2793 caddr_t addr;
2796 2794 ulong_t size;
2797 2795 ssize_t soff;
2798 2796
2799 2797 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_UNLOAD,
2800 2798 "devmap_unload:start dhp=%p offset=%llx len=%lx",
2801 2799 (void *)dhp, offset, len);
2802 2800 DEBUGF(7, (CE_CONT, "devmap_unload: dhp %p offset %llx len %lx\n",
2803 2801 (void *)dhp, offset, len));
2804 2802
2805 2803 soff = (ssize_t)(offset - dhp->dh_uoff);
2806 2804 soff = round_down_p2(soff, PAGESIZE);
2807 2805 if (soff < 0 || soff >= dhp->dh_len)
2808 2806 return (FC_MAKE_ERR(EINVAL));
2809 2807
2810 2808 /*
2811 2809 * Address and size must be page aligned. Len is set to the
2812 2810 * number of bytes in the number of pages that are required to
2813 2811 * support len. Offset is set to the byte offset of the first byte
2814 2812 * of the page that contains offset.
2815 2813 */
2816 2814 len = round_up_p2(len, PAGESIZE);
2817 2815
2818 2816 /*
2819 2817 * If len is == 0, then calculate the size by getting
2820 2818 * the number of bytes from offset to the end of the segment.
2821 2819 */
2822 2820 if (len == 0)
2823 2821 size = dhp->dh_len - soff;
2824 2822 else {
2825 2823 size = len;
2826 2824 if ((soff + size) > dhp->dh_len)
2827 2825 return (FC_MAKE_ERR(EINVAL));
2828 2826 }
2829 2827
2830 2828 /*
2831 2829 * The address is offset bytes from the base address of
2832 2830 * the dhp.
2833 2831 */
2834 2832 addr = (caddr_t)(soff + dhp->dh_uvaddr);
2835 2833
2836 2834 /*
2837 2835 * If large page size was used in hat_devload(),
2838 2836 * the same page size must be used in hat_unload().
2839 2837 */
2840 2838 if (dhp->dh_flags & DEVMAP_FLAG_LARGE) {
2841 2839 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
2842 2840 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
2843 2841 } else {
2844 2842 hat_unload(dhp->dh_seg->s_as->a_hat, addr, size,
2845 2843 HAT_UNLOAD|HAT_UNLOAD_OTHER);
2846 2844 }
2847 2845
2848 2846 return (0);
2849 2847 }
2850 2848
2851 2849 /*
2852 2850 * calculates the optimal page size that will be used for hat_devload().
2853 2851 */
2854 2852 static void
2855 2853 devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr,
2856 2854 size_t *llen, caddr_t *laddr)
2857 2855 {
2858 2856 ulong_t off;
2859 2857 ulong_t pfn;
2860 2858 ulong_t pgsize;
2861 2859 uint_t first = 1;
2862 2860
2863 2861 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GET_LARGE_PGSIZE,
2864 2862 "devmap_get_large_pgsize:start");
2865 2863
2866 2864 /*
2867 2865 * RFE - Code only supports large page mappings for devmem
2868 2866 * This code could be changed in future if we want to support
2869 2867 * large page mappings for kernel exported memory.
2870 2868 */
2871 2869 ASSERT(dhp_is_devmem(dhp));
2872 2870 ASSERT(!(dhp->dh_flags & DEVMAP_MAPPING_INVALID));
2873 2871
2874 2872 *llen = 0;
2875 2873 off = (ulong_t)(addr - dhp->dh_uvaddr);
2876 2874 while ((long)len > 0) {
2877 2875 /*
2878 2876 * get the optimal pfn to minimize address translations.
2879 2877 * devmap_roundup() returns residue bytes for next round
2880 2878 * calculations.
2881 2879 */
2882 2880 len = devmap_roundup(dhp, off, len, &pfn, &pgsize);
2883 2881
2884 2882 if (first) {
2885 2883 *laddr = dhp->dh_uvaddr + ptob(pfn - dhp->dh_pfn);
2886 2884 first = 0;
2887 2885 }
2888 2886
2889 2887 *llen += pgsize;
2890 2888 off = ptob(pfn - dhp->dh_pfn) + pgsize;
2891 2889 }
2892 2890 /* Large page mapping len/addr cover more range than original fault */
2893 2891 ASSERT(*llen >= len && *laddr <= addr);
2894 2892 ASSERT((*laddr + *llen) >= (addr + len));
2895 2893 }
2896 2894
2897 2895 /*
2898 2896 * Initialize the devmap_softlock structure.
2899 2897 */
2900 2898 static struct devmap_softlock *
2901 2899 devmap_softlock_init(dev_t dev, ulong_t id)
2902 2900 {
2903 2901 struct devmap_softlock *slock;
2904 2902 struct devmap_softlock *tmp;
2905 2903
2906 2904 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_INIT,
2907 2905 "devmap_softlock_init:start");
2908 2906
2909 2907 tmp = kmem_zalloc(sizeof (struct devmap_softlock), KM_SLEEP);
2910 2908 mutex_enter(&devmap_slock);
2911 2909
2912 2910 for (slock = devmap_slist; slock != NULL; slock = slock->next)
2913 2911 if ((slock->dev == dev) && (slock->id == id))
2914 2912 break;
2915 2913
2916 2914 if (slock == NULL) {
2917 2915 slock = tmp;
2918 2916 slock->dev = dev;
2919 2917 slock->id = id;
2920 2918 mutex_init(&slock->lock, NULL, MUTEX_DEFAULT, NULL);
2921 2919 cv_init(&slock->cv, NULL, CV_DEFAULT, NULL);
2922 2920 slock->next = devmap_slist;
2923 2921 devmap_slist = slock;
2924 2922 } else
2925 2923 kmem_free(tmp, sizeof (struct devmap_softlock));
2926 2924
2927 2925 mutex_enter(&slock->lock);
2928 2926 slock->refcnt++;
2929 2927 mutex_exit(&slock->lock);
2930 2928 mutex_exit(&devmap_slock);
2931 2929
2932 2930 return (slock);
2933 2931 }
2934 2932
2935 2933 /*
2936 2934 * Wake up processes that sleep on softlocked.
2937 2935 * Free dh_softlock if refcnt is 0.
2938 2936 */
2939 2937 static void
2940 2938 devmap_softlock_rele(devmap_handle_t *dhp)
2941 2939 {
2942 2940 struct devmap_softlock *slock = dhp->dh_softlock;
2943 2941 struct devmap_softlock *tmp;
2944 2942 struct devmap_softlock *parent;
2945 2943
2946 2944 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_RELE,
2947 2945 "devmap_softlock_rele:start");
2948 2946
2949 2947 mutex_enter(&devmap_slock);
2950 2948 mutex_enter(&slock->lock);
2951 2949
2952 2950 ASSERT(slock->refcnt > 0);
2953 2951
2954 2952 slock->refcnt--;
2955 2953
2956 2954 /*
2957 2955 * If no one is using the device, free up the slock data.
2958 2956 */
2959 2957 if (slock->refcnt == 0) {
2960 2958 slock->softlocked = 0;
2961 2959 cv_signal(&slock->cv);
2962 2960
2963 2961 if (devmap_slist == slock)
2964 2962 devmap_slist = slock->next;
2965 2963 else {
2966 2964 parent = devmap_slist;
2967 2965 for (tmp = devmap_slist->next; tmp != NULL;
2968 2966 tmp = tmp->next) {
2969 2967 if (tmp == slock) {
2970 2968 parent->next = tmp->next;
2971 2969 break;
2972 2970 }
2973 2971 parent = tmp;
2974 2972 }
2975 2973 }
2976 2974 mutex_exit(&slock->lock);
2977 2975 mutex_destroy(&slock->lock);
2978 2976 cv_destroy(&slock->cv);
2979 2977 kmem_free(slock, sizeof (struct devmap_softlock));
2980 2978 } else
2981 2979 mutex_exit(&slock->lock);
2982 2980
2983 2981 mutex_exit(&devmap_slock);
2984 2982 }
2985 2983
2986 2984 /*
2987 2985 * Wake up processes that sleep on dh_ctx->locked.
2988 2986 * Free dh_ctx if refcnt is 0.
2989 2987 */
2990 2988 static void
2991 2989 devmap_ctx_rele(devmap_handle_t *dhp)
2992 2990 {
2993 2991 struct devmap_ctx *devctx = dhp->dh_ctx;
2994 2992 struct devmap_ctx *tmp;
2995 2993 struct devmap_ctx *parent;
2996 2994 timeout_id_t tid;
2997 2995
2998 2996 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE,
2999 2997 "devmap_ctx_rele:start");
3000 2998
3001 2999 mutex_enter(&devmapctx_lock);
3002 3000 mutex_enter(&devctx->lock);
3003 3001
3004 3002 ASSERT(devctx->refcnt > 0);
3005 3003
3006 3004 devctx->refcnt--;
3007 3005
3008 3006 /*
3009 3007 * If no one is using the device, free up the devctx data.
3010 3008 */
3011 3009 if (devctx->refcnt == 0) {
3012 3010 /*
3013 3011 * Untimeout any threads using this mapping as they are about
3014 3012 * to go away.
3015 3013 */
3016 3014 if (devctx->timeout != 0) {
3017 3015 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE_CK1,
3018 3016 "devmap_ctx_rele:untimeout ctx->timeout");
3019 3017
3020 3018 tid = devctx->timeout;
3021 3019 mutex_exit(&devctx->lock);
3022 3020 (void) untimeout(tid);
3023 3021 mutex_enter(&devctx->lock);
3024 3022 }
3025 3023
3026 3024 devctx->oncpu = 0;
3027 3025 cv_signal(&devctx->cv);
3028 3026
3029 3027 if (devmapctx_list == devctx)
3030 3028 devmapctx_list = devctx->next;
3031 3029 else {
3032 3030 parent = devmapctx_list;
3033 3031 for (tmp = devmapctx_list->next; tmp != NULL;
3034 3032 tmp = tmp->next) {
3035 3033 if (tmp == devctx) {
3036 3034 parent->next = tmp->next;
3037 3035 break;
3038 3036 }
3039 3037 parent = tmp;
3040 3038 }
3041 3039 }
3042 3040 mutex_exit(&devctx->lock);
3043 3041 mutex_destroy(&devctx->lock);
3044 3042 cv_destroy(&devctx->cv);
3045 3043 kmem_free(devctx, sizeof (struct devmap_ctx));
3046 3044 } else
3047 3045 mutex_exit(&devctx->lock);
3048 3046
3049 3047 mutex_exit(&devmapctx_lock);
3050 3048 }
3051 3049
3052 3050 /*
3053 3051 * devmap_load:
3054 3052 * Marks a segdev segment or pages if offset->offset+len
3055 3053 * is not the entire segment as nointercept and faults in
3056 3054 * the pages in the range offset -> offset+len.
3057 3055 */
3058 3056 int
3059 3057 devmap_load(devmap_cookie_t dhc, offset_t offset, size_t len, uint_t type,
3060 3058 uint_t rw)
3061 3059 {
3062 3060 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3063 3061 struct as *asp = dhp->dh_seg->s_as;
3064 3062 caddr_t addr;
3065 3063 ulong_t size;
3066 3064 ssize_t soff; /* offset from the beginning of the segment */
3067 3065 int rc;
3068 3066
3069 3067 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_LOAD,
3070 3068 "devmap_load:start dhp=%p offset=%llx len=%lx",
3071 3069 (void *)dhp, offset, len);
3072 3070
3073 3071 DEBUGF(7, (CE_CONT, "devmap_load: dhp %p offset %llx len %lx\n",
3074 3072 (void *)dhp, offset, len));
3075 3073
3076 3074 /*
3077 3075 * Hat layer only supports devload to process' context for which
3078 3076 * the as lock is held. Verify here and return error if drivers
3079 3077 * inadvertently call devmap_load on a wrong devmap handle.
3080 3078 */
3081 3079 if ((asp != &kas) && !AS_LOCK_HELD(asp, &asp->a_lock))
3082 3080 return (FC_MAKE_ERR(EINVAL));
3083 3081
3084 3082 soff = (ssize_t)(offset - dhp->dh_uoff);
3085 3083 soff = round_down_p2(soff, PAGESIZE);
3086 3084 if (soff < 0 || soff >= dhp->dh_len)
3087 3085 return (FC_MAKE_ERR(EINVAL));
3088 3086
3089 3087 /*
3090 3088 * Address and size must be page aligned. Len is set to the
3091 3089 * number of bytes in the number of pages that are required to
3092 3090 * support len. Offset is set to the byte offset of the first byte
3093 3091 * of the page that contains offset.
3094 3092 */
3095 3093 len = round_up_p2(len, PAGESIZE);
3096 3094
3097 3095 /*
3098 3096 * If len == 0, then calculate the size by getting
3099 3097 * the number of bytes from offset to the end of the segment.
3100 3098 */
3101 3099 if (len == 0)
3102 3100 size = dhp->dh_len - soff;
3103 3101 else {
3104 3102 size = len;
3105 3103 if ((soff + size) > dhp->dh_len)
3106 3104 return (FC_MAKE_ERR(EINVAL));
3107 3105 }
3108 3106
3109 3107 /*
3110 3108 * The address is offset bytes from the base address of
3111 3109 * the segment.
3112 3110 */
3113 3111 addr = (caddr_t)(soff + dhp->dh_uvaddr);
3114 3112
3115 3113 HOLD_DHP_LOCK(dhp);
3116 3114 rc = segdev_faultpages(asp->a_hat,
3117 3115 dhp->dh_seg, addr, size, type, rw, dhp);
3118 3116 RELE_DHP_LOCK(dhp);
3119 3117 return (rc);
3120 3118 }
3121 3119
3122 3120 int
3123 3121 devmap_setup(dev_t dev, offset_t off, struct as *as, caddr_t *addrp,
3124 3122 size_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3125 3123 {
3126 3124 register devmap_handle_t *dhp;
3127 3125 int (*devmap)(dev_t, devmap_cookie_t, offset_t, size_t,
3128 3126 size_t *, uint_t);
3129 3127 int (*mmap)(dev_t, off_t, int);
3130 3128 struct devmap_callback_ctl *callbackops;
3131 3129 devmap_handle_t *dhp_head = NULL;
3132 3130 devmap_handle_t *dhp_prev = NULL;
3133 3131 devmap_handle_t *dhp_curr;
3134 3132 caddr_t addr;
3135 3133 int map_flag;
3136 3134 int ret;
3137 3135 ulong_t total_len;
3138 3136 size_t map_len;
3139 3137 size_t resid_len = len;
3140 3138 offset_t map_off = off;
3141 3139 struct devmap_softlock *slock = NULL;
3142 3140
3143 3141 #ifdef lint
3144 3142 cred = cred;
3145 3143 #endif
3146 3144
3147 3145 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SETUP,
3148 3146 "devmap_setup:start off=%llx len=%lx", off, len);
3149 3147 DEBUGF(3, (CE_CONT, "devmap_setup: off %llx len %lx\n",
3150 3148 off, len));
3151 3149
3152 3150 devmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_devmap;
3153 3151 mmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap;
3154 3152
3155 3153 /*
3156 3154 * driver must provide devmap(9E) entry point in cb_ops to use the
3157 3155 * devmap framework.
3158 3156 */
3159 3157 if (devmap == NULL || devmap == nulldev || devmap == nodev)
3160 3158 return (EINVAL);
3161 3159
3162 3160 /*
3163 3161 * To protect from an inadvertent entry because the devmap entry point
3164 3162 * is not NULL, return error if D_DEVMAP bit is not set in cb_flag and
3165 3163 * mmap is NULL.
3166 3164 */
3167 3165 map_flag = devopsp[getmajor(dev)]->devo_cb_ops->cb_flag;
3168 3166 if ((map_flag & D_DEVMAP) == 0 && (mmap == NULL || mmap == nulldev))
3169 3167 return (EINVAL);
3170 3168
3171 3169 /*
3172 3170 * devmap allows mmap(2) to map multiple registers.
3173 3171 * one devmap_handle is created for each register mapped.
3174 3172 */
3175 3173 for (total_len = 0; total_len < len; total_len += map_len) {
3176 3174 dhp = kmem_zalloc(sizeof (devmap_handle_t), KM_SLEEP);
3177 3175
3178 3176 if (dhp_prev != NULL)
3179 3177 dhp_prev->dh_next = dhp;
3180 3178 else
3181 3179 dhp_head = dhp;
3182 3180 dhp_prev = dhp;
3183 3181
3184 3182 dhp->dh_prot = prot;
3185 3183 dhp->dh_orig_maxprot = dhp->dh_maxprot = maxprot;
3186 3184 dhp->dh_dev = dev;
3187 3185 dhp->dh_timeout_length = CTX_TIMEOUT_VALUE;
3188 3186 dhp->dh_uoff = map_off;
3189 3187
3190 3188 /*
3191 3189 * Get mapping specific info from
3192 3190 * the driver, such as rnumber, roff, len, callbackops,
3193 3191 * accattrp and, if the mapping is for kernel memory,
3194 3192 * ddi_umem_cookie.
3195 3193 */
3196 3194 if ((ret = cdev_devmap(dev, dhp, map_off,
3197 3195 resid_len, &map_len, get_udatamodel())) != 0) {
3198 3196 free_devmap_handle(dhp_head);
3199 3197 return (ENXIO);
3200 3198 }
3201 3199
3202 3200 if (map_len & PAGEOFFSET) {
3203 3201 free_devmap_handle(dhp_head);
3204 3202 return (EINVAL);
3205 3203 }
3206 3204
3207 3205 callbackops = &dhp->dh_callbackops;
3208 3206
3209 3207 if ((callbackops->devmap_access == NULL) ||
3210 3208 (callbackops->devmap_access == nulldev) ||
3211 3209 (callbackops->devmap_access == nodev)) {
3212 3210 /*
3213 3211 * Normally devmap does not support MAP_PRIVATE unless
3214 3212 * the drivers provide a valid devmap_access routine.
3215 3213 */
3216 3214 if ((flags & MAP_PRIVATE) != 0) {
3217 3215 free_devmap_handle(dhp_head);
3218 3216 return (EINVAL);
3219 3217 }
3220 3218 } else {
3221 3219 /*
3222 3220 * Initialize dhp_softlock and dh_ctx if the drivers
3223 3221 * provide devmap_access.
3224 3222 */
3225 3223 dhp->dh_softlock = devmap_softlock_init(dev,
3226 3224 (ulong_t)callbackops->devmap_access);
3227 3225 dhp->dh_ctx = devmap_ctxinit(dev,
3228 3226 (ulong_t)callbackops->devmap_access);
3229 3227
3230 3228 /*
3231 3229 * segdev_fault can only work when all
3232 3230 * dh_softlock in a multi-dhp mapping
3233 3231 * are same. see comments in segdev_fault
3234 3232 * This code keeps track of the first
3235 3233 * dh_softlock allocated in slock and
3236 3234 * compares all later allocations and if
3237 3235 * not similar, returns an error.
3238 3236 */
3239 3237 if (slock == NULL)
3240 3238 slock = dhp->dh_softlock;
3241 3239 if (slock != dhp->dh_softlock) {
3242 3240 free_devmap_handle(dhp_head);
3243 3241 return (ENOTSUP);
3244 3242 }
3245 3243 }
3246 3244
3247 3245 map_off += map_len;
3248 3246 resid_len -= map_len;
3249 3247 }
3250 3248
3251 3249 /*
3252 3250 * get the user virtual address and establish the mapping between
3253 3251 * uvaddr and device physical address.
3254 3252 */
3255 3253 if ((ret = devmap_device(dhp_head, as, addrp, off, len, flags))
3256 3254 != 0) {
3257 3255 /*
3258 3256 * free devmap handles if error during the mapping.
3259 3257 */
3260 3258 free_devmap_handle(dhp_head);
3261 3259
3262 3260 return (ret);
3263 3261 }
3264 3262
3265 3263 /*
3266 3264 * call the driver's devmap_map callback to do more after the mapping,
3267 3265 * such as to allocate driver private data for context management.
3268 3266 */
3269 3267 dhp = dhp_head;
3270 3268 map_off = off;
3271 3269 addr = *addrp;
3272 3270 while (dhp != NULL) {
3273 3271 callbackops = &dhp->dh_callbackops;
3274 3272 dhp->dh_uvaddr = addr;
3275 3273 dhp_curr = dhp;
3276 3274 if (callbackops->devmap_map != NULL) {
3277 3275 ret = (*callbackops->devmap_map)((devmap_cookie_t)dhp,
3278 3276 dev, flags, map_off,
3279 3277 dhp->dh_len, &dhp->dh_pvtp);
3280 3278 if (ret != 0) {
3281 3279 struct segdev_data *sdp;
3282 3280
3283 3281 /*
3284 3282 * call driver's devmap_unmap entry point
3285 3283 * to free driver resources.
3286 3284 */
3287 3285 dhp = dhp_head;
3288 3286 map_off = off;
3289 3287 while (dhp != dhp_curr) {
3290 3288 callbackops = &dhp->dh_callbackops;
3291 3289 if (callbackops->devmap_unmap != NULL) {
3292 3290 (*callbackops->devmap_unmap)(
3293 3291 dhp, dhp->dh_pvtp,
3294 3292 map_off, dhp->dh_len,
3295 3293 NULL, NULL, NULL, NULL);
3296 3294 }
3297 3295 map_off += dhp->dh_len;
3298 3296 dhp = dhp->dh_next;
3299 3297 }
3300 3298 sdp = dhp_head->dh_seg->s_data;
3301 3299 sdp->devmap_data = NULL;
3302 3300 free_devmap_handle(dhp_head);
3303 3301 return (ENXIO);
3304 3302 }
3305 3303 }
3306 3304 map_off += dhp->dh_len;
3307 3305 addr += dhp->dh_len;
3308 3306 dhp = dhp->dh_next;
3309 3307 }
3310 3308
3311 3309 return (0);
3312 3310 }
3313 3311
3314 3312 int
3315 3313 ddi_devmap_segmap(dev_t dev, off_t off, ddi_as_handle_t as, caddr_t *addrp,
3316 3314 off_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3317 3315 {
3318 3316 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP,
3319 3317 "devmap_segmap:start");
3320 3318 return (devmap_setup(dev, (offset_t)off, (struct as *)as, addrp,
3321 3319 (size_t)len, prot, maxprot, flags, cred));
3322 3320 }
3323 3321
3324 3322 /*
3325 3323 * Called from devmap_devmem_setup/remap to see if can use large pages for
3326 3324 * this device mapping.
3327 3325 * Also calculate the max. page size for this mapping.
3328 3326 * this page size will be used in fault routine for
3329 3327 * optimal page size calculations.
3330 3328 */
3331 3329 static void
3332 3330 devmap_devmem_large_page_setup(devmap_handle_t *dhp)
3333 3331 {
3334 3332 ASSERT(dhp_is_devmem(dhp));
3335 3333 dhp->dh_mmulevel = 0;
3336 3334
3337 3335 /*
3338 3336 * use large page size only if:
3339 3337 * 1. device memory.
3340 3338 * 2. mmu supports multiple page sizes,
3341 3339 * 3. Driver did not disallow it
3342 3340 * 4. dhp length is at least as big as the large pagesize
3343 3341 * 5. the uvaddr and pfn are large pagesize aligned
3344 3342 */
3345 3343 if (page_num_pagesizes() > 1 &&
3346 3344 !(dhp->dh_flags & (DEVMAP_USE_PAGESIZE | DEVMAP_MAPPING_INVALID))) {
3347 3345 ulong_t base;
3348 3346 int level;
3349 3347
3350 3348 base = (ulong_t)ptob(dhp->dh_pfn);
3351 3349 for (level = 1; level < page_num_pagesizes(); level++) {
3352 3350 size_t pgsize = page_get_pagesize(level);
3353 3351 if ((dhp->dh_len < pgsize) ||
3354 3352 (!VA_PA_PGSIZE_ALIGNED((uintptr_t)dhp->dh_uvaddr,
3355 3353 base, pgsize))) {
3356 3354 break;
3357 3355 }
3358 3356 }
3359 3357 dhp->dh_mmulevel = level - 1;
3360 3358 }
3361 3359 if (dhp->dh_mmulevel > 0) {
3362 3360 dhp->dh_flags |= DEVMAP_FLAG_LARGE;
3363 3361 } else {
3364 3362 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3365 3363 }
3366 3364 }
3367 3365
3368 3366 /*
3369 3367 * Called by driver devmap routine to pass device specific info to
3370 3368 * the framework. used for device memory mapping only.
3371 3369 */
3372 3370 int
3373 3371 devmap_devmem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3374 3372 struct devmap_callback_ctl *callbackops, uint_t rnumber, offset_t roff,
3375 3373 size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp)
3376 3374 {
3377 3375 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3378 3376 ddi_acc_handle_t handle;
3379 3377 ddi_map_req_t mr;
3380 3378 ddi_acc_hdl_t *hp;
3381 3379 int err;
3382 3380
3383 3381 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_SETUP,
3384 3382 "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3385 3383 (void *)dhp, roff, rnumber, (uint_t)len);
3386 3384 DEBUGF(2, (CE_CONT, "devmap_devmem_setup: dhp %p offset %llx "
3387 3385 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3388 3386
3389 3387 /*
3390 3388 * First to check if this function has been called for this dhp.
3391 3389 */
3392 3390 if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3393 3391 return (DDI_FAILURE);
3394 3392
3395 3393 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3396 3394 return (DDI_FAILURE);
3397 3395
3398 3396 if (flags & DEVMAP_MAPPING_INVALID) {
3399 3397 /*
3400 3398 * Don't go up the tree to get pfn if the driver specifies
3401 3399 * DEVMAP_MAPPING_INVALID in flags.
3402 3400 *
3403 3401 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3404 3402 * remap permission.
3405 3403 */
3406 3404 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3407 3405 return (DDI_FAILURE);
3408 3406 }
3409 3407 dhp->dh_pfn = PFN_INVALID;
3410 3408 } else {
3411 3409 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3412 3410 if (handle == NULL)
3413 3411 return (DDI_FAILURE);
3414 3412
3415 3413 hp = impl_acc_hdl_get(handle);
3416 3414 hp->ah_vers = VERS_ACCHDL;
3417 3415 hp->ah_dip = dip;
3418 3416 hp->ah_rnumber = rnumber;
3419 3417 hp->ah_offset = roff;
3420 3418 hp->ah_len = len;
3421 3419 if (accattrp != NULL)
3422 3420 hp->ah_acc = *accattrp;
3423 3421
3424 3422 mr.map_op = DDI_MO_MAP_LOCKED;
3425 3423 mr.map_type = DDI_MT_RNUMBER;
3426 3424 mr.map_obj.rnumber = rnumber;
3427 3425 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3428 3426 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3429 3427 mr.map_handlep = hp;
3430 3428 mr.map_vers = DDI_MAP_VERSION;
3431 3429
3432 3430 /*
3433 3431 * up the device tree to get pfn.
3434 3432 * The rootnex_map_regspec() routine in nexus drivers has been
3435 3433 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3436 3434 */
3437 3435 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&dhp->dh_pfn);
3438 3436 dhp->dh_hat_attr = hp->ah_hat_flags;
3439 3437 impl_acc_hdl_free(handle);
3440 3438
3441 3439 if (err)
3442 3440 return (DDI_FAILURE);
3443 3441 }
3444 3442 /* Should not be using devmem setup for memory pages */
3445 3443 ASSERT(!pf_is_memory(dhp->dh_pfn));
3446 3444
3447 3445 /* Only some of the flags bits are settable by the driver */
3448 3446 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3449 3447 dhp->dh_len = ptob(btopr(len));
3450 3448
3451 3449 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3452 3450 dhp->dh_roff = ptob(btop(roff));
3453 3451
3454 3452 /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3455 3453 devmap_devmem_large_page_setup(dhp);
3456 3454 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3457 3455 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3458 3456
3459 3457
3460 3458 if (callbackops != NULL) {
3461 3459 bcopy(callbackops, &dhp->dh_callbackops,
3462 3460 sizeof (struct devmap_callback_ctl));
3463 3461 }
3464 3462
3465 3463 /*
3466 3464 * Initialize dh_lock if we want to do remap.
3467 3465 */
3468 3466 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3469 3467 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3470 3468 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3471 3469 }
3472 3470
3473 3471 dhp->dh_flags |= DEVMAP_SETUP_DONE;
3474 3472
3475 3473 return (DDI_SUCCESS);
3476 3474 }
3477 3475
3478 3476 int
3479 3477 devmap_devmem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3480 3478 uint_t rnumber, offset_t roff, size_t len, uint_t maxprot,
3481 3479 uint_t flags, ddi_device_acc_attr_t *accattrp)
3482 3480 {
3483 3481 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3484 3482 ddi_acc_handle_t handle;
3485 3483 ddi_map_req_t mr;
3486 3484 ddi_acc_hdl_t *hp;
3487 3485 pfn_t pfn;
3488 3486 uint_t hat_flags;
3489 3487 int err;
3490 3488
3491 3489 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_REMAP,
3492 3490 "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3493 3491 (void *)dhp, roff, rnumber, (uint_t)len);
3494 3492 DEBUGF(2, (CE_CONT, "devmap_devmem_remap: dhp %p offset %llx "
3495 3493 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3496 3494
3497 3495 /*
3498 3496 * Return failure if setup has not been done or no remap permission
3499 3497 * has been granted during the setup.
3500 3498 */
3501 3499 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3502 3500 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3503 3501 return (DDI_FAILURE);
3504 3502
3505 3503 /* Only DEVMAP_MAPPING_INVALID flag supported for remap */
3506 3504 if ((flags != 0) && (flags != DEVMAP_MAPPING_INVALID))
3507 3505 return (DDI_FAILURE);
3508 3506
3509 3507 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3510 3508 return (DDI_FAILURE);
3511 3509
3512 3510 if (!(flags & DEVMAP_MAPPING_INVALID)) {
3513 3511 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3514 3512 if (handle == NULL)
3515 3513 return (DDI_FAILURE);
3516 3514 }
3517 3515
3518 3516 HOLD_DHP_LOCK(dhp);
3519 3517
3520 3518 /*
3521 3519 * Unload the old mapping, so next fault will setup the new mappings
3522 3520 * Do this while holding the dhp lock so other faults dont reestablish
3523 3521 * the mappings
3524 3522 */
3525 3523 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3526 3524 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3527 3525
3528 3526 if (flags & DEVMAP_MAPPING_INVALID) {
3529 3527 dhp->dh_flags |= DEVMAP_MAPPING_INVALID;
3530 3528 dhp->dh_pfn = PFN_INVALID;
3531 3529 } else {
3532 3530 /* clear any prior DEVMAP_MAPPING_INVALID flag */
3533 3531 dhp->dh_flags &= ~DEVMAP_MAPPING_INVALID;
3534 3532 hp = impl_acc_hdl_get(handle);
3535 3533 hp->ah_vers = VERS_ACCHDL;
3536 3534 hp->ah_dip = dip;
3537 3535 hp->ah_rnumber = rnumber;
3538 3536 hp->ah_offset = roff;
3539 3537 hp->ah_len = len;
3540 3538 if (accattrp != NULL)
3541 3539 hp->ah_acc = *accattrp;
3542 3540
3543 3541 mr.map_op = DDI_MO_MAP_LOCKED;
3544 3542 mr.map_type = DDI_MT_RNUMBER;
3545 3543 mr.map_obj.rnumber = rnumber;
3546 3544 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3547 3545 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3548 3546 mr.map_handlep = hp;
3549 3547 mr.map_vers = DDI_MAP_VERSION;
3550 3548
3551 3549 /*
3552 3550 * up the device tree to get pfn.
3553 3551 * The rootnex_map_regspec() routine in nexus drivers has been
3554 3552 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3555 3553 */
3556 3554 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&pfn);
3557 3555 hat_flags = hp->ah_hat_flags;
3558 3556 impl_acc_hdl_free(handle);
3559 3557 if (err) {
3560 3558 RELE_DHP_LOCK(dhp);
3561 3559 return (DDI_FAILURE);
3562 3560 }
3563 3561 /*
3564 3562 * Store result of ddi_map first in local variables, as we do
3565 3563 * not want to overwrite the existing dhp with wrong data.
3566 3564 */
3567 3565 dhp->dh_pfn = pfn;
3568 3566 dhp->dh_hat_attr = hat_flags;
3569 3567 }
3570 3568
3571 3569 /* clear the large page size flag */
3572 3570 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3573 3571
3574 3572 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3575 3573 dhp->dh_roff = ptob(btop(roff));
3576 3574
3577 3575 /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3578 3576 devmap_devmem_large_page_setup(dhp);
3579 3577 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3580 3578 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3581 3579
3582 3580 RELE_DHP_LOCK(dhp);
3583 3581 return (DDI_SUCCESS);
3584 3582 }
3585 3583
3586 3584 /*
3587 3585 * called by driver devmap routine to pass kernel virtual address mapping
3588 3586 * info to the framework. used only for kernel memory
3589 3587 * allocated from ddi_umem_alloc().
3590 3588 */
3591 3589 int
3592 3590 devmap_umem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3593 3591 struct devmap_callback_ctl *callbackops, ddi_umem_cookie_t cookie,
3594 3592 offset_t off, size_t len, uint_t maxprot, uint_t flags,
3595 3593 ddi_device_acc_attr_t *accattrp)
3596 3594 {
3597 3595 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3598 3596 struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3599 3597
3600 3598 #ifdef lint
3601 3599 dip = dip;
3602 3600 #endif
3603 3601
3604 3602 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_SETUP,
3605 3603 "devmap_umem_setup:start dhp=%p offset=%llx cookie=%p len=%lx",
3606 3604 (void *)dhp, off, cookie, len);
3607 3605 DEBUGF(2, (CE_CONT, "devmap_umem_setup: dhp %p offset %llx "
3608 3606 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3609 3607
3610 3608 if (cookie == NULL)
3611 3609 return (DDI_FAILURE);
3612 3610
3613 3611 /* For UMEM_TRASH, this restriction is not needed */
3614 3612 if ((off + len) > cp->size)
3615 3613 return (DDI_FAILURE);
3616 3614
3617 3615 /* check if the cache attributes are supported */
3618 3616 if (i_ddi_check_cache_attr(flags) == B_FALSE)
3619 3617 return (DDI_FAILURE);
3620 3618
3621 3619 /*
3622 3620 * First to check if this function has been called for this dhp.
3623 3621 */
3624 3622 if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3625 3623 return (DDI_FAILURE);
3626 3624
3627 3625 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3628 3626 return (DDI_FAILURE);
3629 3627
3630 3628 if (flags & DEVMAP_MAPPING_INVALID) {
3631 3629 /*
3632 3630 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3633 3631 * remap permission.
3634 3632 */
3635 3633 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3636 3634 return (DDI_FAILURE);
3637 3635 }
3638 3636 } else {
3639 3637 dhp->dh_cookie = cookie;
3640 3638 dhp->dh_roff = ptob(btop(off));
3641 3639 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3642 3640 /* set HAT cache attributes */
3643 3641 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3644 3642 /* set HAT endianess attributes */
3645 3643 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3646 3644 }
3647 3645
3648 3646 /*
3649 3647 * The default is _not_ to pass HAT_LOAD_NOCONSIST to hat_devload();
3650 3648 * we pass HAT_LOAD_NOCONSIST _only_ in cases where hat tries to
3651 3649 * create consistent mappings but our intention was to create
3652 3650 * non-consistent mappings.
3653 3651 *
3654 3652 * DEVMEM: hat figures it out it's DEVMEM and creates non-consistent
3655 3653 * mappings.
3656 3654 *
3657 3655 * kernel exported memory: hat figures it out it's memory and always
3658 3656 * creates consistent mappings.
3659 3657 *
3660 3658 * /dev/mem: non-consistent mappings. See comments in common/io/mem.c
3661 3659 *
3662 3660 * /dev/kmem: consistent mappings are created unless they are
3663 3661 * MAP_FIXED. We _explicitly_ tell hat to create non-consistent
3664 3662 * mappings by passing HAT_LOAD_NOCONSIST in case of MAP_FIXED
3665 3663 * mappings of /dev/kmem. See common/io/mem.c
3666 3664 */
3667 3665
3668 3666 /* Only some of the flags bits are settable by the driver */
3669 3667 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3670 3668
3671 3669 dhp->dh_len = ptob(btopr(len));
3672 3670 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3673 3671 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3674 3672
3675 3673 if (callbackops != NULL) {
3676 3674 bcopy(callbackops, &dhp->dh_callbackops,
3677 3675 sizeof (struct devmap_callback_ctl));
3678 3676 }
3679 3677 /*
3680 3678 * Initialize dh_lock if we want to do remap.
3681 3679 */
3682 3680 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3683 3681 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3684 3682 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3685 3683 }
3686 3684
3687 3685 dhp->dh_flags |= DEVMAP_SETUP_DONE;
3688 3686
3689 3687 return (DDI_SUCCESS);
3690 3688 }
3691 3689
3692 3690 int
3693 3691 devmap_umem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3694 3692 ddi_umem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot,
3695 3693 uint_t flags, ddi_device_acc_attr_t *accattrp)
3696 3694 {
3697 3695 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3698 3696 struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3699 3697
3700 3698 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_REMAP,
3701 3699 "devmap_umem_remap:start dhp=%p offset=%llx cookie=%p len=%lx",
3702 3700 (void *)dhp, off, cookie, len);
3703 3701 DEBUGF(2, (CE_CONT, "devmap_umem_remap: dhp %p offset %llx "
3704 3702 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3705 3703
3706 3704 #ifdef lint
3707 3705 dip = dip;
3708 3706 accattrp = accattrp;
3709 3707 #endif
3710 3708 /*
3711 3709 * Reture failure if setup has not been done or no remap permission
3712 3710 * has been granted during the setup.
3713 3711 */
3714 3712 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3715 3713 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3716 3714 return (DDI_FAILURE);
3717 3715
3718 3716 /* No flags supported for remap yet */
3719 3717 if (flags != 0)
3720 3718 return (DDI_FAILURE);
3721 3719
3722 3720 /* check if the cache attributes are supported */
3723 3721 if (i_ddi_check_cache_attr(flags) == B_FALSE)
3724 3722 return (DDI_FAILURE);
3725 3723
3726 3724 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3727 3725 return (DDI_FAILURE);
3728 3726
3729 3727 /* For UMEM_TRASH, this restriction is not needed */
3730 3728 if ((off + len) > cp->size)
3731 3729 return (DDI_FAILURE);
3732 3730
3733 3731 HOLD_DHP_LOCK(dhp);
3734 3732 /*
3735 3733 * Unload the old mapping, so next fault will setup the new mappings
3736 3734 * Do this while holding the dhp lock so other faults dont reestablish
3737 3735 * the mappings
3738 3736 */
3739 3737 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3740 3738 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3741 3739
3742 3740 dhp->dh_cookie = cookie;
3743 3741 dhp->dh_roff = ptob(btop(off));
3744 3742 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3745 3743 /* set HAT cache attributes */
3746 3744 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3747 3745 /* set HAT endianess attributes */
3748 3746 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3749 3747
3750 3748 /* clear the large page size flag */
3751 3749 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3752 3750
3753 3751 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3754 3752 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3755 3753 RELE_DHP_LOCK(dhp);
3756 3754 return (DDI_SUCCESS);
3757 3755 }
3758 3756
3759 3757 /*
3760 3758 * to set timeout value for the driver's context management callback, e.g.
3761 3759 * devmap_access().
3762 3760 */
3763 3761 void
3764 3762 devmap_set_ctx_timeout(devmap_cookie_t dhc, clock_t ticks)
3765 3763 {
3766 3764 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3767 3765
3768 3766 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SET_CTX_TIMEOUT,
3769 3767 "devmap_set_ctx_timeout:start dhp=%p ticks=%x",
3770 3768 (void *)dhp, ticks);
3771 3769 dhp->dh_timeout_length = ticks;
3772 3770 }
3773 3771
3774 3772 int
3775 3773 devmap_default_access(devmap_cookie_t dhp, void *pvtp, offset_t off,
3776 3774 size_t len, uint_t type, uint_t rw)
3777 3775 {
3778 3776 #ifdef lint
3779 3777 pvtp = pvtp;
3780 3778 #endif
3781 3779
3782 3780 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DEFAULT_ACCESS,
3783 3781 "devmap_default_access:start");
3784 3782 return (devmap_load(dhp, off, len, type, rw));
3785 3783 }
3786 3784
3787 3785 /*
3788 3786 * segkmem_alloc() wrapper to allocate memory which is both
3789 3787 * non-relocatable (for DR) and sharelocked, since the rest
3790 3788 * of this segment driver requires it.
3791 3789 */
3792 3790 static void *
3793 3791 devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag)
3794 3792 {
3795 3793 ASSERT(vmp != NULL);
3796 3794 ASSERT(kvseg.s_base != NULL);
3797 3795 vmflag |= (VM_NORELOC | SEGKMEM_SHARELOCKED);
3798 3796 return (segkmem_alloc(vmp, size, vmflag));
3799 3797 }
3800 3798
3801 3799 /*
3802 3800 * This is where things are a bit incestuous with seg_kmem: unlike
3803 3801 * seg_kp, seg_kmem does not keep its pages long-term sharelocked, so
3804 3802 * we need to do a bit of a dance around that to prevent duplication of
3805 3803 * code until we decide to bite the bullet and implement a new kernel
3806 3804 * segment for driver-allocated memory that is exported to user space.
3807 3805 */
3808 3806 static void
3809 3807 devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size)
3810 3808 {
3811 3809 page_t *pp;
3812 3810 caddr_t addr = inaddr;
3813 3811 caddr_t eaddr;
3814 3812 pgcnt_t npages = btopr(size);
3815 3813
3816 3814 ASSERT(vmp != NULL);
3817 3815 ASSERT(kvseg.s_base != NULL);
3818 3816 ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
3819 3817
3820 3818 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
3821 3819
3822 3820 for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
3823 3821 /*
3824 3822 * Use page_find() instead of page_lookup() to find the page
3825 3823 * since we know that it is hashed and has a shared lock.
3826 3824 */
3827 3825 pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr);
3828 3826
3829 3827 if (pp == NULL)
3830 3828 panic("devmap_free_pages: page not found");
3831 3829 if (!page_tryupgrade(pp)) {
3832 3830 page_unlock(pp);
3833 3831 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr,
3834 3832 SE_EXCL);
3835 3833 if (pp == NULL)
3836 3834 panic("devmap_free_pages: page already freed");
3837 3835 }
3838 3836 /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
3839 3837 pp->p_lckcnt = 0;
3840 3838 page_destroy(pp, 0);
3841 3839 }
3842 3840 page_unresv(npages);
3843 3841
3844 3842 if (vmp != NULL)
3845 3843 vmem_free(vmp, inaddr, size);
3846 3844 }
3847 3845
3848 3846 /*
3849 3847 * devmap_umem_alloc_np() replaces kmem_zalloc() as the method for
3850 3848 * allocating non-pageable kmem in response to a ddi_umem_alloc()
3851 3849 * default request. For now we allocate our own pages and we keep
3852 3850 * them long-term sharelocked, since: A) the fault routines expect the
3853 3851 * memory to already be locked; B) pageable umem is already long-term
3854 3852 * locked; C) it's a lot of work to make it otherwise, particularly
3855 3853 * since the nexus layer expects the pages to never fault. An RFE is to
3856 3854 * not keep the pages long-term locked, but instead to be able to
3857 3855 * take faults on them and simply look them up in kvp in case we
3858 3856 * fault on them. Even then, we must take care not to let pageout
3859 3857 * steal them from us since the data must remain resident; if we
3860 3858 * do this we must come up with some way to pin the pages to prevent
3861 3859 * faults while a driver is doing DMA to/from them.
3862 3860 */
3863 3861 static void *
3864 3862 devmap_umem_alloc_np(size_t size, size_t flags)
3865 3863 {
3866 3864 void *buf;
3867 3865 int vmflags = (flags & DDI_UMEM_NOSLEEP)? VM_NOSLEEP : VM_SLEEP;
3868 3866
3869 3867 buf = vmem_alloc(umem_np_arena, size, vmflags);
3870 3868 if (buf != NULL)
3871 3869 bzero(buf, size);
3872 3870 return (buf);
3873 3871 }
3874 3872
3875 3873 static void
3876 3874 devmap_umem_free_np(void *addr, size_t size)
3877 3875 {
3878 3876 vmem_free(umem_np_arena, addr, size);
3879 3877 }
3880 3878
3881 3879 /*
3882 3880 * allocate page aligned kernel memory for exporting to user land.
3883 3881 * The devmap framework will use the cookie allocated by ddi_umem_alloc()
3884 3882 * to find a user virtual address that is in same color as the address
3885 3883 * allocated here.
3886 3884 */
3887 3885 void *
3888 3886 ddi_umem_alloc(size_t size, int flags, ddi_umem_cookie_t *cookie)
3889 3887 {
3890 3888 register size_t len = ptob(btopr(size));
3891 3889 void *buf = NULL;
3892 3890 struct ddi_umem_cookie *cp;
3893 3891 int iflags = 0;
3894 3892
3895 3893 *cookie = NULL;
3896 3894
3897 3895 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_ALLOC,
3898 3896 "devmap_umem_alloc:start");
3899 3897 if (len == 0)
3900 3898 return ((void *)NULL);
3901 3899
3902 3900 /*
3903 3901 * allocate cookie
3904 3902 */
3905 3903 if ((cp = kmem_zalloc(sizeof (struct ddi_umem_cookie),
3906 3904 flags & DDI_UMEM_NOSLEEP ? KM_NOSLEEP : KM_SLEEP)) == NULL) {
3907 3905 ASSERT(flags & DDI_UMEM_NOSLEEP);
3908 3906 return ((void *)NULL);
3909 3907 }
3910 3908
3911 3909 if (flags & DDI_UMEM_PAGEABLE) {
3912 3910 /* Only one of the flags is allowed */
3913 3911 ASSERT(!(flags & DDI_UMEM_TRASH));
3914 3912 /* initialize resource with 0 */
3915 3913 iflags = KPD_ZERO;
3916 3914
3917 3915 /*
3918 3916 * to allocate unlocked pageable memory, use segkp_get() to
3919 3917 * create a segkp segment. Since segkp can only service kas,
3920 3918 * other segment drivers such as segdev have to do
3921 3919 * as_fault(segkp, SOFTLOCK) in its fault routine,
3922 3920 */
3923 3921 if (flags & DDI_UMEM_NOSLEEP)
3924 3922 iflags |= KPD_NOWAIT;
3925 3923
3926 3924 if ((buf = segkp_get(segkp, len, iflags)) == NULL) {
3927 3925 kmem_free(cp, sizeof (struct ddi_umem_cookie));
3928 3926 return ((void *)NULL);
3929 3927 }
3930 3928 cp->type = KMEM_PAGEABLE;
3931 3929 mutex_init(&cp->lock, NULL, MUTEX_DEFAULT, NULL);
3932 3930 cp->locked = 0;
3933 3931 } else if (flags & DDI_UMEM_TRASH) {
3934 3932 /* Only one of the flags is allowed */
3935 3933 ASSERT(!(flags & DDI_UMEM_PAGEABLE));
3936 3934 cp->type = UMEM_TRASH;
3937 3935 buf = NULL;
3938 3936 } else {
3939 3937 if ((buf = devmap_umem_alloc_np(len, flags)) == NULL) {
3940 3938 kmem_free(cp, sizeof (struct ddi_umem_cookie));
3941 3939 return ((void *)NULL);
3942 3940 }
3943 3941
3944 3942 cp->type = KMEM_NON_PAGEABLE;
3945 3943 }
3946 3944
3947 3945 /*
3948 3946 * need to save size here. size will be used when
3949 3947 * we do kmem_free.
3950 3948 */
3951 3949 cp->size = len;
3952 3950 cp->cvaddr = (caddr_t)buf;
3953 3951
3954 3952 *cookie = (void *)cp;
3955 3953 return (buf);
3956 3954 }
3957 3955
3958 3956 void
3959 3957 ddi_umem_free(ddi_umem_cookie_t cookie)
3960 3958 {
3961 3959 struct ddi_umem_cookie *cp;
3962 3960
3963 3961 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_FREE,
3964 3962 "devmap_umem_free:start");
3965 3963
3966 3964 /*
3967 3965 * if cookie is NULL, no effects on the system
3968 3966 */
3969 3967 if (cookie == NULL)
3970 3968 return;
3971 3969
3972 3970 cp = (struct ddi_umem_cookie *)cookie;
3973 3971
3974 3972 switch (cp->type) {
3975 3973 case KMEM_PAGEABLE :
3976 3974 ASSERT(cp->cvaddr != NULL && cp->size != 0);
3977 3975 /*
3978 3976 * Check if there are still any pending faults on the cookie
3979 3977 * while the driver is deleting it,
3980 3978 * XXX - could change to an ASSERT but wont catch errant drivers
3981 3979 */
3982 3980 mutex_enter(&cp->lock);
3983 3981 if (cp->locked) {
3984 3982 mutex_exit(&cp->lock);
3985 3983 panic("ddi_umem_free for cookie with pending faults %p",
3986 3984 (void *)cp);
3987 3985 return;
3988 3986 }
3989 3987
3990 3988 segkp_release(segkp, cp->cvaddr);
3991 3989
3992 3990 /*
3993 3991 * release mutex associated with this cookie.
3994 3992 */
3995 3993 mutex_destroy(&cp->lock);
3996 3994 break;
3997 3995 case KMEM_NON_PAGEABLE :
3998 3996 ASSERT(cp->cvaddr != NULL && cp->size != 0);
3999 3997 devmap_umem_free_np(cp->cvaddr, cp->size);
4000 3998 break;
4001 3999 case UMEM_TRASH :
4002 4000 break;
4003 4001 case UMEM_LOCKED :
4004 4002 /* Callers should use ddi_umem_unlock for this type */
4005 4003 ddi_umem_unlock(cookie);
4006 4004 /* Frees the cookie too */
4007 4005 return;
4008 4006 default:
4009 4007 /* panic so we can diagnose the underlying cause */
4010 4008 panic("ddi_umem_free: illegal cookie type 0x%x\n",
4011 4009 cp->type);
4012 4010 }
4013 4011
4014 4012 kmem_free(cookie, sizeof (struct ddi_umem_cookie));
4015 4013 }
4016 4014
4017 4015
4018 4016 static int
4019 4017 segdev_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
4020 4018 {
|
↓ open down ↓ |
3795 lines elided |
↑ open up ↑ |
4021 4019 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4022 4020
4023 4021 /*
4024 4022 * It looks as if it is always mapped shared
4025 4023 */
4026 4024 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GETMEMID,
4027 4025 "segdev_getmemid:start");
4028 4026 memidp->val[0] = (uintptr_t)VTOCVP(sdp->vp);
4029 4027 memidp->val[1] = sdp->offset + (uintptr_t)(addr - seg->s_base);
4030 4028 return (0);
4031 -}
4032 -
4033 -/*ARGSUSED*/
4034 -static lgrp_mem_policy_info_t *
4035 -segdev_getpolicy(struct seg *seg, caddr_t addr)
4036 -{
4037 - return (NULL);
4038 4029 }
4039 4030
4040 4031 /*ARGSUSED*/
4041 4032 static int
4042 4033 segdev_capable(struct seg *seg, segcapability_t capability)
4043 4034 {
4044 4035 return (0);
4045 4036 }
4046 4037
4047 4038 /*
4048 4039 * ddi_umem_alloc() non-pageable quantum cache max size.
4049 4040 * This is just a SWAG.
4050 4041 */
4051 4042 #define DEVMAP_UMEM_QUANTUM (8*PAGESIZE)
4052 4043
4053 4044 /*
4054 4045 * Initialize seg_dev from boot. This routine sets up the trash page
4055 4046 * and creates the umem_np_arena used to back non-pageable memory
4056 4047 * requests.
4057 4048 */
4058 4049 void
4059 4050 segdev_init(void)
4060 4051 {
4061 4052 struct seg kseg;
4062 4053
4063 4054 umem_np_arena = vmem_create("umem_np", NULL, 0, PAGESIZE,
4064 4055 devmap_alloc_pages, devmap_free_pages, heap_arena,
4065 4056 DEVMAP_UMEM_QUANTUM, VM_SLEEP);
4066 4057
4067 4058 kseg.s_as = &kas;
4068 4059 trashpp = page_create_va(&trashvp, 0, PAGESIZE,
4069 4060 PG_NORELOC | PG_EXCL | PG_WAIT, &kseg, NULL);
4070 4061 if (trashpp == NULL)
4071 4062 panic("segdev_init: failed to create trash page");
4072 4063 pagezero(trashpp, 0, PAGESIZE);
4073 4064 page_downgrade(trashpp);
4074 4065 }
4075 4066
4076 4067 /*
4077 4068 * Invoke platform-dependent support routines so that /proc can have
4078 4069 * the platform code deal with curious hardware.
4079 4070 */
4080 4071 int
4081 4072 segdev_copyfrom(struct seg *seg,
4082 4073 caddr_t uaddr, const void *devaddr, void *kaddr, size_t len)
4083 4074 {
4084 4075 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4085 4076 struct snode *sp = VTOS(VTOCVP(sdp->vp));
4086 4077
4087 4078 return (e_ddi_copyfromdev(sp->s_dip,
4088 4079 (off_t)(uaddr - seg->s_base), devaddr, kaddr, len));
4089 4080 }
4090 4081
4091 4082 int
4092 4083 segdev_copyto(struct seg *seg,
4093 4084 caddr_t uaddr, const void *kaddr, void *devaddr, size_t len)
4094 4085 {
4095 4086 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4096 4087 struct snode *sp = VTOS(VTOCVP(sdp->vp));
4097 4088
4098 4089 return (e_ddi_copytodev(sp->s_dip,
4099 4090 (off_t)(uaddr - seg->s_base), kaddr, devaddr, len));
4100 4091 }
|
↓ open down ↓ |
53 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX