1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
28 /* All Rights Reserved */
29
30 /*
31 * University Copyright- Copyright (c) 1982, 1986, 1988
32 * The Regents of the University of California
33 * All Rights Reserved
34 *
35 * University Acknowledgment- Portions of this document are derived from
36 * software developed by the University of California, Berkeley, and its
37 * contributors.
38 */
39
40 /*
41 * VM - segment of a mapped device.
42 *
43 * This segment driver is used when mapping character special devices.
44 */
45
46 #include <sys/types.h>
47 #include <sys/t_lock.h>
48 #include <sys/sysmacros.h>
49 #include <sys/vtrace.h>
50 #include <sys/systm.h>
51 #include <sys/vmsystm.h>
52 #include <sys/mman.h>
53 #include <sys/errno.h>
54 #include <sys/kmem.h>
55 #include <sys/cmn_err.h>
56 #include <sys/vnode.h>
57 #include <sys/proc.h>
58 #include <sys/conf.h>
59 #include <sys/debug.h>
60 #include <sys/ddidevmap.h>
61 #include <sys/ddi_implfuncs.h>
62 #include <sys/lgrp.h>
63
64 #include <vm/page.h>
65 #include <vm/hat.h>
66 #include <vm/as.h>
67 #include <vm/seg.h>
68 #include <vm/seg_dev.h>
69 #include <vm/seg_kp.h>
70 #include <vm/seg_kmem.h>
71 #include <vm/vpage.h>
72
73 #include <sys/sunddi.h>
74 #include <sys/esunddi.h>
75 #include <sys/fs/snode.h>
76
77
78 #if DEBUG
79 int segdev_debug;
80 #define DEBUGF(level, args) { if (segdev_debug >= (level)) cmn_err args; }
81 #else
82 #define DEBUGF(level, args)
83 #endif
84
85 /* Default timeout for devmap context management */
86 #define CTX_TIMEOUT_VALUE 0
87
88 #define HOLD_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
89 { mutex_enter(&dhp->dh_lock); }
90
91 #define RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
92 { mutex_exit(&dhp->dh_lock); }
93
94 #define round_down_p2(a, s) ((a) & ~((s) - 1))
95 #define round_up_p2(a, s) (((a) + (s) - 1) & ~((s) - 1))
96
97 /*
98 * VA_PA_ALIGNED checks to see if both VA and PA are on pgsize boundary
99 * VA_PA_PGSIZE_ALIGNED check to see if VA is aligned with PA w.r.t. pgsize
100 */
101 #define VA_PA_ALIGNED(uvaddr, paddr, pgsize) \
102 (((uvaddr | paddr) & (pgsize - 1)) == 0)
103 #define VA_PA_PGSIZE_ALIGNED(uvaddr, paddr, pgsize) \
104 (((uvaddr ^ paddr) & (pgsize - 1)) == 0)
105
106 #define vpgtob(n) ((n) * sizeof (struct vpage)) /* For brevity */
107
108 #define VTOCVP(vp) (VTOS(vp)->s_commonvp) /* we "know" it's an snode */
109
110 static struct devmap_ctx *devmapctx_list = NULL;
111 static struct devmap_softlock *devmap_slist = NULL;
112
113 /*
114 * mutex, vnode and page for the page of zeros we use for the trash mappings.
115 * One trash page is allocated on the first ddi_umem_setup call that uses it
116 * XXX Eventually, we may want to combine this with what segnf does when all
117 * hat layers implement HAT_NOFAULT.
118 *
119 * The trash page is used when the backing store for a userland mapping is
120 * removed but the application semantics do not take kindly to a SIGBUS.
121 * In that scenario, the applications pages are mapped to some dummy page
122 * which returns garbage on read and writes go into a common place.
123 * (Perfect for NO_FAULT semantics)
124 * The device driver is responsible to communicating to the app with some
125 * other mechanism that such remapping has happened and the app should take
126 * corrective action.
127 * We can also use an anonymous memory page as there is no requirement to
128 * keep the page locked, however this complicates the fault code. RFE.
129 */
130 static struct vnode trashvp;
131 static struct page *trashpp;
132
133 /* Non-pageable kernel memory is allocated from the umem_np_arena. */
134 static vmem_t *umem_np_arena;
135
136 /* Set the cookie to a value we know will never be a valid umem_cookie */
137 #define DEVMAP_DEVMEM_COOKIE ((ddi_umem_cookie_t)0x1)
138
139 /*
140 * Macros to check if type of devmap handle
141 */
142 #define cookie_is_devmem(c) \
143 ((c) == (struct ddi_umem_cookie *)DEVMAP_DEVMEM_COOKIE)
144
145 #define cookie_is_pmem(c) \
146 ((c) == (struct ddi_umem_cookie *)DEVMAP_PMEM_COOKIE)
147
148 #define cookie_is_kpmem(c) (!cookie_is_devmem(c) && !cookie_is_pmem(c) &&\
149 ((c)->type == KMEM_PAGEABLE))
150
151 #define dhp_is_devmem(dhp) \
152 (cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
153
154 #define dhp_is_pmem(dhp) \
155 (cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
156
157 #define dhp_is_kpmem(dhp) \
158 (cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
159
160 /*
161 * Private seg op routines.
162 */
163 static int segdev_dup(struct seg *, struct seg *);
164 static int segdev_unmap(struct seg *, caddr_t, size_t);
165 static void segdev_free(struct seg *);
166 static faultcode_t segdev_fault(struct hat *, struct seg *, caddr_t, size_t,
167 enum fault_type, enum seg_rw);
168 static faultcode_t segdev_faulta(struct seg *, caddr_t);
169 static int segdev_setprot(struct seg *, caddr_t, size_t, uint_t);
170 static int segdev_checkprot(struct seg *, caddr_t, size_t, uint_t);
171 static void segdev_badop(void);
172 static int segdev_sync(struct seg *, caddr_t, size_t, int, uint_t);
173 static size_t segdev_incore(struct seg *, caddr_t, size_t, char *);
174 static int segdev_lockop(struct seg *, caddr_t, size_t, int, int,
175 ulong_t *, size_t);
176 static int segdev_getprot(struct seg *, caddr_t, size_t, uint_t *);
177 static u_offset_t segdev_getoffset(struct seg *, caddr_t);
178 static int segdev_gettype(struct seg *, caddr_t);
179 static int segdev_getvp(struct seg *, caddr_t, struct vnode **);
180 static int segdev_advise(struct seg *, caddr_t, size_t, uint_t);
181 static void segdev_dump(struct seg *);
182 static int segdev_pagelock(struct seg *, caddr_t, size_t,
183 struct page ***, enum lock_type, enum seg_rw);
184 static int segdev_setpagesize(struct seg *, caddr_t, size_t, uint_t);
185 static int segdev_getmemid(struct seg *, caddr_t, memid_t *);
186 static lgrp_mem_policy_info_t *segdev_getpolicy(struct seg *, caddr_t);
187 static int segdev_capable(struct seg *, segcapability_t);
188
189 /*
190 * XXX this struct is used by rootnex_map_fault to identify
191 * the segment it has been passed. So if you make it
192 * "static" you'll need to fix rootnex_map_fault.
193 */
194 struct seg_ops segdev_ops = {
195 .dup = segdev_dup,
196 .unmap = segdev_unmap,
197 .free = segdev_free,
198 .fault = segdev_fault,
199 .faulta = segdev_faulta,
200 .setprot = segdev_setprot,
201 .checkprot = segdev_checkprot,
202 .kluster = (int (*)())segdev_badop,
203 .sync = segdev_sync,
204 .incore = segdev_incore,
205 .lockop = segdev_lockop,
206 .getprot = segdev_getprot,
207 .getoffset = segdev_getoffset,
208 .gettype = segdev_gettype,
209 .getvp = segdev_getvp,
210 .advise = segdev_advise,
211 .dump = segdev_dump,
212 .pagelock = segdev_pagelock,
213 .setpagesize = segdev_setpagesize,
214 .getmemid = segdev_getmemid,
215 .getpolicy = segdev_getpolicy,
216 .capable = segdev_capable,
217 };
218
219 /*
220 * Private segdev support routines
221 */
222 static struct segdev_data *sdp_alloc(void);
223
224 static void segdev_softunlock(struct hat *, struct seg *, caddr_t,
225 size_t, enum seg_rw);
226
227 static faultcode_t segdev_faultpage(struct hat *, struct seg *, caddr_t,
228 struct vpage *, enum fault_type, enum seg_rw, devmap_handle_t *);
229
230 static faultcode_t segdev_faultpages(struct hat *, struct seg *, caddr_t,
231 size_t, enum fault_type, enum seg_rw, devmap_handle_t *);
232
233 static struct devmap_ctx *devmap_ctxinit(dev_t, ulong_t);
234 static struct devmap_softlock *devmap_softlock_init(dev_t, ulong_t);
235 static void devmap_softlock_rele(devmap_handle_t *);
236 static void devmap_ctx_rele(devmap_handle_t *);
237
238 static void devmap_ctxto(void *);
239
240 static devmap_handle_t *devmap_find_handle(devmap_handle_t *dhp_head,
241 caddr_t addr);
242
243 static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
244 ulong_t *opfn, ulong_t *pagesize);
245
246 static void free_devmap_handle(devmap_handle_t *dhp);
247
248 static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
249 struct seg *newseg);
250
251 static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp);
252
253 static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len);
254
255 static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr);
256
257 static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
258 offset_t off, size_t len, uint_t flags);
259
260 static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len,
261 caddr_t addr, size_t *llen, caddr_t *laddr);
262
263 static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len);
264
265 static void *devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag);
266 static void devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size);
267
268 static void *devmap_umem_alloc_np(size_t size, size_t flags);
269 static void devmap_umem_free_np(void *addr, size_t size);
270
271 /*
272 * routines to lock and unlock underlying segkp segment for
273 * KMEM_PAGEABLE type cookies.
274 */
275 static faultcode_t acquire_kpmem_lock(struct ddi_umem_cookie *, size_t);
276 static void release_kpmem_lock(struct ddi_umem_cookie *, size_t);
277
278 /*
279 * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
280 * drivers with devmap_access callbacks
281 */
282 static int devmap_softlock_enter(struct devmap_softlock *, size_t,
283 enum fault_type);
284 static void devmap_softlock_exit(struct devmap_softlock *, size_t,
285 enum fault_type);
286
287 static kmutex_t devmapctx_lock;
288
289 static kmutex_t devmap_slock;
290
291 /*
292 * Initialize the thread callbacks and thread private data.
293 */
294 static struct devmap_ctx *
295 devmap_ctxinit(dev_t dev, ulong_t id)
296 {
297 struct devmap_ctx *devctx;
298 struct devmap_ctx *tmp;
299 dev_info_t *dip;
300
301 tmp = kmem_zalloc(sizeof (struct devmap_ctx), KM_SLEEP);
302
303 mutex_enter(&devmapctx_lock);
304
305 dip = e_ddi_hold_devi_by_dev(dev, 0);
306 ASSERT(dip != NULL);
307 ddi_release_devi(dip);
308
309 for (devctx = devmapctx_list; devctx != NULL; devctx = devctx->next)
310 if ((devctx->dip == dip) && (devctx->id == id))
311 break;
312
313 if (devctx == NULL) {
314 devctx = tmp;
315 devctx->dip = dip;
316 devctx->id = id;
317 mutex_init(&devctx->lock, NULL, MUTEX_DEFAULT, NULL);
318 cv_init(&devctx->cv, NULL, CV_DEFAULT, NULL);
319 devctx->next = devmapctx_list;
320 devmapctx_list = devctx;
321 } else
322 kmem_free(tmp, sizeof (struct devmap_ctx));
323
324 mutex_enter(&devctx->lock);
325 devctx->refcnt++;
326 mutex_exit(&devctx->lock);
327 mutex_exit(&devmapctx_lock);
328
329 return (devctx);
330 }
331
332 /*
333 * Timeout callback called if a CPU has not given up the device context
334 * within dhp->dh_timeout_length ticks
335 */
336 static void
337 devmap_ctxto(void *data)
338 {
339 struct devmap_ctx *devctx = data;
340
341 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_CTXTO,
342 "devmap_ctxto:timeout expired, devctx=%p", (void *)devctx);
343 mutex_enter(&devctx->lock);
344 /*
345 * Set oncpu = 0 so the next mapping trying to get the device context
346 * can.
347 */
348 devctx->oncpu = 0;
349 devctx->timeout = 0;
350 cv_signal(&devctx->cv);
351 mutex_exit(&devctx->lock);
352 }
353
354 /*
355 * Create a device segment.
356 */
357 int
358 segdev_create(struct seg *seg, void *argsp)
359 {
360 struct segdev_data *sdp;
361 struct segdev_crargs *a = (struct segdev_crargs *)argsp;
362 devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data;
363 int error;
364
365 /*
366 * Since the address space is "write" locked, we
367 * don't need the segment lock to protect "segdev" data.
368 */
369 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
370
371 hat_map(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_MAP);
372
373 sdp = sdp_alloc();
374
375 sdp->mapfunc = a->mapfunc;
376 sdp->offset = a->offset;
377 sdp->prot = a->prot;
378 sdp->maxprot = a->maxprot;
379 sdp->type = a->type;
380 sdp->pageprot = 0;
381 sdp->softlockcnt = 0;
382 sdp->vpage = NULL;
383
384 if (sdp->mapfunc == NULL)
385 sdp->devmap_data = dhp;
386 else
387 sdp->devmap_data = dhp = NULL;
388
389 sdp->hat_flags = a->hat_flags;
390 sdp->hat_attr = a->hat_attr;
391
392 /*
393 * Currently, hat_flags supports only HAT_LOAD_NOCONSIST
394 */
395 ASSERT(!(sdp->hat_flags & ~HAT_LOAD_NOCONSIST));
396
397 /*
398 * Hold shadow vnode -- segdev only deals with
399 * character (VCHR) devices. We use the common
400 * vp to hang pages on.
401 */
402 sdp->vp = specfind(a->dev, VCHR);
403 ASSERT(sdp->vp != NULL);
404
405 seg->s_ops = &segdev_ops;
406 seg->s_data = sdp;
407
408 while (dhp != NULL) {
409 dhp->dh_seg = seg;
410 dhp = dhp->dh_next;
411 }
412
413 /*
414 * Inform the vnode of the new mapping.
415 */
416 /*
417 * It is ok to use pass sdp->maxprot to ADDMAP rather than to use
418 * dhp specific maxprot because spec_addmap does not use maxprot.
419 */
420 error = VOP_ADDMAP(VTOCVP(sdp->vp), sdp->offset,
421 seg->s_as, seg->s_base, seg->s_size,
422 sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
423
424 if (error != 0) {
425 sdp->devmap_data = NULL;
426 hat_unload(seg->s_as->a_hat, seg->s_base, seg->s_size,
427 HAT_UNLOAD_UNMAP);
428 } else {
429 /*
430 * Mappings of /dev/null don't count towards the VSZ of a
431 * process. Mappings of /dev/null have no mapping type.
432 */
433 if ((segop_gettype(seg, seg->s_base) & (MAP_SHARED |
434 MAP_PRIVATE)) == 0) {
435 seg->s_as->a_resvsize -= seg->s_size;
436 }
437 }
438
439 return (error);
440 }
441
442 static struct segdev_data *
443 sdp_alloc(void)
444 {
445 struct segdev_data *sdp;
446
447 sdp = kmem_zalloc(sizeof (struct segdev_data), KM_SLEEP);
448 rw_init(&sdp->lock, NULL, RW_DEFAULT, NULL);
449
450 return (sdp);
451 }
452
453 /*
454 * Duplicate seg and return new segment in newseg.
455 */
456 static int
457 segdev_dup(struct seg *seg, struct seg *newseg)
458 {
459 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
460 struct segdev_data *newsdp;
461 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
462 size_t npages;
463 int ret;
464
465 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DUP,
466 "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg);
467
468 DEBUGF(3, (CE_CONT, "segdev_dup: dhp %p seg %p\n",
469 (void *)dhp, (void *)seg));
470
471 /*
472 * Since the address space is "write" locked, we
473 * don't need the segment lock to protect "segdev" data.
474 */
475 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
476
477 newsdp = sdp_alloc();
478
479 newseg->s_ops = seg->s_ops;
480 newseg->s_data = (void *)newsdp;
481
482 VN_HOLD(sdp->vp);
483 newsdp->vp = sdp->vp;
484 newsdp->mapfunc = sdp->mapfunc;
485 newsdp->offset = sdp->offset;
486 newsdp->pageprot = sdp->pageprot;
487 newsdp->prot = sdp->prot;
488 newsdp->maxprot = sdp->maxprot;
489 newsdp->type = sdp->type;
490 newsdp->hat_attr = sdp->hat_attr;
491 newsdp->hat_flags = sdp->hat_flags;
492 newsdp->softlockcnt = 0;
493
494 /*
495 * Initialize per page data if the segment we are
496 * dup'ing has per page information.
497 */
498 npages = seg_pages(newseg);
499
500 if (sdp->vpage != NULL) {
501 size_t nbytes = vpgtob(npages);
502
503 newsdp->vpage = kmem_zalloc(nbytes, KM_SLEEP);
504 bcopy(sdp->vpage, newsdp->vpage, nbytes);
505 } else
506 newsdp->vpage = NULL;
507
508 /*
509 * duplicate devmap handles
510 */
511 if (dhp != NULL) {
512 ret = devmap_handle_dup(dhp,
513 (devmap_handle_t **)&newsdp->devmap_data, newseg);
514 if (ret != 0) {
515 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DUP_CK1,
516 "segdev_dup:ret1 ret=%x, dhp=%p seg=%p",
517 ret, (void *)dhp, (void *)seg);
518 DEBUGF(1, (CE_CONT,
519 "segdev_dup: ret %x dhp %p seg %p\n",
520 ret, (void *)dhp, (void *)seg));
521 return (ret);
522 }
523 }
524
525 /*
526 * Inform the common vnode of the new mapping.
527 */
528 return (VOP_ADDMAP(VTOCVP(newsdp->vp),
529 newsdp->offset, newseg->s_as,
530 newseg->s_base, newseg->s_size, newsdp->prot,
531 newsdp->maxprot, sdp->type, CRED(), NULL));
532 }
533
534 /*
535 * duplicate devmap handles
536 */
537 static int
538 devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
539 struct seg *newseg)
540 {
541 devmap_handle_t *newdhp_save = NULL;
542 devmap_handle_t *newdhp = NULL;
543 struct devmap_callback_ctl *callbackops;
544
545 while (dhp != NULL) {
546 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
547
548 /* Need to lock the original dhp while copying if REMAP */
549 HOLD_DHP_LOCK(dhp);
550 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
551 RELE_DHP_LOCK(dhp);
552 newdhp->dh_seg = newseg;
553 newdhp->dh_next = NULL;
554 if (newdhp_save != NULL)
555 newdhp_save->dh_next = newdhp;
556 else
557 *new_dhp = newdhp;
558 newdhp_save = newdhp;
559
560 callbackops = &newdhp->dh_callbackops;
561
562 if (dhp->dh_softlock != NULL)
563 newdhp->dh_softlock = devmap_softlock_init(
564 newdhp->dh_dev,
565 (ulong_t)callbackops->devmap_access);
566 if (dhp->dh_ctx != NULL)
567 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
568 (ulong_t)callbackops->devmap_access);
569
570 /*
571 * Initialize dh_lock if we want to do remap.
572 */
573 if (newdhp->dh_flags & DEVMAP_ALLOW_REMAP) {
574 mutex_init(&newdhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
575 newdhp->dh_flags |= DEVMAP_LOCK_INITED;
576 }
577
578 if (callbackops->devmap_dup != NULL) {
579 int ret;
580
581 /*
582 * Call the dup callback so that the driver can
583 * duplicate its private data.
584 */
585 ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp,
586 (devmap_cookie_t *)newdhp, &newdhp->dh_pvtp);
587
588 if (ret != 0) {
589 /*
590 * We want to free up this segment as the driver
591 * has indicated that we can't dup it. But we
592 * don't want to call the drivers, devmap_unmap,
593 * callback function as the driver does not
594 * think this segment exists. The caller of
595 * devmap_dup will call seg_free on newseg
596 * as it was the caller that allocated the
597 * segment.
598 */
599 DEBUGF(1, (CE_CONT, "devmap_handle_dup ERROR: "
600 "newdhp %p dhp %p\n", (void *)newdhp,
601 (void *)dhp));
602 callbackops->devmap_unmap = NULL;
603 return (ret);
604 }
605 }
606
607 dhp = dhp->dh_next;
608 }
609
610 return (0);
611 }
612
613 /*
614 * Split a segment at addr for length len.
615 */
616 /*ARGSUSED*/
617 static int
618 segdev_unmap(struct seg *seg, caddr_t addr, size_t len)
619 {
620 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
621 register struct segdev_data *nsdp;
622 register struct seg *nseg;
623 register size_t opages; /* old segment size in pages */
624 register size_t npages; /* new segment size in pages */
625 register size_t dpages; /* pages being deleted (unmapped) */
626 register size_t nbytes;
627 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
628 devmap_handle_t *dhpp;
629 devmap_handle_t *newdhp;
630 struct devmap_callback_ctl *callbackops;
631 caddr_t nbase;
632 offset_t off;
633 ulong_t nsize;
634 size_t mlen, sz;
635
636 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP,
637 "segdev_unmap:start dhp=%p, seg=%p addr=%p len=%lx",
638 (void *)dhp, (void *)seg, (void *)addr, len);
639
640 DEBUGF(3, (CE_CONT, "segdev_unmap: dhp %p seg %p addr %p len %lx\n",
641 (void *)dhp, (void *)seg, (void *)addr, len));
642
643 /*
644 * Since the address space is "write" locked, we
645 * don't need the segment lock to protect "segdev" data.
646 */
647 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
648
649 if ((sz = sdp->softlockcnt) > 0) {
650 /*
651 * Fail the unmap if pages are SOFTLOCKed through this mapping.
652 * softlockcnt is protected from change by the as write lock.
653 */
654 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK1,
655 "segdev_unmap:error softlockcnt = %ld", sz);
656 DEBUGF(1, (CE_CONT, "segdev_unmap: softlockcnt %ld\n", sz));
657 return (EAGAIN);
658 }
659
660 /*
661 * Check for bad sizes
662 */
663 if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size ||
664 (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET))
665 panic("segdev_unmap");
666
667 if (dhp != NULL) {
668 devmap_handle_t *tdhp;
669 /*
670 * If large page size was used in hat_devload(),
671 * the same page size must be used in hat_unload().
672 */
673 dhpp = tdhp = devmap_find_handle(dhp, addr);
674 while (tdhp != NULL) {
675 if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
676 break;
677 }
678 tdhp = tdhp->dh_next;
679 }
680 if (tdhp != NULL) { /* found a dhp using large pages */
681 size_t slen = len;
682 size_t mlen;
683 size_t soff;
684
685 soff = (ulong_t)(addr - dhpp->dh_uvaddr);
686 while (slen != 0) {
687 mlen = MIN(slen, (dhpp->dh_len - soff));
688 hat_unload(seg->s_as->a_hat, dhpp->dh_uvaddr,
689 dhpp->dh_len, HAT_UNLOAD_UNMAP);
690 dhpp = dhpp->dh_next;
691 ASSERT(slen >= mlen);
692 slen -= mlen;
693 soff = 0;
694 }
695 } else
696 hat_unload(seg->s_as->a_hat, addr, len,
697 HAT_UNLOAD_UNMAP);
698 } else {
699 /*
700 * Unload any hardware translations in the range
701 * to be taken out.
702 */
703 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP);
704 }
705
706 /*
707 * get the user offset which will used in the driver callbacks
708 */
709 off = sdp->offset + (offset_t)(addr - seg->s_base);
710
711 /*
712 * Inform the vnode of the unmapping.
713 */
714 ASSERT(sdp->vp != NULL);
715 (void) VOP_DELMAP(VTOCVP(sdp->vp), off, seg->s_as, addr, len,
716 sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
717
718 /*
719 * Check for entire segment
720 */
721 if (addr == seg->s_base && len == seg->s_size) {
722 seg_free(seg);
723 return (0);
724 }
725
726 opages = seg_pages(seg);
727 dpages = btop(len);
728 npages = opages - dpages;
729
730 /*
731 * Check for beginning of segment
732 */
733 if (addr == seg->s_base) {
734 if (sdp->vpage != NULL) {
735 register struct vpage *ovpage;
736
737 ovpage = sdp->vpage; /* keep pointer to vpage */
738
739 nbytes = vpgtob(npages);
740 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
741 bcopy(&ovpage[dpages], sdp->vpage, nbytes);
742
743 /* free up old vpage */
744 kmem_free(ovpage, vpgtob(opages));
745 }
746
747 /*
748 * free devmap handles from the beginning of the mapping.
749 */
750 if (dhp != NULL)
751 devmap_handle_unmap_head(dhp, len);
752
753 sdp->offset += (offset_t)len;
754
755 seg->s_base += len;
756 seg->s_size -= len;
757
758 return (0);
759 }
760
761 /*
762 * Check for end of segment
763 */
764 if (addr + len == seg->s_base + seg->s_size) {
765 if (sdp->vpage != NULL) {
766 register struct vpage *ovpage;
767
768 ovpage = sdp->vpage; /* keep pointer to vpage */
769
770 nbytes = vpgtob(npages);
771 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
772 bcopy(ovpage, sdp->vpage, nbytes);
773
774 /* free up old vpage */
775 kmem_free(ovpage, vpgtob(opages));
776 }
777 seg->s_size -= len;
778
779 /*
780 * free devmap handles from addr to the end of the mapping.
781 */
782 if (dhp != NULL)
783 devmap_handle_unmap_tail(dhp, addr);
784
785 return (0);
786 }
787
788 /*
789 * The section to go is in the middle of the segment,
790 * have to make it into two segments. nseg is made for
791 * the high end while seg is cut down at the low end.
792 */
793 nbase = addr + len; /* new seg base */
794 nsize = (seg->s_base + seg->s_size) - nbase; /* new seg size */
795 seg->s_size = addr - seg->s_base; /* shrink old seg */
796 nseg = seg_alloc(seg->s_as, nbase, nsize);
797 if (nseg == NULL)
798 panic("segdev_unmap seg_alloc");
799
800 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK2,
801 "segdev_unmap: seg=%p nseg=%p", (void *)seg, (void *)nseg);
802 DEBUGF(3, (CE_CONT, "segdev_unmap: segdev_dup seg %p nseg %p\n",
803 (void *)seg, (void *)nseg));
804 nsdp = sdp_alloc();
805
806 nseg->s_ops = seg->s_ops;
807 nseg->s_data = (void *)nsdp;
808
809 VN_HOLD(sdp->vp);
810 nsdp->mapfunc = sdp->mapfunc;
811 nsdp->offset = sdp->offset + (offset_t)(nseg->s_base - seg->s_base);
812 nsdp->vp = sdp->vp;
813 nsdp->pageprot = sdp->pageprot;
814 nsdp->prot = sdp->prot;
815 nsdp->maxprot = sdp->maxprot;
816 nsdp->type = sdp->type;
817 nsdp->hat_attr = sdp->hat_attr;
818 nsdp->hat_flags = sdp->hat_flags;
819 nsdp->softlockcnt = 0;
820
821 /*
822 * Initialize per page data if the segment we are
823 * dup'ing has per page information.
824 */
825 if (sdp->vpage != NULL) {
826 /* need to split vpage into two arrays */
827 register size_t nnbytes;
828 register size_t nnpages;
829 register struct vpage *ovpage;
830
831 ovpage = sdp->vpage; /* keep pointer to vpage */
832
833 npages = seg_pages(seg); /* seg has shrunk */
834 nbytes = vpgtob(npages);
835 nnpages = seg_pages(nseg);
836 nnbytes = vpgtob(nnpages);
837
838 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
839 bcopy(ovpage, sdp->vpage, nbytes);
840
841 nsdp->vpage = kmem_alloc(nnbytes, KM_SLEEP);
842 bcopy(&ovpage[npages + dpages], nsdp->vpage, nnbytes);
843
844 /* free up old vpage */
845 kmem_free(ovpage, vpgtob(opages));
846 } else
847 nsdp->vpage = NULL;
848
849 /*
850 * unmap dhps.
851 */
852 if (dhp == NULL) {
853 nsdp->devmap_data = NULL;
854 return (0);
855 }
856 while (dhp != NULL) {
857 callbackops = &dhp->dh_callbackops;
858 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK3,
859 "segdev_unmap: dhp=%p addr=%p", dhp, addr);
860 DEBUGF(3, (CE_CONT, "unmap: dhp %p addr %p uvaddr %p len %lx\n",
861 (void *)dhp, (void *)addr,
862 (void *)dhp->dh_uvaddr, dhp->dh_len));
863
864 if (addr == (dhp->dh_uvaddr + dhp->dh_len)) {
865 dhpp = dhp->dh_next;
866 dhp->dh_next = NULL;
867 dhp = dhpp;
868 } else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) {
869 dhp = dhp->dh_next;
870 } else if (addr > dhp->dh_uvaddr &&
871 (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) {
872 /*
873 * <addr, addr+len> is enclosed by dhp.
874 * create a newdhp that begins at addr+len and
875 * ends at dhp->dh_uvaddr+dhp->dh_len.
876 */
877 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
878 HOLD_DHP_LOCK(dhp);
879 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
880 RELE_DHP_LOCK(dhp);
881 newdhp->dh_seg = nseg;
882 newdhp->dh_next = dhp->dh_next;
883 if (dhp->dh_softlock != NULL)
884 newdhp->dh_softlock = devmap_softlock_init(
885 newdhp->dh_dev,
886 (ulong_t)callbackops->devmap_access);
887 if (dhp->dh_ctx != NULL)
888 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
889 (ulong_t)callbackops->devmap_access);
890 if (newdhp->dh_flags & DEVMAP_LOCK_INITED) {
891 mutex_init(&newdhp->dh_lock,
892 NULL, MUTEX_DEFAULT, NULL);
893 }
894 if (callbackops->devmap_unmap != NULL)
895 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
896 off, len, dhp, &dhp->dh_pvtp,
897 newdhp, &newdhp->dh_pvtp);
898 mlen = len + (addr - dhp->dh_uvaddr);
899 devmap_handle_reduce_len(newdhp, mlen);
900 nsdp->devmap_data = newdhp;
901 /* XX Changing len should recalculate LARGE flag */
902 dhp->dh_len = addr - dhp->dh_uvaddr;
903 dhpp = dhp->dh_next;
904 dhp->dh_next = NULL;
905 dhp = dhpp;
906 } else if ((addr > dhp->dh_uvaddr) &&
907 ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) {
908 mlen = dhp->dh_len + dhp->dh_uvaddr - addr;
909 /*
910 * <addr, addr+len> spans over dhps.
911 */
912 if (callbackops->devmap_unmap != NULL)
913 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
914 off, mlen, (devmap_cookie_t *)dhp,
915 &dhp->dh_pvtp, NULL, NULL);
916 /* XX Changing len should recalculate LARGE flag */
917 dhp->dh_len = addr - dhp->dh_uvaddr;
918 dhpp = dhp->dh_next;
919 dhp->dh_next = NULL;
920 dhp = dhpp;
921 nsdp->devmap_data = dhp;
922 } else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) {
923 /*
924 * dhp is enclosed by <addr, addr+len>.
925 */
926 dhp->dh_seg = nseg;
927 nsdp->devmap_data = dhp;
928 dhp = devmap_handle_unmap(dhp);
929 nsdp->devmap_data = dhp; /* XX redundant? */
930 } else if (((addr + len) > dhp->dh_uvaddr) &&
931 ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) {
932 mlen = addr + len - dhp->dh_uvaddr;
933 if (callbackops->devmap_unmap != NULL)
934 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
935 dhp->dh_uoff, mlen, NULL,
936 NULL, dhp, &dhp->dh_pvtp);
937 devmap_handle_reduce_len(dhp, mlen);
938 nsdp->devmap_data = dhp;
939 dhp->dh_seg = nseg;
940 dhp = dhp->dh_next;
941 } else {
942 dhp->dh_seg = nseg;
943 dhp = dhp->dh_next;
944 }
945 }
946 return (0);
947 }
948
949 /*
950 * Utility function handles reducing the length of a devmap handle during unmap
951 * Note that is only used for unmapping the front portion of the handler,
952 * i.e., we are bumping up the offset/pfn etc up by len
953 * Do not use if reducing length at the tail.
954 */
955 static void
956 devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len)
957 {
958 struct ddi_umem_cookie *cp;
959 struct devmap_pmem_cookie *pcp;
960 /*
961 * adjust devmap handle fields
962 */
963 ASSERT(len < dhp->dh_len);
964
965 /* Make sure only page-aligned changes are done */
966 ASSERT((len & PAGEOFFSET) == 0);
967
968 dhp->dh_len -= len;
969 dhp->dh_uoff += (offset_t)len;
970 dhp->dh_roff += (offset_t)len;
971 dhp->dh_uvaddr += len;
972 /* Need to grab dhp lock if REMAP */
973 HOLD_DHP_LOCK(dhp);
974 cp = dhp->dh_cookie;
975 if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) {
976 if (cookie_is_devmem(cp)) {
977 dhp->dh_pfn += btop(len);
978 } else if (cookie_is_pmem(cp)) {
979 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
980 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
981 dhp->dh_roff < ptob(pcp->dp_npages));
982 } else {
983 ASSERT(dhp->dh_roff < cp->size);
984 ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
985 dhp->dh_cvaddr < (cp->cvaddr + cp->size));
986 ASSERT((dhp->dh_cvaddr + len) <=
987 (cp->cvaddr + cp->size));
988
989 dhp->dh_cvaddr += len;
990 }
991 }
992 /* XXX - Should recalculate the DEVMAP_FLAG_LARGE after changes */
993 RELE_DHP_LOCK(dhp);
994 }
995
996 /*
997 * Free devmap handle, dhp.
998 * Return the next devmap handle on the linked list.
999 */
1000 static devmap_handle_t *
1001 devmap_handle_unmap(devmap_handle_t *dhp)
1002 {
1003 struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops;
1004 struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data;
1005 devmap_handle_t *dhpp = (devmap_handle_t *)sdp->devmap_data;
1006
1007 ASSERT(dhp != NULL);
1008
1009 /*
1010 * before we free up dhp, call the driver's devmap_unmap entry point
1011 * to free resources allocated for this dhp.
1012 */
1013 if (callbackops->devmap_unmap != NULL) {
1014 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff,
1015 dhp->dh_len, NULL, NULL, NULL, NULL);
1016 }
1017
1018 if (dhpp == dhp) { /* releasing first dhp, change sdp data */
1019 sdp->devmap_data = dhp->dh_next;
1020 } else {
1021 while (dhpp->dh_next != dhp) {
1022 dhpp = dhpp->dh_next;
1023 }
1024 dhpp->dh_next = dhp->dh_next;
1025 }
1026 dhpp = dhp->dh_next; /* return value is next dhp in chain */
1027
1028 if (dhp->dh_softlock != NULL)
1029 devmap_softlock_rele(dhp);
1030
1031 if (dhp->dh_ctx != NULL)
1032 devmap_ctx_rele(dhp);
1033
1034 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1035 mutex_destroy(&dhp->dh_lock);
1036 }
1037 kmem_free(dhp, sizeof (devmap_handle_t));
1038
1039 return (dhpp);
1040 }
1041
1042 /*
1043 * Free complete devmap handles from dhp for len bytes
1044 * dhp can be either the first handle or a subsequent handle
1045 */
1046 static void
1047 devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len)
1048 {
1049 struct devmap_callback_ctl *callbackops;
1050
1051 /*
1052 * free the devmap handles covered by len.
1053 */
1054 while (len >= dhp->dh_len) {
1055 len -= dhp->dh_len;
1056 dhp = devmap_handle_unmap(dhp);
1057 }
1058 if (len != 0) { /* partial unmap at head of first remaining dhp */
1059 callbackops = &dhp->dh_callbackops;
1060
1061 /*
1062 * Call the unmap callback so the drivers can make
1063 * adjustment on its private data.
1064 */
1065 if (callbackops->devmap_unmap != NULL)
1066 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
1067 dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp);
1068 devmap_handle_reduce_len(dhp, len);
1069 }
1070 }
1071
1072 /*
1073 * Free devmap handles to truncate the mapping after addr
1074 * RFE: Simpler to pass in dhp pointing at correct dhp (avoid find again)
1075 * Also could then use the routine in middle unmap case too
1076 */
1077 static void
1078 devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr)
1079 {
1080 register struct seg *seg = dhp->dh_seg;
1081 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1082 register devmap_handle_t *dhph = (devmap_handle_t *)sdp->devmap_data;
1083 struct devmap_callback_ctl *callbackops;
1084 register devmap_handle_t *dhpp;
1085 size_t maplen;
1086 ulong_t off;
1087 size_t len;
1088
1089 maplen = (size_t)(addr - dhp->dh_uvaddr);
1090 dhph = devmap_find_handle(dhph, addr);
1091
1092 while (dhph != NULL) {
1093 if (maplen == 0) {
1094 dhph = devmap_handle_unmap(dhph);
1095 } else {
1096 callbackops = &dhph->dh_callbackops;
1097 len = dhph->dh_len - maplen;
1098 off = (ulong_t)sdp->offset + (addr - seg->s_base);
1099 /*
1100 * Call the unmap callback so the driver
1101 * can make adjustments on its private data.
1102 */
1103 if (callbackops->devmap_unmap != NULL)
1104 (*callbackops->devmap_unmap)(dhph,
1105 dhph->dh_pvtp, off, len,
1106 (devmap_cookie_t *)dhph,
1107 &dhph->dh_pvtp, NULL, NULL);
1108 /* XXX Reducing len needs to recalculate LARGE flag */
1109 dhph->dh_len = maplen;
1110 maplen = 0;
1111 dhpp = dhph->dh_next;
1112 dhph->dh_next = NULL;
1113 dhph = dhpp;
1114 }
1115 } /* end while */
1116 }
1117
1118 /*
1119 * Free a segment.
1120 */
1121 static void
1122 segdev_free(struct seg *seg)
1123 {
1124 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1125 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
1126
1127 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FREE,
1128 "segdev_free: dhp=%p seg=%p", (void *)dhp, (void *)seg);
1129 DEBUGF(3, (CE_CONT, "segdev_free: dhp %p seg %p\n",
1130 (void *)dhp, (void *)seg));
1131
1132 /*
1133 * Since the address space is "write" locked, we
1134 * don't need the segment lock to protect "segdev" data.
1135 */
1136 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
1137
1138 while (dhp != NULL)
1139 dhp = devmap_handle_unmap(dhp);
1140
1141 VN_RELE(sdp->vp);
1142 if (sdp->vpage != NULL)
1143 kmem_free(sdp->vpage, vpgtob(seg_pages(seg)));
1144
1145 rw_destroy(&sdp->lock);
1146 kmem_free(sdp, sizeof (*sdp));
1147 }
1148
1149 static void
1150 free_devmap_handle(devmap_handle_t *dhp)
1151 {
1152 register devmap_handle_t *dhpp;
1153
1154 /*
1155 * free up devmap handle
1156 */
1157 while (dhp != NULL) {
1158 dhpp = dhp->dh_next;
1159 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1160 mutex_destroy(&dhp->dh_lock);
1161 }
1162
1163 if (dhp->dh_softlock != NULL)
1164 devmap_softlock_rele(dhp);
1165
1166 if (dhp->dh_ctx != NULL)
1167 devmap_ctx_rele(dhp);
1168
1169 kmem_free(dhp, sizeof (devmap_handle_t));
1170 dhp = dhpp;
1171 }
1172 }
1173
1174 /*
1175 * routines to lock and unlock underlying segkp segment for
1176 * KMEM_PAGEABLE type cookies.
1177 * segkp only allows a single pending F_SOFTLOCK
1178 * we keep track of number of locks in the cookie so we can
1179 * have multiple pending faults and manage the calls to segkp.
1180 * RFE: if segkp supports either pagelock or can support multiple
1181 * calls to F_SOFTLOCK, then these routines can go away.
1182 * If pagelock, segdev_faultpage can fault on a page by page basis
1183 * and simplifies the code quite a bit.
1184 * if multiple calls allowed but not partial ranges, then need for
1185 * cookie->lock and locked count goes away, code can call as_fault directly
1186 */
1187 static faultcode_t
1188 acquire_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1189 {
1190 int err = 0;
1191 ASSERT(cookie_is_kpmem(cookie));
1192 /*
1193 * Fault in pages in segkp with F_SOFTLOCK.
1194 * We want to hold the lock until all pages have been loaded.
1195 * segkp only allows single caller to hold SOFTLOCK, so cookie
1196 * holds a count so we dont call into segkp multiple times
1197 */
1198 mutex_enter(&cookie->lock);
1199
1200 /*
1201 * Check for overflow in locked field
1202 */
1203 if ((UINT32_MAX - cookie->locked) < npages) {
1204 err = FC_MAKE_ERR(ENOMEM);
1205 } else if (cookie->locked == 0) {
1206 /* First time locking */
1207 err = as_fault(kas.a_hat, &kas, cookie->cvaddr,
1208 cookie->size, F_SOFTLOCK, PROT_READ|PROT_WRITE);
1209 }
1210 if (!err) {
1211 cookie->locked += npages;
1212 }
1213 mutex_exit(&cookie->lock);
1214 return (err);
1215 }
1216
1217 static void
1218 release_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1219 {
1220 mutex_enter(&cookie->lock);
1221 ASSERT(cookie_is_kpmem(cookie));
1222 ASSERT(cookie->locked >= npages);
1223 cookie->locked -= (uint_t)npages;
1224 if (cookie->locked == 0) {
1225 /* Last unlock */
1226 if (as_fault(kas.a_hat, &kas, cookie->cvaddr,
1227 cookie->size, F_SOFTUNLOCK, PROT_READ|PROT_WRITE))
1228 panic("segdev releasing kpmem lock %p", (void *)cookie);
1229 }
1230 mutex_exit(&cookie->lock);
1231 }
1232
1233 /*
1234 * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
1235 * drivers with devmap_access callbacks
1236 * slock->softlocked basically works like a rw lock
1237 * -ve counts => F_SOFTLOCK in progress
1238 * +ve counts => F_INVAL/F_PROT in progress
1239 * We allow only one F_SOFTLOCK at a time
1240 * but can have multiple pending F_INVAL/F_PROT calls
1241 *
1242 * This routine waits using cv_wait_sig so killing processes is more graceful
1243 * Returns EINTR if coming out of this routine due to a signal, 0 otherwise
1244 */
1245 static int devmap_softlock_enter(
1246 struct devmap_softlock *slock,
1247 size_t npages,
1248 enum fault_type type)
1249 {
1250 if (npages == 0)
1251 return (0);
1252 mutex_enter(&(slock->lock));
1253 switch (type) {
1254 case F_SOFTLOCK :
1255 while (slock->softlocked) {
1256 if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1257 /* signalled */
1258 mutex_exit(&(slock->lock));
1259 return (EINTR);
1260 }
1261 }
1262 slock->softlocked -= npages; /* -ve count => locked */
1263 break;
1264 case F_INVAL :
1265 case F_PROT :
1266 while (slock->softlocked < 0)
1267 if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1268 /* signalled */
1269 mutex_exit(&(slock->lock));
1270 return (EINTR);
1271 }
1272 slock->softlocked += npages; /* +ve count => f_invals */
1273 break;
1274 default:
1275 ASSERT(0);
1276 }
1277 mutex_exit(&(slock->lock));
1278 return (0);
1279 }
1280
1281 static void devmap_softlock_exit(
1282 struct devmap_softlock *slock,
1283 size_t npages,
1284 enum fault_type type)
1285 {
1286 if (slock == NULL)
1287 return;
1288 mutex_enter(&(slock->lock));
1289 switch (type) {
1290 case F_SOFTLOCK :
1291 ASSERT(-slock->softlocked >= npages);
1292 slock->softlocked += npages; /* -ve count is softlocked */
1293 if (slock->softlocked == 0)
1294 cv_signal(&slock->cv);
1295 break;
1296 case F_INVAL :
1297 case F_PROT:
1298 ASSERT(slock->softlocked >= npages);
1299 slock->softlocked -= npages;
1300 if (slock->softlocked == 0)
1301 cv_signal(&slock->cv);
1302 break;
1303 default:
1304 ASSERT(0);
1305 }
1306 mutex_exit(&(slock->lock));
1307 }
1308
1309 /*
1310 * Do a F_SOFTUNLOCK call over the range requested.
1311 * The range must have already been F_SOFTLOCK'ed.
1312 * The segment lock should be held, (but not the segment private lock?)
1313 * The softunlock code below does not adjust for large page sizes
1314 * assumes the caller already did any addr/len adjustments for
1315 * pagesize mappings before calling.
1316 */
1317 /*ARGSUSED*/
1318 static void
1319 segdev_softunlock(
1320 struct hat *hat, /* the hat */
1321 struct seg *seg, /* seg_dev of interest */
1322 caddr_t addr, /* base address of range */
1323 size_t len, /* number of bytes */
1324 enum seg_rw rw) /* type of access at fault */
1325 {
1326 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1327 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1328
1329 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SOFTUNLOCK,
1330 "segdev_softunlock:dhp_head=%p sdp=%p addr=%p len=%lx",
1331 dhp_head, sdp, addr, len);
1332 DEBUGF(3, (CE_CONT, "segdev_softunlock: dhp %p lockcnt %lx "
1333 "addr %p len %lx\n",
1334 (void *)dhp_head, sdp->softlockcnt, (void *)addr, len));
1335
1336 hat_unlock(hat, addr, len);
1337
1338 if (dhp_head != NULL) {
1339 devmap_handle_t *dhp;
1340 size_t mlen;
1341 size_t tlen = len;
1342 ulong_t off;
1343
1344 dhp = devmap_find_handle(dhp_head, addr);
1345 ASSERT(dhp != NULL);
1346
1347 off = (ulong_t)(addr - dhp->dh_uvaddr);
1348 while (tlen != 0) {
1349 mlen = MIN(tlen, (dhp->dh_len - off));
1350
1351 /*
1352 * unlock segkp memory, locked during F_SOFTLOCK
1353 */
1354 if (dhp_is_kpmem(dhp)) {
1355 release_kpmem_lock(
1356 (struct ddi_umem_cookie *)dhp->dh_cookie,
1357 btopr(mlen));
1358 }
1359
1360 /*
1361 * Do the softlock accounting for devmap_access
1362 */
1363 if (dhp->dh_callbackops.devmap_access != NULL) {
1364 devmap_softlock_exit(dhp->dh_softlock,
1365 btopr(mlen), F_SOFTLOCK);
1366 }
1367
1368 tlen -= mlen;
1369 dhp = dhp->dh_next;
1370 off = 0;
1371 }
1372 }
1373
1374 mutex_enter(&freemem_lock);
1375 ASSERT(sdp->softlockcnt >= btopr(len));
1376 sdp->softlockcnt -= btopr(len);
1377 mutex_exit(&freemem_lock);
1378 if (sdp->softlockcnt == 0) {
1379 /*
1380 * All SOFTLOCKS are gone. Wakeup any waiting
1381 * unmappers so they can try again to unmap.
1382 * Check for waiters first without the mutex
1383 * held so we don't always grab the mutex on
1384 * softunlocks.
1385 */
1386 if (AS_ISUNMAPWAIT(seg->s_as)) {
1387 mutex_enter(&seg->s_as->a_contents);
1388 if (AS_ISUNMAPWAIT(seg->s_as)) {
1389 AS_CLRUNMAPWAIT(seg->s_as);
1390 cv_broadcast(&seg->s_as->a_cv);
1391 }
1392 mutex_exit(&seg->s_as->a_contents);
1393 }
1394 }
1395
1396 }
1397
1398 /*
1399 * Handle fault for a single page.
1400 * Done in a separate routine so we can handle errors more easily.
1401 * This routine is called only from segdev_faultpages()
1402 * when looping over the range of addresses requested. The segment lock is held.
1403 */
1404 static faultcode_t
1405 segdev_faultpage(
1406 struct hat *hat, /* the hat */
1407 struct seg *seg, /* seg_dev of interest */
1408 caddr_t addr, /* address in as */
1409 struct vpage *vpage, /* pointer to vpage for seg, addr */
1410 enum fault_type type, /* type of fault */
1411 enum seg_rw rw, /* type of access at fault */
1412 devmap_handle_t *dhp) /* devmap handle if any for this page */
1413 {
1414 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1415 uint_t prot;
1416 pfn_t pfnum = PFN_INVALID;
1417 u_offset_t offset;
1418 uint_t hat_flags;
1419 dev_info_t *dip;
1420
1421 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE,
1422 "segdev_faultpage: dhp=%p seg=%p addr=%p", dhp, seg, addr);
1423 DEBUGF(8, (CE_CONT, "segdev_faultpage: dhp %p seg %p addr %p \n",
1424 (void *)dhp, (void *)seg, (void *)addr));
1425
1426 /*
1427 * Initialize protection value for this page.
1428 * If we have per page protection values check it now.
1429 */
1430 if (sdp->pageprot) {
1431 uint_t protchk;
1432
1433 switch (rw) {
1434 case S_READ:
1435 protchk = PROT_READ;
1436 break;
1437 case S_WRITE:
1438 protchk = PROT_WRITE;
1439 break;
1440 case S_EXEC:
1441 protchk = PROT_EXEC;
1442 break;
1443 case S_OTHER:
1444 default:
1445 protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1446 break;
1447 }
1448
1449 prot = VPP_PROT(vpage);
1450 if ((prot & protchk) == 0)
1451 return (FC_PROT); /* illegal access type */
1452 } else {
1453 prot = sdp->prot;
1454 /* caller has already done segment level protection check */
1455 }
1456
1457 if (type == F_SOFTLOCK) {
1458 mutex_enter(&freemem_lock);
1459 sdp->softlockcnt++;
1460 mutex_exit(&freemem_lock);
1461 }
1462
1463 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
1464 offset = sdp->offset + (u_offset_t)(addr - seg->s_base);
1465 /*
1466 * In the devmap framework, sdp->mapfunc is set to NULL. we can get
1467 * pfnum from dhp->dh_pfn (at beginning of segment) and offset from
1468 * seg->s_base.
1469 */
1470 if (dhp == NULL) {
1471 /* If segment has devmap_data, then dhp should be non-NULL */
1472 ASSERT(sdp->devmap_data == NULL);
1473 pfnum = (pfn_t)cdev_mmap(sdp->mapfunc, sdp->vp->v_rdev,
1474 (off_t)offset, prot);
1475 prot |= sdp->hat_attr;
1476 } else {
1477 ulong_t off;
1478 struct ddi_umem_cookie *cp;
1479 struct devmap_pmem_cookie *pcp;
1480
1481 /* ensure the dhp passed in contains addr. */
1482 ASSERT(dhp == devmap_find_handle(
1483 (devmap_handle_t *)sdp->devmap_data, addr));
1484
1485 off = addr - dhp->dh_uvaddr;
1486
1487 /*
1488 * This routine assumes that the caller makes sure that the
1489 * fields in dhp used below are unchanged due to remap during
1490 * this call. Caller does HOLD_DHP_LOCK if neeed
1491 */
1492 cp = dhp->dh_cookie;
1493 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1494 pfnum = PFN_INVALID;
1495 } else if (cookie_is_devmem(cp)) {
1496 pfnum = dhp->dh_pfn + btop(off);
1497 } else if (cookie_is_pmem(cp)) {
1498 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
1499 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
1500 dhp->dh_roff < ptob(pcp->dp_npages));
1501 pfnum = page_pptonum(
1502 pcp->dp_pparray[btop(off + dhp->dh_roff)]);
1503 } else {
1504 ASSERT(dhp->dh_roff < cp->size);
1505 ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
1506 dhp->dh_cvaddr < (cp->cvaddr + cp->size));
1507 ASSERT((dhp->dh_cvaddr + off) <=
1508 (cp->cvaddr + cp->size));
1509 ASSERT((dhp->dh_cvaddr + off + PAGESIZE) <=
1510 (cp->cvaddr + cp->size));
1511
1512 switch (cp->type) {
1513 case UMEM_LOCKED :
1514 if (cp->pparray != NULL) {
1515 ASSERT((dhp->dh_roff &
1516 PAGEOFFSET) == 0);
1517 pfnum = page_pptonum(
1518 cp->pparray[btop(off +
1519 dhp->dh_roff)]);
1520 } else {
1521 pfnum = hat_getpfnum(
1522 ((proc_t *)cp->procp)->p_as->a_hat,
1523 cp->cvaddr + off);
1524 }
1525 break;
1526 case UMEM_TRASH :
1527 pfnum = page_pptonum(trashpp);
1528 /*
1529 * We should set hat_flags to HAT_NOFAULT also
1530 * However, not all hat layers implement this
1531 */
1532 break;
1533 case KMEM_PAGEABLE:
1534 case KMEM_NON_PAGEABLE:
1535 pfnum = hat_getpfnum(kas.a_hat,
1536 dhp->dh_cvaddr + off);
1537 break;
1538 default :
1539 pfnum = PFN_INVALID;
1540 break;
1541 }
1542 }
1543 prot |= dhp->dh_hat_attr;
1544 }
1545 if (pfnum == PFN_INVALID) {
1546 return (FC_MAKE_ERR(EFAULT));
1547 }
1548 /* prot should already be OR'ed in with hat_attributes if needed */
1549
1550 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE_CK1,
1551 "segdev_faultpage: pfnum=%lx memory=%x prot=%x flags=%x",
1552 pfnum, pf_is_memory(pfnum), prot, hat_flags);
1553 DEBUGF(9, (CE_CONT, "segdev_faultpage: pfnum %lx memory %x "
1554 "prot %x flags %x\n", pfnum, pf_is_memory(pfnum), prot, hat_flags));
1555
1556 if (pf_is_memory(pfnum) || (dhp != NULL)) {
1557 /*
1558 * It's not _really_ required here to pass sdp->hat_flags
1559 * to hat_devload even though we do it.
1560 * This is because hat figures it out DEVMEM mappings
1561 * are non-consistent, anyway.
1562 */
1563 hat_devload(hat, addr, PAGESIZE, pfnum,
1564 prot, hat_flags | sdp->hat_flags);
1565 return (0);
1566 }
1567
1568 /*
1569 * Fall through to the case where devmap is not used and need to call
1570 * up the device tree to set up the mapping
1571 */
1572
1573 dip = VTOS(VTOCVP(sdp->vp))->s_dip;
1574 ASSERT(dip);
1575
1576 /*
1577 * When calling ddi_map_fault, we do not OR in sdp->hat_attr
1578 * This is because this calls drivers which may not expect
1579 * prot to have any other values than PROT_ALL
1580 * The root nexus driver has a hack to peek into the segment
1581 * structure and then OR in sdp->hat_attr.
1582 * XX In case the bus_ops interfaces are ever revisited
1583 * we need to fix this. prot should include other hat attributes
1584 */
1585 if (ddi_map_fault(dip, hat, seg, addr, NULL, pfnum, prot & PROT_ALL,
1586 (uint_t)(type == F_SOFTLOCK)) != DDI_SUCCESS) {
1587 return (FC_MAKE_ERR(EFAULT));
1588 }
1589 return (0);
1590 }
1591
1592 static faultcode_t
1593 segdev_fault(
1594 struct hat *hat, /* the hat */
1595 struct seg *seg, /* the seg_dev of interest */
1596 caddr_t addr, /* the address of the fault */
1597 size_t len, /* the length of the range */
1598 enum fault_type type, /* type of fault */
1599 enum seg_rw rw) /* type of access at fault */
1600 {
1601 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1602 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1603 devmap_handle_t *dhp;
1604 struct devmap_softlock *slock = NULL;
1605 ulong_t slpage = 0;
1606 ulong_t off;
1607 caddr_t maddr = addr;
1608 int err;
1609 int err_is_faultcode = 0;
1610
1611 TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_FAULT,
1612 "segdev_fault: dhp_head=%p seg=%p addr=%p len=%lx type=%x",
1613 (void *)dhp_head, (void *)seg, (void *)addr, len, type);
1614 DEBUGF(7, (CE_CONT, "segdev_fault: dhp_head %p seg %p "
1615 "addr %p len %lx type %x\n",
1616 (void *)dhp_head, (void *)seg, (void *)addr, len, type));
1617
1618 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
1619
1620 /* Handle non-devmap case */
1621 if (dhp_head == NULL)
1622 return (segdev_faultpages(hat, seg, addr, len, type, rw, NULL));
1623
1624 /* Find devmap handle */
1625 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
1626 return (FC_NOMAP);
1627
1628 /*
1629 * The seg_dev driver does not implement copy-on-write,
1630 * and always loads translations with maximal allowed permissions
1631 * but we got an fault trying to access the device.
1632 * Servicing the fault is not going to result in any better result
1633 * RFE: If we want devmap_access callbacks to be involved in F_PROT
1634 * faults, then the code below is written for that
1635 * Pending resolution of the following:
1636 * - determine if the F_INVAL/F_SOFTLOCK syncing
1637 * is needed for F_PROT also or not. The code below assumes it does
1638 * - If driver sees F_PROT and calls devmap_load with same type,
1639 * then segdev_faultpages will fail with FC_PROT anyway, need to
1640 * change that so calls from devmap_load to segdev_faultpages for
1641 * F_PROT type are retagged to F_INVAL.
1642 * RFE: Today we dont have drivers that use devmap and want to handle
1643 * F_PROT calls. The code in segdev_fault* is written to allow
1644 * this case but is not tested. A driver that needs this capability
1645 * should be able to remove the short-circuit case; resolve the
1646 * above issues and "should" work.
1647 */
1648 if (type == F_PROT) {
1649 return (FC_PROT);
1650 }
1651
1652 /*
1653 * Loop through dhp list calling devmap_access or segdev_faultpages for
1654 * each devmap handle.
1655 * drivers which implement devmap_access can interpose on faults and do
1656 * device-appropriate special actions before calling devmap_load.
1657 */
1658
1659 /*
1660 * Unfortunately, this simple loop has turned out to expose a variety
1661 * of complex problems which results in the following convoluted code.
1662 *
1663 * First, a desire to handle a serialization of F_SOFTLOCK calls
1664 * to the driver within the framework.
1665 * This results in a dh_softlock structure that is on a per device
1666 * (or device instance) basis and serializes devmap_access calls.
1667 * Ideally we would need to do this for underlying
1668 * memory/device regions that are being faulted on
1669 * but that is hard to identify and with REMAP, harder
1670 * Second, a desire to serialize F_INVAL(and F_PROT) calls w.r.t.
1671 * to F_SOFTLOCK calls to the driver.
1672 * These serializations are to simplify the driver programmer model.
1673 * To support these two features, the code first goes through the
1674 * devmap handles and counts the pages (slpage) that are covered
1675 * by devmap_access callbacks.
1676 * This part ends with a devmap_softlock_enter call
1677 * which allows only one F_SOFTLOCK active on a device instance,
1678 * but multiple F_INVAL/F_PROTs can be active except when a
1679 * F_SOFTLOCK is active
1680 *
1681 * Next, we dont short-circuit the fault code upfront to call
1682 * segdev_softunlock for F_SOFTUNLOCK, because we must use
1683 * the same length when we softlock and softunlock.
1684 *
1685 * -Hat layers may not support softunlocking lengths less than the
1686 * original length when there is large page support.
1687 * -kpmem locking is dependent on keeping the lengths same.
1688 * -if drivers handled F_SOFTLOCK, they probably also expect to
1689 * see an F_SOFTUNLOCK of the same length
1690 * Hence, if extending lengths during softlock,
1691 * softunlock has to make the same adjustments and goes through
1692 * the same loop calling segdev_faultpages/segdev_softunlock
1693 * But some of the synchronization and error handling is different
1694 */
1695
1696 if (type != F_SOFTUNLOCK) {
1697 devmap_handle_t *dhpp = dhp;
1698 size_t slen = len;
1699
1700 /*
1701 * Calculate count of pages that are :
1702 * a) within the (potentially extended) fault region
1703 * b) AND covered by devmap handle with devmap_access
1704 */
1705 off = (ulong_t)(addr - dhpp->dh_uvaddr);
1706 while (slen != 0) {
1707 size_t mlen;
1708
1709 /*
1710 * Softlocking on a region that allows remap is
1711 * unsupported due to unresolved locking issues
1712 * XXX: unclear what these are?
1713 * One potential is that if there is a pending
1714 * softlock, then a remap should not be allowed
1715 * until the unlock is done. This is easily
1716 * fixed by returning error in devmap*remap on
1717 * checking the dh->dh_softlock->softlocked value
1718 */
1719 if ((type == F_SOFTLOCK) &&
1720 (dhpp->dh_flags & DEVMAP_ALLOW_REMAP)) {
1721 return (FC_NOSUPPORT);
1722 }
1723
1724 mlen = MIN(slen, (dhpp->dh_len - off));
1725 if (dhpp->dh_callbackops.devmap_access) {
1726 size_t llen;
1727 caddr_t laddr;
1728 /*
1729 * use extended length for large page mappings
1730 */
1731 HOLD_DHP_LOCK(dhpp);
1732 if ((sdp->pageprot == 0) &&
1733 (dhpp->dh_flags & DEVMAP_FLAG_LARGE)) {
1734 devmap_get_large_pgsize(dhpp,
1735 mlen, maddr, &llen, &laddr);
1736 } else {
1737 llen = mlen;
1738 }
1739 RELE_DHP_LOCK(dhpp);
1740
1741 slpage += btopr(llen);
1742 slock = dhpp->dh_softlock;
1743 }
1744 maddr += mlen;
1745 ASSERT(slen >= mlen);
1746 slen -= mlen;
1747 dhpp = dhpp->dh_next;
1748 off = 0;
1749 }
1750 /*
1751 * synchonize with other faulting threads and wait till safe
1752 * devmap_softlock_enter might return due to signal in cv_wait
1753 *
1754 * devmap_softlock_enter has to be called outside of while loop
1755 * to prevent a deadlock if len spans over multiple dhps.
1756 * dh_softlock is based on device instance and if multiple dhps
1757 * use the same device instance, the second dhp's LOCK call
1758 * will hang waiting on the first to complete.
1759 * devmap_setup verifies that slocks in a dhp_chain are same.
1760 * RFE: this deadlock only hold true for F_SOFTLOCK. For
1761 * F_INVAL/F_PROT, since we now allow multiple in parallel,
1762 * we could have done the softlock_enter inside the loop
1763 * and supported multi-dhp mappings with dissimilar devices
1764 */
1765 if (err = devmap_softlock_enter(slock, slpage, type))
1766 return (FC_MAKE_ERR(err));
1767 }
1768
1769 /* reset 'maddr' to the start addr of the range of fault. */
1770 maddr = addr;
1771
1772 /* calculate the offset corresponds to 'addr' in the first dhp. */
1773 off = (ulong_t)(addr - dhp->dh_uvaddr);
1774
1775 /*
1776 * The fault length may span over multiple dhps.
1777 * Loop until the total length is satisfied.
1778 */
1779 while (len != 0) {
1780 size_t llen;
1781 size_t mlen;
1782 caddr_t laddr;
1783
1784 /*
1785 * mlen is the smaller of 'len' and the length
1786 * from addr to the end of mapping defined by dhp.
1787 */
1788 mlen = MIN(len, (dhp->dh_len - off));
1789
1790 HOLD_DHP_LOCK(dhp);
1791 /*
1792 * Pass the extended length and address to devmap_access
1793 * if large pagesize is used for loading address translations.
1794 */
1795 if ((sdp->pageprot == 0) &&
1796 (dhp->dh_flags & DEVMAP_FLAG_LARGE)) {
1797 devmap_get_large_pgsize(dhp, mlen, maddr,
1798 &llen, &laddr);
1799 ASSERT(maddr == addr || laddr == maddr);
1800 } else {
1801 llen = mlen;
1802 laddr = maddr;
1803 }
1804
1805 if (dhp->dh_callbackops.devmap_access != NULL) {
1806 offset_t aoff;
1807
1808 aoff = sdp->offset + (offset_t)(laddr - seg->s_base);
1809
1810 /*
1811 * call driver's devmap_access entry point which will
1812 * call devmap_load/contextmgmt to load the translations
1813 *
1814 * We drop the dhp_lock before calling access so
1815 * drivers can call devmap_*_remap within access
1816 */
1817 RELE_DHP_LOCK(dhp);
1818
1819 err = (*dhp->dh_callbackops.devmap_access)(
1820 dhp, (void *)dhp->dh_pvtp, aoff, llen, type, rw);
1821 } else {
1822 /*
1823 * If no devmap_access entry point, then load mappings
1824 * hold dhp_lock across faultpages if REMAP
1825 */
1826 err = segdev_faultpages(hat, seg, laddr, llen,
1827 type, rw, dhp);
1828 err_is_faultcode = 1;
1829 RELE_DHP_LOCK(dhp);
1830 }
1831
1832 if (err) {
1833 if ((type == F_SOFTLOCK) && (maddr > addr)) {
1834 /*
1835 * If not first dhp, use
1836 * segdev_fault(F_SOFTUNLOCK) for prior dhps
1837 * While this is recursion, it is incorrect to
1838 * call just segdev_softunlock
1839 * if we are using either large pages
1840 * or devmap_access. It will be more right
1841 * to go through the same loop as above
1842 * rather than call segdev_softunlock directly
1843 * It will use the right lenghths as well as
1844 * call into the driver devmap_access routines.
1845 */
1846 size_t done = (size_t)(maddr - addr);
1847 (void) segdev_fault(hat, seg, addr, done,
1848 F_SOFTUNLOCK, S_OTHER);
1849 /*
1850 * reduce slpage by number of pages
1851 * released by segdev_softunlock
1852 */
1853 ASSERT(slpage >= btopr(done));
1854 devmap_softlock_exit(slock,
1855 slpage - btopr(done), type);
1856 } else {
1857 devmap_softlock_exit(slock, slpage, type);
1858 }
1859
1860
1861 /*
1862 * Segdev_faultpages() already returns a faultcode,
1863 * hence, result from segdev_faultpages() should be
1864 * returned directly.
1865 */
1866 if (err_is_faultcode)
1867 return (err);
1868 return (FC_MAKE_ERR(err));
1869 }
1870
1871 maddr += mlen;
1872 ASSERT(len >= mlen);
1873 len -= mlen;
1874 dhp = dhp->dh_next;
1875 off = 0;
1876
1877 ASSERT(!dhp || len == 0 || maddr == dhp->dh_uvaddr);
1878 }
1879 /*
1880 * release the softlock count at end of fault
1881 * For F_SOFTLOCk this is done in the later F_SOFTUNLOCK
1882 */
1883 if ((type == F_INVAL) || (type == F_PROT))
1884 devmap_softlock_exit(slock, slpage, type);
1885 return (0);
1886 }
1887
1888 /*
1889 * segdev_faultpages
1890 *
1891 * Used to fault in seg_dev segment pages. Called by segdev_fault or devmap_load
1892 * This routine assumes that the callers makes sure that the fields
1893 * in dhp used below are not changed due to remap during this call.
1894 * Caller does HOLD_DHP_LOCK if neeed
1895 * This routine returns a faultcode_t as a return value for segdev_fault.
1896 */
1897 static faultcode_t
1898 segdev_faultpages(
1899 struct hat *hat, /* the hat */
1900 struct seg *seg, /* the seg_dev of interest */
1901 caddr_t addr, /* the address of the fault */
1902 size_t len, /* the length of the range */
1903 enum fault_type type, /* type of fault */
1904 enum seg_rw rw, /* type of access at fault */
1905 devmap_handle_t *dhp) /* devmap handle */
1906 {
1907 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1908 register caddr_t a;
1909 struct vpage *vpage;
1910 struct ddi_umem_cookie *kpmem_cookie = NULL;
1911 int err;
1912
1913 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGES,
1914 "segdev_faultpages: dhp=%p seg=%p addr=%p len=%lx",
1915 (void *)dhp, (void *)seg, (void *)addr, len);
1916 DEBUGF(5, (CE_CONT, "segdev_faultpages: "
1917 "dhp %p seg %p addr %p len %lx\n",
1918 (void *)dhp, (void *)seg, (void *)addr, len));
1919
1920 /*
1921 * The seg_dev driver does not implement copy-on-write,
1922 * and always loads translations with maximal allowed permissions
1923 * but we got an fault trying to access the device.
1924 * Servicing the fault is not going to result in any better result
1925 * XXX: If we want to allow devmap_access to handle F_PROT calls,
1926 * This code should be removed and let the normal fault handling
1927 * take care of finding the error
1928 */
1929 if (type == F_PROT) {
1930 return (FC_PROT);
1931 }
1932
1933 if (type == F_SOFTUNLOCK) {
1934 segdev_softunlock(hat, seg, addr, len, rw);
1935 return (0);
1936 }
1937
1938 /*
1939 * For kernel pageable memory, fault/lock segkp pages
1940 * We hold this until the completion of this
1941 * fault (INVAL/PROT) or till unlock (SOFTLOCK).
1942 */
1943 if ((dhp != NULL) && dhp_is_kpmem(dhp)) {
1944 kpmem_cookie = (struct ddi_umem_cookie *)dhp->dh_cookie;
1945 if (err = acquire_kpmem_lock(kpmem_cookie, btopr(len)))
1946 return (err);
1947 }
1948
1949 /*
1950 * If we have the same protections for the entire segment,
1951 * insure that the access being attempted is legitimate.
1952 */
1953 rw_enter(&sdp->lock, RW_READER);
1954 if (sdp->pageprot == 0) {
1955 uint_t protchk;
1956
1957 switch (rw) {
1958 case S_READ:
1959 protchk = PROT_READ;
1960 break;
1961 case S_WRITE:
1962 protchk = PROT_WRITE;
1963 break;
1964 case S_EXEC:
1965 protchk = PROT_EXEC;
1966 break;
1967 case S_OTHER:
1968 default:
1969 protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1970 break;
1971 }
1972
1973 if ((sdp->prot & protchk) == 0) {
1974 rw_exit(&sdp->lock);
1975 /* undo kpmem locking */
1976 if (kpmem_cookie != NULL) {
1977 release_kpmem_lock(kpmem_cookie, btopr(len));
1978 }
1979 return (FC_PROT); /* illegal access type */
1980 }
1981 }
1982
1983 /*
1984 * we do a single hat_devload for the range if
1985 * - devmap framework (dhp is not NULL),
1986 * - pageprot == 0, i.e., no per-page protection set and
1987 * - is device pages, irrespective of whether we are using large pages
1988 */
1989 if ((sdp->pageprot == 0) && (dhp != NULL) && dhp_is_devmem(dhp)) {
1990 pfn_t pfnum;
1991 uint_t hat_flags;
1992
1993 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1994 rw_exit(&sdp->lock);
1995 return (FC_NOMAP);
1996 }
1997
1998 if (type == F_SOFTLOCK) {
1999 mutex_enter(&freemem_lock);
2000 sdp->softlockcnt += btopr(len);
2001 mutex_exit(&freemem_lock);
2002 }
2003
2004 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
2005 pfnum = dhp->dh_pfn + btop((uintptr_t)(addr - dhp->dh_uvaddr));
2006 ASSERT(!pf_is_memory(pfnum));
2007
2008 hat_devload(hat, addr, len, pfnum, sdp->prot | dhp->dh_hat_attr,
2009 hat_flags | sdp->hat_flags);
2010 rw_exit(&sdp->lock);
2011 return (0);
2012 }
2013
2014 /* Handle cases where we have to loop through fault handling per-page */
2015
2016 if (sdp->vpage == NULL)
2017 vpage = NULL;
2018 else
2019 vpage = &sdp->vpage[seg_page(seg, addr)];
2020
2021 /* loop over the address range handling each fault */
2022 for (a = addr; a < addr + len; a += PAGESIZE) {
2023 if (err = segdev_faultpage(hat, seg, a, vpage, type, rw, dhp)) {
2024 break;
2025 }
2026 if (vpage != NULL)
2027 vpage++;
2028 }
2029 rw_exit(&sdp->lock);
2030 if (err && (type == F_SOFTLOCK)) { /* error handling for F_SOFTLOCK */
2031 size_t done = (size_t)(a - addr); /* pages fault successfully */
2032 if (done > 0) {
2033 /* use softunlock for those pages */
2034 segdev_softunlock(hat, seg, addr, done, S_OTHER);
2035 }
2036 if (kpmem_cookie != NULL) {
2037 /* release kpmem lock for rest of pages */
2038 ASSERT(len >= done);
2039 release_kpmem_lock(kpmem_cookie, btopr(len - done));
2040 }
2041 } else if ((kpmem_cookie != NULL) && (type != F_SOFTLOCK)) {
2042 /* for non-SOFTLOCK cases, release kpmem */
2043 release_kpmem_lock(kpmem_cookie, btopr(len));
2044 }
2045 return (err);
2046 }
2047
2048 /*
2049 * Asynchronous page fault. We simply do nothing since this
2050 * entry point is not supposed to load up the translation.
2051 */
2052 /*ARGSUSED*/
2053 static faultcode_t
2054 segdev_faulta(struct seg *seg, caddr_t addr)
2055 {
2056 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FAULTA,
2057 "segdev_faulta: seg=%p addr=%p", (void *)seg, (void *)addr);
2058 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2059
2060 return (0);
2061 }
2062
2063 static int
2064 segdev_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2065 {
2066 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2067 register devmap_handle_t *dhp;
2068 register struct vpage *vp, *evp;
2069 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
2070 ulong_t off;
2071 size_t mlen, sz;
2072
2073 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT,
2074 "segdev_setprot:start seg=%p addr=%p len=%lx prot=%x",
2075 (void *)seg, (void *)addr, len, prot);
2076 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2077
2078 if ((sz = sdp->softlockcnt) > 0 && dhp_head != NULL) {
2079 /*
2080 * Fail the setprot if pages are SOFTLOCKed through this
2081 * mapping.
2082 * Softlockcnt is protected from change by the as read lock.
2083 */
2084 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT_CK1,
2085 "segdev_setprot:error softlockcnt=%lx", sz);
2086 DEBUGF(1, (CE_CONT, "segdev_setprot: softlockcnt %ld\n", sz));
2087 return (EAGAIN);
2088 }
2089
2090 if (dhp_head != NULL) {
2091 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
2092 return (EINVAL);
2093
2094 /*
2095 * check if violate maxprot.
2096 */
2097 off = (ulong_t)(addr - dhp->dh_uvaddr);
2098 mlen = len;
2099 while (dhp) {
2100 if ((dhp->dh_maxprot & prot) != prot)
2101 return (EACCES); /* violated maxprot */
2102
2103 if (mlen > (dhp->dh_len - off)) {
2104 mlen -= dhp->dh_len - off;
2105 dhp = dhp->dh_next;
2106 off = 0;
2107 } else
2108 break;
2109 }
2110 } else {
2111 if ((sdp->maxprot & prot) != prot)
2112 return (EACCES);
2113 }
2114
2115 rw_enter(&sdp->lock, RW_WRITER);
2116 if (addr == seg->s_base && len == seg->s_size && sdp->pageprot == 0) {
2117 if (sdp->prot == prot) {
2118 rw_exit(&sdp->lock);
2119 return (0); /* all done */
2120 }
2121 sdp->prot = (uchar_t)prot;
2122 } else {
2123 sdp->pageprot = 1;
2124 if (sdp->vpage == NULL) {
2125 /*
2126 * First time through setting per page permissions,
2127 * initialize all the vpage structures to prot
2128 */
2129 sdp->vpage = kmem_zalloc(vpgtob(seg_pages(seg)),
2130 KM_SLEEP);
2131 evp = &sdp->vpage[seg_pages(seg)];
2132 for (vp = sdp->vpage; vp < evp; vp++)
2133 VPP_SETPROT(vp, sdp->prot);
2134 }
2135 /*
2136 * Now go change the needed vpages protections.
2137 */
2138 evp = &sdp->vpage[seg_page(seg, addr + len)];
2139 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++)
2140 VPP_SETPROT(vp, prot);
2141 }
2142 rw_exit(&sdp->lock);
2143
2144 if (dhp_head != NULL) {
2145 devmap_handle_t *tdhp;
2146 /*
2147 * If large page size was used in hat_devload(),
2148 * the same page size must be used in hat_unload().
2149 */
2150 dhp = tdhp = devmap_find_handle(dhp_head, addr);
2151 while (tdhp != NULL) {
2152 if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
2153 break;
2154 }
2155 tdhp = tdhp->dh_next;
2156 }
2157 if (tdhp) {
2158 size_t slen = len;
2159 size_t mlen;
2160 size_t soff;
2161
2162 soff = (ulong_t)(addr - dhp->dh_uvaddr);
2163 while (slen != 0) {
2164 mlen = MIN(slen, (dhp->dh_len - soff));
2165 hat_unload(seg->s_as->a_hat, dhp->dh_uvaddr,
2166 dhp->dh_len, HAT_UNLOAD);
2167 dhp = dhp->dh_next;
2168 ASSERT(slen >= mlen);
2169 slen -= mlen;
2170 soff = 0;
2171 }
2172 return (0);
2173 }
2174 }
2175
2176 if ((prot & ~PROT_USER) == PROT_NONE) {
2177 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD);
2178 } else {
2179 /*
2180 * RFE: the segment should keep track of all attributes
2181 * allowing us to remove the deprecated hat_chgprot
2182 * and use hat_chgattr.
2183 */
2184 hat_chgprot(seg->s_as->a_hat, addr, len, prot);
2185 }
2186
2187 return (0);
2188 }
2189
2190 static int
2191 segdev_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2192 {
2193 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2194 struct vpage *vp, *evp;
2195
2196 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_CHECKPROT,
2197 "segdev_checkprot:start seg=%p addr=%p len=%lx prot=%x",
2198 (void *)seg, (void *)addr, len, prot);
2199 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2200
2201 /*
2202 * If segment protection can be used, simply check against them
2203 */
2204 rw_enter(&sdp->lock, RW_READER);
2205 if (sdp->pageprot == 0) {
2206 register int err;
2207
2208 err = ((sdp->prot & prot) != prot) ? EACCES : 0;
2209 rw_exit(&sdp->lock);
2210 return (err);
2211 }
2212
2213 /*
2214 * Have to check down to the vpage level
2215 */
2216 evp = &sdp->vpage[seg_page(seg, addr + len)];
2217 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++) {
2218 if ((VPP_PROT(vp) & prot) != prot) {
2219 rw_exit(&sdp->lock);
2220 return (EACCES);
2221 }
2222 }
2223 rw_exit(&sdp->lock);
2224 return (0);
2225 }
2226
2227 static int
2228 segdev_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
2229 {
2230 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2231 size_t pgno;
2232
2233 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_GETPROT,
2234 "segdev_getprot:start seg=%p addr=%p len=%lx protv=%p",
2235 (void *)seg, (void *)addr, len, (void *)protv);
2236 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2237
2238 pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
2239 if (pgno != 0) {
2240 rw_enter(&sdp->lock, RW_READER);
2241 if (sdp->pageprot == 0) {
2242 do {
2243 protv[--pgno] = sdp->prot;
2244 } while (pgno != 0);
2245 } else {
2246 size_t pgoff = seg_page(seg, addr);
2247
2248 do {
2249 pgno--;
2250 protv[pgno] =
2251 VPP_PROT(&sdp->vpage[pgno + pgoff]);
2252 } while (pgno != 0);
2253 }
2254 rw_exit(&sdp->lock);
2255 }
2256 return (0);
2257 }
2258
2259 static u_offset_t
2260 segdev_getoffset(register struct seg *seg, caddr_t addr)
2261 {
2262 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2263
2264 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETOFFSET,
2265 "segdev_getoffset:start seg=%p addr=%p", (void *)seg, (void *)addr);
2266
2267 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2268
2269 return ((u_offset_t)sdp->offset + (addr - seg->s_base));
2270 }
2271
2272 /*ARGSUSED*/
2273 static int
2274 segdev_gettype(register struct seg *seg, caddr_t addr)
2275 {
2276 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2277
2278 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETTYPE,
2279 "segdev_gettype:start seg=%p addr=%p", (void *)seg, (void *)addr);
2280
2281 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2282
2283 return (sdp->type);
2284 }
2285
2286
2287 /*ARGSUSED*/
2288 static int
2289 segdev_getvp(register struct seg *seg, caddr_t addr, struct vnode **vpp)
2290 {
2291 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2292
2293 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETVP,
2294 "segdev_getvp:start seg=%p addr=%p", (void *)seg, (void *)addr);
2295
2296 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2297
2298 /*
2299 * Note that this vp is the common_vp of the device, where the
2300 * pages are hung ..
2301 */
2302 *vpp = VTOCVP(sdp->vp);
2303
2304 return (0);
2305 }
2306
2307 static void
2308 segdev_badop(void)
2309 {
2310 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGDEV_BADOP,
2311 "segdev_badop:start");
2312 panic("segdev_badop");
2313 /*NOTREACHED*/
2314 }
2315
2316 /*
2317 * segdev pages are not in the cache, and thus can't really be controlled.
2318 * Hence, syncs are simply always successful.
2319 */
2320 /*ARGSUSED*/
2321 static int
2322 segdev_sync(struct seg *seg, caddr_t addr, size_t len, int attr, uint_t flags)
2323 {
2324 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SYNC, "segdev_sync:start");
2325
2326 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2327
2328 return (0);
2329 }
2330
2331 /*
2332 * segdev pages are always "in core".
2333 */
2334 /*ARGSUSED*/
2335 static size_t
2336 segdev_incore(struct seg *seg, caddr_t addr, size_t len, char *vec)
2337 {
2338 size_t v = 0;
2339
2340 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_INCORE, "segdev_incore:start");
2341
2342 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2343
2344 for (len = (len + PAGEOFFSET) & PAGEMASK; len; len -= PAGESIZE,
2345 v += PAGESIZE)
2346 *vec++ = 1;
2347 return (v);
2348 }
2349
2350 /*
2351 * segdev pages are not in the cache, and thus can't really be controlled.
2352 * Hence, locks are simply always successful.
2353 */
2354 /*ARGSUSED*/
2355 static int
2356 segdev_lockop(struct seg *seg, caddr_t addr,
2357 size_t len, int attr, int op, ulong_t *lockmap, size_t pos)
2358 {
2359 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_LOCKOP, "segdev_lockop:start");
2360
2361 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2362
2363 return (0);
2364 }
2365
2366 /*
2367 * segdev pages are not in the cache, and thus can't really be controlled.
2368 * Hence, advise is simply always successful.
2369 */
2370 /*ARGSUSED*/
2371 static int
2372 segdev_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
2373 {
2374 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_ADVISE, "segdev_advise:start");
2375
2376 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2377
2378 return (0);
2379 }
2380
2381 /*
2382 * segdev pages are not dumped, so we just return
2383 */
2384 /*ARGSUSED*/
2385 static void
2386 segdev_dump(struct seg *seg)
2387 {}
2388
2389 /*
2390 * ddi_segmap_setup: Used by drivers who wish specify mapping attributes
2391 * for a segment. Called from a drivers segmap(9E)
2392 * routine.
2393 */
2394 /*ARGSUSED*/
2395 int
2396 ddi_segmap_setup(dev_t dev, off_t offset, struct as *as, caddr_t *addrp,
2397 off_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cred,
2398 ddi_device_acc_attr_t *accattrp, uint_t rnumber)
2399 {
2400 struct segdev_crargs dev_a;
2401 int (*mapfunc)(dev_t dev, off_t off, int prot);
2402 uint_t hat_attr;
2403 pfn_t pfn;
2404 int error, i;
2405
2406 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP_SETUP,
2407 "ddi_segmap_setup:start");
2408
2409 if ((mapfunc = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap) == nodev)
2410 return (ENODEV);
2411
2412 /*
2413 * Character devices that support the d_mmap
2414 * interface can only be mmap'ed shared.
2415 */
2416 if ((flags & MAP_TYPE) != MAP_SHARED)
2417 return (EINVAL);
2418
2419 /*
2420 * Check that this region is indeed mappable on this platform.
2421 * Use the mapping function.
2422 */
2423 if (ddi_device_mapping_check(dev, accattrp, rnumber, &hat_attr) == -1)
2424 return (ENXIO);
2425
2426 /*
2427 * Check to ensure that the entire range is
2428 * legal and we are not trying to map in
2429 * more than the device will let us.
2430 */
2431 for (i = 0; i < len; i += PAGESIZE) {
2432 if (i == 0) {
2433 /*
2434 * Save the pfn at offset here. This pfn will be
2435 * used later to get user address.
2436 */
2437 if ((pfn = (pfn_t)cdev_mmap(mapfunc, dev, offset,
2438 maxprot)) == PFN_INVALID)
2439 return (ENXIO);
2440 } else {
2441 if (cdev_mmap(mapfunc, dev, offset + i, maxprot) ==
2442 PFN_INVALID)
2443 return (ENXIO);
2444 }
2445 }
2446
2447 as_rangelock(as);
2448 /* Pick an address w/o worrying about any vac alignment constraints. */
2449 error = choose_addr(as, addrp, len, ptob(pfn), ADDR_NOVACALIGN, flags);
2450 if (error != 0) {
2451 as_rangeunlock(as);
2452 return (error);
2453 }
2454
2455 dev_a.mapfunc = mapfunc;
2456 dev_a.dev = dev;
2457 dev_a.offset = (offset_t)offset;
2458 dev_a.type = flags & MAP_TYPE;
2459 dev_a.prot = (uchar_t)prot;
2460 dev_a.maxprot = (uchar_t)maxprot;
2461 dev_a.hat_attr = hat_attr;
2462 dev_a.hat_flags = 0;
2463 dev_a.devmap_data = NULL;
2464
2465 error = as_map(as, *addrp, len, segdev_create, &dev_a);
2466 as_rangeunlock(as);
2467 return (error);
2468
2469 }
2470
2471 /*ARGSUSED*/
2472 static int
2473 segdev_pagelock(struct seg *seg, caddr_t addr, size_t len,
2474 struct page ***ppp, enum lock_type type, enum seg_rw rw)
2475 {
2476 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_PAGELOCK,
2477 "segdev_pagelock:start");
2478 return (ENOTSUP);
2479 }
2480
2481 /*ARGSUSED*/
2482 static int
2483 segdev_setpagesize(struct seg *seg, caddr_t addr, size_t len,
2484 uint_t szc)
2485 {
2486 return (ENOTSUP);
2487 }
2488
2489 /*
2490 * devmap_device: Used by devmap framework to establish mapping
2491 * called by devmap_seup(9F) during map setup time.
2492 */
2493 /*ARGSUSED*/
2494 static int
2495 devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
2496 offset_t off, size_t len, uint_t flags)
2497 {
2498 devmap_handle_t *rdhp, *maxdhp;
2499 struct segdev_crargs dev_a;
2500 int err;
2501 uint_t maxprot = PROT_ALL;
2502 offset_t offset = 0;
2503 pfn_t pfn;
2504 struct devmap_pmem_cookie *pcp;
2505
2506 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVICE,
2507 "devmap_device:start dhp=%p addr=%p off=%llx, len=%lx",
2508 (void *)dhp, (void *)addr, off, len);
2509
2510 DEBUGF(2, (CE_CONT, "devmap_device: dhp %p addr %p off %llx len %lx\n",
2511 (void *)dhp, (void *)addr, off, len));
2512
2513 as_rangelock(as);
2514 if ((flags & MAP_FIXED) == 0) {
2515 offset_t aligned_off;
2516
2517 rdhp = maxdhp = dhp;
2518 while (rdhp != NULL) {
2519 maxdhp = (maxdhp->dh_len > rdhp->dh_len) ?
2520 maxdhp : rdhp;
2521 rdhp = rdhp->dh_next;
2522 maxprot |= dhp->dh_maxprot;
2523 }
2524 offset = maxdhp->dh_uoff - dhp->dh_uoff;
2525
2526 /*
2527 * Use the dhp that has the
2528 * largest len to get user address.
2529 */
2530 /*
2531 * If MAPPING_INVALID, cannot use dh_pfn/dh_cvaddr,
2532 * use 0 which is as good as any other.
2533 */
2534 if (maxdhp->dh_flags & DEVMAP_MAPPING_INVALID) {
2535 aligned_off = (offset_t)0;
2536 } else if (dhp_is_devmem(maxdhp)) {
2537 aligned_off = (offset_t)ptob(maxdhp->dh_pfn) - offset;
2538 } else if (dhp_is_pmem(maxdhp)) {
2539 pcp = (struct devmap_pmem_cookie *)maxdhp->dh_pcookie;
2540 pfn = page_pptonum(
2541 pcp->dp_pparray[btop(maxdhp->dh_roff)]);
2542 aligned_off = (offset_t)ptob(pfn) - offset;
2543 } else {
2544 aligned_off = (offset_t)(uintptr_t)maxdhp->dh_cvaddr -
2545 offset;
2546 }
2547
2548 /*
2549 * Pick an address aligned to dh_cookie.
2550 * for kernel memory/user memory, cookie is cvaddr.
2551 * for device memory, cookie is physical address.
2552 */
2553 map_addr(addr, len, aligned_off, 1, flags);
2554 if (*addr == NULL) {
2555 as_rangeunlock(as);
2556 return (ENOMEM);
2557 }
2558 } else {
2559 /*
2560 * User-specified address; blow away any previous mappings.
2561 */
2562 (void) as_unmap(as, *addr, len);
2563 }
2564
2565 dev_a.mapfunc = NULL;
2566 dev_a.dev = dhp->dh_dev;
2567 dev_a.type = flags & MAP_TYPE;
2568 dev_a.offset = off;
2569 /*
2570 * sdp->maxprot has the least restrict protection of all dhps.
2571 */
2572 dev_a.maxprot = maxprot;
2573 dev_a.prot = dhp->dh_prot;
2574 /*
2575 * devmap uses dhp->dh_hat_attr for hat.
2576 */
2577 dev_a.hat_flags = 0;
2578 dev_a.hat_attr = 0;
2579 dev_a.devmap_data = (void *)dhp;
2580
2581 err = as_map(as, *addr, len, segdev_create, &dev_a);
2582 as_rangeunlock(as);
2583 return (err);
2584 }
2585
2586 int
2587 devmap_do_ctxmgt(devmap_cookie_t dhc, void *pvtp, offset_t off, size_t len,
2588 uint_t type, uint_t rw, int (*ctxmgt)(devmap_cookie_t, void *, offset_t,
2589 size_t, uint_t, uint_t))
2590 {
2591 register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2592 struct devmap_ctx *devctx;
2593 int do_timeout = 0;
2594 int ret;
2595
2596 #ifdef lint
2597 pvtp = pvtp;
2598 #endif
2599
2600 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT,
2601 "devmap_do_ctxmgt:start dhp=%p off=%llx, len=%lx",
2602 (void *)dhp, off, len);
2603 DEBUGF(7, (CE_CONT, "devmap_do_ctxmgt: dhp %p off %llx len %lx\n",
2604 (void *)dhp, off, len));
2605
2606 if (ctxmgt == NULL)
2607 return (FC_HWERR);
2608
2609 devctx = dhp->dh_ctx;
2610
2611 /*
2612 * If we are on an MP system with more than one cpu running
2613 * and if a thread on some CPU already has the context, wait
2614 * for it to finish if there is a hysteresis timeout.
2615 *
2616 * We call cv_wait() instead of cv_wait_sig() because
2617 * it does not matter much if it returned due to a signal
2618 * or due to a cv_signal() or cv_broadcast(). In either event
2619 * we need to complete the mapping otherwise the processes
2620 * will die with a SEGV.
2621 */
2622 if ((dhp->dh_timeout_length > 0) && (ncpus > 1)) {
2623 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK1,
2624 "devmap_do_ctxmgt:doing hysteresis, devctl %p dhp %p",
2625 devctx, dhp);
2626 do_timeout = 1;
2627 mutex_enter(&devctx->lock);
2628 while (devctx->oncpu)
2629 cv_wait(&devctx->cv, &devctx->lock);
2630 devctx->oncpu = 1;
2631 mutex_exit(&devctx->lock);
2632 }
2633
2634 /*
2635 * Call the contextmgt callback so that the driver can handle
2636 * the fault.
2637 */
2638 ret = (*ctxmgt)(dhp, dhp->dh_pvtp, off, len, type, rw);
2639
2640 /*
2641 * If devmap_access() returned -1, then there was a hardware
2642 * error so we need to convert the return value to something
2643 * that trap() will understand. Otherwise, the return value
2644 * is already a fault code generated by devmap_unload()
2645 * or devmap_load().
2646 */
2647 if (ret) {
2648 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK2,
2649 "devmap_do_ctxmgt: ret=%x dhp=%p devctx=%p",
2650 ret, dhp, devctx);
2651 DEBUGF(1, (CE_CONT, "devmap_do_ctxmgt: ret %x dhp %p\n",
2652 ret, (void *)dhp));
2653 if (devctx->oncpu) {
2654 mutex_enter(&devctx->lock);
2655 devctx->oncpu = 0;
2656 cv_signal(&devctx->cv);
2657 mutex_exit(&devctx->lock);
2658 }
2659 return (FC_HWERR);
2660 }
2661
2662 /*
2663 * Setup the timeout if we need to
2664 */
2665 if (do_timeout) {
2666 mutex_enter(&devctx->lock);
2667 if (dhp->dh_timeout_length > 0) {
2668 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK3,
2669 "devmap_do_ctxmgt:timeout set");
2670 devctx->timeout = timeout(devmap_ctxto,
2671 devctx, dhp->dh_timeout_length);
2672 } else {
2673 /*
2674 * We don't want to wait so set oncpu to
2675 * 0 and wake up anyone waiting.
2676 */
2677 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK4,
2678 "devmap_do_ctxmgt:timeout not set");
2679 devctx->oncpu = 0;
2680 cv_signal(&devctx->cv);
2681 }
2682 mutex_exit(&devctx->lock);
2683 }
2684
2685 return (DDI_SUCCESS);
2686 }
2687
2688 /*
2689 * end of mapping
2690 * poff fault_offset |
2691 * base | | |
2692 * | | | |
2693 * V V V V
2694 * +-----------+---------------+-------+---------+-------+
2695 * ^ ^ ^ ^
2696 * |<--- offset--->|<-len->| |
2697 * |<--- dh_len(size of mapping) --->|
2698 * |<-- pg -->|
2699 * -->|rlen|<--
2700 */
2701 static ulong_t
2702 devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
2703 ulong_t *opfn, ulong_t *pagesize)
2704 {
2705 register int level;
2706 ulong_t pg;
2707 ulong_t poff;
2708 ulong_t base;
2709 caddr_t uvaddr;
2710 long rlen;
2711
2712 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP,
2713 "devmap_roundup:start dhp=%p off=%lx len=%lx",
2714 (void *)dhp, offset, len);
2715 DEBUGF(2, (CE_CONT, "devmap_roundup: dhp %p off %lx len %lx\n",
2716 (void *)dhp, offset, len));
2717
2718 /*
2719 * get the max. pagesize that is aligned within the range
2720 * <dh_pfn, dh_pfn+offset>.
2721 *
2722 * The calculations below use physical address to ddetermine
2723 * the page size to use. The same calculations can use the
2724 * virtual address to determine the page size.
2725 */
2726 base = (ulong_t)ptob(dhp->dh_pfn);
2727 for (level = dhp->dh_mmulevel; level >= 0; level--) {
2728 pg = page_get_pagesize(level);
2729 poff = ((base + offset) & ~(pg - 1));
2730 uvaddr = dhp->dh_uvaddr + (poff - base);
2731 if ((poff >= base) &&
2732 ((poff + pg) <= (base + dhp->dh_len)) &&
2733 VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg))
2734 break;
2735 }
2736
2737 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK1,
2738 "devmap_roundup: base=%lx poff=%lx dhp=%p",
2739 base, poff, dhp);
2740 DEBUGF(2, (CE_CONT, "devmap_roundup: base %lx poff %lx pfn %lx\n",
2741 base, poff, dhp->dh_pfn));
2742
2743 ASSERT(VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg));
2744 ASSERT(level >= 0);
2745
2746 *pagesize = pg;
2747 *opfn = dhp->dh_pfn + btop(poff - base);
2748
2749 rlen = len + offset - (poff - base + pg);
2750
2751 ASSERT(rlen < (long)len);
2752
2753 TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK2,
2754 "devmap_roundup:ret dhp=%p level=%x rlen=%lx psiz=%p opfn=%p",
2755 (void *)dhp, level, rlen, pagesize, opfn);
2756 DEBUGF(1, (CE_CONT, "devmap_roundup: dhp %p "
2757 "level %x rlen %lx psize %lx opfn %lx\n",
2758 (void *)dhp, level, rlen, *pagesize, *opfn));
2759
2760 return ((ulong_t)((rlen > 0) ? rlen : 0));
2761 }
2762
2763 /*
2764 * find the dhp that contains addr.
2765 */
2766 static devmap_handle_t *
2767 devmap_find_handle(devmap_handle_t *dhp_head, caddr_t addr)
2768 {
2769 devmap_handle_t *dhp;
2770
2771 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_FIND_HANDLE,
2772 "devmap_find_handle:start");
2773
2774 dhp = dhp_head;
2775 while (dhp) {
2776 if (addr >= dhp->dh_uvaddr &&
2777 addr < (dhp->dh_uvaddr + dhp->dh_len))
2778 return (dhp);
2779 dhp = dhp->dh_next;
2780 }
2781
2782 return ((devmap_handle_t *)NULL);
2783 }
2784
2785 /*
2786 * devmap_unload:
2787 * Marks a segdev segment or pages if offset->offset+len
2788 * is not the entire segment as intercept and unloads the
2789 * pages in the range offset -> offset+len.
2790 */
2791 int
2792 devmap_unload(devmap_cookie_t dhc, offset_t offset, size_t len)
2793 {
2794 register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2795 caddr_t addr;
2796 ulong_t size;
2797 ssize_t soff;
2798
2799 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_UNLOAD,
2800 "devmap_unload:start dhp=%p offset=%llx len=%lx",
2801 (void *)dhp, offset, len);
2802 DEBUGF(7, (CE_CONT, "devmap_unload: dhp %p offset %llx len %lx\n",
2803 (void *)dhp, offset, len));
2804
2805 soff = (ssize_t)(offset - dhp->dh_uoff);
2806 soff = round_down_p2(soff, PAGESIZE);
2807 if (soff < 0 || soff >= dhp->dh_len)
2808 return (FC_MAKE_ERR(EINVAL));
2809
2810 /*
2811 * Address and size must be page aligned. Len is set to the
2812 * number of bytes in the number of pages that are required to
2813 * support len. Offset is set to the byte offset of the first byte
2814 * of the page that contains offset.
2815 */
2816 len = round_up_p2(len, PAGESIZE);
2817
2818 /*
2819 * If len is == 0, then calculate the size by getting
2820 * the number of bytes from offset to the end of the segment.
2821 */
2822 if (len == 0)
2823 size = dhp->dh_len - soff;
2824 else {
2825 size = len;
2826 if ((soff + size) > dhp->dh_len)
2827 return (FC_MAKE_ERR(EINVAL));
2828 }
2829
2830 /*
2831 * The address is offset bytes from the base address of
2832 * the dhp.
2833 */
2834 addr = (caddr_t)(soff + dhp->dh_uvaddr);
2835
2836 /*
2837 * If large page size was used in hat_devload(),
2838 * the same page size must be used in hat_unload().
2839 */
2840 if (dhp->dh_flags & DEVMAP_FLAG_LARGE) {
2841 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
2842 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
2843 } else {
2844 hat_unload(dhp->dh_seg->s_as->a_hat, addr, size,
2845 HAT_UNLOAD|HAT_UNLOAD_OTHER);
2846 }
2847
2848 return (0);
2849 }
2850
2851 /*
2852 * calculates the optimal page size that will be used for hat_devload().
2853 */
2854 static void
2855 devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr,
2856 size_t *llen, caddr_t *laddr)
2857 {
2858 ulong_t off;
2859 ulong_t pfn;
2860 ulong_t pgsize;
2861 uint_t first = 1;
2862
2863 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GET_LARGE_PGSIZE,
2864 "devmap_get_large_pgsize:start");
2865
2866 /*
2867 * RFE - Code only supports large page mappings for devmem
2868 * This code could be changed in future if we want to support
2869 * large page mappings for kernel exported memory.
2870 */
2871 ASSERT(dhp_is_devmem(dhp));
2872 ASSERT(!(dhp->dh_flags & DEVMAP_MAPPING_INVALID));
2873
2874 *llen = 0;
2875 off = (ulong_t)(addr - dhp->dh_uvaddr);
2876 while ((long)len > 0) {
2877 /*
2878 * get the optimal pfn to minimize address translations.
2879 * devmap_roundup() returns residue bytes for next round
2880 * calculations.
2881 */
2882 len = devmap_roundup(dhp, off, len, &pfn, &pgsize);
2883
2884 if (first) {
2885 *laddr = dhp->dh_uvaddr + ptob(pfn - dhp->dh_pfn);
2886 first = 0;
2887 }
2888
2889 *llen += pgsize;
2890 off = ptob(pfn - dhp->dh_pfn) + pgsize;
2891 }
2892 /* Large page mapping len/addr cover more range than original fault */
2893 ASSERT(*llen >= len && *laddr <= addr);
2894 ASSERT((*laddr + *llen) >= (addr + len));
2895 }
2896
2897 /*
2898 * Initialize the devmap_softlock structure.
2899 */
2900 static struct devmap_softlock *
2901 devmap_softlock_init(dev_t dev, ulong_t id)
2902 {
2903 struct devmap_softlock *slock;
2904 struct devmap_softlock *tmp;
2905
2906 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_INIT,
2907 "devmap_softlock_init:start");
2908
2909 tmp = kmem_zalloc(sizeof (struct devmap_softlock), KM_SLEEP);
2910 mutex_enter(&devmap_slock);
2911
2912 for (slock = devmap_slist; slock != NULL; slock = slock->next)
2913 if ((slock->dev == dev) && (slock->id == id))
2914 break;
2915
2916 if (slock == NULL) {
2917 slock = tmp;
2918 slock->dev = dev;
2919 slock->id = id;
2920 mutex_init(&slock->lock, NULL, MUTEX_DEFAULT, NULL);
2921 cv_init(&slock->cv, NULL, CV_DEFAULT, NULL);
2922 slock->next = devmap_slist;
2923 devmap_slist = slock;
2924 } else
2925 kmem_free(tmp, sizeof (struct devmap_softlock));
2926
2927 mutex_enter(&slock->lock);
2928 slock->refcnt++;
2929 mutex_exit(&slock->lock);
2930 mutex_exit(&devmap_slock);
2931
2932 return (slock);
2933 }
2934
2935 /*
2936 * Wake up processes that sleep on softlocked.
2937 * Free dh_softlock if refcnt is 0.
2938 */
2939 static void
2940 devmap_softlock_rele(devmap_handle_t *dhp)
2941 {
2942 struct devmap_softlock *slock = dhp->dh_softlock;
2943 struct devmap_softlock *tmp;
2944 struct devmap_softlock *parent;
2945
2946 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_RELE,
2947 "devmap_softlock_rele:start");
2948
2949 mutex_enter(&devmap_slock);
2950 mutex_enter(&slock->lock);
2951
2952 ASSERT(slock->refcnt > 0);
2953
2954 slock->refcnt--;
2955
2956 /*
2957 * If no one is using the device, free up the slock data.
2958 */
2959 if (slock->refcnt == 0) {
2960 slock->softlocked = 0;
2961 cv_signal(&slock->cv);
2962
2963 if (devmap_slist == slock)
2964 devmap_slist = slock->next;
2965 else {
2966 parent = devmap_slist;
2967 for (tmp = devmap_slist->next; tmp != NULL;
2968 tmp = tmp->next) {
2969 if (tmp == slock) {
2970 parent->next = tmp->next;
2971 break;
2972 }
2973 parent = tmp;
2974 }
2975 }
2976 mutex_exit(&slock->lock);
2977 mutex_destroy(&slock->lock);
2978 cv_destroy(&slock->cv);
2979 kmem_free(slock, sizeof (struct devmap_softlock));
2980 } else
2981 mutex_exit(&slock->lock);
2982
2983 mutex_exit(&devmap_slock);
2984 }
2985
2986 /*
2987 * Wake up processes that sleep on dh_ctx->locked.
2988 * Free dh_ctx if refcnt is 0.
2989 */
2990 static void
2991 devmap_ctx_rele(devmap_handle_t *dhp)
2992 {
2993 struct devmap_ctx *devctx = dhp->dh_ctx;
2994 struct devmap_ctx *tmp;
2995 struct devmap_ctx *parent;
2996 timeout_id_t tid;
2997
2998 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE,
2999 "devmap_ctx_rele:start");
3000
3001 mutex_enter(&devmapctx_lock);
3002 mutex_enter(&devctx->lock);
3003
3004 ASSERT(devctx->refcnt > 0);
3005
3006 devctx->refcnt--;
3007
3008 /*
3009 * If no one is using the device, free up the devctx data.
3010 */
3011 if (devctx->refcnt == 0) {
3012 /*
3013 * Untimeout any threads using this mapping as they are about
3014 * to go away.
3015 */
3016 if (devctx->timeout != 0) {
3017 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE_CK1,
3018 "devmap_ctx_rele:untimeout ctx->timeout");
3019
3020 tid = devctx->timeout;
3021 mutex_exit(&devctx->lock);
3022 (void) untimeout(tid);
3023 mutex_enter(&devctx->lock);
3024 }
3025
3026 devctx->oncpu = 0;
3027 cv_signal(&devctx->cv);
3028
3029 if (devmapctx_list == devctx)
3030 devmapctx_list = devctx->next;
3031 else {
3032 parent = devmapctx_list;
3033 for (tmp = devmapctx_list->next; tmp != NULL;
3034 tmp = tmp->next) {
3035 if (tmp == devctx) {
3036 parent->next = tmp->next;
3037 break;
3038 }
3039 parent = tmp;
3040 }
3041 }
3042 mutex_exit(&devctx->lock);
3043 mutex_destroy(&devctx->lock);
3044 cv_destroy(&devctx->cv);
3045 kmem_free(devctx, sizeof (struct devmap_ctx));
3046 } else
3047 mutex_exit(&devctx->lock);
3048
3049 mutex_exit(&devmapctx_lock);
3050 }
3051
3052 /*
3053 * devmap_load:
3054 * Marks a segdev segment or pages if offset->offset+len
3055 * is not the entire segment as nointercept and faults in
3056 * the pages in the range offset -> offset+len.
3057 */
3058 int
3059 devmap_load(devmap_cookie_t dhc, offset_t offset, size_t len, uint_t type,
3060 uint_t rw)
3061 {
3062 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3063 struct as *asp = dhp->dh_seg->s_as;
3064 caddr_t addr;
3065 ulong_t size;
3066 ssize_t soff; /* offset from the beginning of the segment */
3067 int rc;
3068
3069 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_LOAD,
3070 "devmap_load:start dhp=%p offset=%llx len=%lx",
3071 (void *)dhp, offset, len);
3072
3073 DEBUGF(7, (CE_CONT, "devmap_load: dhp %p offset %llx len %lx\n",
3074 (void *)dhp, offset, len));
3075
3076 /*
3077 * Hat layer only supports devload to process' context for which
3078 * the as lock is held. Verify here and return error if drivers
3079 * inadvertently call devmap_load on a wrong devmap handle.
3080 */
3081 if ((asp != &kas) && !AS_LOCK_HELD(asp, &asp->a_lock))
3082 return (FC_MAKE_ERR(EINVAL));
3083
3084 soff = (ssize_t)(offset - dhp->dh_uoff);
3085 soff = round_down_p2(soff, PAGESIZE);
3086 if (soff < 0 || soff >= dhp->dh_len)
3087 return (FC_MAKE_ERR(EINVAL));
3088
3089 /*
3090 * Address and size must be page aligned. Len is set to the
3091 * number of bytes in the number of pages that are required to
3092 * support len. Offset is set to the byte offset of the first byte
3093 * of the page that contains offset.
3094 */
3095 len = round_up_p2(len, PAGESIZE);
3096
3097 /*
3098 * If len == 0, then calculate the size by getting
3099 * the number of bytes from offset to the end of the segment.
3100 */
3101 if (len == 0)
3102 size = dhp->dh_len - soff;
3103 else {
3104 size = len;
3105 if ((soff + size) > dhp->dh_len)
3106 return (FC_MAKE_ERR(EINVAL));
3107 }
3108
3109 /*
3110 * The address is offset bytes from the base address of
3111 * the segment.
3112 */
3113 addr = (caddr_t)(soff + dhp->dh_uvaddr);
3114
3115 HOLD_DHP_LOCK(dhp);
3116 rc = segdev_faultpages(asp->a_hat,
3117 dhp->dh_seg, addr, size, type, rw, dhp);
3118 RELE_DHP_LOCK(dhp);
3119 return (rc);
3120 }
3121
3122 int
3123 devmap_setup(dev_t dev, offset_t off, struct as *as, caddr_t *addrp,
3124 size_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3125 {
3126 register devmap_handle_t *dhp;
3127 int (*devmap)(dev_t, devmap_cookie_t, offset_t, size_t,
3128 size_t *, uint_t);
3129 int (*mmap)(dev_t, off_t, int);
3130 struct devmap_callback_ctl *callbackops;
3131 devmap_handle_t *dhp_head = NULL;
3132 devmap_handle_t *dhp_prev = NULL;
3133 devmap_handle_t *dhp_curr;
3134 caddr_t addr;
3135 int map_flag;
3136 int ret;
3137 ulong_t total_len;
3138 size_t map_len;
3139 size_t resid_len = len;
3140 offset_t map_off = off;
3141 struct devmap_softlock *slock = NULL;
3142
3143 #ifdef lint
3144 cred = cred;
3145 #endif
3146
3147 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SETUP,
3148 "devmap_setup:start off=%llx len=%lx", off, len);
3149 DEBUGF(3, (CE_CONT, "devmap_setup: off %llx len %lx\n",
3150 off, len));
3151
3152 devmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_devmap;
3153 mmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap;
3154
3155 /*
3156 * driver must provide devmap(9E) entry point in cb_ops to use the
3157 * devmap framework.
3158 */
3159 if (devmap == NULL || devmap == nulldev || devmap == nodev)
3160 return (EINVAL);
3161
3162 /*
3163 * To protect from an inadvertent entry because the devmap entry point
3164 * is not NULL, return error if D_DEVMAP bit is not set in cb_flag and
3165 * mmap is NULL.
3166 */
3167 map_flag = devopsp[getmajor(dev)]->devo_cb_ops->cb_flag;
3168 if ((map_flag & D_DEVMAP) == 0 && (mmap == NULL || mmap == nulldev))
3169 return (EINVAL);
3170
3171 /*
3172 * devmap allows mmap(2) to map multiple registers.
3173 * one devmap_handle is created for each register mapped.
3174 */
3175 for (total_len = 0; total_len < len; total_len += map_len) {
3176 dhp = kmem_zalloc(sizeof (devmap_handle_t), KM_SLEEP);
3177
3178 if (dhp_prev != NULL)
3179 dhp_prev->dh_next = dhp;
3180 else
3181 dhp_head = dhp;
3182 dhp_prev = dhp;
3183
3184 dhp->dh_prot = prot;
3185 dhp->dh_orig_maxprot = dhp->dh_maxprot = maxprot;
3186 dhp->dh_dev = dev;
3187 dhp->dh_timeout_length = CTX_TIMEOUT_VALUE;
3188 dhp->dh_uoff = map_off;
3189
3190 /*
3191 * Get mapping specific info from
3192 * the driver, such as rnumber, roff, len, callbackops,
3193 * accattrp and, if the mapping is for kernel memory,
3194 * ddi_umem_cookie.
3195 */
3196 if ((ret = cdev_devmap(dev, dhp, map_off,
3197 resid_len, &map_len, get_udatamodel())) != 0) {
3198 free_devmap_handle(dhp_head);
3199 return (ENXIO);
3200 }
3201
3202 if (map_len & PAGEOFFSET) {
3203 free_devmap_handle(dhp_head);
3204 return (EINVAL);
3205 }
3206
3207 callbackops = &dhp->dh_callbackops;
3208
3209 if ((callbackops->devmap_access == NULL) ||
3210 (callbackops->devmap_access == nulldev) ||
3211 (callbackops->devmap_access == nodev)) {
3212 /*
3213 * Normally devmap does not support MAP_PRIVATE unless
3214 * the drivers provide a valid devmap_access routine.
3215 */
3216 if ((flags & MAP_PRIVATE) != 0) {
3217 free_devmap_handle(dhp_head);
3218 return (EINVAL);
3219 }
3220 } else {
3221 /*
3222 * Initialize dhp_softlock and dh_ctx if the drivers
3223 * provide devmap_access.
3224 */
3225 dhp->dh_softlock = devmap_softlock_init(dev,
3226 (ulong_t)callbackops->devmap_access);
3227 dhp->dh_ctx = devmap_ctxinit(dev,
3228 (ulong_t)callbackops->devmap_access);
3229
3230 /*
3231 * segdev_fault can only work when all
3232 * dh_softlock in a multi-dhp mapping
3233 * are same. see comments in segdev_fault
3234 * This code keeps track of the first
3235 * dh_softlock allocated in slock and
3236 * compares all later allocations and if
3237 * not similar, returns an error.
3238 */
3239 if (slock == NULL)
3240 slock = dhp->dh_softlock;
3241 if (slock != dhp->dh_softlock) {
3242 free_devmap_handle(dhp_head);
3243 return (ENOTSUP);
3244 }
3245 }
3246
3247 map_off += map_len;
3248 resid_len -= map_len;
3249 }
3250
3251 /*
3252 * get the user virtual address and establish the mapping between
3253 * uvaddr and device physical address.
3254 */
3255 if ((ret = devmap_device(dhp_head, as, addrp, off, len, flags))
3256 != 0) {
3257 /*
3258 * free devmap handles if error during the mapping.
3259 */
3260 free_devmap_handle(dhp_head);
3261
3262 return (ret);
3263 }
3264
3265 /*
3266 * call the driver's devmap_map callback to do more after the mapping,
3267 * such as to allocate driver private data for context management.
3268 */
3269 dhp = dhp_head;
3270 map_off = off;
3271 addr = *addrp;
3272 while (dhp != NULL) {
3273 callbackops = &dhp->dh_callbackops;
3274 dhp->dh_uvaddr = addr;
3275 dhp_curr = dhp;
3276 if (callbackops->devmap_map != NULL) {
3277 ret = (*callbackops->devmap_map)((devmap_cookie_t)dhp,
3278 dev, flags, map_off,
3279 dhp->dh_len, &dhp->dh_pvtp);
3280 if (ret != 0) {
3281 struct segdev_data *sdp;
3282
3283 /*
3284 * call driver's devmap_unmap entry point
3285 * to free driver resources.
3286 */
3287 dhp = dhp_head;
3288 map_off = off;
3289 while (dhp != dhp_curr) {
3290 callbackops = &dhp->dh_callbackops;
3291 if (callbackops->devmap_unmap != NULL) {
3292 (*callbackops->devmap_unmap)(
3293 dhp, dhp->dh_pvtp,
3294 map_off, dhp->dh_len,
3295 NULL, NULL, NULL, NULL);
3296 }
3297 map_off += dhp->dh_len;
3298 dhp = dhp->dh_next;
3299 }
3300 sdp = dhp_head->dh_seg->s_data;
3301 sdp->devmap_data = NULL;
3302 free_devmap_handle(dhp_head);
3303 return (ENXIO);
3304 }
3305 }
3306 map_off += dhp->dh_len;
3307 addr += dhp->dh_len;
3308 dhp = dhp->dh_next;
3309 }
3310
3311 return (0);
3312 }
3313
3314 int
3315 ddi_devmap_segmap(dev_t dev, off_t off, ddi_as_handle_t as, caddr_t *addrp,
3316 off_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3317 {
3318 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP,
3319 "devmap_segmap:start");
3320 return (devmap_setup(dev, (offset_t)off, (struct as *)as, addrp,
3321 (size_t)len, prot, maxprot, flags, cred));
3322 }
3323
3324 /*
3325 * Called from devmap_devmem_setup/remap to see if can use large pages for
3326 * this device mapping.
3327 * Also calculate the max. page size for this mapping.
3328 * this page size will be used in fault routine for
3329 * optimal page size calculations.
3330 */
3331 static void
3332 devmap_devmem_large_page_setup(devmap_handle_t *dhp)
3333 {
3334 ASSERT(dhp_is_devmem(dhp));
3335 dhp->dh_mmulevel = 0;
3336
3337 /*
3338 * use large page size only if:
3339 * 1. device memory.
3340 * 2. mmu supports multiple page sizes,
3341 * 3. Driver did not disallow it
3342 * 4. dhp length is at least as big as the large pagesize
3343 * 5. the uvaddr and pfn are large pagesize aligned
3344 */
3345 if (page_num_pagesizes() > 1 &&
3346 !(dhp->dh_flags & (DEVMAP_USE_PAGESIZE | DEVMAP_MAPPING_INVALID))) {
3347 ulong_t base;
3348 int level;
3349
3350 base = (ulong_t)ptob(dhp->dh_pfn);
3351 for (level = 1; level < page_num_pagesizes(); level++) {
3352 size_t pgsize = page_get_pagesize(level);
3353 if ((dhp->dh_len < pgsize) ||
3354 (!VA_PA_PGSIZE_ALIGNED((uintptr_t)dhp->dh_uvaddr,
3355 base, pgsize))) {
3356 break;
3357 }
3358 }
3359 dhp->dh_mmulevel = level - 1;
3360 }
3361 if (dhp->dh_mmulevel > 0) {
3362 dhp->dh_flags |= DEVMAP_FLAG_LARGE;
3363 } else {
3364 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3365 }
3366 }
3367
3368 /*
3369 * Called by driver devmap routine to pass device specific info to
3370 * the framework. used for device memory mapping only.
3371 */
3372 int
3373 devmap_devmem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3374 struct devmap_callback_ctl *callbackops, uint_t rnumber, offset_t roff,
3375 size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp)
3376 {
3377 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3378 ddi_acc_handle_t handle;
3379 ddi_map_req_t mr;
3380 ddi_acc_hdl_t *hp;
3381 int err;
3382
3383 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_SETUP,
3384 "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3385 (void *)dhp, roff, rnumber, (uint_t)len);
3386 DEBUGF(2, (CE_CONT, "devmap_devmem_setup: dhp %p offset %llx "
3387 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3388
3389 /*
3390 * First to check if this function has been called for this dhp.
3391 */
3392 if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3393 return (DDI_FAILURE);
3394
3395 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3396 return (DDI_FAILURE);
3397
3398 if (flags & DEVMAP_MAPPING_INVALID) {
3399 /*
3400 * Don't go up the tree to get pfn if the driver specifies
3401 * DEVMAP_MAPPING_INVALID in flags.
3402 *
3403 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3404 * remap permission.
3405 */
3406 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3407 return (DDI_FAILURE);
3408 }
3409 dhp->dh_pfn = PFN_INVALID;
3410 } else {
3411 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3412 if (handle == NULL)
3413 return (DDI_FAILURE);
3414
3415 hp = impl_acc_hdl_get(handle);
3416 hp->ah_vers = VERS_ACCHDL;
3417 hp->ah_dip = dip;
3418 hp->ah_rnumber = rnumber;
3419 hp->ah_offset = roff;
3420 hp->ah_len = len;
3421 if (accattrp != NULL)
3422 hp->ah_acc = *accattrp;
3423
3424 mr.map_op = DDI_MO_MAP_LOCKED;
3425 mr.map_type = DDI_MT_RNUMBER;
3426 mr.map_obj.rnumber = rnumber;
3427 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3428 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3429 mr.map_handlep = hp;
3430 mr.map_vers = DDI_MAP_VERSION;
3431
3432 /*
3433 * up the device tree to get pfn.
3434 * The rootnex_map_regspec() routine in nexus drivers has been
3435 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3436 */
3437 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&dhp->dh_pfn);
3438 dhp->dh_hat_attr = hp->ah_hat_flags;
3439 impl_acc_hdl_free(handle);
3440
3441 if (err)
3442 return (DDI_FAILURE);
3443 }
3444 /* Should not be using devmem setup for memory pages */
3445 ASSERT(!pf_is_memory(dhp->dh_pfn));
3446
3447 /* Only some of the flags bits are settable by the driver */
3448 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3449 dhp->dh_len = ptob(btopr(len));
3450
3451 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3452 dhp->dh_roff = ptob(btop(roff));
3453
3454 /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3455 devmap_devmem_large_page_setup(dhp);
3456 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3457 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3458
3459
3460 if (callbackops != NULL) {
3461 bcopy(callbackops, &dhp->dh_callbackops,
3462 sizeof (struct devmap_callback_ctl));
3463 }
3464
3465 /*
3466 * Initialize dh_lock if we want to do remap.
3467 */
3468 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3469 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3470 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3471 }
3472
3473 dhp->dh_flags |= DEVMAP_SETUP_DONE;
3474
3475 return (DDI_SUCCESS);
3476 }
3477
3478 int
3479 devmap_devmem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3480 uint_t rnumber, offset_t roff, size_t len, uint_t maxprot,
3481 uint_t flags, ddi_device_acc_attr_t *accattrp)
3482 {
3483 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3484 ddi_acc_handle_t handle;
3485 ddi_map_req_t mr;
3486 ddi_acc_hdl_t *hp;
3487 pfn_t pfn;
3488 uint_t hat_flags;
3489 int err;
3490
3491 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_REMAP,
3492 "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3493 (void *)dhp, roff, rnumber, (uint_t)len);
3494 DEBUGF(2, (CE_CONT, "devmap_devmem_remap: dhp %p offset %llx "
3495 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3496
3497 /*
3498 * Return failure if setup has not been done or no remap permission
3499 * has been granted during the setup.
3500 */
3501 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3502 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3503 return (DDI_FAILURE);
3504
3505 /* Only DEVMAP_MAPPING_INVALID flag supported for remap */
3506 if ((flags != 0) && (flags != DEVMAP_MAPPING_INVALID))
3507 return (DDI_FAILURE);
3508
3509 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3510 return (DDI_FAILURE);
3511
3512 if (!(flags & DEVMAP_MAPPING_INVALID)) {
3513 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3514 if (handle == NULL)
3515 return (DDI_FAILURE);
3516 }
3517
3518 HOLD_DHP_LOCK(dhp);
3519
3520 /*
3521 * Unload the old mapping, so next fault will setup the new mappings
3522 * Do this while holding the dhp lock so other faults dont reestablish
3523 * the mappings
3524 */
3525 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3526 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3527
3528 if (flags & DEVMAP_MAPPING_INVALID) {
3529 dhp->dh_flags |= DEVMAP_MAPPING_INVALID;
3530 dhp->dh_pfn = PFN_INVALID;
3531 } else {
3532 /* clear any prior DEVMAP_MAPPING_INVALID flag */
3533 dhp->dh_flags &= ~DEVMAP_MAPPING_INVALID;
3534 hp = impl_acc_hdl_get(handle);
3535 hp->ah_vers = VERS_ACCHDL;
3536 hp->ah_dip = dip;
3537 hp->ah_rnumber = rnumber;
3538 hp->ah_offset = roff;
3539 hp->ah_len = len;
3540 if (accattrp != NULL)
3541 hp->ah_acc = *accattrp;
3542
3543 mr.map_op = DDI_MO_MAP_LOCKED;
3544 mr.map_type = DDI_MT_RNUMBER;
3545 mr.map_obj.rnumber = rnumber;
3546 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3547 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3548 mr.map_handlep = hp;
3549 mr.map_vers = DDI_MAP_VERSION;
3550
3551 /*
3552 * up the device tree to get pfn.
3553 * The rootnex_map_regspec() routine in nexus drivers has been
3554 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3555 */
3556 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&pfn);
3557 hat_flags = hp->ah_hat_flags;
3558 impl_acc_hdl_free(handle);
3559 if (err) {
3560 RELE_DHP_LOCK(dhp);
3561 return (DDI_FAILURE);
3562 }
3563 /*
3564 * Store result of ddi_map first in local variables, as we do
3565 * not want to overwrite the existing dhp with wrong data.
3566 */
3567 dhp->dh_pfn = pfn;
3568 dhp->dh_hat_attr = hat_flags;
3569 }
3570
3571 /* clear the large page size flag */
3572 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3573
3574 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3575 dhp->dh_roff = ptob(btop(roff));
3576
3577 /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3578 devmap_devmem_large_page_setup(dhp);
3579 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3580 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3581
3582 RELE_DHP_LOCK(dhp);
3583 return (DDI_SUCCESS);
3584 }
3585
3586 /*
3587 * called by driver devmap routine to pass kernel virtual address mapping
3588 * info to the framework. used only for kernel memory
3589 * allocated from ddi_umem_alloc().
3590 */
3591 int
3592 devmap_umem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3593 struct devmap_callback_ctl *callbackops, ddi_umem_cookie_t cookie,
3594 offset_t off, size_t len, uint_t maxprot, uint_t flags,
3595 ddi_device_acc_attr_t *accattrp)
3596 {
3597 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3598 struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3599
3600 #ifdef lint
3601 dip = dip;
3602 #endif
3603
3604 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_SETUP,
3605 "devmap_umem_setup:start dhp=%p offset=%llx cookie=%p len=%lx",
3606 (void *)dhp, off, cookie, len);
3607 DEBUGF(2, (CE_CONT, "devmap_umem_setup: dhp %p offset %llx "
3608 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3609
3610 if (cookie == NULL)
3611 return (DDI_FAILURE);
3612
3613 /* For UMEM_TRASH, this restriction is not needed */
3614 if ((off + len) > cp->size)
3615 return (DDI_FAILURE);
3616
3617 /* check if the cache attributes are supported */
3618 if (i_ddi_check_cache_attr(flags) == B_FALSE)
3619 return (DDI_FAILURE);
3620
3621 /*
3622 * First to check if this function has been called for this dhp.
3623 */
3624 if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3625 return (DDI_FAILURE);
3626
3627 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3628 return (DDI_FAILURE);
3629
3630 if (flags & DEVMAP_MAPPING_INVALID) {
3631 /*
3632 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3633 * remap permission.
3634 */
3635 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3636 return (DDI_FAILURE);
3637 }
3638 } else {
3639 dhp->dh_cookie = cookie;
3640 dhp->dh_roff = ptob(btop(off));
3641 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3642 /* set HAT cache attributes */
3643 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3644 /* set HAT endianess attributes */
3645 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3646 }
3647
3648 /*
3649 * The default is _not_ to pass HAT_LOAD_NOCONSIST to hat_devload();
3650 * we pass HAT_LOAD_NOCONSIST _only_ in cases where hat tries to
3651 * create consistent mappings but our intention was to create
3652 * non-consistent mappings.
3653 *
3654 * DEVMEM: hat figures it out it's DEVMEM and creates non-consistent
3655 * mappings.
3656 *
3657 * kernel exported memory: hat figures it out it's memory and always
3658 * creates consistent mappings.
3659 *
3660 * /dev/mem: non-consistent mappings. See comments in common/io/mem.c
3661 *
3662 * /dev/kmem: consistent mappings are created unless they are
3663 * MAP_FIXED. We _explicitly_ tell hat to create non-consistent
3664 * mappings by passing HAT_LOAD_NOCONSIST in case of MAP_FIXED
3665 * mappings of /dev/kmem. See common/io/mem.c
3666 */
3667
3668 /* Only some of the flags bits are settable by the driver */
3669 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3670
3671 dhp->dh_len = ptob(btopr(len));
3672 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3673 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3674
3675 if (callbackops != NULL) {
3676 bcopy(callbackops, &dhp->dh_callbackops,
3677 sizeof (struct devmap_callback_ctl));
3678 }
3679 /*
3680 * Initialize dh_lock if we want to do remap.
3681 */
3682 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3683 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3684 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3685 }
3686
3687 dhp->dh_flags |= DEVMAP_SETUP_DONE;
3688
3689 return (DDI_SUCCESS);
3690 }
3691
3692 int
3693 devmap_umem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3694 ddi_umem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot,
3695 uint_t flags, ddi_device_acc_attr_t *accattrp)
3696 {
3697 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3698 struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3699
3700 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_REMAP,
3701 "devmap_umem_remap:start dhp=%p offset=%llx cookie=%p len=%lx",
3702 (void *)dhp, off, cookie, len);
3703 DEBUGF(2, (CE_CONT, "devmap_umem_remap: dhp %p offset %llx "
3704 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3705
3706 #ifdef lint
3707 dip = dip;
3708 accattrp = accattrp;
3709 #endif
3710 /*
3711 * Reture failure if setup has not been done or no remap permission
3712 * has been granted during the setup.
3713 */
3714 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3715 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3716 return (DDI_FAILURE);
3717
3718 /* No flags supported for remap yet */
3719 if (flags != 0)
3720 return (DDI_FAILURE);
3721
3722 /* check if the cache attributes are supported */
3723 if (i_ddi_check_cache_attr(flags) == B_FALSE)
3724 return (DDI_FAILURE);
3725
3726 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3727 return (DDI_FAILURE);
3728
3729 /* For UMEM_TRASH, this restriction is not needed */
3730 if ((off + len) > cp->size)
3731 return (DDI_FAILURE);
3732
3733 HOLD_DHP_LOCK(dhp);
3734 /*
3735 * Unload the old mapping, so next fault will setup the new mappings
3736 * Do this while holding the dhp lock so other faults dont reestablish
3737 * the mappings
3738 */
3739 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3740 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3741
3742 dhp->dh_cookie = cookie;
3743 dhp->dh_roff = ptob(btop(off));
3744 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3745 /* set HAT cache attributes */
3746 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3747 /* set HAT endianess attributes */
3748 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3749
3750 /* clear the large page size flag */
3751 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3752
3753 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3754 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3755 RELE_DHP_LOCK(dhp);
3756 return (DDI_SUCCESS);
3757 }
3758
3759 /*
3760 * to set timeout value for the driver's context management callback, e.g.
3761 * devmap_access().
3762 */
3763 void
3764 devmap_set_ctx_timeout(devmap_cookie_t dhc, clock_t ticks)
3765 {
3766 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3767
3768 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SET_CTX_TIMEOUT,
3769 "devmap_set_ctx_timeout:start dhp=%p ticks=%x",
3770 (void *)dhp, ticks);
3771 dhp->dh_timeout_length = ticks;
3772 }
3773
3774 int
3775 devmap_default_access(devmap_cookie_t dhp, void *pvtp, offset_t off,
3776 size_t len, uint_t type, uint_t rw)
3777 {
3778 #ifdef lint
3779 pvtp = pvtp;
3780 #endif
3781
3782 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DEFAULT_ACCESS,
3783 "devmap_default_access:start");
3784 return (devmap_load(dhp, off, len, type, rw));
3785 }
3786
3787 /*
3788 * segkmem_alloc() wrapper to allocate memory which is both
3789 * non-relocatable (for DR) and sharelocked, since the rest
3790 * of this segment driver requires it.
3791 */
3792 static void *
3793 devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag)
3794 {
3795 ASSERT(vmp != NULL);
3796 ASSERT(kvseg.s_base != NULL);
3797 vmflag |= (VM_NORELOC | SEGKMEM_SHARELOCKED);
3798 return (segkmem_alloc(vmp, size, vmflag));
3799 }
3800
3801 /*
3802 * This is where things are a bit incestuous with seg_kmem: unlike
3803 * seg_kp, seg_kmem does not keep its pages long-term sharelocked, so
3804 * we need to do a bit of a dance around that to prevent duplication of
3805 * code until we decide to bite the bullet and implement a new kernel
3806 * segment for driver-allocated memory that is exported to user space.
3807 */
3808 static void
3809 devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size)
3810 {
3811 page_t *pp;
3812 caddr_t addr = inaddr;
3813 caddr_t eaddr;
3814 pgcnt_t npages = btopr(size);
3815
3816 ASSERT(vmp != NULL);
3817 ASSERT(kvseg.s_base != NULL);
3818 ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
3819
3820 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
3821
3822 for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
3823 /*
3824 * Use page_find() instead of page_lookup() to find the page
3825 * since we know that it is hashed and has a shared lock.
3826 */
3827 pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr);
3828
3829 if (pp == NULL)
3830 panic("devmap_free_pages: page not found");
3831 if (!page_tryupgrade(pp)) {
3832 page_unlock(pp);
3833 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr,
3834 SE_EXCL);
3835 if (pp == NULL)
3836 panic("devmap_free_pages: page already freed");
3837 }
3838 /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
3839 pp->p_lckcnt = 0;
3840 page_destroy(pp, 0);
3841 }
3842 page_unresv(npages);
3843
3844 if (vmp != NULL)
3845 vmem_free(vmp, inaddr, size);
3846 }
3847
3848 /*
3849 * devmap_umem_alloc_np() replaces kmem_zalloc() as the method for
3850 * allocating non-pageable kmem in response to a ddi_umem_alloc()
3851 * default request. For now we allocate our own pages and we keep
3852 * them long-term sharelocked, since: A) the fault routines expect the
3853 * memory to already be locked; B) pageable umem is already long-term
3854 * locked; C) it's a lot of work to make it otherwise, particularly
3855 * since the nexus layer expects the pages to never fault. An RFE is to
3856 * not keep the pages long-term locked, but instead to be able to
3857 * take faults on them and simply look them up in kvp in case we
3858 * fault on them. Even then, we must take care not to let pageout
3859 * steal them from us since the data must remain resident; if we
3860 * do this we must come up with some way to pin the pages to prevent
3861 * faults while a driver is doing DMA to/from them.
3862 */
3863 static void *
3864 devmap_umem_alloc_np(size_t size, size_t flags)
3865 {
3866 void *buf;
3867 int vmflags = (flags & DDI_UMEM_NOSLEEP)? VM_NOSLEEP : VM_SLEEP;
3868
3869 buf = vmem_alloc(umem_np_arena, size, vmflags);
3870 if (buf != NULL)
3871 bzero(buf, size);
3872 return (buf);
3873 }
3874
3875 static void
3876 devmap_umem_free_np(void *addr, size_t size)
3877 {
3878 vmem_free(umem_np_arena, addr, size);
3879 }
3880
3881 /*
3882 * allocate page aligned kernel memory for exporting to user land.
3883 * The devmap framework will use the cookie allocated by ddi_umem_alloc()
3884 * to find a user virtual address that is in same color as the address
3885 * allocated here.
3886 */
3887 void *
3888 ddi_umem_alloc(size_t size, int flags, ddi_umem_cookie_t *cookie)
3889 {
3890 register size_t len = ptob(btopr(size));
3891 void *buf = NULL;
3892 struct ddi_umem_cookie *cp;
3893 int iflags = 0;
3894
3895 *cookie = NULL;
3896
3897 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_ALLOC,
3898 "devmap_umem_alloc:start");
3899 if (len == 0)
3900 return ((void *)NULL);
3901
3902 /*
3903 * allocate cookie
3904 */
3905 if ((cp = kmem_zalloc(sizeof (struct ddi_umem_cookie),
3906 flags & DDI_UMEM_NOSLEEP ? KM_NOSLEEP : KM_SLEEP)) == NULL) {
3907 ASSERT(flags & DDI_UMEM_NOSLEEP);
3908 return ((void *)NULL);
3909 }
3910
3911 if (flags & DDI_UMEM_PAGEABLE) {
3912 /* Only one of the flags is allowed */
3913 ASSERT(!(flags & DDI_UMEM_TRASH));
3914 /* initialize resource with 0 */
3915 iflags = KPD_ZERO;
3916
3917 /*
3918 * to allocate unlocked pageable memory, use segkp_get() to
3919 * create a segkp segment. Since segkp can only service kas,
3920 * other segment drivers such as segdev have to do
3921 * as_fault(segkp, SOFTLOCK) in its fault routine,
3922 */
3923 if (flags & DDI_UMEM_NOSLEEP)
3924 iflags |= KPD_NOWAIT;
3925
3926 if ((buf = segkp_get(segkp, len, iflags)) == NULL) {
3927 kmem_free(cp, sizeof (struct ddi_umem_cookie));
3928 return ((void *)NULL);
3929 }
3930 cp->type = KMEM_PAGEABLE;
3931 mutex_init(&cp->lock, NULL, MUTEX_DEFAULT, NULL);
3932 cp->locked = 0;
3933 } else if (flags & DDI_UMEM_TRASH) {
3934 /* Only one of the flags is allowed */
3935 ASSERT(!(flags & DDI_UMEM_PAGEABLE));
3936 cp->type = UMEM_TRASH;
3937 buf = NULL;
3938 } else {
3939 if ((buf = devmap_umem_alloc_np(len, flags)) == NULL) {
3940 kmem_free(cp, sizeof (struct ddi_umem_cookie));
3941 return ((void *)NULL);
3942 }
3943
3944 cp->type = KMEM_NON_PAGEABLE;
3945 }
3946
3947 /*
3948 * need to save size here. size will be used when
3949 * we do kmem_free.
3950 */
3951 cp->size = len;
3952 cp->cvaddr = (caddr_t)buf;
3953
3954 *cookie = (void *)cp;
3955 return (buf);
3956 }
3957
3958 void
3959 ddi_umem_free(ddi_umem_cookie_t cookie)
3960 {
3961 struct ddi_umem_cookie *cp;
3962
3963 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_FREE,
3964 "devmap_umem_free:start");
3965
3966 /*
3967 * if cookie is NULL, no effects on the system
3968 */
3969 if (cookie == NULL)
3970 return;
3971
3972 cp = (struct ddi_umem_cookie *)cookie;
3973
3974 switch (cp->type) {
3975 case KMEM_PAGEABLE :
3976 ASSERT(cp->cvaddr != NULL && cp->size != 0);
3977 /*
3978 * Check if there are still any pending faults on the cookie
3979 * while the driver is deleting it,
3980 * XXX - could change to an ASSERT but wont catch errant drivers
3981 */
3982 mutex_enter(&cp->lock);
3983 if (cp->locked) {
3984 mutex_exit(&cp->lock);
3985 panic("ddi_umem_free for cookie with pending faults %p",
3986 (void *)cp);
3987 return;
3988 }
3989
3990 segkp_release(segkp, cp->cvaddr);
3991
3992 /*
3993 * release mutex associated with this cookie.
3994 */
3995 mutex_destroy(&cp->lock);
3996 break;
3997 case KMEM_NON_PAGEABLE :
3998 ASSERT(cp->cvaddr != NULL && cp->size != 0);
3999 devmap_umem_free_np(cp->cvaddr, cp->size);
4000 break;
4001 case UMEM_TRASH :
4002 break;
4003 case UMEM_LOCKED :
4004 /* Callers should use ddi_umem_unlock for this type */
4005 ddi_umem_unlock(cookie);
4006 /* Frees the cookie too */
4007 return;
4008 default:
4009 /* panic so we can diagnose the underlying cause */
4010 panic("ddi_umem_free: illegal cookie type 0x%x\n",
4011 cp->type);
4012 }
4013
4014 kmem_free(cookie, sizeof (struct ddi_umem_cookie));
4015 }
4016
4017
4018 static int
4019 segdev_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
4020 {
4021 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4022
4023 /*
4024 * It looks as if it is always mapped shared
4025 */
4026 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GETMEMID,
4027 "segdev_getmemid:start");
4028 memidp->val[0] = (uintptr_t)VTOCVP(sdp->vp);
4029 memidp->val[1] = sdp->offset + (uintptr_t)(addr - seg->s_base);
4030 return (0);
4031 }
4032
4033 /*ARGSUSED*/
4034 static lgrp_mem_policy_info_t *
4035 segdev_getpolicy(struct seg *seg, caddr_t addr)
4036 {
4037 return (NULL);
4038 }
4039
4040 /*ARGSUSED*/
4041 static int
4042 segdev_capable(struct seg *seg, segcapability_t capability)
4043 {
4044 return (0);
4045 }
4046
4047 /*
4048 * ddi_umem_alloc() non-pageable quantum cache max size.
4049 * This is just a SWAG.
4050 */
4051 #define DEVMAP_UMEM_QUANTUM (8*PAGESIZE)
4052
4053 /*
4054 * Initialize seg_dev from boot. This routine sets up the trash page
4055 * and creates the umem_np_arena used to back non-pageable memory
4056 * requests.
4057 */
4058 void
4059 segdev_init(void)
4060 {
4061 struct seg kseg;
4062
4063 umem_np_arena = vmem_create("umem_np", NULL, 0, PAGESIZE,
4064 devmap_alloc_pages, devmap_free_pages, heap_arena,
4065 DEVMAP_UMEM_QUANTUM, VM_SLEEP);
4066
4067 kseg.s_as = &kas;
4068 trashpp = page_create_va(&trashvp, 0, PAGESIZE,
4069 PG_NORELOC | PG_EXCL | PG_WAIT, &kseg, NULL);
4070 if (trashpp == NULL)
4071 panic("segdev_init: failed to create trash page");
4072 pagezero(trashpp, 0, PAGESIZE);
4073 page_downgrade(trashpp);
4074 }
4075
4076 /*
4077 * Invoke platform-dependent support routines so that /proc can have
4078 * the platform code deal with curious hardware.
4079 */
4080 int
4081 segdev_copyfrom(struct seg *seg,
4082 caddr_t uaddr, const void *devaddr, void *kaddr, size_t len)
4083 {
4084 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4085 struct snode *sp = VTOS(VTOCVP(sdp->vp));
4086
4087 return (e_ddi_copyfromdev(sp->s_dip,
4088 (off_t)(uaddr - seg->s_base), devaddr, kaddr, len));
4089 }
4090
4091 int
4092 segdev_copyto(struct seg *seg,
4093 caddr_t uaddr, const void *kaddr, void *devaddr, size_t len)
4094 {
4095 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4096 struct snode *sp = VTOS(VTOCVP(sdp->vp));
4097
4098 return (e_ddi_copytodev(sp->s_dip,
4099 (off_t)(uaddr - seg->s_base), kaddr, devaddr, len));
4100 }