Print this page
patch as-lock-macro-simplification

*** 358,368 **** as_findseg(struct as *as, caddr_t addr, int tail) { struct seg *seg = as->a_seglast; avl_index_t where; ! ASSERT(AS_LOCK_HELD(as, &as->a_lock)); if (seg != NULL && seg->s_base <= addr && addr < seg->s_base + seg->s_size) return (seg); --- 358,368 ---- as_findseg(struct as *as, caddr_t addr, int tail) { struct seg *seg = as->a_seglast; avl_index_t where; ! ASSERT(AS_LOCK_HELD(as)); if (seg != NULL && seg->s_base <= addr && addr < seg->s_base + seg->s_size) return (seg);
*** 420,430 **** struct seg *seg; caddr_t addr; caddr_t eaddr; avl_index_t where; ! ASSERT(AS_WRITE_HELD(as, &as->a_lock)); as->a_updatedir = 1; /* inform /proc */ gethrestime(&as->a_updatetime); if (as->a_lastgaphl != NULL) { --- 420,430 ---- struct seg *seg; caddr_t addr; caddr_t eaddr; avl_index_t where; ! ASSERT(AS_WRITE_HELD(as)); as->a_updatedir = 1; /* inform /proc */ gethrestime(&as->a_updatetime); if (as->a_lastgaphl != NULL) {
*** 502,512 **** struct seg * as_removeseg(struct as *as, struct seg *seg) { avl_tree_t *t; ! ASSERT(AS_WRITE_HELD(as, &as->a_lock)); as->a_updatedir = 1; /* inform /proc */ gethrestime(&as->a_updatetime); if (seg == NULL) --- 502,512 ---- struct seg * as_removeseg(struct as *as, struct seg *seg) { avl_tree_t *t; ! ASSERT(AS_WRITE_HELD(as)); as->a_updatedir = 1; /* inform /proc */ gethrestime(&as->a_updatetime); if (seg == NULL)
*** 542,552 **** struct seg * as_segat(struct as *as, caddr_t addr) { struct seg *seg = as->a_seglast; ! ASSERT(AS_LOCK_HELD(as, &as->a_lock)); if (seg != NULL && seg->s_base <= addr && addr < seg->s_base + seg->s_size) return (seg); --- 542,552 ---- struct seg * as_segat(struct as *as, caddr_t addr) { struct seg *seg = as->a_seglast; ! ASSERT(AS_LOCK_HELD(as)); if (seg != NULL && seg->s_base <= addr && addr < seg->s_base + seg->s_size) return (seg);
*** 665,677 **** as->a_userlimit = (caddr_t)USERLIMIT; as->a_lastgap = NULL; as->a_lastgaphl = NULL; as->a_callbacks = NULL; ! AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); as->a_hat = hat_alloc(as); /* create hat for default system mmu */ ! AS_LOCK_EXIT(as, &as->a_lock); as->a_xhat = NULL; return (as); } --- 665,677 ---- as->a_userlimit = (caddr_t)USERLIMIT; as->a_lastgap = NULL; as->a_lastgaphl = NULL; as->a_callbacks = NULL; ! AS_LOCK_ENTER(as, RW_WRITER); as->a_hat = hat_alloc(as); /* create hat for default system mmu */ ! AS_LOCK_EXIT(as); as->a_xhat = NULL; return (as); }
*** 701,711 **** /* This will prevent new XHATs from attaching to as */ if (!called) AS_SETBUSY(as); mutex_exit(&as->a_contents); ! AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); if (!called) { called = 1; hat_free_start(hat); if (as->a_xhat != NULL) --- 701,711 ---- /* This will prevent new XHATs from attaching to as */ if (!called) AS_SETBUSY(as); mutex_exit(&as->a_contents); ! AS_LOCK_ENTER(as, RW_WRITER); if (!called) { called = 1; hat_free_start(hat); if (as->a_xhat != NULL)
*** 718,738 **** retry: err = SEGOP_UNMAP(seg, seg->s_base, seg->s_size); if (err == EAGAIN) { mutex_enter(&as->a_contents); if (as->a_callbacks) { ! AS_LOCK_EXIT(as, &as->a_lock); } else if (!AS_ISNOUNMAPWAIT(as)) { /* * Memory is currently locked. Wait for a * cv_signal that it has been unlocked, then * try the operation again. */ if (AS_ISUNMAPWAIT(as) == 0) cv_broadcast(&as->a_cv); AS_SETUNMAPWAIT(as); ! AS_LOCK_EXIT(as, &as->a_lock); while (AS_ISUNMAPWAIT(as)) cv_wait(&as->a_cv, &as->a_contents); } else { /* * We may have raced with --- 718,738 ---- retry: err = SEGOP_UNMAP(seg, seg->s_base, seg->s_size); if (err == EAGAIN) { mutex_enter(&as->a_contents); if (as->a_callbacks) { ! AS_LOCK_EXIT(as); } else if (!AS_ISNOUNMAPWAIT(as)) { /* * Memory is currently locked. Wait for a * cv_signal that it has been unlocked, then * try the operation again. */ if (AS_ISUNMAPWAIT(as) == 0) cv_broadcast(&as->a_cv); AS_SETUNMAPWAIT(as); ! AS_LOCK_EXIT(as); while (AS_ISUNMAPWAIT(as)) cv_wait(&as->a_cv, &as->a_contents); } else { /* * We may have raced with
*** 759,769 **** } } hat_free_end(hat); if (as->a_xhat != NULL) xhat_free_end_all(as); ! AS_LOCK_EXIT(as, &as->a_lock); /* /proc stuff */ ASSERT(avl_numnodes(&as->a_wpage) == 0); if (as->a_objectdir) { kmem_free(as->a_objectdir, as->a_sizedir * sizeof (vnode_t *)); --- 759,769 ---- } } hat_free_end(hat); if (as->a_xhat != NULL) xhat_free_end_all(as); ! AS_LOCK_EXIT(as); /* /proc stuff */ ASSERT(avl_numnodes(&as->a_wpage) == 0); if (as->a_objectdir) { kmem_free(as->a_objectdir, as->a_sizedir * sizeof (vnode_t *));
*** 784,800 **** struct as *newas; struct seg *seg, *newseg; size_t purgesize = 0; int error; ! AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); as_clearwatch(as); newas = as_alloc(); newas->a_userlimit = as->a_userlimit; newas->a_proc = forkedproc; ! AS_LOCK_ENTER(newas, &newas->a_lock, RW_WRITER); /* This will prevent new XHATs from attaching */ mutex_enter(&as->a_contents); AS_SETBUSY(as); mutex_exit(&as->a_contents); --- 784,800 ---- struct as *newas; struct seg *seg, *newseg; size_t purgesize = 0; int error; ! AS_LOCK_ENTER(as, RW_WRITER); as_clearwatch(as); newas = as_alloc(); newas->a_userlimit = as->a_userlimit; newas->a_proc = forkedproc; ! AS_LOCK_ENTER(newas, RW_WRITER); /* This will prevent new XHATs from attaching */ mutex_enter(&as->a_contents); AS_SETBUSY(as); mutex_exit(&as->a_contents);
*** 811,826 **** continue; } newseg = seg_alloc(newas, seg->s_base, seg->s_size); if (newseg == NULL) { ! AS_LOCK_EXIT(newas, &newas->a_lock); as_setwatch(as); mutex_enter(&as->a_contents); AS_CLRBUSY(as); mutex_exit(&as->a_contents); ! AS_LOCK_EXIT(as, &as->a_lock); as_free(newas); return (-1); } if ((error = SEGOP_DUP(seg, newseg)) != 0) { /* --- 811,826 ---- continue; } newseg = seg_alloc(newas, seg->s_base, seg->s_size); if (newseg == NULL) { ! AS_LOCK_EXIT(newas); as_setwatch(as); mutex_enter(&as->a_contents); AS_CLRBUSY(as); mutex_exit(&as->a_contents); ! AS_LOCK_EXIT(as); as_free(newas); return (-1); } if ((error = SEGOP_DUP(seg, newseg)) != 0) { /*
*** 830,842 **** */ as_setwatch(as); mutex_enter(&as->a_contents); AS_CLRBUSY(as); mutex_exit(&as->a_contents); ! AS_LOCK_EXIT(as, &as->a_lock); seg_free(newseg); ! AS_LOCK_EXIT(newas, &newas->a_lock); as_free(newas); return (error); } newas->a_size += seg->s_size; } --- 830,842 ---- */ as_setwatch(as); mutex_enter(&as->a_contents); AS_CLRBUSY(as); mutex_exit(&as->a_contents); ! AS_LOCK_EXIT(as); seg_free(newseg); ! AS_LOCK_EXIT(newas); as_free(newas); return (error); } newas->a_size += seg->s_size; }
*** 847,863 **** error |= xhat_dup_all(as, newas, NULL, 0, HAT_DUP_ALL); mutex_enter(&newas->a_contents); AS_CLRBUSY(newas); mutex_exit(&newas->a_contents); ! AS_LOCK_EXIT(newas, &newas->a_lock); as_setwatch(as); mutex_enter(&as->a_contents); AS_CLRBUSY(as); mutex_exit(&as->a_contents); ! AS_LOCK_EXIT(as, &as->a_lock); if (error != 0) { as_free(newas); return (error); } forkedproc->p_as = newas; --- 847,863 ---- error |= xhat_dup_all(as, newas, NULL, 0, HAT_DUP_ALL); mutex_enter(&newas->a_contents); AS_CLRBUSY(newas); mutex_exit(&newas->a_contents); ! AS_LOCK_EXIT(newas); as_setwatch(as); mutex_enter(&as->a_contents); AS_CLRBUSY(as); mutex_exit(&as->a_contents); ! AS_LOCK_EXIT(as); if (error != 0) { as_free(newas); return (error); } forkedproc->p_as = newas;
*** 957,967 **** * FC_NOSUPPORT. */ seg = segkmap; as_lock_held = 0; } else { ! AS_LOCK_ENTER(as, &as->a_lock, RW_READER); if (is_xhat && avl_numnodes(&as->a_wpage) != 0) { /* * Grab and hold the writers' lock on the as * if the fault is to a watched page. * This will keep CPUs from "peeking" at the --- 957,967 ---- * FC_NOSUPPORT. */ seg = segkmap; as_lock_held = 0; } else { ! AS_LOCK_ENTER(as, RW_READER); if (is_xhat && avl_numnodes(&as->a_wpage) != 0) { /* * Grab and hold the writers' lock on the as * if the fault is to a watched page. * This will keep CPUs from "peeking" at the
*** 971,987 **** * * We could check whether faulted address * is within a watched page and only then grab * the writer lock, but this is simpler. */ ! AS_LOCK_EXIT(as, &as->a_lock); ! AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); } seg = as_segat(as, raddr); if (seg == NULL) { ! AS_LOCK_EXIT(as, &as->a_lock); if ((lwp != NULL) && (!is_xhat)) lwp->lwp_nostop--; return (FC_NOMAP); } --- 971,987 ---- * * We could check whether faulted address * is within a watched page and only then grab * the writer lock, but this is simpler. */ ! AS_LOCK_EXIT(as); ! AS_LOCK_ENTER(as, RW_WRITER); } seg = as_segat(as, raddr); if (seg == NULL) { ! AS_LOCK_EXIT(as); if ((lwp != NULL) && (!is_xhat)) lwp->lwp_nostop--; return (FC_NOMAP); }
*** 1058,1068 **** (void) SEGOP_FAULT(hat, seg, addrsav, ssize, F_SOFTUNLOCK, S_OTHER); } } if (as_lock_held) ! AS_LOCK_EXIT(as, &as->a_lock); if ((lwp != NULL) && (!is_xhat)) lwp->lwp_nostop--; /* * If the lower levels returned EDEADLK for a fault, --- 1058,1068 ---- (void) SEGOP_FAULT(hat, seg, addrsav, ssize, F_SOFTUNLOCK, S_OTHER); } } if (as_lock_held) ! AS_LOCK_EXIT(as); if ((lwp != NULL) && (!is_xhat)) lwp->lwp_nostop--; /* * If the lower levels returned EDEADLK for a fault,
*** 1106,1119 **** raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - (size_t)raddr; ! AS_LOCK_ENTER(as, &as->a_lock, RW_READER); seg = as_segat(as, raddr); if (seg == NULL) { ! AS_LOCK_EXIT(as, &as->a_lock); if (lwp != NULL) lwp->lwp_nostop--; return (FC_NOMAP); } --- 1106,1119 ---- raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - (size_t)raddr; ! AS_LOCK_ENTER(as, RW_READER); seg = as_segat(as, raddr); if (seg == NULL) { ! AS_LOCK_EXIT(as); if (lwp != NULL) lwp->lwp_nostop--; return (FC_NOMAP); }
*** 1127,1137 **** } res = SEGOP_FAULTA(seg, raddr); if (res != 0) break; } ! AS_LOCK_EXIT(as, &as->a_lock); if (lwp != NULL) lwp->lwp_nostop--; /* * If the lower levels returned EDEADLK for a fault, * It means that we should retry the fault. Let's wait --- 1127,1137 ---- } res = SEGOP_FAULTA(seg, raddr); if (res != 0) break; } ! AS_LOCK_EXIT(as); if (lwp != NULL) lwp->lwp_nostop--; /* * If the lower levels returned EDEADLK for a fault, * It means that we should retry the fault. Let's wait
*** 1187,1206 **** * after it has changed the segment list so we therefore keep * locking as a writer. Since these opeartions should be rare * want to only lock as a writer when necessary. */ if (writer || avl_numnodes(&as->a_wpage) != 0) { ! AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); } else { ! AS_LOCK_ENTER(as, &as->a_lock, RW_READER); } as_clearwatchprot(as, raddr, rsize); seg = as_segat(as, raddr); if (seg == NULL) { as_setwatch(as); ! AS_LOCK_EXIT(as, &as->a_lock); return (ENOMEM); } for (; rsize != 0; rsize -= ssize, raddr += ssize) { if (raddr >= seg->s_base + seg->s_size) { --- 1187,1206 ---- * after it has changed the segment list so we therefore keep * locking as a writer. Since these opeartions should be rare * want to only lock as a writer when necessary. */ if (writer || avl_numnodes(&as->a_wpage) != 0) { ! AS_LOCK_ENTER(as, RW_WRITER); } else { ! AS_LOCK_ENTER(as, RW_READER); } as_clearwatchprot(as, raddr, rsize); seg = as_segat(as, raddr); if (seg == NULL) { as_setwatch(as); ! AS_LOCK_EXIT(as); return (ENOMEM); } for (; rsize != 0; rsize -= ssize, raddr += ssize) { if (raddr >= seg->s_base + seg->s_size) {
*** 1221,1241 **** error = EAGAIN; break; } if (error == IE_RETRY) { ! AS_LOCK_EXIT(as, &as->a_lock); writer = 1; goto setprot_top; } if (error == EAGAIN) { /* * Make sure we have a_lock as writer. */ if (writer == 0) { ! AS_LOCK_EXIT(as, &as->a_lock); writer = 1; goto setprot_top; } /* --- 1221,1241 ---- error = EAGAIN; break; } if (error == IE_RETRY) { ! AS_LOCK_EXIT(as); writer = 1; goto setprot_top; } if (error == EAGAIN) { /* * Make sure we have a_lock as writer. */ if (writer == 0) { ! AS_LOCK_EXIT(as); writer = 1; goto setprot_top; } /*
*** 1272,1288 **** */ mutex_enter(&as->a_contents); if (as->a_callbacks && (cb = as_find_callback(as, AS_SETPROT_EVENT, seg->s_base, seg->s_size))) { ! AS_LOCK_EXIT(as, &as->a_lock); as_execute_callback(as, cb, AS_SETPROT_EVENT); } else if (!AS_ISNOUNMAPWAIT(as)) { if (AS_ISUNMAPWAIT(as) == 0) cv_broadcast(&as->a_cv); AS_SETUNMAPWAIT(as); ! AS_LOCK_EXIT(as, &as->a_lock); while (AS_ISUNMAPWAIT(as)) cv_wait(&as->a_cv, &as->a_contents); } else { /* * We may have raced with --- 1272,1288 ---- */ mutex_enter(&as->a_contents); if (as->a_callbacks && (cb = as_find_callback(as, AS_SETPROT_EVENT, seg->s_base, seg->s_size))) { ! AS_LOCK_EXIT(as); as_execute_callback(as, cb, AS_SETPROT_EVENT); } else if (!AS_ISNOUNMAPWAIT(as)) { if (AS_ISUNMAPWAIT(as) == 0) cv_broadcast(&as->a_cv); AS_SETUNMAPWAIT(as); ! AS_LOCK_EXIT(as); while (AS_ISUNMAPWAIT(as)) cv_wait(&as->a_cv, &as->a_contents); } else { /* * We may have raced with
*** 1306,1316 **** if (error != 0) { as_setwatch(as); } else { as_setwatchprot(as, saveraddr, saversize, prot); } ! AS_LOCK_EXIT(as, &as->a_lock); return (error); } /* * Check to make sure that the interval [addr, addr + size) --- 1306,1316 ---- if (error != 0) { as_setwatch(as); } else { as_setwatchprot(as, saveraddr, saversize, prot); } ! AS_LOCK_EXIT(as); return (error); } /* * Check to make sure that the interval [addr, addr + size)
*** 1340,1357 **** * However, if the address space has watchpoints present, * we must acquire the writer lock on the address space for * the benefit of as_clearwatchprot() and as_setwatchprot(). */ if (avl_numnodes(&as->a_wpage) != 0) ! AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); else ! AS_LOCK_ENTER(as, &as->a_lock, RW_READER); as_clearwatchprot(as, raddr, rsize); seg = as_segat(as, raddr); if (seg == NULL) { as_setwatch(as); ! AS_LOCK_EXIT(as, &as->a_lock); return (ENOMEM); } for (; rsize != 0; rsize -= ssize, raddr += ssize) { if (raddr >= seg->s_base + seg->s_size) { --- 1340,1357 ---- * However, if the address space has watchpoints present, * we must acquire the writer lock on the address space for * the benefit of as_clearwatchprot() and as_setwatchprot(). */ if (avl_numnodes(&as->a_wpage) != 0) ! AS_LOCK_ENTER(as, RW_WRITER); else ! AS_LOCK_ENTER(as, RW_READER); as_clearwatchprot(as, raddr, rsize); seg = as_segat(as, raddr); if (seg == NULL) { as_setwatch(as); ! AS_LOCK_EXIT(as); return (ENOMEM); } for (; rsize != 0; rsize -= ssize, raddr += ssize) { if (raddr >= seg->s_base + seg->s_size) {
*** 1369,1379 **** error = SEGOP_CHECKPROT(seg, raddr, ssize, prot); if (error != 0) break; } as_setwatch(as); ! AS_LOCK_EXIT(as, &as->a_lock); return (error); } int as_unmap(struct as *as, caddr_t addr, size_t size) --- 1369,1379 ---- error = SEGOP_CHECKPROT(seg, raddr, ssize, prot); if (error != 0) break; } as_setwatch(as); ! AS_LOCK_EXIT(as); return (error); } int as_unmap(struct as *as, caddr_t addr, size_t size)
*** 1387,1397 **** top: raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); eaddr = (caddr_t)(((uintptr_t)(addr + size) + PAGEOFFSET) & (uintptr_t)PAGEMASK); ! AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); as->a_updatedir = 1; /* inform /proc */ gethrestime(&as->a_updatetime); /* --- 1387,1397 ---- top: raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); eaddr = (caddr_t)(((uintptr_t)(addr + size) + PAGEOFFSET) & (uintptr_t)PAGEMASK); ! AS_LOCK_ENTER(as, RW_WRITER); as->a_updatedir = 1; /* inform /proc */ gethrestime(&as->a_updatetime); /*
*** 1468,1484 **** */ mutex_enter(&as->a_contents); if (as->a_callbacks && (cb = as_find_callback(as, AS_UNMAP_EVENT, seg->s_base, seg->s_size))) { ! AS_LOCK_EXIT(as, &as->a_lock); as_execute_callback(as, cb, AS_UNMAP_EVENT); } else if (!AS_ISNOUNMAPWAIT(as)) { if (AS_ISUNMAPWAIT(as) == 0) cv_broadcast(&as->a_cv); AS_SETUNMAPWAIT(as); ! AS_LOCK_EXIT(as, &as->a_lock); while (AS_ISUNMAPWAIT(as)) cv_wait(&as->a_cv, &as->a_contents); } else { /* * We may have raced with --- 1468,1484 ---- */ mutex_enter(&as->a_contents); if (as->a_callbacks && (cb = as_find_callback(as, AS_UNMAP_EVENT, seg->s_base, seg->s_size))) { ! AS_LOCK_EXIT(as); as_execute_callback(as, cb, AS_UNMAP_EVENT); } else if (!AS_ISNOUNMAPWAIT(as)) { if (AS_ISUNMAPWAIT(as) == 0) cv_broadcast(&as->a_cv); AS_SETUNMAPWAIT(as); ! AS_LOCK_EXIT(as); while (AS_ISUNMAPWAIT(as)) cv_wait(&as->a_cv, &as->a_contents); } else { /* * We may have raced with
*** 1495,1518 **** goto retry; } mutex_exit(&as->a_contents); goto top; } else if (err == IE_RETRY) { ! AS_LOCK_EXIT(as, &as->a_lock); goto top; } else if (err) { as_setwatch(as); ! AS_LOCK_EXIT(as, &as->a_lock); return (-1); } as->a_size -= ssize; if (rsize) as->a_resvsize -= rsize; raddr += ssize; } ! AS_LOCK_EXIT(as, &as->a_lock); return (0); } static int as_map_segvn_segs(struct as *as, caddr_t addr, size_t size, uint_t szcvec, --- 1495,1518 ---- goto retry; } mutex_exit(&as->a_contents); goto top; } else if (err == IE_RETRY) { ! AS_LOCK_EXIT(as); goto top; } else if (err) { as_setwatch(as); ! AS_LOCK_EXIT(as); return (-1); } as->a_size -= ssize; if (rsize) as->a_resvsize -= rsize; raddr += ssize; } ! AS_LOCK_EXIT(as); return (0); } static int as_map_segvn_segs(struct as *as, caddr_t addr, size_t size, uint_t szcvec,
*** 1527,1537 **** struct seg *seg; size_t pgsz; int do_off = (vn_a->vp != NULL || vn_a->amp != NULL); uint_t save_szcvec; ! ASSERT(AS_WRITE_HELD(as, &as->a_lock)); ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); ASSERT(IS_P2ALIGNED(size, PAGESIZE)); ASSERT(vn_a->vp == NULL || vn_a->amp == NULL); if (!do_off) { vn_a->offset = 0; --- 1527,1537 ---- struct seg *seg; size_t pgsz; int do_off = (vn_a->vp != NULL || vn_a->amp != NULL); uint_t save_szcvec; ! ASSERT(AS_WRITE_HELD(as)); ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); ASSERT(IS_P2ALIGNED(size, PAGESIZE)); ASSERT(vn_a->vp == NULL || vn_a->amp == NULL); if (!do_off) { vn_a->offset = 0;
*** 1641,1651 **** struct vattr va; u_offset_t eoff; size_t save_size = 0; extern size_t textrepl_size_thresh; ! ASSERT(AS_WRITE_HELD(as, &as->a_lock)); ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); ASSERT(IS_P2ALIGNED(size, PAGESIZE)); ASSERT(vn_a->vp != NULL); ASSERT(vn_a->amp == NULL); --- 1641,1651 ---- struct vattr va; u_offset_t eoff; size_t save_size = 0; extern size_t textrepl_size_thresh; ! ASSERT(AS_WRITE_HELD(as)); ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); ASSERT(IS_P2ALIGNED(size, PAGESIZE)); ASSERT(vn_a->vp != NULL); ASSERT(vn_a->amp == NULL);
*** 1730,1740 **** } } szcvec = map_pgszcvec(addr, size, vn_a->amp == NULL ? (uintptr_t)addr : (uintptr_t)P2ROUNDUP(vn_a->offset, PAGESIZE), (vn_a->flags & MAP_TEXT), type, 0); ! ASSERT(AS_WRITE_HELD(as, &as->a_lock)); ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); ASSERT(IS_P2ALIGNED(size, PAGESIZE)); ASSERT(vn_a->vp == NULL); return (as_map_segvn_segs(as, addr, size, szcvec, --- 1730,1740 ---- } } szcvec = map_pgszcvec(addr, size, vn_a->amp == NULL ? (uintptr_t)addr : (uintptr_t)P2ROUNDUP(vn_a->offset, PAGESIZE), (vn_a->flags & MAP_TEXT), type, 0); ! ASSERT(AS_WRITE_HELD(as)); ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); ASSERT(IS_P2ALIGNED(size, PAGESIZE)); ASSERT(vn_a->vp == NULL); return (as_map_segvn_segs(as, addr, size, szcvec,
*** 1742,1752 **** } int as_map(struct as *as, caddr_t addr, size_t size, int (*crfp)(), void *argsp) { ! AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); return (as_map_locked(as, addr, size, crfp, argsp)); } int as_map_locked(struct as *as, caddr_t addr, size_t size, int (*crfp)(), --- 1742,1752 ---- } int as_map(struct as *as, caddr_t addr, size_t size, int (*crfp)(), void *argsp) { ! AS_LOCK_ENTER(as, RW_WRITER); return (as_map_locked(as, addr, size, crfp, argsp)); } int as_map_locked(struct as *as, caddr_t addr, size_t size, int (*crfp)(),
*** 1766,1784 **** /* * check for wrap around */ if ((raddr + rsize < raddr) || (as->a_size > (ULONG_MAX - size))) { ! AS_LOCK_EXIT(as, &as->a_lock); return (ENOMEM); } as->a_updatedir = 1; /* inform /proc */ gethrestime(&as->a_updatetime); if (as != &kas && as->a_size + rsize > (size_t)p->p_vmem_ctl) { ! AS_LOCK_EXIT(as, &as->a_lock); (void) rctl_action(rctlproc_legacy[RLIMIT_VMEM], p->p_rctls, p, RCA_UNSAFE_ALL); return (ENOMEM); --- 1766,1784 ---- /* * check for wrap around */ if ((raddr + rsize < raddr) || (as->a_size > (ULONG_MAX - size))) { ! AS_LOCK_EXIT(as); return (ENOMEM); } as->a_updatedir = 1; /* inform /proc */ gethrestime(&as->a_updatetime); if (as != &kas && as->a_size + rsize > (size_t)p->p_vmem_ctl) { ! AS_LOCK_EXIT(as); (void) rctl_action(rctlproc_legacy[RLIMIT_VMEM], p->p_rctls, p, RCA_UNSAFE_ALL); return (ENOMEM);
*** 1786,1822 **** if (AS_MAP_CHECK_VNODE_LPOOB(crfp, argsp)) { crargs = *(struct segvn_crargs *)argsp; error = as_map_vnsegs(as, raddr, rsize, crfp, &crargs, &unmap); if (error != 0) { ! AS_LOCK_EXIT(as, &as->a_lock); if (unmap) { (void) as_unmap(as, addr, size); } return (error); } } else if (AS_MAP_CHECK_ANON_LPOOB(crfp, argsp)) { crargs = *(struct segvn_crargs *)argsp; error = as_map_ansegs(as, raddr, rsize, crfp, &crargs, &unmap); if (error != 0) { ! AS_LOCK_EXIT(as, &as->a_lock); if (unmap) { (void) as_unmap(as, addr, size); } return (error); } } else { seg = seg_alloc(as, addr, size); if (seg == NULL) { ! AS_LOCK_EXIT(as, &as->a_lock); return (ENOMEM); } error = (*crfp)(seg, argsp); if (error != 0) { seg_free(seg); ! AS_LOCK_EXIT(as, &as->a_lock); return (error); } /* * Add size now so as_unmap will work if as_ctl fails. */ --- 1786,1822 ---- if (AS_MAP_CHECK_VNODE_LPOOB(crfp, argsp)) { crargs = *(struct segvn_crargs *)argsp; error = as_map_vnsegs(as, raddr, rsize, crfp, &crargs, &unmap); if (error != 0) { ! AS_LOCK_EXIT(as); if (unmap) { (void) as_unmap(as, addr, size); } return (error); } } else if (AS_MAP_CHECK_ANON_LPOOB(crfp, argsp)) { crargs = *(struct segvn_crargs *)argsp; error = as_map_ansegs(as, raddr, rsize, crfp, &crargs, &unmap); if (error != 0) { ! AS_LOCK_EXIT(as); if (unmap) { (void) as_unmap(as, addr, size); } return (error); } } else { seg = seg_alloc(as, addr, size); if (seg == NULL) { ! AS_LOCK_EXIT(as); return (ENOMEM); } error = (*crfp)(seg, argsp); if (error != 0) { seg_free(seg); ! AS_LOCK_EXIT(as); return (error); } /* * Add size now so as_unmap will work if as_ctl fails. */
*** 1831,1847 **** * establish memory locks for the new segment. */ mutex_enter(&as->a_contents); if (AS_ISPGLCK(as)) { mutex_exit(&as->a_contents); ! AS_LOCK_EXIT(as, &as->a_lock); error = as_ctl(as, addr, size, MC_LOCK, 0, 0, NULL, 0); if (error != 0) (void) as_unmap(as, addr, size); } else { mutex_exit(&as->a_contents); ! AS_LOCK_EXIT(as, &as->a_lock); } return (error); } --- 1831,1847 ---- * establish memory locks for the new segment. */ mutex_enter(&as->a_contents); if (AS_ISPGLCK(as)) { mutex_exit(&as->a_contents); ! AS_LOCK_EXIT(as); error = as_ctl(as, addr, size, MC_LOCK, 0, 0, NULL, 0); if (error != 0) (void) as_unmap(as, addr, size); } else { mutex_exit(&as->a_contents); ! AS_LOCK_EXIT(as); } return (error); }
*** 1862,1881 **** * no need to grab a_contents mutex for this check */ if ((as->a_flags & AS_NEEDSPURGE) == 0) return; ! AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); next_seg = NULL; seg = AS_SEGFIRST(as); while (seg != NULL) { next_seg = AS_SEGNEXT(as, seg); if (seg->s_flags & S_PURGE) SEGOP_UNMAP(seg, seg->s_base, seg->s_size); seg = next_seg; } ! AS_LOCK_EXIT(as, &as->a_lock); mutex_enter(&as->a_contents); as->a_flags &= ~AS_NEEDSPURGE; mutex_exit(&as->a_contents); } --- 1862,1881 ---- * no need to grab a_contents mutex for this check */ if ((as->a_flags & AS_NEEDSPURGE) == 0) return; ! AS_LOCK_ENTER(as, RW_WRITER); next_seg = NULL; seg = AS_SEGFIRST(as); while (seg != NULL) { next_seg = AS_SEGNEXT(as, seg); if (seg->s_flags & S_PURGE) SEGOP_UNMAP(seg, seg->s_base, seg->s_size); seg = next_seg; } ! AS_LOCK_EXIT(as); mutex_enter(&as->a_contents); as->a_flags &= ~AS_NEEDSPURGE; mutex_exit(&as->a_contents); }
*** 1934,1951 **** */ minlen += align; minlen += 2 * redzone; redzone = 0; ! AS_LOCK_ENTER(as, &as->a_lock, RW_READER); if (AS_SEGFIRST(as) == NULL) { if (valid_va_range_aligned(basep, lenp, minlen, flags & AH_DIR, align, redzone, off)) { ! AS_LOCK_EXIT(as, &as->a_lock); return (0); } else { ! AS_LOCK_EXIT(as, &as->a_lock); *basep = save_base; *lenp = save_len; return (-1); } } --- 1934,1951 ---- */ minlen += align; minlen += 2 * redzone; redzone = 0; ! AS_LOCK_ENTER(as, RW_READER); if (AS_SEGFIRST(as) == NULL) { if (valid_va_range_aligned(basep, lenp, minlen, flags & AH_DIR, align, redzone, off)) { ! AS_LOCK_EXIT(as); return (0); } else { ! AS_LOCK_EXIT(as); *basep = save_base; *lenp = save_len; return (-1); } }
*** 2022,2032 **** as->a_lastgap = hseg; if (hseg != NULL) as->a_lastgaphl = hseg; else as->a_lastgaphl = lseg; ! AS_LOCK_EXIT(as, &as->a_lock); return (0); } cont: /* * Move to the next hole. --- 2022,2032 ---- as->a_lastgap = hseg; if (hseg != NULL) as->a_lastgaphl = hseg; else as->a_lastgaphl = lseg; ! AS_LOCK_EXIT(as); return (0); } cont: /* * Move to the next hole.
*** 2049,2059 **** redzone = save_redzone; goto retry; } *basep = save_base; *lenp = save_len; ! AS_LOCK_EXIT(as, &as->a_lock); return (-1); } /* * Find a hole of at least size minlen within [*basep, *basep + *lenp). --- 2049,2059 ---- redzone = save_redzone; goto retry; } *basep = save_base; *lenp = save_len; ! AS_LOCK_EXIT(as); return (-1); } /* * Find a hole of at least size minlen within [*basep, *basep + *lenp).
*** 2091,2112 **** extern struct seg_ops segspt_shmops; /* needs a header file */ struct seg *seg; caddr_t addr, eaddr; caddr_t segend; ! AS_LOCK_ENTER(as, &as->a_lock, RW_READER); addr = *basep; eaddr = addr + *lenp; seg = as_findseg(as, addr, 0); if (seg != NULL) addr = MAX(seg->s_base, addr); for (;;) { if (seg == NULL || addr >= eaddr || eaddr <= seg->s_base) { ! AS_LOCK_EXIT(as, &as->a_lock); return (EINVAL); } if (seg->s_ops == &segvn_ops) { segend = seg->s_base + seg->s_size; --- 2091,2112 ---- extern struct seg_ops segspt_shmops; /* needs a header file */ struct seg *seg; caddr_t addr, eaddr; caddr_t segend; ! AS_LOCK_ENTER(as, RW_READER); addr = *basep; eaddr = addr + *lenp; seg = as_findseg(as, addr, 0); if (seg != NULL) addr = MAX(seg->s_base, addr); for (;;) { if (seg == NULL || addr >= eaddr || eaddr <= seg->s_base) { ! AS_LOCK_EXIT(as); return (EINVAL); } if (seg->s_ops == &segvn_ops) { segend = seg->s_base + seg->s_size;
*** 2134,2144 **** if (segend > eaddr) *lenp = eaddr - addr; else *lenp = segend - addr; ! AS_LOCK_EXIT(as, &as->a_lock); return (0); } /* * Swap the pages associated with the address space as out to --- 2134,2144 ---- if (segend > eaddr) *lenp = eaddr - addr; else *lenp = segend - addr; ! AS_LOCK_EXIT(as); return (0); } /* * Swap the pages associated with the address space as out to
*** 2162,2172 **** * swap out such processes in the first place... */ if (as == NULL) return (0); ! AS_LOCK_ENTER(as, &as->a_lock, RW_READER); /* Prevent XHATs from attaching */ mutex_enter(&as->a_contents); AS_SETBUSY(as); mutex_exit(&as->a_contents); --- 2162,2172 ---- * swap out such processes in the first place... */ if (as == NULL) return (0); ! AS_LOCK_ENTER(as, RW_READER); /* Prevent XHATs from attaching */ mutex_enter(&as->a_contents); AS_SETBUSY(as); mutex_exit(&as->a_contents);
*** 2201,2211 **** * the process was picked for swapout. */ if ((ov != NULL) && (ov->swapout != NULL)) swpcnt += SEGOP_SWAPOUT(seg); } ! AS_LOCK_EXIT(as, &as->a_lock); return (swpcnt); } /* * Determine whether data from the mappings in interval [addr, addr + size) --- 2201,2211 ---- * the process was picked for swapout. */ if ((ov != NULL) && (ov->swapout != NULL)) swpcnt += SEGOP_SWAPOUT(seg); } ! AS_LOCK_EXIT(as); return (swpcnt); } /* * Determine whether data from the mappings in interval [addr, addr + size)
*** 2228,2241 **** (size_t)raddr; if (raddr + rsize < raddr) /* check for wraparound */ return (ENOMEM); ! AS_LOCK_ENTER(as, &as->a_lock, RW_READER); seg = as_segat(as, raddr); if (seg == NULL) { ! AS_LOCK_EXIT(as, &as->a_lock); return (-1); } for (; rsize != 0; rsize -= ssize, raddr += ssize) { if (raddr >= seg->s_base + seg->s_size) { --- 2228,2241 ---- (size_t)raddr; if (raddr + rsize < raddr) /* check for wraparound */ return (ENOMEM); ! AS_LOCK_ENTER(as, RW_READER); seg = as_segat(as, raddr); if (seg == NULL) { ! AS_LOCK_EXIT(as); return (-1); } for (; rsize != 0; rsize -= ssize, raddr += ssize) { if (raddr >= seg->s_base + seg->s_size) {
*** 2254,2264 **** error = -1; break; } vec += btopr(ssize); } ! AS_LOCK_EXIT(as, &as->a_lock); return (error); } static void as_segunlock(struct seg *seg, caddr_t addr, int attr, --- 2254,2264 ---- error = -1; break; } vec += btopr(ssize); } ! AS_LOCK_EXIT(as); return (error); } static void as_segunlock(struct seg *seg, caddr_t addr, int attr,
*** 2324,2336 **** ulong_t *mlock_map; /* pointer to bitmap used */ /* to represent the locked */ /* pages. */ retry: if (error == IE_RETRY) ! AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); else ! AS_LOCK_ENTER(as, &as->a_lock, RW_READER); /* * If these are address space lock/unlock operations, loop over * all segments in the address space, as appropriate. */ --- 2324,2336 ---- ulong_t *mlock_map; /* pointer to bitmap used */ /* to represent the locked */ /* pages. */ retry: if (error == IE_RETRY) ! AS_LOCK_ENTER(as, RW_WRITER); else ! AS_LOCK_ENTER(as, RW_READER); /* * If these are address space lock/unlock operations, loop over * all segments in the address space, as appropriate. */
*** 2344,2360 **** mutex_enter(&as->a_contents); AS_SETPGLCK(as); mutex_exit(&as->a_contents); } if ((arg & MCL_CURRENT) == 0) { ! AS_LOCK_EXIT(as, &as->a_lock); return (0); } seg = AS_SEGFIRST(as); if (seg == NULL) { ! AS_LOCK_EXIT(as, &as->a_lock); return (0); } do { raddr = (caddr_t)((uintptr_t)seg->s_base & --- 2344,2360 ---- mutex_enter(&as->a_contents); AS_SETPGLCK(as); mutex_exit(&as->a_contents); } if ((arg & MCL_CURRENT) == 0) { ! AS_LOCK_EXIT(as); return (0); } seg = AS_SEGFIRST(as); if (seg == NULL) { ! AS_LOCK_EXIT(as); return (0); } do { raddr = (caddr_t)((uintptr_t)seg->s_base &
*** 2364,2374 **** } while ((seg = AS_SEGNEXT(as, seg)) != NULL); mlock_size = BT_BITOUL(btopr(rlen)); if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size * sizeof (ulong_t), KM_NOSLEEP)) == NULL) { ! AS_LOCK_EXIT(as, &as->a_lock); return (EAGAIN); } for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) { error = SEGOP_LOCKOP(seg, seg->s_base, --- 2364,2374 ---- } while ((seg = AS_SEGNEXT(as, seg)) != NULL); mlock_size = BT_BITOUL(btopr(rlen)); if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size * sizeof (ulong_t), KM_NOSLEEP)) == NULL) { ! AS_LOCK_EXIT(as); return (EAGAIN); } for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) { error = SEGOP_LOCKOP(seg, seg->s_base,
*** 2390,2400 **** idx += npages; } } kmem_free(mlock_map, mlock_size * sizeof (ulong_t)); ! AS_LOCK_EXIT(as, &as->a_lock); goto lockerr; } else if (func == MC_UNLOCKAS) { mutex_enter(&as->a_contents); AS_CLRPGLCK(as); mutex_exit(&as->a_contents); --- 2390,2400 ---- idx += npages; } } kmem_free(mlock_map, mlock_size * sizeof (ulong_t)); ! AS_LOCK_EXIT(as); goto lockerr; } else if (func == MC_UNLOCKAS) { mutex_enter(&as->a_contents); AS_CLRPGLCK(as); mutex_exit(&as->a_contents);
*** 2404,2414 **** seg->s_size, attr, MC_UNLOCK, NULL, 0); if (error != 0) break; } ! AS_LOCK_EXIT(as, &as->a_lock); goto lockerr; } /* * Normalize addresses and sizes. --- 2404,2414 ---- seg->s_size, attr, MC_UNLOCK, NULL, 0); if (error != 0) break; } ! AS_LOCK_EXIT(as); goto lockerr; } /* * Normalize addresses and sizes.
*** 2416,2442 **** initraddr = raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); initrsize = rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - (size_t)raddr; if (raddr + rsize < raddr) { /* check for wraparound */ ! AS_LOCK_EXIT(as, &as->a_lock); return (ENOMEM); } /* * Get initial segment. */ if ((seg = as_segat(as, raddr)) == NULL) { ! AS_LOCK_EXIT(as, &as->a_lock); return (ENOMEM); } if (func == MC_LOCK) { mlock_size = BT_BITOUL(btopr(rsize)); if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size * sizeof (ulong_t), KM_NOSLEEP)) == NULL) { ! AS_LOCK_EXIT(as, &as->a_lock); return (EAGAIN); } } /* --- 2416,2442 ---- initraddr = raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); initrsize = rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - (size_t)raddr; if (raddr + rsize < raddr) { /* check for wraparound */ ! AS_LOCK_EXIT(as); return (ENOMEM); } /* * Get initial segment. */ if ((seg = as_segat(as, raddr)) == NULL) { ! AS_LOCK_EXIT(as); return (ENOMEM); } if (func == MC_LOCK) { mlock_size = BT_BITOUL(btopr(rsize)); if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size * sizeof (ulong_t), KM_NOSLEEP)) == NULL) { ! AS_LOCK_EXIT(as); return (EAGAIN); } } /*
*** 2457,2467 **** as_unlockerr(as, attr, mlock_map, initraddr, initrsize - rsize); kmem_free(mlock_map, mlock_size * sizeof (ulong_t)); } ! AS_LOCK_EXIT(as, &as->a_lock); return (ENOMEM); } } if ((raddr + rsize) > (seg->s_base + seg->s_size)) ssize = seg->s_base + seg->s_size - raddr; --- 2457,2467 ---- as_unlockerr(as, attr, mlock_map, initraddr, initrsize - rsize); kmem_free(mlock_map, mlock_size * sizeof (ulong_t)); } ! AS_LOCK_EXIT(as); return (ENOMEM); } } if ((raddr + rsize) > (seg->s_base + seg->s_size)) ssize = seg->s_base + seg->s_size - raddr;
*** 2478,2488 **** * objects. */ case MC_SYNC: if (error = SEGOP_SYNC(seg, raddr, ssize, attr, (uint_t)arg)) { ! AS_LOCK_EXIT(as, &as->a_lock); return (error); } break; /* --- 2478,2488 ---- * objects. */ case MC_SYNC: if (error = SEGOP_SYNC(seg, raddr, ssize, attr, (uint_t)arg)) { ! AS_LOCK_EXIT(as); return (error); } break; /*
*** 2493,2503 **** attr, func, mlock_map, pos)) { as_unlockerr(as, attr, mlock_map, initraddr, initrsize - rsize + ssize); kmem_free(mlock_map, mlock_size * sizeof (ulong_t)); ! AS_LOCK_EXIT(as, &as->a_lock); goto lockerr; } break; /* --- 2493,2503 ---- attr, func, mlock_map, pos)) { as_unlockerr(as, attr, mlock_map, initraddr, initrsize - rsize + ssize); kmem_free(mlock_map, mlock_size * sizeof (ulong_t)); ! AS_LOCK_EXIT(as); goto lockerr; } break; /*
*** 2522,2549 **** /* * Need to acquire writers lock, so * have to drop readers lock and start * all over again */ ! AS_LOCK_EXIT(as, &as->a_lock); goto retry; } else if (error == IE_REATTACH) { /* * Find segment for current address * because current segment just got * split or concatenated */ seg = as_segat(as, raddr); if (seg == NULL) { ! AS_LOCK_EXIT(as, &as->a_lock); return (ENOMEM); } } else { /* * Regular error */ ! AS_LOCK_EXIT(as, &as->a_lock); return (error); } } break; --- 2522,2549 ---- /* * Need to acquire writers lock, so * have to drop readers lock and start * all over again */ ! AS_LOCK_EXIT(as); goto retry; } else if (error == IE_REATTACH) { /* * Find segment for current address * because current segment just got * split or concatenated */ seg = as_segat(as, raddr); if (seg == NULL) { ! AS_LOCK_EXIT(as); return (ENOMEM); } } else { /* * Regular error */ ! AS_LOCK_EXIT(as); return (error); } } break;
*** 2553,2563 **** } else { error = SEGOP_INHERIT(seg, raddr, ssize, SEGP_INH_ZERO); } if (error != 0) { ! AS_LOCK_EXIT(as, &as->a_lock); return (error); } break; /* --- 2553,2563 ---- } else { error = SEGOP_INHERIT(seg, raddr, ssize, SEGP_INH_ZERO); } if (error != 0) { ! AS_LOCK_EXIT(as); return (error); } break; /*
*** 2572,2582 **** raddr += ssize; } if (func == MC_LOCK) kmem_free(mlock_map, mlock_size * sizeof (ulong_t)); ! AS_LOCK_EXIT(as, &as->a_lock); return (0); lockerr: /* * If the lower levels returned EDEADLK for a segment lockop, --- 2572,2582 ---- raddr += ssize; } if (func == MC_LOCK) kmem_free(mlock_map, mlock_size * sizeof (ulong_t)); ! AS_LOCK_EXIT(as); return (0); lockerr: /* * If the lower levels returned EDEADLK for a segment lockop,
*** 2637,2647 **** caddr_t eaddr; faultcode_t fault_err = 0; pgcnt_t pl_off; extern struct seg_ops segspt_shmops; ! ASSERT(AS_LOCK_HELD(as, &as->a_lock)); ASSERT(seg != NULL); ASSERT(addr >= seg->s_base && addr < seg->s_base + seg->s_size); ASSERT(addr + size > seg->s_base + seg->s_size); ASSERT(IS_P2ALIGNED(size, PAGESIZE)); ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); --- 2637,2647 ---- caddr_t eaddr; faultcode_t fault_err = 0; pgcnt_t pl_off; extern struct seg_ops segspt_shmops; ! ASSERT(AS_LOCK_HELD(as)); ASSERT(seg != NULL); ASSERT(addr >= seg->s_base && addr < seg->s_base + seg->s_size); ASSERT(addr + size > seg->s_base + seg->s_size); ASSERT(IS_P2ALIGNED(size, PAGESIZE)); ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
*** 2654,2664 **** for (; size != 0; size -= ssize, addr += ssize) { if (addr >= seg->s_base + seg->s_size) { seg = AS_SEGNEXT(as, seg); if (seg == NULL || addr != seg->s_base) { ! AS_LOCK_EXIT(as, &as->a_lock); return (EFAULT); } /* * Do a quick check if subsequent segments * will most likely support pagelock. --- 2654,2664 ---- for (; size != 0; size -= ssize, addr += ssize) { if (addr >= seg->s_base + seg->s_size) { seg = AS_SEGNEXT(as, seg); if (seg == NULL || addr != seg->s_base) { ! AS_LOCK_EXIT(as); return (EFAULT); } /* * Do a quick check if subsequent segments * will most likely support pagelock.
*** 2666,2680 **** if (seg->s_ops == &segvn_ops) { vnode_t *vp; if (SEGOP_GETVP(seg, addr, &vp) != 0 || vp != NULL) { ! AS_LOCK_EXIT(as, &as->a_lock); goto slow; } } else if (seg->s_ops != &segspt_shmops) { ! AS_LOCK_EXIT(as, &as->a_lock); goto slow; } segcnt++; } if (addr + size > seg->s_base + seg->s_size) { --- 2666,2680 ---- if (seg->s_ops == &segvn_ops) { vnode_t *vp; if (SEGOP_GETVP(seg, addr, &vp) != 0 || vp != NULL) { ! AS_LOCK_EXIT(as); goto slow; } } else if (seg->s_ops != &segspt_shmops) { ! AS_LOCK_EXIT(as); goto slow; } segcnt++; } if (addr + size > seg->s_base + seg->s_size) {
*** 2715,2725 **** btop(ssize) * sizeof (page_t *)); pl_off += btop(ssize); } if (size == 0) { ! AS_LOCK_EXIT(as, &as->a_lock); ASSERT(cnt == segcnt - 1); *ppp = plist; return (0); } --- 2715,2725 ---- btop(ssize) * sizeof (page_t *)); pl_off += btop(ssize); } if (size == 0) { ! AS_LOCK_EXIT(as); ASSERT(cnt == segcnt - 1); *ppp = plist; return (0); }
*** 2749,2759 **** ASSERT(*pl != NULL); (void) SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl, L_PAGEUNLOCK, rw); } ! AS_LOCK_EXIT(as, &as->a_lock); kmem_free(plist, (npages + segcnt) * sizeof (page_t *)); if (error != ENOTSUP && error != EFAULT) { return (error); --- 2749,2759 ---- ASSERT(*pl != NULL); (void) SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl, L_PAGEUNLOCK, rw); } ! AS_LOCK_EXIT(as); kmem_free(plist, (npages + segcnt) * sizeof (page_t *)); if (error != ENOTSUP && error != EFAULT) { return (error);
*** 2798,2820 **** /* * if the request crosses two segments let * as_fault handle it. */ ! AS_LOCK_ENTER(as, &as->a_lock, RW_READER); seg = as_segat(as, raddr); if (seg == NULL) { ! AS_LOCK_EXIT(as, &as->a_lock); return (EFAULT); } ASSERT(raddr >= seg->s_base && raddr < seg->s_base + seg->s_size); if (raddr + rsize > seg->s_base + seg->s_size) { return (as_pagelock_segs(as, seg, ppp, raddr, rsize, rw)); } if (raddr + rsize <= raddr) { ! AS_LOCK_EXIT(as, &as->a_lock); return (EFAULT); } TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_START, "seg_lock_1_start: raddr %p rsize %ld", raddr, rsize); --- 2798,2820 ---- /* * if the request crosses two segments let * as_fault handle it. */ ! AS_LOCK_ENTER(as, RW_READER); seg = as_segat(as, raddr); if (seg == NULL) { ! AS_LOCK_EXIT(as); return (EFAULT); } ASSERT(raddr >= seg->s_base && raddr < seg->s_base + seg->s_size); if (raddr + rsize > seg->s_base + seg->s_size) { return (as_pagelock_segs(as, seg, ppp, raddr, rsize, rw)); } if (raddr + rsize <= raddr) { ! AS_LOCK_EXIT(as); return (EFAULT); } TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_START, "seg_lock_1_start: raddr %p rsize %ld", raddr, rsize);
*** 2824,2834 **** */ err = SEGOP_PAGELOCK(seg, raddr, rsize, ppp, L_PAGELOCK, rw); TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_END, "seg_lock_1_end"); ! AS_LOCK_EXIT(as, &as->a_lock); if (err == 0 || (err != ENOTSUP && err != EFAULT)) { return (err); } --- 2824,2834 ---- */ err = SEGOP_PAGELOCK(seg, raddr, rsize, ppp, L_PAGELOCK, rw); TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_END, "seg_lock_1_end"); ! AS_LOCK_EXIT(as); if (err == 0 || (err != ENOTSUP && err != EFAULT)) { return (err); }
*** 2862,2872 **** caddr_t eaddr = addr + size; pgcnt_t npages = btop(size); size_t ssize; page_t **pl; ! ASSERT(AS_LOCK_HELD(as, &as->a_lock)); ASSERT(seg != NULL); ASSERT(addr >= seg->s_base && addr < seg->s_base + seg->s_size); ASSERT(addr + size > seg->s_base + seg->s_size); ASSERT(IS_P2ALIGNED(size, PAGESIZE)); ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); --- 2862,2872 ---- caddr_t eaddr = addr + size; pgcnt_t npages = btop(size); size_t ssize; page_t **pl; ! ASSERT(AS_LOCK_HELD(as)); ASSERT(seg != NULL); ASSERT(addr >= seg->s_base && addr < seg->s_base + seg->s_size); ASSERT(addr + size > seg->s_base + seg->s_size); ASSERT(IS_P2ALIGNED(size, PAGESIZE)); ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
*** 2887,2897 **** ASSERT(*pl != NULL); (void) SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl, L_PAGEUNLOCK, rw); } ASSERT(cnt > 0); ! AS_LOCK_EXIT(as, &as->a_lock); cnt++; kmem_free(plist, (npages + cnt) * sizeof (page_t *)); } --- 2887,2897 ---- ASSERT(*pl != NULL); (void) SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl, L_PAGEUNLOCK, rw); } ASSERT(cnt > 0); ! AS_LOCK_EXIT(as); cnt++; kmem_free(plist, (npages + cnt) * sizeof (page_t *)); }
*** 2920,2930 **** raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - (size_t)raddr; ! AS_LOCK_ENTER(as, &as->a_lock, RW_READER); seg = as_segat(as, raddr); ASSERT(seg != NULL); TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_UNLOCK_START, "seg_unlock_start: raddr %p rsize %ld", raddr, rsize); --- 2920,2930 ---- raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - (size_t)raddr; ! AS_LOCK_ENTER(as, RW_READER); seg = as_segat(as, raddr); ASSERT(seg != NULL); TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_UNLOCK_START, "seg_unlock_start: raddr %p rsize %ld", raddr, rsize);
*** 2934,2944 **** SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGEUNLOCK, rw); } else { as_pageunlock_segs(as, seg, raddr, rsize, pp, rw); return; } ! AS_LOCK_EXIT(as, &as->a_lock); TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_END, "as_pageunlock_end"); } int as_setpagesize(struct as *as, caddr_t addr, size_t size, uint_t szc, --- 2934,2944 ---- SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGEUNLOCK, rw); } else { as_pageunlock_segs(as, seg, raddr, rsize, pp, rw); return; } ! AS_LOCK_EXIT(as); TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_END, "as_pageunlock_end"); } int as_setpagesize(struct as *as, caddr_t addr, size_t size, uint_t szc,
*** 2960,2975 **** rsize = size; if (raddr + rsize < raddr) /* check for wraparound */ return (ENOMEM); ! AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); as_clearwatchprot(as, raddr, rsize); seg = as_segat(as, raddr); if (seg == NULL) { as_setwatch(as); ! AS_LOCK_EXIT(as, &as->a_lock); return (ENOMEM); } for (; rsize != 0; rsize -= ssize, raddr += ssize) { if (raddr >= seg->s_base + seg->s_size) { --- 2960,2975 ---- rsize = size; if (raddr + rsize < raddr) /* check for wraparound */ return (ENOMEM); ! AS_LOCK_ENTER(as, RW_WRITER); as_clearwatchprot(as, raddr, rsize); seg = as_segat(as, raddr); if (seg == NULL) { as_setwatch(as); ! AS_LOCK_EXIT(as); return (ENOMEM); } for (; rsize != 0; rsize -= ssize, raddr += ssize) { if (raddr >= seg->s_base + seg->s_size) {
*** 2992,3002 **** error = EAGAIN; break; } if (error == IE_RETRY) { ! AS_LOCK_EXIT(as, &as->a_lock); goto setpgsz_top; } if (error == ENOTSUP) { error = EINVAL; --- 2992,3002 ---- error = EAGAIN; break; } if (error == IE_RETRY) { ! AS_LOCK_EXIT(as); goto setpgsz_top; } if (error == ENOTSUP) { error = EINVAL;
*** 3032,3042 **** if (!AS_ISNOUNMAPWAIT(as)) { if (AS_ISUNMAPWAIT(as) == 0) { cv_broadcast(&as->a_cv); } AS_SETUNMAPWAIT(as); ! AS_LOCK_EXIT(as, &as->a_lock); while (AS_ISUNMAPWAIT(as)) { cv_wait(&as->a_cv, &as->a_contents); } } else { /* --- 3032,3042 ---- if (!AS_ISNOUNMAPWAIT(as)) { if (AS_ISUNMAPWAIT(as) == 0) { cv_broadcast(&as->a_cv); } AS_SETUNMAPWAIT(as); ! AS_LOCK_EXIT(as); while (AS_ISUNMAPWAIT(as)) { cv_wait(&as->a_cv, &as->a_contents); } } else { /*
*** 3058,3068 **** } else if (error != 0) { break; } } as_setwatch(as); ! AS_LOCK_EXIT(as, &as->a_lock); return (error); } /* * as_iset3_default_lpsize() just calls SEGOP_SETPAGESIZE() on all segments --- 3058,3068 ---- } else if (error != 0) { break; } } as_setwatch(as); ! AS_LOCK_EXIT(as); return (error); } /* * as_iset3_default_lpsize() just calls SEGOP_SETPAGESIZE() on all segments
*** 3074,3084 **** { struct seg *seg; size_t ssize; int error; ! ASSERT(AS_WRITE_HELD(as, &as->a_lock)); seg = as_segat(as, raddr); if (seg == NULL) { panic("as_iset3_default_lpsize: no seg"); } --- 3074,3084 ---- { struct seg *seg; size_t ssize; int error; ! ASSERT(AS_WRITE_HELD(as)); seg = as_segat(as, raddr); if (seg == NULL) { panic("as_iset3_default_lpsize: no seg"); }
*** 3131,3141 **** uint_t szcvec) { int error; int retry; ! ASSERT(AS_WRITE_HELD(as, &as->a_lock)); for (;;) { error = as_iset3_default_lpsize(as, addr, size, szc, &retry); if (error == EINVAL && retry) { szcvec &= ~(1 << szc); --- 3131,3141 ---- uint_t szcvec) { int error; int retry; ! ASSERT(AS_WRITE_HELD(as)); for (;;) { error = as_iset3_default_lpsize(as, addr, size, szc, &retry); if (error == EINVAL && retry) { szcvec &= ~(1 << szc);
*** 3163,3173 **** caddr_t setaddr = raddr; size_t setsize = 0; int set; int error; ! ASSERT(AS_WRITE_HELD(as, &as->a_lock)); seg = as_segat(as, raddr); if (seg == NULL) { panic("as_iset1_default_lpsize: no seg"); } --- 3163,3173 ---- caddr_t setaddr = raddr; size_t setsize = 0; int set; int error; ! ASSERT(AS_WRITE_HELD(as)); seg = as_segat(as, raddr); if (seg == NULL) { panic("as_iset1_default_lpsize: no seg"); }
*** 3231,3241 **** caddr_t eaddr; size_t segsize; size_t pgsz; uint_t save_szcvec; ! ASSERT(AS_WRITE_HELD(as, &as->a_lock)); ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); ASSERT(IS_P2ALIGNED(size, PAGESIZE)); szcvec &= ~1; if (szcvec <= 1) { /* skip if base page size */ --- 3231,3241 ---- caddr_t eaddr; size_t segsize; size_t pgsz; uint_t save_szcvec; ! ASSERT(AS_WRITE_HELD(as)); ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); ASSERT(IS_P2ALIGNED(size, PAGESIZE)); szcvec &= ~1; if (szcvec <= 1) { /* skip if base page size */
*** 3323,3349 **** int segvn; if (size == 0) return (0); ! AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); again: error = 0; raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - (size_t)raddr; if (raddr + rsize < raddr) { /* check for wraparound */ ! AS_LOCK_EXIT(as, &as->a_lock); return (ENOMEM); } as_clearwatchprot(as, raddr, rsize); seg = as_segat(as, raddr); if (seg == NULL) { as_setwatch(as); ! AS_LOCK_EXIT(as, &as->a_lock); return (ENOMEM); } if (seg->s_ops == &segvn_ops) { rtype = SEGOP_GETTYPE(seg, addr); rflags = rtype & (MAP_TEXT | MAP_INITDATA); --- 3323,3349 ---- int segvn; if (size == 0) return (0); ! AS_LOCK_ENTER(as, RW_WRITER); again: error = 0; raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - (size_t)raddr; if (raddr + rsize < raddr) { /* check for wraparound */ ! AS_LOCK_EXIT(as); return (ENOMEM); } as_clearwatchprot(as, raddr, rsize); seg = as_segat(as, raddr); if (seg == NULL) { as_setwatch(as); ! AS_LOCK_EXIT(as); return (ENOMEM); } if (seg->s_ops == &segvn_ops) { rtype = SEGOP_GETTYPE(seg, addr); rflags = rtype & (MAP_TEXT | MAP_INITDATA);
*** 3424,3439 **** if (!AS_ISNOUNMAPWAIT(as)) { if (AS_ISUNMAPWAIT(as) == 0) { cv_broadcast(&as->a_cv); } AS_SETUNMAPWAIT(as); ! AS_LOCK_EXIT(as, &as->a_lock); while (AS_ISUNMAPWAIT(as)) { cv_wait(&as->a_cv, &as->a_contents); } mutex_exit(&as->a_contents); ! AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); } else { /* * We may have raced with * segvn_reclaim()/segspt_reclaim(). In this case * clean nounmapwait flag and retry since softlockcnt --- 3424,3439 ---- if (!AS_ISNOUNMAPWAIT(as)) { if (AS_ISUNMAPWAIT(as) == 0) { cv_broadcast(&as->a_cv); } AS_SETUNMAPWAIT(as); ! AS_LOCK_EXIT(as); while (AS_ISUNMAPWAIT(as)) { cv_wait(&as->a_cv, &as->a_contents); } mutex_exit(&as->a_contents); ! AS_LOCK_ENTER(as, RW_WRITER); } else { /* * We may have raced with * segvn_reclaim()/segspt_reclaim(). In this case * clean nounmapwait flag and retry since softlockcnt
*** 3447,3457 **** } goto again; } as_setwatch(as); ! AS_LOCK_EXIT(as, &as->a_lock); return (error); } /* * Setup all of the uninitialized watched pages that we can. --- 3447,3457 ---- } goto again; } as_setwatch(as); ! AS_LOCK_EXIT(as); return (error); } /* * Setup all of the uninitialized watched pages that we can.
*** 3466,3476 **** int err, retrycnt; if (avl_numnodes(&as->a_wpage) == 0) return; ! ASSERT(AS_WRITE_HELD(as, &as->a_lock)); for (pwp = avl_first(&as->a_wpage); pwp != NULL; pwp = AVL_NEXT(&as->a_wpage, pwp)) { retrycnt = 0; retry: --- 3466,3476 ---- int err, retrycnt; if (avl_numnodes(&as->a_wpage) == 0) return; ! ASSERT(AS_WRITE_HELD(as)); for (pwp = avl_first(&as->a_wpage); pwp != NULL; pwp = AVL_NEXT(&as->a_wpage, pwp)) { retrycnt = 0; retry:
*** 3513,3523 **** int err, retrycnt; if (avl_numnodes(&as->a_wpage) == 0) return; ! ASSERT(AS_WRITE_HELD(as, &as->a_lock)); for (pwp = avl_first(&as->a_wpage); pwp != NULL; pwp = AVL_NEXT(&as->a_wpage, pwp)) { retrycnt = 0; retry: --- 3513,3523 ---- int err, retrycnt; if (avl_numnodes(&as->a_wpage) == 0) return; ! ASSERT(AS_WRITE_HELD(as)); for (pwp = avl_first(&as->a_wpage); pwp != NULL; pwp = AVL_NEXT(&as->a_wpage, pwp)) { retrycnt = 0; retry:
*** 3555,3565 **** avl_index_t where; if (avl_numnodes(&as->a_wpage) == 0) return; ! ASSERT(AS_WRITE_HELD(as, &as->a_lock)); tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL) pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER); --- 3555,3565 ---- avl_index_t where; if (avl_numnodes(&as->a_wpage) == 0) return; ! ASSERT(AS_WRITE_HELD(as)); tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL) pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER);
*** 3614,3624 **** tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL) pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER); ! ASSERT(AS_WRITE_HELD(as, &as->a_lock)); while (pwp != NULL && pwp->wp_vaddr < eaddr) { if ((prot = pwp->wp_oprot) != 0) { retrycnt = 0; --- 3614,3624 ---- tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL) pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER); ! ASSERT(AS_WRITE_HELD(as)); while (pwp != NULL && pwp->wp_vaddr < eaddr) { if ((prot = pwp->wp_oprot) != 0) { retrycnt = 0;
*** 3669,3692 **** as_getmemid(struct as *as, caddr_t addr, memid_t *memidp) { struct seg *seg; int sts; ! AS_LOCK_ENTER(as, &as->a_lock, RW_READER); seg = as_segat(as, addr); if (seg == NULL) { ! AS_LOCK_EXIT(as, &as->a_lock); return (EFAULT); } /* * catch old drivers which may not support getmemid */ if (seg->s_ops->getmemid == NULL) { ! AS_LOCK_EXIT(as, &as->a_lock); return (ENODEV); } sts = SEGOP_GETMEMID(seg, addr, memidp); ! AS_LOCK_EXIT(as, &as->a_lock); return (sts); } --- 3669,3692 ---- as_getmemid(struct as *as, caddr_t addr, memid_t *memidp) { struct seg *seg; int sts; ! AS_LOCK_ENTER(as, RW_READER); seg = as_segat(as, addr); if (seg == NULL) { ! AS_LOCK_EXIT(as); return (EFAULT); } /* * catch old drivers which may not support getmemid */ if (seg->s_ops->getmemid == NULL) { ! AS_LOCK_EXIT(as); return (ENODEV); } sts = SEGOP_GETMEMID(seg, addr, memidp); ! AS_LOCK_EXIT(as); return (sts); }