1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  * Copyright 2015, Joyent, Inc.  All rights reserved.
  25  */
  26 
  27 /*      Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T     */
  28 /*        All Rights Reserved   */
  29 
  30 /*
  31  * University Copyright- Copyright (c) 1982, 1986, 1988
  32  * The Regents of the University of California
  33  * All Rights Reserved
  34  *
  35  * University Acknowledgment- Portions of this document are derived from
  36  * software developed by the University of California, Berkeley, and its
  37  * contributors.
  38  */
  39 
  40 /*
  41  * VM - address spaces.
  42  */
  43 
  44 #include <sys/types.h>
  45 #include <sys/t_lock.h>
  46 #include <sys/param.h>
  47 #include <sys/errno.h>
  48 #include <sys/systm.h>
  49 #include <sys/mman.h>
  50 #include <sys/sysmacros.h>
  51 #include <sys/cpuvar.h>
  52 #include <sys/sysinfo.h>
  53 #include <sys/kmem.h>
  54 #include <sys/vnode.h>
  55 #include <sys/vmsystm.h>
  56 #include <sys/cmn_err.h>
  57 #include <sys/debug.h>
  58 #include <sys/tnf_probe.h>
  59 #include <sys/vtrace.h>
  60 
  61 #include <vm/hat.h>
  62 #include <vm/as.h>
  63 #include <vm/seg.h>
  64 #include <vm/seg_vn.h>
  65 #include <vm/seg_dev.h>
  66 #include <vm/seg_kmem.h>
  67 #include <vm/seg_map.h>
  68 #include <vm/seg_spt.h>
  69 #include <vm/page.h>
  70 
  71 clock_t deadlk_wait = 1; /* number of ticks to wait before retrying */
  72 
  73 static struct kmem_cache *as_cache;
  74 
  75 static void as_setwatchprot(struct as *, caddr_t, size_t, uint_t);
  76 static void as_clearwatchprot(struct as *, caddr_t, size_t);
  77 int as_map_locked(struct as *, caddr_t, size_t, int ((*)()), void *);
  78 
  79 
  80 /*
  81  * Verifying the segment lists is very time-consuming; it may not be
  82  * desirable always to define VERIFY_SEGLIST when DEBUG is set.
  83  */
  84 #ifdef DEBUG
  85 #define VERIFY_SEGLIST
  86 int do_as_verify = 0;
  87 #endif
  88 
  89 /*
  90  * Allocate a new callback data structure entry and fill in the events of
  91  * interest, the address range of interest, and the callback argument.
  92  * Link the entry on the as->a_callbacks list. A callback entry for the
  93  * entire address space may be specified with vaddr = 0 and size = -1.
  94  *
  95  * CALLERS RESPONSIBILITY: If not calling from within the process context for
  96  * the specified as, the caller must guarantee persistence of the specified as
  97  * for the duration of this function (eg. pages being locked within the as
  98  * will guarantee persistence).
  99  */
 100 int
 101 as_add_callback(struct as *as, void (*cb_func)(), void *arg, uint_t events,
 102                 caddr_t vaddr, size_t size, int sleepflag)
 103 {
 104         struct as_callback      *current_head, *cb;
 105         caddr_t                 saddr;
 106         size_t                  rsize;
 107 
 108         /* callback function and an event are mandatory */
 109         if ((cb_func == NULL) || ((events & AS_ALL_EVENT) == 0))
 110                 return (EINVAL);
 111 
 112         /* Adding a callback after as_free has been called is not allowed */
 113         if (as == &kas)
 114                 return (ENOMEM);
 115 
 116         /*
 117          * vaddr = 0 and size = -1 is used to indicate that the callback range
 118          * is the entire address space so no rounding is done in that case.
 119          */
 120         if (size != -1) {
 121                 saddr = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK);
 122                 rsize = (((size_t)(vaddr + size) + PAGEOFFSET) & PAGEMASK) -
 123                     (size_t)saddr;
 124                 /* check for wraparound */
 125                 if (saddr + rsize < saddr)
 126                         return (ENOMEM);
 127         } else {
 128                 if (vaddr != 0)
 129                         return (EINVAL);
 130                 saddr = vaddr;
 131                 rsize = size;
 132         }
 133 
 134         /* Allocate and initialize a callback entry */
 135         cb = kmem_zalloc(sizeof (struct as_callback), sleepflag);
 136         if (cb == NULL)
 137                 return (EAGAIN);
 138 
 139         cb->ascb_func = cb_func;
 140         cb->ascb_arg = arg;
 141         cb->ascb_events = events;
 142         cb->ascb_saddr = saddr;
 143         cb->ascb_len = rsize;
 144 
 145         /* Add the entry to the list */
 146         mutex_enter(&as->a_contents);
 147         current_head = as->a_callbacks;
 148         as->a_callbacks = cb;
 149         cb->ascb_next = current_head;
 150 
 151         /*
 152          * The call to this function may lose in a race with
 153          * a pertinent event - eg. a thread does long term memory locking
 154          * but before the callback is added another thread executes as_unmap.
 155          * A broadcast here resolves that.
 156          */
 157         if ((cb->ascb_events & AS_UNMAPWAIT_EVENT) && AS_ISUNMAPWAIT(as)) {
 158                 AS_CLRUNMAPWAIT(as);
 159                 cv_broadcast(&as->a_cv);
 160         }
 161 
 162         mutex_exit(&as->a_contents);
 163         return (0);
 164 }
 165 
 166 /*
 167  * Search the callback list for an entry which pertains to arg.
 168  *
 169  * This is called from within the client upon completion of the callback.
 170  * RETURN VALUES:
 171  *      AS_CALLBACK_DELETED  (callback entry found and deleted)
 172  *      AS_CALLBACK_NOTFOUND (no callback entry found - this is ok)
 173  *      AS_CALLBACK_DELETE_DEFERRED (callback is in process, delete of this
 174  *                      entry will be made in as_do_callbacks)
 175  *
 176  * If as_delete_callback encounters a matching entry with AS_CALLBACK_CALLED
 177  * set, it indicates that as_do_callbacks is processing this entry.  The
 178  * AS_ALL_EVENT events are cleared in the entry, and a broadcast is made
 179  * to unblock as_do_callbacks, in case it is blocked.
 180  *
 181  * CALLERS RESPONSIBILITY: If not calling from within the process context for
 182  * the specified as, the caller must guarantee persistence of the specified as
 183  * for the duration of this function (eg. pages being locked within the as
 184  * will guarantee persistence).
 185  */
 186 uint_t
 187 as_delete_callback(struct as *as, void *arg)
 188 {
 189         struct as_callback **prevcb = &as->a_callbacks;
 190         struct as_callback *cb;
 191         uint_t rc = AS_CALLBACK_NOTFOUND;
 192 
 193         mutex_enter(&as->a_contents);
 194         for (cb = as->a_callbacks; cb; prevcb = &cb->ascb_next, cb = *prevcb) {
 195                 if (cb->ascb_arg != arg)
 196                         continue;
 197 
 198                 /*
 199                  * If the events indicate AS_CALLBACK_CALLED, just clear
 200                  * AS_ALL_EVENT in the events field and wakeup the thread
 201                  * that may be waiting in as_do_callbacks.  as_do_callbacks
 202                  * will take care of removing this entry from the list.  In
 203                  * that case, return AS_CALLBACK_DELETE_DEFERRED.  Otherwise
 204                  * (AS_CALLBACK_CALLED not set), just remove it from the
 205                  * list, return the memory and return AS_CALLBACK_DELETED.
 206                  */
 207                 if ((cb->ascb_events & AS_CALLBACK_CALLED) != 0) {
 208                         /* leave AS_CALLBACK_CALLED */
 209                         cb->ascb_events &= ~AS_ALL_EVENT;
 210                         rc = AS_CALLBACK_DELETE_DEFERRED;
 211                         cv_broadcast(&as->a_cv);
 212                 } else {
 213                         *prevcb = cb->ascb_next;
 214                         kmem_free(cb, sizeof (struct as_callback));
 215                         rc = AS_CALLBACK_DELETED;
 216                 }
 217                 break;
 218         }
 219         mutex_exit(&as->a_contents);
 220         return (rc);
 221 }
 222 
 223 /*
 224  * Searches the as callback list for a matching entry.
 225  * Returns a pointer to the first matching callback, or NULL if
 226  * nothing is found.
 227  * This function never sleeps so it is ok to call it with more
 228  * locks held but the (required) a_contents mutex.
 229  *
 230  * See also comment on as_do_callbacks below.
 231  */
 232 static struct as_callback *
 233 as_find_callback(struct as *as, uint_t events, caddr_t event_addr,
 234                         size_t event_len)
 235 {
 236         struct as_callback      *cb;
 237 
 238         ASSERT(MUTEX_HELD(&as->a_contents));
 239         for (cb = as->a_callbacks; cb != NULL; cb = cb->ascb_next) {
 240                 /*
 241                  * If the callback has not already been called, then
 242                  * check if events or address range pertains.  An event_len
 243                  * of zero means do an unconditional callback.
 244                  */
 245                 if (((cb->ascb_events & AS_CALLBACK_CALLED) != 0) ||
 246                     ((event_len != 0) && (((cb->ascb_events & events) == 0) ||
 247                     (event_addr + event_len < cb->ascb_saddr) ||
 248                     (event_addr > (cb->ascb_saddr + cb->ascb_len))))) {
 249                         continue;
 250                 }
 251                 break;
 252         }
 253         return (cb);
 254 }
 255 
 256 /*
 257  * Executes a given callback and removes it from the callback list for
 258  * this address space.
 259  * This function may sleep so the caller must drop all locks except
 260  * a_contents before calling this func.
 261  *
 262  * See also comments on as_do_callbacks below.
 263  */
 264 static void
 265 as_execute_callback(struct as *as, struct as_callback *cb,
 266                                 uint_t events)
 267 {
 268         struct as_callback **prevcb;
 269         void    *cb_arg;
 270 
 271         ASSERT(MUTEX_HELD(&as->a_contents) && (cb->ascb_events & events));
 272         cb->ascb_events |= AS_CALLBACK_CALLED;
 273         mutex_exit(&as->a_contents);
 274         (*cb->ascb_func)(as, cb->ascb_arg, events);
 275         mutex_enter(&as->a_contents);
 276         /*
 277          * the callback function is required to delete the callback
 278          * when the callback function determines it is OK for
 279          * this thread to continue. as_delete_callback will clear
 280          * the AS_ALL_EVENT in the events field when it is deleted.
 281          * If the callback function called as_delete_callback,
 282          * events will already be cleared and there will be no blocking.
 283          */
 284         while ((cb->ascb_events & events) != 0) {
 285                 cv_wait(&as->a_cv, &as->a_contents);
 286         }
 287         /*
 288          * This entry needs to be taken off the list. Normally, the
 289          * callback func itself does that, but unfortunately the list
 290          * may have changed while the callback was running because the
 291          * a_contents mutex was dropped and someone else other than the
 292          * callback func itself could have called as_delete_callback,
 293          * so we have to search to find this entry again.  The entry
 294          * must have AS_CALLBACK_CALLED, and have the same 'arg'.
 295          */
 296         cb_arg = cb->ascb_arg;
 297         prevcb = &as->a_callbacks;
 298         for (cb = as->a_callbacks; cb != NULL;
 299             prevcb = &cb->ascb_next, cb = *prevcb) {
 300                 if (((cb->ascb_events & AS_CALLBACK_CALLED) == 0) ||
 301                     (cb_arg != cb->ascb_arg)) {
 302                         continue;
 303                 }
 304                 *prevcb = cb->ascb_next;
 305                 kmem_free(cb, sizeof (struct as_callback));
 306                 break;
 307         }
 308 }
 309 
 310 /*
 311  * Check the callback list for a matching event and intersection of
 312  * address range. If there is a match invoke the callback.  Skip an entry if:
 313  *    - a callback is already in progress for this entry (AS_CALLBACK_CALLED)
 314  *    - not event of interest
 315  *    - not address range of interest
 316  *
 317  * An event_len of zero indicates a request for an unconditional callback
 318  * (regardless of event), only the AS_CALLBACK_CALLED is checked.  The
 319  * a_contents lock must be dropped before a callback, so only one callback
 320  * can be done before returning. Return -1 (true) if a callback was
 321  * executed and removed from the list, else return 0 (false).
 322  *
 323  * The logically separate parts, i.e. finding a matching callback and
 324  * executing a given callback have been separated into two functions
 325  * so that they can be called with different sets of locks held beyond
 326  * the always-required a_contents. as_find_callback does not sleep so
 327  * it is ok to call it if more locks than a_contents (i.e. the a_lock
 328  * rwlock) are held. as_execute_callback on the other hand may sleep
 329  * so all locks beyond a_contents must be dropped by the caller if one
 330  * does not want to end comatose.
 331  */
 332 static int
 333 as_do_callbacks(struct as *as, uint_t events, caddr_t event_addr,
 334                         size_t event_len)
 335 {
 336         struct as_callback *cb;
 337 
 338         if ((cb = as_find_callback(as, events, event_addr, event_len))) {
 339                 as_execute_callback(as, cb, events);
 340                 return (-1);
 341         }
 342         return (0);
 343 }
 344 
 345 /*
 346  * Search for the segment containing addr. If a segment containing addr
 347  * exists, that segment is returned.  If no such segment exists, and
 348  * the list spans addresses greater than addr, then the first segment
 349  * whose base is greater than addr is returned; otherwise, NULL is
 350  * returned unless tail is true, in which case the last element of the
 351  * list is returned.
 352  *
 353  * a_seglast is used to cache the last found segment for repeated
 354  * searches to the same addr (which happens frequently).
 355  */
 356 struct seg *
 357 as_findseg(struct as *as, caddr_t addr, int tail)
 358 {
 359         struct seg *seg = as->a_seglast;
 360         avl_index_t where;
 361 
 362         ASSERT(AS_LOCK_HELD(as));
 363 
 364         if (seg != NULL &&
 365             seg->s_base <= addr &&
 366             addr < seg->s_base + seg->s_size)
 367                 return (seg);
 368 
 369         seg = avl_find(&as->a_segtree, &addr, &where);
 370         if (seg != NULL)
 371                 return (as->a_seglast = seg);
 372 
 373         seg = avl_nearest(&as->a_segtree, where, AVL_AFTER);
 374         if (seg == NULL && tail)
 375                 seg = avl_last(&as->a_segtree);
 376         return (as->a_seglast = seg);
 377 }
 378 
 379 #ifdef VERIFY_SEGLIST
 380 /*
 381  * verify that the linked list is coherent
 382  */
 383 static void
 384 as_verify(struct as *as)
 385 {
 386         struct seg *seg, *seglast, *p, *n;
 387         uint_t nsegs = 0;
 388 
 389         if (do_as_verify == 0)
 390                 return;
 391 
 392         seglast = as->a_seglast;
 393 
 394         for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
 395                 ASSERT(seg->s_as == as);
 396                 p = AS_SEGPREV(as, seg);
 397                 n = AS_SEGNEXT(as, seg);
 398                 ASSERT(p == NULL || p->s_as == as);
 399                 ASSERT(p == NULL || p->s_base < seg->s_base);
 400                 ASSERT(n == NULL || n->s_base > seg->s_base);
 401                 ASSERT(n != NULL || seg == avl_last(&as->a_segtree));
 402                 if (seg == seglast)
 403                         seglast = NULL;
 404                 nsegs++;
 405         }
 406         ASSERT(seglast == NULL);
 407         ASSERT(avl_numnodes(&as->a_segtree) == nsegs);
 408 }
 409 #endif /* VERIFY_SEGLIST */
 410 
 411 /*
 412  * Add a new segment to the address space. The avl_find()
 413  * may be expensive so we attempt to use last segment accessed
 414  * in as_gap() as an insertion point.
 415  */
 416 int
 417 as_addseg(struct as  *as, struct seg *newseg)
 418 {
 419         struct seg *seg;
 420         caddr_t addr;
 421         caddr_t eaddr;
 422         avl_index_t where;
 423 
 424         ASSERT(AS_WRITE_HELD(as));
 425 
 426         as->a_updatedir = 1; /* inform /proc */
 427         gethrestime(&as->a_updatetime);
 428 
 429         if (as->a_lastgaphl != NULL) {
 430                 struct seg *hseg = NULL;
 431                 struct seg *lseg = NULL;
 432 
 433                 if (as->a_lastgaphl->s_base > newseg->s_base) {
 434                         hseg = as->a_lastgaphl;
 435                         lseg = AVL_PREV(&as->a_segtree, hseg);
 436                 } else {
 437                         lseg = as->a_lastgaphl;
 438                         hseg = AVL_NEXT(&as->a_segtree, lseg);
 439                 }
 440 
 441                 if (hseg && lseg && lseg->s_base < newseg->s_base &&
 442                     hseg->s_base > newseg->s_base) {
 443                         avl_insert_here(&as->a_segtree, newseg, lseg,
 444                             AVL_AFTER);
 445                         as->a_lastgaphl = NULL;
 446                         as->a_seglast = newseg;
 447                         return (0);
 448                 }
 449                 as->a_lastgaphl = NULL;
 450         }
 451 
 452         addr = newseg->s_base;
 453         eaddr = addr + newseg->s_size;
 454 again:
 455 
 456         seg = avl_find(&as->a_segtree, &addr, &where);
 457 
 458         if (seg == NULL)
 459                 seg = avl_nearest(&as->a_segtree, where, AVL_AFTER);
 460 
 461         if (seg == NULL)
 462                 seg = avl_last(&as->a_segtree);
 463 
 464         if (seg != NULL) {
 465                 caddr_t base = seg->s_base;
 466 
 467                 /*
 468                  * If top of seg is below the requested address, then
 469                  * the insertion point is at the end of the linked list,
 470                  * and seg points to the tail of the list.  Otherwise,
 471                  * the insertion point is immediately before seg.
 472                  */
 473                 if (base + seg->s_size > addr) {
 474                         if (addr >= base || eaddr > base) {
 475 #ifdef __sparc
 476                                 extern struct seg_ops segnf_ops;
 477 
 478                                 /*
 479                                  * no-fault segs must disappear if overlaid.
 480                                  * XXX need new segment type so
 481                                  * we don't have to check s_ops
 482                                  */
 483                                 if (seg->s_ops == &segnf_ops) {
 484                                         seg_unmap(seg);
 485                                         goto again;
 486                                 }
 487 #endif
 488                                 return (-1);    /* overlapping segment */
 489                         }
 490                 }
 491         }
 492         as->a_seglast = newseg;
 493         avl_insert(&as->a_segtree, newseg, where);
 494 
 495 #ifdef VERIFY_SEGLIST
 496         as_verify(as);
 497 #endif
 498         return (0);
 499 }
 500 
 501 struct seg *
 502 as_removeseg(struct as *as, struct seg *seg)
 503 {
 504         avl_tree_t *t;
 505 
 506         ASSERT(AS_WRITE_HELD(as));
 507 
 508         as->a_updatedir = 1; /* inform /proc */
 509         gethrestime(&as->a_updatetime);
 510 
 511         if (seg == NULL)
 512                 return (NULL);
 513 
 514         t = &as->a_segtree;
 515         if (as->a_seglast == seg)
 516                 as->a_seglast = NULL;
 517         as->a_lastgaphl = NULL;
 518 
 519         /*
 520          * if this segment is at an address higher than
 521          * a_lastgap, set a_lastgap to the next segment (NULL if last segment)
 522          */
 523         if (as->a_lastgap &&
 524             (seg == as->a_lastgap || seg->s_base > as->a_lastgap->s_base))
 525                 as->a_lastgap = AVL_NEXT(t, seg);
 526 
 527         /*
 528          * remove the segment from the seg tree
 529          */
 530         avl_remove(t, seg);
 531 
 532 #ifdef VERIFY_SEGLIST
 533         as_verify(as);
 534 #endif
 535         return (seg);
 536 }
 537 
 538 /*
 539  * Find a segment containing addr.
 540  */
 541 struct seg *
 542 as_segat(struct as *as, caddr_t addr)
 543 {
 544         struct seg *seg = as->a_seglast;
 545 
 546         ASSERT(AS_LOCK_HELD(as));
 547 
 548         if (seg != NULL && seg->s_base <= addr &&
 549             addr < seg->s_base + seg->s_size)
 550                 return (seg);
 551 
 552         seg = avl_find(&as->a_segtree, &addr, NULL);
 553         return (seg);
 554 }
 555 
 556 /*
 557  * Serialize all searches for holes in an address space to
 558  * prevent two or more threads from allocating the same virtual
 559  * address range.  The address space must not be "read/write"
 560  * locked by the caller since we may block.
 561  */
 562 void
 563 as_rangelock(struct as *as)
 564 {
 565         mutex_enter(&as->a_contents);
 566         while (AS_ISCLAIMGAP(as))
 567                 cv_wait(&as->a_cv, &as->a_contents);
 568         AS_SETCLAIMGAP(as);
 569         mutex_exit(&as->a_contents);
 570 }
 571 
 572 /*
 573  * Release hold on a_state & AS_CLAIMGAP and signal any other blocked threads.
 574  */
 575 void
 576 as_rangeunlock(struct as *as)
 577 {
 578         mutex_enter(&as->a_contents);
 579         AS_CLRCLAIMGAP(as);
 580         cv_signal(&as->a_cv);
 581         mutex_exit(&as->a_contents);
 582 }
 583 
 584 /*
 585  * compar segments (or just an address) by segment address range
 586  */
 587 static int
 588 as_segcompar(const void *x, const void *y)
 589 {
 590         struct seg *a = (struct seg *)x;
 591         struct seg *b = (struct seg *)y;
 592 
 593         if (a->s_base < b->s_base)
 594                 return (-1);
 595         if (a->s_base >= b->s_base + b->s_size)
 596                 return (1);
 597         return (0);
 598 }
 599 
 600 
 601 void
 602 as_avlinit(struct as *as)
 603 {
 604         avl_create(&as->a_segtree, as_segcompar, sizeof (struct seg),
 605             offsetof(struct seg, s_tree));
 606         avl_create(&as->a_wpage, wp_compare, sizeof (struct watched_page),
 607             offsetof(struct watched_page, wp_link));
 608 }
 609 
 610 /*ARGSUSED*/
 611 static int
 612 as_constructor(void *buf, void *cdrarg, int kmflags)
 613 {
 614         struct as *as = buf;
 615 
 616         mutex_init(&as->a_contents, NULL, MUTEX_DEFAULT, NULL);
 617         cv_init(&as->a_cv, NULL, CV_DEFAULT, NULL);
 618         rw_init(&as->a_lock, NULL, RW_DEFAULT, NULL);
 619         as_avlinit(as);
 620         return (0);
 621 }
 622 
 623 /*ARGSUSED1*/
 624 static void
 625 as_destructor(void *buf, void *cdrarg)
 626 {
 627         struct as *as = buf;
 628 
 629         avl_destroy(&as->a_segtree);
 630         mutex_destroy(&as->a_contents);
 631         cv_destroy(&as->a_cv);
 632         rw_destroy(&as->a_lock);
 633 }
 634 
 635 void
 636 as_init(void)
 637 {
 638         as_cache = kmem_cache_create("as_cache", sizeof (struct as), 0,
 639             as_constructor, as_destructor, NULL, NULL, NULL, 0);
 640 }
 641 
 642 /*
 643  * Allocate and initialize an address space data structure.
 644  * We call hat_alloc to allow any machine dependent
 645  * information in the hat structure to be initialized.
 646  */
 647 struct as *
 648 as_alloc(void)
 649 {
 650         struct as *as;
 651 
 652         as = kmem_cache_alloc(as_cache, KM_SLEEP);
 653 
 654         as->a_flags          = 0;
 655         as->a_vbits          = 0;
 656         as->a_hrm            = NULL;
 657         as->a_seglast                = NULL;
 658         as->a_size           = 0;
 659         as->a_resvsize               = 0;
 660         as->a_updatedir              = 0;
 661         gethrestime(&as->a_updatetime);
 662         as->a_objectdir              = NULL;
 663         as->a_sizedir                = 0;
 664         as->a_userlimit              = (caddr_t)USERLIMIT;
 665         as->a_lastgap                = NULL;
 666         as->a_lastgaphl              = NULL;
 667         as->a_callbacks              = NULL;
 668 
 669         AS_LOCK_ENTER(as, RW_WRITER);
 670         as->a_hat = hat_alloc(as);   /* create hat for default system mmu */
 671         AS_LOCK_EXIT(as);
 672 
 673         return (as);
 674 }
 675 
 676 /*
 677  * Free an address space data structure.
 678  * Need to free the hat first and then
 679  * all the segments on this as and finally
 680  * the space for the as struct itself.
 681  */
 682 void
 683 as_free(struct as *as)
 684 {
 685         struct hat *hat = as->a_hat;
 686         struct seg *seg, *next;
 687         boolean_t free_started = B_FALSE;
 688 
 689 top:
 690         /*
 691          * Invoke ALL callbacks. as_do_callbacks will do one callback
 692          * per call, and not return (-1) until the callback has completed.
 693          * When as_do_callbacks returns zero, all callbacks have completed.
 694          */
 695         mutex_enter(&as->a_contents);
 696         while (as->a_callbacks && as_do_callbacks(as, AS_ALL_EVENT, 0, 0))
 697                 ;
 698 
 699         mutex_exit(&as->a_contents);
 700         AS_LOCK_ENTER(as, RW_WRITER);
 701 
 702         if (!free_started) {
 703                 free_started = B_TRUE;
 704                 hat_free_start(hat);
 705         }
 706         for (seg = AS_SEGFIRST(as); seg != NULL; seg = next) {
 707                 int err;
 708 
 709                 next = AS_SEGNEXT(as, seg);
 710 retry:
 711                 err = SEGOP_UNMAP(seg, seg->s_base, seg->s_size);
 712                 if (err == EAGAIN) {
 713                         mutex_enter(&as->a_contents);
 714                         if (as->a_callbacks) {
 715                                 AS_LOCK_EXIT(as);
 716                         } else if (!AS_ISNOUNMAPWAIT(as)) {
 717                                 /*
 718                                  * Memory is currently locked. Wait for a
 719                                  * cv_signal that it has been unlocked, then
 720                                  * try the operation again.
 721                                  */
 722                                 if (AS_ISUNMAPWAIT(as) == 0)
 723                                         cv_broadcast(&as->a_cv);
 724                                 AS_SETUNMAPWAIT(as);
 725                                 AS_LOCK_EXIT(as);
 726                                 while (AS_ISUNMAPWAIT(as))
 727                                         cv_wait(&as->a_cv, &as->a_contents);
 728                         } else {
 729                                 /*
 730                                  * We may have raced with
 731                                  * segvn_reclaim()/segspt_reclaim(). In this
 732                                  * case clean nounmapwait flag and retry since
 733                                  * softlockcnt in this segment may be already
 734                                  * 0.  We don't drop as writer lock so our
 735                                  * number of retries without sleeping should
 736                                  * be very small. See segvn_reclaim() for
 737                                  * more comments.
 738                                  */
 739                                 AS_CLRNOUNMAPWAIT(as);
 740                                 mutex_exit(&as->a_contents);
 741                                 goto retry;
 742                         }
 743                         mutex_exit(&as->a_contents);
 744                         goto top;
 745                 } else {
 746                         /*
 747                          * We do not expect any other error return at this
 748                          * time. This is similar to an ASSERT in seg_unmap()
 749                          */
 750                         ASSERT(err == 0);
 751                 }
 752         }
 753         hat_free_end(hat);
 754         AS_LOCK_EXIT(as);
 755 
 756         /* /proc stuff */
 757         ASSERT(avl_numnodes(&as->a_wpage) == 0);
 758         if (as->a_objectdir) {
 759                 kmem_free(as->a_objectdir, as->a_sizedir * sizeof (vnode_t *));
 760                 as->a_objectdir = NULL;
 761                 as->a_sizedir = 0;
 762         }
 763 
 764         /*
 765          * Free the struct as back to kmem.  Assert it has no segments.
 766          */
 767         ASSERT(avl_numnodes(&as->a_segtree) == 0);
 768         kmem_cache_free(as_cache, as);
 769 }
 770 
 771 int
 772 as_dup(struct as *as, struct proc *forkedproc)
 773 {
 774         struct as *newas;
 775         struct seg *seg, *newseg;
 776         size_t  purgesize = 0;
 777         int error;
 778 
 779         AS_LOCK_ENTER(as, RW_WRITER);
 780         as_clearwatch(as);
 781         newas = as_alloc();
 782         newas->a_userlimit = as->a_userlimit;
 783         newas->a_proc = forkedproc;
 784 
 785         AS_LOCK_ENTER(newas, RW_WRITER);
 786 
 787         (void) hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_SRD);
 788 
 789         for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
 790 
 791                 if (seg->s_flags & S_PURGE) {
 792                         purgesize += seg->s_size;
 793                         continue;
 794                 }
 795 
 796                 newseg = seg_alloc(newas, seg->s_base, seg->s_size);
 797                 if (newseg == NULL) {
 798                         AS_LOCK_EXIT(newas);
 799                         as_setwatch(as);
 800                         AS_LOCK_EXIT(as);
 801                         as_free(newas);
 802                         return (-1);
 803                 }
 804                 if ((error = SEGOP_DUP(seg, newseg)) != 0) {
 805                         /*
 806                          * We call seg_free() on the new seg
 807                          * because the segment is not set up
 808                          * completely; i.e. it has no ops.
 809                          */
 810                         as_setwatch(as);
 811                         AS_LOCK_EXIT(as);
 812                         seg_free(newseg);
 813                         AS_LOCK_EXIT(newas);
 814                         as_free(newas);
 815                         return (error);
 816                 }
 817                 newas->a_size += seg->s_size;
 818         }
 819         newas->a_resvsize = as->a_resvsize - purgesize;
 820 
 821         error = hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_ALL);
 822 
 823         AS_LOCK_EXIT(newas);
 824 
 825         as_setwatch(as);
 826         AS_LOCK_EXIT(as);
 827         if (error != 0) {
 828                 as_free(newas);
 829                 return (error);
 830         }
 831         forkedproc->p_as = newas;
 832         return (0);
 833 }
 834 
 835 /*
 836  * Handle a ``fault'' at addr for size bytes.
 837  */
 838 faultcode_t
 839 as_fault(struct hat *hat, struct as *as, caddr_t addr, size_t size,
 840         enum fault_type type, enum seg_rw rw)
 841 {
 842         struct seg *seg;
 843         caddr_t raddr;                  /* rounded down addr */
 844         size_t rsize;                   /* rounded up size */
 845         size_t ssize;
 846         faultcode_t res = 0;
 847         caddr_t addrsav;
 848         struct seg *segsav;
 849         int as_lock_held;
 850         klwp_t *lwp = ttolwp(curthread);
 851 
 852 
 853 
 854 retry:
 855         /*
 856          * Indicate that the lwp is not to be stopped while waiting for a
 857          * pagefault.  This is to avoid deadlock while debugging a process
 858          * via /proc over NFS (in particular).
 859          */
 860         if (lwp != NULL)
 861                 lwp->lwp_nostop++;
 862 
 863         /*
 864          * same length must be used when we softlock and softunlock.  We
 865          * don't support softunlocking lengths less than the original length
 866          * when there is largepage support.  See seg_dev.c for more
 867          * comments.
 868          */
 869         switch (type) {
 870 
 871         case F_SOFTLOCK:
 872                 CPU_STATS_ADD_K(vm, softlock, 1);
 873                 break;
 874 
 875         case F_SOFTUNLOCK:
 876                 break;
 877 
 878         case F_PROT:
 879                 CPU_STATS_ADD_K(vm, prot_fault, 1);
 880                 break;
 881 
 882         case F_INVAL:
 883                 CPU_STATS_ENTER_K();
 884                 CPU_STATS_ADDQ(CPU, vm, as_fault, 1);
 885                 if (as == &kas)
 886                         CPU_STATS_ADDQ(CPU, vm, kernel_asflt, 1);
 887                 CPU_STATS_EXIT_K();
 888                 break;
 889         }
 890 
 891         /* Kernel probe */
 892         TNF_PROBE_3(address_fault, "vm pagefault", /* CSTYLED */,
 893             tnf_opaque, address,        addr,
 894             tnf_fault_type,     fault_type,     type,
 895             tnf_seg_access,     access,         rw);
 896 
 897         raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
 898         rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
 899             (size_t)raddr;
 900 
 901         /*
 902          * XXX -- Don't grab the as lock for segkmap. We should grab it for
 903          * correctness, but then we could be stuck holding this lock for
 904          * a LONG time if the fault needs to be resolved on a slow
 905          * filesystem, and then no-one will be able to exec new commands,
 906          * as exec'ing requires the write lock on the as.
 907          */
 908         if (as == &kas && segkmap && segkmap->s_base <= raddr &&
 909             raddr + size < segkmap->s_base + segkmap->s_size) {
 910                 seg = segkmap;
 911                 as_lock_held = 0;
 912         } else {
 913                 AS_LOCK_ENTER(as, RW_READER);
 914 
 915                 seg = as_segat(as, raddr);
 916                 if (seg == NULL) {
 917                         AS_LOCK_EXIT(as);
 918                         if (lwp != NULL)
 919                                 lwp->lwp_nostop--;
 920                         return (FC_NOMAP);
 921                 }
 922 
 923                 as_lock_held = 1;
 924         }
 925 
 926         addrsav = raddr;
 927         segsav = seg;
 928 
 929         for (; rsize != 0; rsize -= ssize, raddr += ssize) {
 930                 if (raddr >= seg->s_base + seg->s_size) {
 931                         seg = AS_SEGNEXT(as, seg);
 932                         if (seg == NULL || raddr != seg->s_base) {
 933                                 res = FC_NOMAP;
 934                                 break;
 935                         }
 936                 }
 937                 if (raddr + rsize > seg->s_base + seg->s_size)
 938                         ssize = seg->s_base + seg->s_size - raddr;
 939                 else
 940                         ssize = rsize;
 941 
 942                 res = SEGOP_FAULT(hat, seg, raddr, ssize, type, rw);
 943                 if (res != 0)
 944                         break;
 945         }
 946 
 947         /*
 948          * If we were SOFTLOCKing and encountered a failure,
 949          * we must SOFTUNLOCK the range we already did. (Maybe we
 950          * should just panic if we are SOFTLOCKing or even SOFTUNLOCKing
 951          * right here...)
 952          */
 953         if (res != 0 && type == F_SOFTLOCK) {
 954                 for (seg = segsav; addrsav < raddr; addrsav += ssize) {
 955                         if (addrsav >= seg->s_base + seg->s_size)
 956                                 seg = AS_SEGNEXT(as, seg);
 957                         ASSERT(seg != NULL);
 958                         /*
 959                          * Now call the fault routine again to perform the
 960                          * unlock using S_OTHER instead of the rw variable
 961                          * since we never got a chance to touch the pages.
 962                          */
 963                         if (raddr > seg->s_base + seg->s_size)
 964                                 ssize = seg->s_base + seg->s_size - addrsav;
 965                         else
 966                                 ssize = raddr - addrsav;
 967                         (void) SEGOP_FAULT(hat, seg, addrsav, ssize,
 968                             F_SOFTUNLOCK, S_OTHER);
 969                 }
 970         }
 971         if (as_lock_held)
 972                 AS_LOCK_EXIT(as);
 973         if (lwp != NULL)
 974                 lwp->lwp_nostop--;
 975 
 976         /*
 977          * If the lower levels returned EDEADLK for a fault,
 978          * It means that we should retry the fault.  Let's wait
 979          * a bit also to let the deadlock causing condition clear.
 980          * This is part of a gross hack to work around a design flaw
 981          * in the ufs/sds logging code and should go away when the
 982          * logging code is re-designed to fix the problem. See bug
 983          * 4125102 for details of the problem.
 984          */
 985         if (FC_ERRNO(res) == EDEADLK) {
 986                 delay(deadlk_wait);
 987                 res = 0;
 988                 goto retry;
 989         }
 990         return (res);
 991 }
 992 
 993 
 994 
 995 /*
 996  * Asynchronous ``fault'' at addr for size bytes.
 997  */
 998 faultcode_t
 999 as_faulta(struct as *as, caddr_t addr, size_t size)
1000 {
1001         struct seg *seg;
1002         caddr_t raddr;                  /* rounded down addr */
1003         size_t rsize;                   /* rounded up size */
1004         faultcode_t res = 0;
1005         klwp_t *lwp = ttolwp(curthread);
1006 
1007 retry:
1008         /*
1009          * Indicate that the lwp is not to be stopped while waiting
1010          * for a pagefault.  This is to avoid deadlock while debugging
1011          * a process via /proc over NFS (in particular).
1012          */
1013         if (lwp != NULL)
1014                 lwp->lwp_nostop++;
1015 
1016         raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1017         rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1018             (size_t)raddr;
1019 
1020         AS_LOCK_ENTER(as, RW_READER);
1021         seg = as_segat(as, raddr);
1022         if (seg == NULL) {
1023                 AS_LOCK_EXIT(as);
1024                 if (lwp != NULL)
1025                         lwp->lwp_nostop--;
1026                 return (FC_NOMAP);
1027         }
1028 
1029         for (; rsize != 0; rsize -= PAGESIZE, raddr += PAGESIZE) {
1030                 if (raddr >= seg->s_base + seg->s_size) {
1031                         seg = AS_SEGNEXT(as, seg);
1032                         if (seg == NULL || raddr != seg->s_base) {
1033                                 res = FC_NOMAP;
1034                                 break;
1035                         }
1036                 }
1037                 res = SEGOP_FAULTA(seg, raddr);
1038                 if (res != 0)
1039                         break;
1040         }
1041         AS_LOCK_EXIT(as);
1042         if (lwp != NULL)
1043                 lwp->lwp_nostop--;
1044         /*
1045          * If the lower levels returned EDEADLK for a fault,
1046          * It means that we should retry the fault.  Let's wait
1047          * a bit also to let the deadlock causing condition clear.
1048          * This is part of a gross hack to work around a design flaw
1049          * in the ufs/sds logging code and should go away when the
1050          * logging code is re-designed to fix the problem. See bug
1051          * 4125102 for details of the problem.
1052          */
1053         if (FC_ERRNO(res) == EDEADLK) {
1054                 delay(deadlk_wait);
1055                 res = 0;
1056                 goto retry;
1057         }
1058         return (res);
1059 }
1060 
1061 /*
1062  * Set the virtual mapping for the interval from [addr : addr + size)
1063  * in address space `as' to have the specified protection.
1064  * It is ok for the range to cross over several segments,
1065  * as long as they are contiguous.
1066  */
1067 int
1068 as_setprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
1069 {
1070         struct seg *seg;
1071         struct as_callback *cb;
1072         size_t ssize;
1073         caddr_t raddr;                  /* rounded down addr */
1074         size_t rsize;                   /* rounded up size */
1075         int error = 0, writer = 0;
1076         caddr_t saveraddr;
1077         size_t saversize;
1078 
1079 setprot_top:
1080         raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1081         rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1082             (size_t)raddr;
1083 
1084         if (raddr + rsize < raddr)           /* check for wraparound */
1085                 return (ENOMEM);
1086 
1087         saveraddr = raddr;
1088         saversize = rsize;
1089 
1090         /*
1091          * Normally we only lock the as as a reader. But
1092          * if due to setprot the segment driver needs to split
1093          * a segment it will return IE_RETRY. Therefore we re-acquire
1094          * the as lock as a writer so the segment driver can change
1095          * the seg list. Also the segment driver will return IE_RETRY
1096          * after it has changed the segment list so we therefore keep
1097          * locking as a writer. Since these opeartions should be rare
1098          * want to only lock as a writer when necessary.
1099          */
1100         if (writer || avl_numnodes(&as->a_wpage) != 0) {
1101                 AS_LOCK_ENTER(as, RW_WRITER);
1102         } else {
1103                 AS_LOCK_ENTER(as, RW_READER);
1104         }
1105 
1106         as_clearwatchprot(as, raddr, rsize);
1107         seg = as_segat(as, raddr);
1108         if (seg == NULL) {
1109                 as_setwatch(as);
1110                 AS_LOCK_EXIT(as);
1111                 return (ENOMEM);
1112         }
1113 
1114         for (; rsize != 0; rsize -= ssize, raddr += ssize) {
1115                 if (raddr >= seg->s_base + seg->s_size) {
1116                         seg = AS_SEGNEXT(as, seg);
1117                         if (seg == NULL || raddr != seg->s_base) {
1118                                 error = ENOMEM;
1119                                 break;
1120                         }
1121                 }
1122                 if ((raddr + rsize) > (seg->s_base + seg->s_size))
1123                         ssize = seg->s_base + seg->s_size - raddr;
1124                 else
1125                         ssize = rsize;
1126 retry:
1127                 error = SEGOP_SETPROT(seg, raddr, ssize, prot);
1128 
1129                 if (error == IE_NOMEM) {
1130                         error = EAGAIN;
1131                         break;
1132                 }
1133 
1134                 if (error == IE_RETRY) {
1135                         AS_LOCK_EXIT(as);
1136                         writer = 1;
1137                         goto setprot_top;
1138                 }
1139 
1140                 if (error == EAGAIN) {
1141                         /*
1142                          * Make sure we have a_lock as writer.
1143                          */
1144                         if (writer == 0) {
1145                                 AS_LOCK_EXIT(as);
1146                                 writer = 1;
1147                                 goto setprot_top;
1148                         }
1149 
1150                         /*
1151                          * Memory is currently locked.  It must be unlocked
1152                          * before this operation can succeed through a retry.
1153                          * The possible reasons for locked memory and
1154                          * corresponding strategies for unlocking are:
1155                          * (1) Normal I/O
1156                          *      wait for a signal that the I/O operation
1157                          *      has completed and the memory is unlocked.
1158                          * (2) Asynchronous I/O
1159                          *      The aio subsystem does not unlock pages when
1160                          *      the I/O is completed. Those pages are unlocked
1161                          *      when the application calls aiowait/aioerror.
1162                          *      So, to prevent blocking forever, cv_broadcast()
1163                          *      is done to wake up aio_cleanup_thread.
1164                          *      Subsequently, segvn_reclaim will be called, and
1165                          *      that will do AS_CLRUNMAPWAIT() and wake us up.
1166                          * (3) Long term page locking:
1167                          *      Drivers intending to have pages locked for a
1168                          *      period considerably longer than for normal I/O
1169                          *      (essentially forever) may have registered for a
1170                          *      callback so they may unlock these pages on
1171                          *      request. This is needed to allow this operation
1172                          *      to succeed. Each entry on the callback list is
1173                          *      examined. If the event or address range pertains
1174                          *      the callback is invoked (unless it already is in
1175                          *      progress). The a_contents lock must be dropped
1176                          *      before the callback, so only one callback can
1177                          *      be done at a time. Go to the top and do more
1178                          *      until zero is returned. If zero is returned,
1179                          *      either there were no callbacks for this event
1180                          *      or they were already in progress.
1181                          */
1182                         mutex_enter(&as->a_contents);
1183                         if (as->a_callbacks &&
1184                             (cb = as_find_callback(as, AS_SETPROT_EVENT,
1185                             seg->s_base, seg->s_size))) {
1186                                 AS_LOCK_EXIT(as);
1187                                 as_execute_callback(as, cb, AS_SETPROT_EVENT);
1188                         } else if (!AS_ISNOUNMAPWAIT(as)) {
1189                                 if (AS_ISUNMAPWAIT(as) == 0)
1190                                         cv_broadcast(&as->a_cv);
1191                                 AS_SETUNMAPWAIT(as);
1192                                 AS_LOCK_EXIT(as);
1193                                 while (AS_ISUNMAPWAIT(as))
1194                                         cv_wait(&as->a_cv, &as->a_contents);
1195                         } else {
1196                                 /*
1197                                  * We may have raced with
1198                                  * segvn_reclaim()/segspt_reclaim(). In this
1199                                  * case clean nounmapwait flag and retry since
1200                                  * softlockcnt in this segment may be already
1201                                  * 0.  We don't drop as writer lock so our
1202                                  * number of retries without sleeping should
1203                                  * be very small. See segvn_reclaim() for
1204                                  * more comments.
1205                                  */
1206                                 AS_CLRNOUNMAPWAIT(as);
1207                                 mutex_exit(&as->a_contents);
1208                                 goto retry;
1209                         }
1210                         mutex_exit(&as->a_contents);
1211                         goto setprot_top;
1212                 } else if (error != 0)
1213                         break;
1214         }
1215         if (error != 0) {
1216                 as_setwatch(as);
1217         } else {
1218                 as_setwatchprot(as, saveraddr, saversize, prot);
1219         }
1220         AS_LOCK_EXIT(as);
1221         return (error);
1222 }
1223 
1224 /*
1225  * Check to make sure that the interval [addr, addr + size)
1226  * in address space `as' has at least the specified protection.
1227  * It is ok for the range to cross over several segments, as long
1228  * as they are contiguous.
1229  */
1230 int
1231 as_checkprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
1232 {
1233         struct seg *seg;
1234         size_t ssize;
1235         caddr_t raddr;                  /* rounded down addr */
1236         size_t rsize;                   /* rounded up size */
1237         int error = 0;
1238 
1239         raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1240         rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1241             (size_t)raddr;
1242 
1243         if (raddr + rsize < raddr)           /* check for wraparound */
1244                 return (ENOMEM);
1245 
1246         /*
1247          * This is ugly as sin...
1248          * Normally, we only acquire the address space readers lock.
1249          * However, if the address space has watchpoints present,
1250          * we must acquire the writer lock on the address space for
1251          * the benefit of as_clearwatchprot() and as_setwatchprot().
1252          */
1253         if (avl_numnodes(&as->a_wpage) != 0)
1254                 AS_LOCK_ENTER(as, RW_WRITER);
1255         else
1256                 AS_LOCK_ENTER(as, RW_READER);
1257         as_clearwatchprot(as, raddr, rsize);
1258         seg = as_segat(as, raddr);
1259         if (seg == NULL) {
1260                 as_setwatch(as);
1261                 AS_LOCK_EXIT(as);
1262                 return (ENOMEM);
1263         }
1264 
1265         for (; rsize != 0; rsize -= ssize, raddr += ssize) {
1266                 if (raddr >= seg->s_base + seg->s_size) {
1267                         seg = AS_SEGNEXT(as, seg);
1268                         if (seg == NULL || raddr != seg->s_base) {
1269                                 error = ENOMEM;
1270                                 break;
1271                         }
1272                 }
1273                 if ((raddr + rsize) > (seg->s_base + seg->s_size))
1274                         ssize = seg->s_base + seg->s_size - raddr;
1275                 else
1276                         ssize = rsize;
1277 
1278                 error = SEGOP_CHECKPROT(seg, raddr, ssize, prot);
1279                 if (error != 0)
1280                         break;
1281         }
1282         as_setwatch(as);
1283         AS_LOCK_EXIT(as);
1284         return (error);
1285 }
1286 
1287 int
1288 as_unmap(struct as *as, caddr_t addr, size_t size)
1289 {
1290         struct seg *seg, *seg_next;
1291         struct as_callback *cb;
1292         caddr_t raddr, eaddr;
1293         size_t ssize, rsize = 0;
1294         int err;
1295 
1296 top:
1297         raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1298         eaddr = (caddr_t)(((uintptr_t)(addr + size) + PAGEOFFSET) &
1299             (uintptr_t)PAGEMASK);
1300 
1301         AS_LOCK_ENTER(as, RW_WRITER);
1302 
1303         as->a_updatedir = 1; /* inform /proc */
1304         gethrestime(&as->a_updatetime);
1305 
1306         /*
1307          * Use as_findseg to find the first segment in the range, then
1308          * step through the segments in order, following s_next.
1309          */
1310         as_clearwatchprot(as, raddr, eaddr - raddr);
1311 
1312         for (seg = as_findseg(as, raddr, 0); seg != NULL; seg = seg_next) {
1313                 if (eaddr <= seg->s_base)
1314                         break;          /* eaddr was in a gap; all done */
1315 
1316                 /* this is implied by the test above */
1317                 ASSERT(raddr < eaddr);
1318 
1319                 if (raddr < seg->s_base)
1320                         raddr = seg->s_base;         /* raddr was in a gap */
1321 
1322                 if (eaddr > (seg->s_base + seg->s_size))
1323                         ssize = seg->s_base + seg->s_size - raddr;
1324                 else
1325                         ssize = eaddr - raddr;
1326 
1327                 /*
1328                  * Save next segment pointer since seg can be
1329                  * destroyed during the segment unmap operation.
1330                  */
1331                 seg_next = AS_SEGNEXT(as, seg);
1332 
1333                 /*
1334                  * We didn't count /dev/null mappings, so ignore them here.
1335                  * We'll handle MAP_NORESERVE cases in segvn_unmap(). (Again,
1336                  * we have to do this check here while we have seg.)
1337                  */
1338                 rsize = 0;
1339                 if (!SEG_IS_DEVNULL_MAPPING(seg) &&
1340                     !SEG_IS_PARTIAL_RESV(seg))
1341                         rsize = ssize;
1342 
1343 retry:
1344                 err = SEGOP_UNMAP(seg, raddr, ssize);
1345                 if (err == EAGAIN) {
1346                         /*
1347                          * Memory is currently locked.  It must be unlocked
1348                          * before this operation can succeed through a retry.
1349                          * The possible reasons for locked memory and
1350                          * corresponding strategies for unlocking are:
1351                          * (1) Normal I/O
1352                          *      wait for a signal that the I/O operation
1353                          *      has completed and the memory is unlocked.
1354                          * (2) Asynchronous I/O
1355                          *      The aio subsystem does not unlock pages when
1356                          *      the I/O is completed. Those pages are unlocked
1357                          *      when the application calls aiowait/aioerror.
1358                          *      So, to prevent blocking forever, cv_broadcast()
1359                          *      is done to wake up aio_cleanup_thread.
1360                          *      Subsequently, segvn_reclaim will be called, and
1361                          *      that will do AS_CLRUNMAPWAIT() and wake us up.
1362                          * (3) Long term page locking:
1363                          *      Drivers intending to have pages locked for a
1364                          *      period considerably longer than for normal I/O
1365                          *      (essentially forever) may have registered for a
1366                          *      callback so they may unlock these pages on
1367                          *      request. This is needed to allow this operation
1368                          *      to succeed. Each entry on the callback list is
1369                          *      examined. If the event or address range pertains
1370                          *      the callback is invoked (unless it already is in
1371                          *      progress). The a_contents lock must be dropped
1372                          *      before the callback, so only one callback can
1373                          *      be done at a time. Go to the top and do more
1374                          *      until zero is returned. If zero is returned,
1375                          *      either there were no callbacks for this event
1376                          *      or they were already in progress.
1377                          */
1378                         mutex_enter(&as->a_contents);
1379                         if (as->a_callbacks &&
1380                             (cb = as_find_callback(as, AS_UNMAP_EVENT,
1381                             seg->s_base, seg->s_size))) {
1382                                 AS_LOCK_EXIT(as);
1383                                 as_execute_callback(as, cb, AS_UNMAP_EVENT);
1384                         } else if (!AS_ISNOUNMAPWAIT(as)) {
1385                                 if (AS_ISUNMAPWAIT(as) == 0)
1386                                         cv_broadcast(&as->a_cv);
1387                                 AS_SETUNMAPWAIT(as);
1388                                 AS_LOCK_EXIT(as);
1389                                 while (AS_ISUNMAPWAIT(as))
1390                                         cv_wait(&as->a_cv, &as->a_contents);
1391                         } else {
1392                                 /*
1393                                  * We may have raced with
1394                                  * segvn_reclaim()/segspt_reclaim(). In this
1395                                  * case clean nounmapwait flag and retry since
1396                                  * softlockcnt in this segment may be already
1397                                  * 0.  We don't drop as writer lock so our
1398                                  * number of retries without sleeping should
1399                                  * be very small. See segvn_reclaim() for
1400                                  * more comments.
1401                                  */
1402                                 AS_CLRNOUNMAPWAIT(as);
1403                                 mutex_exit(&as->a_contents);
1404                                 goto retry;
1405                         }
1406                         mutex_exit(&as->a_contents);
1407                         goto top;
1408                 } else if (err == IE_RETRY) {
1409                         AS_LOCK_EXIT(as);
1410                         goto top;
1411                 } else if (err) {
1412                         as_setwatch(as);
1413                         AS_LOCK_EXIT(as);
1414                         return (-1);
1415                 }
1416 
1417                 as->a_size -= ssize;
1418                 if (rsize)
1419                         as->a_resvsize -= rsize;
1420                 raddr += ssize;
1421         }
1422         AS_LOCK_EXIT(as);
1423         return (0);
1424 }
1425 
1426 static int
1427 as_map_segvn_segs(struct as *as, caddr_t addr, size_t size, uint_t szcvec,
1428     int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated)
1429 {
1430         uint_t szc;
1431         uint_t nszc;
1432         int error;
1433         caddr_t a;
1434         caddr_t eaddr;
1435         size_t segsize;
1436         struct seg *seg;
1437         size_t pgsz;
1438         int do_off = (vn_a->vp != NULL || vn_a->amp != NULL);
1439         uint_t save_szcvec;
1440 
1441         ASSERT(AS_WRITE_HELD(as));
1442         ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
1443         ASSERT(IS_P2ALIGNED(size, PAGESIZE));
1444         ASSERT(vn_a->vp == NULL || vn_a->amp == NULL);
1445         if (!do_off) {
1446                 vn_a->offset = 0;
1447         }
1448 
1449         if (szcvec <= 1) {
1450                 seg = seg_alloc(as, addr, size);
1451                 if (seg == NULL) {
1452                         return (ENOMEM);
1453                 }
1454                 vn_a->szc = 0;
1455                 error = (*crfp)(seg, vn_a);
1456                 if (error != 0) {
1457                         seg_free(seg);
1458                 } else {
1459                         as->a_size += size;
1460                         as->a_resvsize += size;
1461                 }
1462                 return (error);
1463         }
1464 
1465         eaddr = addr + size;
1466         save_szcvec = szcvec;
1467         szcvec >>= 1;
1468         szc = 0;
1469         nszc = 0;
1470         while (szcvec) {
1471                 if ((szcvec & 0x1) == 0) {
1472                         nszc++;
1473                         szcvec >>= 1;
1474                         continue;
1475                 }
1476                 nszc++;
1477                 pgsz = page_get_pagesize(nszc);
1478                 a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
1479                 if (a != addr) {
1480                         ASSERT(a < eaddr);
1481                         segsize = a - addr;
1482                         seg = seg_alloc(as, addr, segsize);
1483                         if (seg == NULL) {
1484                                 return (ENOMEM);
1485                         }
1486                         vn_a->szc = szc;
1487                         error = (*crfp)(seg, vn_a);
1488                         if (error != 0) {
1489                                 seg_free(seg);
1490                                 return (error);
1491                         }
1492                         as->a_size += segsize;
1493                         as->a_resvsize += segsize;
1494                         *segcreated = 1;
1495                         if (do_off) {
1496                                 vn_a->offset += segsize;
1497                         }
1498                         addr = a;
1499                 }
1500                 szc = nszc;
1501                 szcvec >>= 1;
1502         }
1503 
1504         ASSERT(addr < eaddr);
1505         szcvec = save_szcvec | 1; /* add 8K pages */
1506         while (szcvec) {
1507                 a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
1508                 ASSERT(a >= addr);
1509                 if (a != addr) {
1510                         segsize = a - addr;
1511                         seg = seg_alloc(as, addr, segsize);
1512                         if (seg == NULL) {
1513                                 return (ENOMEM);
1514                         }
1515                         vn_a->szc = szc;
1516                         error = (*crfp)(seg, vn_a);
1517                         if (error != 0) {
1518                                 seg_free(seg);
1519                                 return (error);
1520                         }
1521                         as->a_size += segsize;
1522                         as->a_resvsize += segsize;
1523                         *segcreated = 1;
1524                         if (do_off) {
1525                                 vn_a->offset += segsize;
1526                         }
1527                         addr = a;
1528                 }
1529                 szcvec &= ~(1 << szc);
1530                 if (szcvec) {
1531                         szc = highbit(szcvec) - 1;
1532                         pgsz = page_get_pagesize(szc);
1533                 }
1534         }
1535         ASSERT(addr == eaddr);
1536 
1537         return (0);
1538 }
1539 
1540 static int
1541 as_map_vnsegs(struct as *as, caddr_t addr, size_t size,
1542     int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated)
1543 {
1544         uint_t mapflags = vn_a->flags & (MAP_TEXT | MAP_INITDATA);
1545         int type = (vn_a->type == MAP_SHARED) ? MAPPGSZC_SHM : MAPPGSZC_PRIVM;
1546         uint_t szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, mapflags,
1547             type, 0);
1548         int error;
1549         struct seg *seg;
1550         struct vattr va;
1551         u_offset_t eoff;
1552         size_t save_size = 0;
1553         extern size_t textrepl_size_thresh;
1554 
1555         ASSERT(AS_WRITE_HELD(as));
1556         ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
1557         ASSERT(IS_P2ALIGNED(size, PAGESIZE));
1558         ASSERT(vn_a->vp != NULL);
1559         ASSERT(vn_a->amp == NULL);
1560 
1561 again:
1562         if (szcvec <= 1) {
1563                 seg = seg_alloc(as, addr, size);
1564                 if (seg == NULL) {
1565                         return (ENOMEM);
1566                 }
1567                 vn_a->szc = 0;
1568                 error = (*crfp)(seg, vn_a);
1569                 if (error != 0) {
1570                         seg_free(seg);
1571                 } else {
1572                         as->a_size += size;
1573                         as->a_resvsize += size;
1574                 }
1575                 return (error);
1576         }
1577 
1578         va.va_mask = AT_SIZE;
1579         if (VOP_GETATTR(vn_a->vp, &va, ATTR_HINT, vn_a->cred, NULL) != 0) {
1580                 szcvec = 0;
1581                 goto again;
1582         }
1583         eoff = vn_a->offset & PAGEMASK;
1584         if (eoff >= va.va_size) {
1585                 szcvec = 0;
1586                 goto again;
1587         }
1588         eoff += size;
1589         if (btopr(va.va_size) < btopr(eoff)) {
1590                 save_size = size;
1591                 size = va.va_size - (vn_a->offset & PAGEMASK);
1592                 size = P2ROUNDUP_TYPED(size, PAGESIZE, size_t);
1593                 szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, mapflags,
1594                     type, 0);
1595                 if (szcvec <= 1) {
1596                         size = save_size;
1597                         goto again;
1598                 }
1599         }
1600 
1601         if (size > textrepl_size_thresh) {
1602                 vn_a->flags |= _MAP_TEXTREPL;
1603         }
1604         error = as_map_segvn_segs(as, addr, size, szcvec, crfp, vn_a,
1605             segcreated);
1606         if (error != 0) {
1607                 return (error);
1608         }
1609         if (save_size) {
1610                 addr += size;
1611                 size = save_size - size;
1612                 szcvec = 0;
1613                 goto again;
1614         }
1615         return (0);
1616 }
1617 
1618 /*
1619  * as_map_ansegs: shared or private anonymous memory.  Note that the flags
1620  * passed to map_pgszvec cannot be MAP_INITDATA, for anon.
1621  */
1622 static int
1623 as_map_ansegs(struct as *as, caddr_t addr, size_t size,
1624     int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated)
1625 {
1626         uint_t szcvec;
1627         uchar_t type;
1628 
1629         ASSERT(vn_a->type == MAP_SHARED || vn_a->type == MAP_PRIVATE);
1630         if (vn_a->type == MAP_SHARED) {
1631                 type = MAPPGSZC_SHM;
1632         } else if (vn_a->type == MAP_PRIVATE) {
1633                 if (vn_a->szc == AS_MAP_HEAP) {
1634                         type = MAPPGSZC_HEAP;
1635                 } else if (vn_a->szc == AS_MAP_STACK) {
1636                         type = MAPPGSZC_STACK;
1637                 } else {
1638                         type = MAPPGSZC_PRIVM;
1639                 }
1640         }
1641         szcvec = map_pgszcvec(addr, size, vn_a->amp == NULL ?
1642             (uintptr_t)addr : (uintptr_t)P2ROUNDUP(vn_a->offset, PAGESIZE),
1643             (vn_a->flags & MAP_TEXT), type, 0);
1644         ASSERT(AS_WRITE_HELD(as));
1645         ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
1646         ASSERT(IS_P2ALIGNED(size, PAGESIZE));
1647         ASSERT(vn_a->vp == NULL);
1648 
1649         return (as_map_segvn_segs(as, addr, size, szcvec,
1650             crfp, vn_a, segcreated));
1651 }
1652 
1653 int
1654 as_map(struct as *as, caddr_t addr, size_t size, int (*crfp)(), void *argsp)
1655 {
1656         AS_LOCK_ENTER(as, RW_WRITER);
1657         return (as_map_locked(as, addr, size, crfp, argsp));
1658 }
1659 
1660 int
1661 as_map_locked(struct as *as, caddr_t addr, size_t size, int (*crfp)(),
1662                 void *argsp)
1663 {
1664         struct seg *seg = NULL;
1665         caddr_t raddr;                  /* rounded down addr */
1666         size_t rsize;                   /* rounded up size */
1667         int error;
1668         int unmap = 0;
1669         struct proc *p = curproc;
1670         struct segvn_crargs crargs;
1671 
1672         raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1673         rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1674             (size_t)raddr;
1675 
1676         /*
1677          * check for wrap around
1678          */
1679         if ((raddr + rsize < raddr) || (as->a_size > (ULONG_MAX - size))) {
1680                 AS_LOCK_EXIT(as);
1681                 return (ENOMEM);
1682         }
1683 
1684         as->a_updatedir = 1; /* inform /proc */
1685         gethrestime(&as->a_updatetime);
1686 
1687         if (as != &kas && as->a_size + rsize > (size_t)p->p_vmem_ctl) {
1688                 AS_LOCK_EXIT(as);
1689 
1690                 (void) rctl_action(rctlproc_legacy[RLIMIT_VMEM], p->p_rctls, p,
1691                     RCA_UNSAFE_ALL);
1692 
1693                 return (ENOMEM);
1694         }
1695 
1696         if (AS_MAP_CHECK_VNODE_LPOOB(crfp, argsp)) {
1697                 crargs = *(struct segvn_crargs *)argsp;
1698                 error = as_map_vnsegs(as, raddr, rsize, crfp, &crargs, &unmap);
1699                 if (error != 0) {
1700                         AS_LOCK_EXIT(as);
1701                         if (unmap) {
1702                                 (void) as_unmap(as, addr, size);
1703                         }
1704                         return (error);
1705                 }
1706         } else if (AS_MAP_CHECK_ANON_LPOOB(crfp, argsp)) {
1707                 crargs = *(struct segvn_crargs *)argsp;
1708                 error = as_map_ansegs(as, raddr, rsize, crfp, &crargs, &unmap);
1709                 if (error != 0) {
1710                         AS_LOCK_EXIT(as);
1711                         if (unmap) {
1712                                 (void) as_unmap(as, addr, size);
1713                         }
1714                         return (error);
1715                 }
1716         } else {
1717                 seg = seg_alloc(as, addr, size);
1718                 if (seg == NULL) {
1719                         AS_LOCK_EXIT(as);
1720                         return (ENOMEM);
1721                 }
1722 
1723                 error = (*crfp)(seg, argsp);
1724                 if (error != 0) {
1725                         seg_free(seg);
1726                         AS_LOCK_EXIT(as);
1727                         return (error);
1728                 }
1729                 /*
1730                  * Add size now so as_unmap will work if as_ctl fails.
1731                  */
1732                 as->a_size += rsize;
1733                 as->a_resvsize += rsize;
1734         }
1735 
1736         as_setwatch(as);
1737 
1738         /*
1739          * If the address space is locked,
1740          * establish memory locks for the new segment.
1741          */
1742         mutex_enter(&as->a_contents);
1743         if (AS_ISPGLCK(as)) {
1744                 mutex_exit(&as->a_contents);
1745                 AS_LOCK_EXIT(as);
1746                 error = as_ctl(as, addr, size, MC_LOCK, 0, 0, NULL, 0);
1747                 if (error != 0)
1748                         (void) as_unmap(as, addr, size);
1749         } else {
1750                 mutex_exit(&as->a_contents);
1751                 AS_LOCK_EXIT(as);
1752         }
1753         return (error);
1754 }
1755 
1756 
1757 /*
1758  * Delete all segments in the address space marked with S_PURGE.
1759  * This is currently used for Sparc V9 nofault ASI segments (seg_nf.c).
1760  * These segments are deleted as a first step before calls to as_gap(), so
1761  * that they don't affect mmap() or shmat().
1762  */
1763 void
1764 as_purge(struct as *as)
1765 {
1766         struct seg *seg;
1767         struct seg *next_seg;
1768 
1769         /*
1770          * the setting of NEEDSPURGE is protect by as_rangelock(), so
1771          * no need to grab a_contents mutex for this check
1772          */
1773         if ((as->a_flags & AS_NEEDSPURGE) == 0)
1774                 return;
1775 
1776         AS_LOCK_ENTER(as, RW_WRITER);
1777         next_seg = NULL;
1778         seg = AS_SEGFIRST(as);
1779         while (seg != NULL) {
1780                 next_seg = AS_SEGNEXT(as, seg);
1781                 if (seg->s_flags & S_PURGE)
1782                         SEGOP_UNMAP(seg, seg->s_base, seg->s_size);
1783                 seg = next_seg;
1784         }
1785         AS_LOCK_EXIT(as);
1786 
1787         mutex_enter(&as->a_contents);
1788         as->a_flags &= ~AS_NEEDSPURGE;
1789         mutex_exit(&as->a_contents);
1790 }
1791 
1792 /*
1793  * Find a hole within [*basep, *basep + *lenp), which contains a mappable
1794  * range of addresses at least "minlen" long, where the base of the range is
1795  * at "off" phase from an "align" boundary and there is space for a
1796  * "redzone"-sized redzone on eithe rside of the range.  Thus,
1797  * if align was 4M and off was 16k, the user wants a hole which will start
1798  * 16k into a 4M page.
1799  *
1800  * If flags specifies AH_HI, the hole will have the highest possible address
1801  * in the range.  We use the as->a_lastgap field to figure out where to
1802  * start looking for a gap.
1803  *
1804  * Otherwise, the gap will have the lowest possible address.
1805  *
1806  * If flags specifies AH_CONTAIN, the hole will contain the address addr.
1807  *
1808  * If an adequate hole is found, *basep and *lenp are set to reflect the part of
1809  * the hole that is within range, and 0 is returned. On failure, -1 is returned.
1810  *
1811  * NOTE: This routine is not correct when base+len overflows caddr_t.
1812  */
1813 int
1814 as_gap_aligned(struct as *as, size_t minlen, caddr_t *basep, size_t *lenp,
1815     uint_t flags, caddr_t addr, size_t align, size_t redzone, size_t off)
1816 {
1817         caddr_t lobound = *basep;
1818         caddr_t hibound = lobound + *lenp;
1819         struct seg *lseg, *hseg;
1820         caddr_t lo, hi;
1821         int forward;
1822         caddr_t save_base;
1823         size_t save_len;
1824         size_t save_minlen;
1825         size_t save_redzone;
1826         int fast_path = 1;
1827 
1828         save_base = *basep;
1829         save_len = *lenp;
1830         save_minlen = minlen;
1831         save_redzone = redzone;
1832 
1833         /*
1834          * For the first pass/fast_path, just add align and redzone into
1835          * minlen since if we get an allocation, we can guarantee that it
1836          * will fit the alignment and redzone requested.
1837          * This increases the chance that hibound will be adjusted to
1838          * a_lastgap->s_base which will likely allow us to find an
1839          * acceptable hole in the address space quicker.
1840          * If we can't find a hole with this fast_path, then we look for
1841          * smaller holes in which the alignment and offset may allow
1842          * the allocation to fit.
1843          */
1844         minlen += align;
1845         minlen += 2 * redzone;
1846         redzone = 0;
1847 
1848         AS_LOCK_ENTER(as, RW_READER);
1849         if (AS_SEGFIRST(as) == NULL) {
1850                 if (valid_va_range_aligned(basep, lenp, minlen, flags & AH_DIR,
1851                     align, redzone, off)) {
1852                         AS_LOCK_EXIT(as);
1853                         return (0);
1854                 } else {
1855                         AS_LOCK_EXIT(as);
1856                         *basep = save_base;
1857                         *lenp = save_len;
1858                         return (-1);
1859                 }
1860         }
1861 
1862 retry:
1863         /*
1864          * Set up to iterate over all the inter-segment holes in the given
1865          * direction.  lseg is NULL for the lowest-addressed hole and hseg is
1866          * NULL for the highest-addressed hole.  If moving backwards, we reset
1867          * sseg to denote the highest-addressed segment.
1868          */
1869         forward = (flags & AH_DIR) == AH_LO;
1870         if (forward) {
1871                 hseg = as_findseg(as, lobound, 1);
1872                 lseg = AS_SEGPREV(as, hseg);
1873         } else {
1874 
1875                 /*
1876                  * If allocating at least as much as the last allocation,
1877                  * use a_lastgap's base as a better estimate of hibound.
1878                  */
1879                 if (as->a_lastgap &&
1880                     minlen >= as->a_lastgap->s_size &&
1881                     hibound >= as->a_lastgap->s_base)
1882                         hibound = as->a_lastgap->s_base;
1883 
1884                 hseg = as_findseg(as, hibound, 1);
1885                 if (hseg->s_base + hseg->s_size < hibound) {
1886                         lseg = hseg;
1887                         hseg = NULL;
1888                 } else {
1889                         lseg = AS_SEGPREV(as, hseg);
1890                 }
1891         }
1892 
1893         for (;;) {
1894                 /*
1895                  * Set lo and hi to the hole's boundaries.  (We should really
1896                  * use MAXADDR in place of hibound in the expression below,
1897                  * but can't express it easily; using hibound in its place is
1898                  * harmless.)
1899                  */
1900                 lo = (lseg == NULL) ? 0 : lseg->s_base + lseg->s_size;
1901                 hi = (hseg == NULL) ? hibound : hseg->s_base;
1902                 /*
1903                  * If the iteration has moved past the interval from lobound
1904                  * to hibound it's pointless to continue.
1905                  */
1906                 if ((forward && lo > hibound) || (!forward && hi < lobound))
1907                         break;
1908                 else if (lo > hibound || hi < lobound)
1909                         goto cont;
1910                 /*
1911                  * Candidate hole lies at least partially within the allowable
1912                  * range.  Restrict it to fall completely within that range,
1913                  * i.e., to [max(lo, lobound), min(hi, hibound)].
1914                  */
1915                 if (lo < lobound)
1916                         lo = lobound;
1917                 if (hi > hibound)
1918                         hi = hibound;
1919                 /*
1920                  * Verify that the candidate hole is big enough and meets
1921                  * hardware constraints.  If the hole is too small, no need
1922                  * to do the further checks since they will fail.
1923                  */
1924                 *basep = lo;
1925                 *lenp = hi - lo;
1926                 if (*lenp >= minlen && valid_va_range_aligned(basep, lenp,
1927                     minlen, forward ? AH_LO : AH_HI, align, redzone, off) &&
1928                     ((flags & AH_CONTAIN) == 0 ||
1929                     (*basep <= addr && *basep + *lenp > addr))) {
1930                         if (!forward)
1931                                 as->a_lastgap = hseg;
1932                         if (hseg != NULL)
1933                                 as->a_lastgaphl = hseg;
1934                         else
1935                                 as->a_lastgaphl = lseg;
1936                         AS_LOCK_EXIT(as);
1937                         return (0);
1938                 }
1939         cont:
1940                 /*
1941                  * Move to the next hole.
1942                  */
1943                 if (forward) {
1944                         lseg = hseg;
1945                         if (lseg == NULL)
1946                                 break;
1947                         hseg = AS_SEGNEXT(as, hseg);
1948                 } else {
1949                         hseg = lseg;
1950                         if (hseg == NULL)
1951                                 break;
1952                         lseg = AS_SEGPREV(as, lseg);
1953                 }
1954         }
1955         if (fast_path && (align != 0 || save_redzone != 0)) {
1956                 fast_path = 0;
1957                 minlen = save_minlen;
1958                 redzone = save_redzone;
1959                 goto retry;
1960         }
1961         *basep = save_base;
1962         *lenp = save_len;
1963         AS_LOCK_EXIT(as);
1964         return (-1);
1965 }
1966 
1967 /*
1968  * Find a hole of at least size minlen within [*basep, *basep + *lenp).
1969  *
1970  * If flags specifies AH_HI, the hole will have the highest possible address
1971  * in the range.  We use the as->a_lastgap field to figure out where to
1972  * start looking for a gap.
1973  *
1974  * Otherwise, the gap will have the lowest possible address.
1975  *
1976  * If flags specifies AH_CONTAIN, the hole will contain the address addr.
1977  *
1978  * If an adequate hole is found, base and len are set to reflect the part of
1979  * the hole that is within range, and 0 is returned, otherwise,
1980  * -1 is returned.
1981  *
1982  * NOTE: This routine is not correct when base+len overflows caddr_t.
1983  */
1984 int
1985 as_gap(struct as *as, size_t minlen, caddr_t *basep, size_t *lenp, uint_t flags,
1986     caddr_t addr)
1987 {
1988 
1989         return (as_gap_aligned(as, minlen, basep, lenp, flags, addr, 0, 0, 0));
1990 }
1991 
1992 /*
1993  * Return the next range within [base, base + len) that is backed
1994  * with "real memory".  Skip holes and non-seg_vn segments.
1995  * We're lazy and only return one segment at a time.
1996  */
1997 int
1998 as_memory(struct as *as, caddr_t *basep, size_t *lenp)
1999 {
2000         extern struct seg_ops segspt_shmops;    /* needs a header file */
2001         struct seg *seg;
2002         caddr_t addr, eaddr;
2003         caddr_t segend;
2004 
2005         AS_LOCK_ENTER(as, RW_READER);
2006 
2007         addr = *basep;
2008         eaddr = addr + *lenp;
2009 
2010         seg = as_findseg(as, addr, 0);
2011         if (seg != NULL)
2012                 addr = MAX(seg->s_base, addr);
2013 
2014         for (;;) {
2015                 if (seg == NULL || addr >= eaddr || eaddr <= seg->s_base) {
2016                         AS_LOCK_EXIT(as);
2017                         return (EINVAL);
2018                 }
2019 
2020                 if (seg->s_ops == &segvn_ops) {
2021                         segend = seg->s_base + seg->s_size;
2022                         break;
2023                 }
2024 
2025                 /*
2026                  * We do ISM by looking into the private data
2027                  * to determine the real size of the segment.
2028                  */
2029                 if (seg->s_ops == &segspt_shmops) {
2030                         segend = seg->s_base + spt_realsize(seg);
2031                         if (addr < segend)
2032                                 break;
2033                 }
2034 
2035                 seg = AS_SEGNEXT(as, seg);
2036 
2037                 if (seg != NULL)
2038                         addr = seg->s_base;
2039         }
2040 
2041         *basep = addr;
2042 
2043         if (segend > eaddr)
2044                 *lenp = eaddr - addr;
2045         else
2046                 *lenp = segend - addr;
2047 
2048         AS_LOCK_EXIT(as);
2049         return (0);
2050 }
2051 
2052 /*
2053  * Determine whether data from the mappings in interval [addr, addr + size)
2054  * are in the primary memory (core) cache.
2055  */
2056 int
2057 as_incore(struct as *as, caddr_t addr,
2058     size_t size, char *vec, size_t *sizep)
2059 {
2060         struct seg *seg;
2061         size_t ssize;
2062         caddr_t raddr;          /* rounded down addr */
2063         size_t rsize;           /* rounded up size */
2064         size_t isize;                   /* iteration size */
2065         int error = 0;          /* result, assume success */
2066 
2067         *sizep = 0;
2068         raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2069         rsize = ((((size_t)addr + size) + PAGEOFFSET) & PAGEMASK) -
2070             (size_t)raddr;
2071 
2072         if (raddr + rsize < raddr)           /* check for wraparound */
2073                 return (ENOMEM);
2074 
2075         AS_LOCK_ENTER(as, RW_READER);
2076         seg = as_segat(as, raddr);
2077         if (seg == NULL) {
2078                 AS_LOCK_EXIT(as);
2079                 return (-1);
2080         }
2081 
2082         for (; rsize != 0; rsize -= ssize, raddr += ssize) {
2083                 if (raddr >= seg->s_base + seg->s_size) {
2084                         seg = AS_SEGNEXT(as, seg);
2085                         if (seg == NULL || raddr != seg->s_base) {
2086                                 error = -1;
2087                                 break;
2088                         }
2089                 }
2090                 if ((raddr + rsize) > (seg->s_base + seg->s_size))
2091                         ssize = seg->s_base + seg->s_size - raddr;
2092                 else
2093                         ssize = rsize;
2094                 *sizep += isize = SEGOP_INCORE(seg, raddr, ssize, vec);
2095                 if (isize != ssize) {
2096                         error = -1;
2097                         break;
2098                 }
2099                 vec += btopr(ssize);
2100         }
2101         AS_LOCK_EXIT(as);
2102         return (error);
2103 }
2104 
2105 static void
2106 as_segunlock(struct seg *seg, caddr_t addr, int attr,
2107         ulong_t *bitmap, size_t position, size_t npages)
2108 {
2109         caddr_t range_start;
2110         size_t  pos1 = position;
2111         size_t  pos2;
2112         size_t  size;
2113         size_t  end_pos = npages + position;
2114 
2115         while (bt_range(bitmap, &pos1, &pos2, end_pos)) {
2116                 size = ptob((pos2 - pos1));
2117                 range_start = (caddr_t)((uintptr_t)addr +
2118                     ptob(pos1 - position));
2119 
2120                 (void) SEGOP_LOCKOP(seg, range_start, size, attr, MC_UNLOCK,
2121                     (ulong_t *)NULL, (size_t)NULL);
2122                 pos1 = pos2;
2123         }
2124 }
2125 
2126 static void
2127 as_unlockerr(struct as *as, int attr, ulong_t *mlock_map,
2128         caddr_t raddr, size_t rsize)
2129 {
2130         struct seg *seg = as_segat(as, raddr);
2131         size_t ssize;
2132 
2133         while (rsize != 0) {
2134                 if (raddr >= seg->s_base + seg->s_size)
2135                         seg = AS_SEGNEXT(as, seg);
2136 
2137                 if ((raddr + rsize) > (seg->s_base + seg->s_size))
2138                         ssize = seg->s_base + seg->s_size - raddr;
2139                 else
2140                         ssize = rsize;
2141 
2142                 as_segunlock(seg, raddr, attr, mlock_map, 0, btopr(ssize));
2143 
2144                 rsize -= ssize;
2145                 raddr += ssize;
2146         }
2147 }
2148 
2149 /*
2150  * Cache control operations over the interval [addr, addr + size) in
2151  * address space "as".
2152  */
2153 /*ARGSUSED*/
2154 int
2155 as_ctl(struct as *as, caddr_t addr, size_t size, int func, int attr,
2156     uintptr_t arg, ulong_t *lock_map, size_t pos)
2157 {
2158         struct seg *seg;        /* working segment */
2159         caddr_t raddr;          /* rounded down addr */
2160         caddr_t initraddr;      /* saved initial rounded down addr */
2161         size_t rsize;           /* rounded up size */
2162         size_t initrsize;       /* saved initial rounded up size */
2163         size_t ssize;           /* size of seg */
2164         int error = 0;                  /* result */
2165         size_t mlock_size;      /* size of bitmap */
2166         ulong_t *mlock_map;     /* pointer to bitmap used */
2167                                 /* to represent the locked */
2168                                 /* pages. */
2169 retry:
2170         if (error == IE_RETRY)
2171                 AS_LOCK_ENTER(as, RW_WRITER);
2172         else
2173                 AS_LOCK_ENTER(as, RW_READER);
2174 
2175         /*
2176          * If these are address space lock/unlock operations, loop over
2177          * all segments in the address space, as appropriate.
2178          */
2179         if (func == MC_LOCKAS) {
2180                 size_t npages, idx;
2181                 size_t rlen = 0;        /* rounded as length */
2182 
2183                 idx = pos;
2184 
2185                 if (arg & MCL_FUTURE) {
2186                         mutex_enter(&as->a_contents);
2187                         AS_SETPGLCK(as);
2188                         mutex_exit(&as->a_contents);
2189                 }
2190                 if ((arg & MCL_CURRENT) == 0) {
2191                         AS_LOCK_EXIT(as);
2192                         return (0);
2193                 }
2194 
2195                 seg = AS_SEGFIRST(as);
2196                 if (seg == NULL) {
2197                         AS_LOCK_EXIT(as);
2198                         return (0);
2199                 }
2200 
2201                 do {
2202                         raddr = (caddr_t)((uintptr_t)seg->s_base &
2203                             (uintptr_t)PAGEMASK);
2204                         rlen += (((uintptr_t)(seg->s_base + seg->s_size) +
2205                             PAGEOFFSET) & PAGEMASK) - (uintptr_t)raddr;
2206                 } while ((seg = AS_SEGNEXT(as, seg)) != NULL);
2207 
2208                 mlock_size = BT_BITOUL(btopr(rlen));
2209                 if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size *
2210                     sizeof (ulong_t), KM_NOSLEEP)) == NULL) {
2211                                 AS_LOCK_EXIT(as);
2212                                 return (EAGAIN);
2213                 }
2214 
2215                 for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) {
2216                         error = SEGOP_LOCKOP(seg, seg->s_base,
2217                             seg->s_size, attr, MC_LOCK, mlock_map, pos);
2218                         if (error != 0)
2219                                 break;
2220                         pos += seg_pages(seg);
2221                 }
2222 
2223                 if (error) {
2224                         for (seg = AS_SEGFIRST(as); seg != NULL;
2225                             seg = AS_SEGNEXT(as, seg)) {
2226 
2227                                 raddr = (caddr_t)((uintptr_t)seg->s_base &
2228                                     (uintptr_t)PAGEMASK);
2229                                 npages = seg_pages(seg);
2230                                 as_segunlock(seg, raddr, attr, mlock_map,
2231                                     idx, npages);
2232                                 idx += npages;
2233                         }
2234                 }
2235 
2236                 kmem_free(mlock_map, mlock_size * sizeof (ulong_t));
2237                 AS_LOCK_EXIT(as);
2238                 goto lockerr;
2239         } else if (func == MC_UNLOCKAS) {
2240                 mutex_enter(&as->a_contents);
2241                 AS_CLRPGLCK(as);
2242                 mutex_exit(&as->a_contents);
2243 
2244                 for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) {
2245                         error = SEGOP_LOCKOP(seg, seg->s_base,
2246                             seg->s_size, attr, MC_UNLOCK, NULL, 0);
2247                         if (error != 0)
2248                                 break;
2249                 }
2250 
2251                 AS_LOCK_EXIT(as);
2252                 goto lockerr;
2253         }
2254 
2255         /*
2256          * Normalize addresses and sizes.
2257          */
2258         initraddr = raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2259         initrsize = rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2260             (size_t)raddr;
2261 
2262         if (raddr + rsize < raddr) {         /* check for wraparound */
2263                 AS_LOCK_EXIT(as);
2264                 return (ENOMEM);
2265         }
2266 
2267         /*
2268          * Get initial segment.
2269          */
2270         if ((seg = as_segat(as, raddr)) == NULL) {
2271                 AS_LOCK_EXIT(as);
2272                 return (ENOMEM);
2273         }
2274 
2275         if (func == MC_LOCK) {
2276                 mlock_size = BT_BITOUL(btopr(rsize));
2277                 if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size *
2278                     sizeof (ulong_t), KM_NOSLEEP)) == NULL) {
2279                                 AS_LOCK_EXIT(as);
2280                                 return (EAGAIN);
2281                 }
2282         }
2283 
2284         /*
2285          * Loop over all segments.  If a hole in the address range is
2286          * discovered, then fail.  For each segment, perform the appropriate
2287          * control operation.
2288          */
2289         while (rsize != 0) {
2290 
2291                 /*
2292                  * Make sure there's no hole, calculate the portion
2293                  * of the next segment to be operated over.
2294                  */
2295                 if (raddr >= seg->s_base + seg->s_size) {
2296                         seg = AS_SEGNEXT(as, seg);
2297                         if (seg == NULL || raddr != seg->s_base) {
2298                                 if (func == MC_LOCK) {
2299                                         as_unlockerr(as, attr, mlock_map,
2300                                             initraddr, initrsize - rsize);
2301                                         kmem_free(mlock_map,
2302                                             mlock_size * sizeof (ulong_t));
2303                                 }
2304                                 AS_LOCK_EXIT(as);
2305                                 return (ENOMEM);
2306                         }
2307                 }
2308                 if ((raddr + rsize) > (seg->s_base + seg->s_size))
2309                         ssize = seg->s_base + seg->s_size - raddr;
2310                 else
2311                         ssize = rsize;
2312 
2313                 /*
2314                  * Dispatch on specific function.
2315                  */
2316                 switch (func) {
2317 
2318                 /*
2319                  * Synchronize cached data from mappings with backing
2320                  * objects.
2321                  */
2322                 case MC_SYNC:
2323                         if (error = SEGOP_SYNC(seg, raddr, ssize,
2324                             attr, (uint_t)arg)) {
2325                                 AS_LOCK_EXIT(as);
2326                                 return (error);
2327                         }
2328                         break;
2329 
2330                 /*
2331                  * Lock pages in memory.
2332                  */
2333                 case MC_LOCK:
2334                         if (error = SEGOP_LOCKOP(seg, raddr, ssize,
2335                             attr, func, mlock_map, pos)) {
2336                                 as_unlockerr(as, attr, mlock_map, initraddr,
2337                                     initrsize - rsize + ssize);
2338                                 kmem_free(mlock_map, mlock_size *
2339                                     sizeof (ulong_t));
2340                                 AS_LOCK_EXIT(as);
2341                                 goto lockerr;
2342                         }
2343                         break;
2344 
2345                 /*
2346                  * Unlock mapped pages.
2347                  */
2348                 case MC_UNLOCK:
2349                         (void) SEGOP_LOCKOP(seg, raddr, ssize, attr, func,
2350                             (ulong_t *)NULL, (size_t)NULL);
2351                         break;
2352 
2353                 /*
2354                  * Store VM advise for mapped pages in segment layer.
2355                  */
2356                 case MC_ADVISE:
2357                         error = SEGOP_ADVISE(seg, raddr, ssize, (uint_t)arg);
2358 
2359                         /*
2360                          * Check for regular errors and special retry error
2361                          */
2362                         if (error) {
2363                                 if (error == IE_RETRY) {
2364                                         /*
2365                                          * Need to acquire writers lock, so
2366                                          * have to drop readers lock and start
2367                                          * all over again
2368                                          */
2369                                         AS_LOCK_EXIT(as);
2370                                         goto retry;
2371                                 } else if (error == IE_REATTACH) {
2372                                         /*
2373                                          * Find segment for current address
2374                                          * because current segment just got
2375                                          * split or concatenated
2376                                          */
2377                                         seg = as_segat(as, raddr);
2378                                         if (seg == NULL) {
2379                                                 AS_LOCK_EXIT(as);
2380                                                 return (ENOMEM);
2381                                         }
2382                                 } else {
2383                                         /*
2384                                          * Regular error
2385                                          */
2386                                         AS_LOCK_EXIT(as);
2387                                         return (error);
2388                                 }
2389                         }
2390                         break;
2391 
2392                 case MC_INHERIT_ZERO:
2393                         if (seg->s_ops->inherit == NULL) {
2394                                 error = ENOTSUP;
2395                         } else {
2396                                 error = SEGOP_INHERIT(seg, raddr, ssize,
2397                                     SEGP_INH_ZERO);
2398                         }
2399                         if (error != 0) {
2400                                 AS_LOCK_EXIT(as);
2401                                 return (error);
2402                         }
2403                         break;
2404 
2405                 /*
2406                  * Can't happen.
2407                  */
2408                 default:
2409                         panic("as_ctl: bad operation %d", func);
2410                         /*NOTREACHED*/
2411                 }
2412 
2413                 rsize -= ssize;
2414                 raddr += ssize;
2415         }
2416 
2417         if (func == MC_LOCK)
2418                 kmem_free(mlock_map, mlock_size * sizeof (ulong_t));
2419         AS_LOCK_EXIT(as);
2420         return (0);
2421 lockerr:
2422 
2423         /*
2424          * If the lower levels returned EDEADLK for a segment lockop,
2425          * it means that we should retry the operation.  Let's wait
2426          * a bit also to let the deadlock causing condition clear.
2427          * This is part of a gross hack to work around a design flaw
2428          * in the ufs/sds logging code and should go away when the
2429          * logging code is re-designed to fix the problem. See bug
2430          * 4125102 for details of the problem.
2431          */
2432         if (error == EDEADLK) {
2433                 delay(deadlk_wait);
2434                 error = 0;
2435                 goto retry;
2436         }
2437         return (error);
2438 }
2439 
2440 int
2441 fc_decode(faultcode_t fault_err)
2442 {
2443         int error = 0;
2444 
2445         switch (FC_CODE(fault_err)) {
2446         case FC_OBJERR:
2447                 error = FC_ERRNO(fault_err);
2448                 break;
2449         case FC_PROT:
2450                 error = EACCES;
2451                 break;
2452         default:
2453                 error = EFAULT;
2454                 break;
2455         }
2456         return (error);
2457 }
2458 
2459 /*
2460  * Pagelock pages from a range that spans more than 1 segment.  Obtain shadow
2461  * lists from each segment and copy them to one contiguous shadow list (plist)
2462  * as expected by the caller.  Save pointers to per segment shadow lists at
2463  * the tail of plist so that they can be used during as_pageunlock().
2464  */
2465 static int
2466 as_pagelock_segs(struct as *as, struct seg *seg, struct page ***ppp,
2467     caddr_t addr, size_t size, enum seg_rw rw)
2468 {
2469         caddr_t sv_addr = addr;
2470         size_t sv_size = size;
2471         struct seg *sv_seg = seg;
2472         ulong_t segcnt = 1;
2473         ulong_t cnt;
2474         size_t ssize;
2475         pgcnt_t npages = btop(size);
2476         page_t **plist;
2477         page_t **pl;
2478         int error;
2479         caddr_t eaddr;
2480         faultcode_t fault_err = 0;
2481         pgcnt_t pl_off;
2482         extern struct seg_ops segspt_shmops;
2483 
2484         ASSERT(AS_LOCK_HELD(as));
2485         ASSERT(seg != NULL);
2486         ASSERT(addr >= seg->s_base && addr < seg->s_base + seg->s_size);
2487         ASSERT(addr + size > seg->s_base + seg->s_size);
2488         ASSERT(IS_P2ALIGNED(size, PAGESIZE));
2489         ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
2490 
2491         /*
2492          * Count the number of segments covered by the range we are about to
2493          * lock. The segment count is used to size the shadow list we return
2494          * back to the caller.
2495          */
2496         for (; size != 0; size -= ssize, addr += ssize) {
2497                 if (addr >= seg->s_base + seg->s_size) {
2498 
2499                         seg = AS_SEGNEXT(as, seg);
2500                         if (seg == NULL || addr != seg->s_base) {
2501                                 AS_LOCK_EXIT(as);
2502                                 return (EFAULT);
2503                         }
2504                         /*
2505                          * Do a quick check if subsequent segments
2506                          * will most likely support pagelock.
2507                          */
2508                         if (seg->s_ops == &segvn_ops) {
2509                                 vnode_t *vp;
2510 
2511                                 if (SEGOP_GETVP(seg, addr, &vp) != 0 ||
2512                                     vp != NULL) {
2513                                         AS_LOCK_EXIT(as);
2514                                         goto slow;
2515                                 }
2516                         } else if (seg->s_ops != &segspt_shmops) {
2517                                 AS_LOCK_EXIT(as);
2518                                 goto slow;
2519                         }
2520                         segcnt++;
2521                 }
2522                 if (addr + size > seg->s_base + seg->s_size) {
2523                         ssize = seg->s_base + seg->s_size - addr;
2524                 } else {
2525                         ssize = size;
2526                 }
2527         }
2528         ASSERT(segcnt > 1);
2529 
2530         plist = kmem_zalloc((npages + segcnt) * sizeof (page_t *), KM_SLEEP);
2531 
2532         addr = sv_addr;
2533         size = sv_size;
2534         seg = sv_seg;
2535 
2536         for (cnt = 0, pl_off = 0; size != 0; size -= ssize, addr += ssize) {
2537                 if (addr >= seg->s_base + seg->s_size) {
2538                         seg = AS_SEGNEXT(as, seg);
2539                         ASSERT(seg != NULL && addr == seg->s_base);
2540                         cnt++;
2541                         ASSERT(cnt < segcnt);
2542                 }
2543                 if (addr + size > seg->s_base + seg->s_size) {
2544                         ssize = seg->s_base + seg->s_size - addr;
2545                 } else {
2546                         ssize = size;
2547                 }
2548                 pl = &plist[npages + cnt];
2549                 error = SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl,
2550                     L_PAGELOCK, rw);
2551                 if (error) {
2552                         break;
2553                 }
2554                 ASSERT(plist[npages + cnt] != NULL);
2555                 ASSERT(pl_off + btop(ssize) <= npages);
2556                 bcopy(plist[npages + cnt], &plist[pl_off],
2557                     btop(ssize) * sizeof (page_t *));
2558                 pl_off += btop(ssize);
2559         }
2560 
2561         if (size == 0) {
2562                 AS_LOCK_EXIT(as);
2563                 ASSERT(cnt == segcnt - 1);
2564                 *ppp = plist;
2565                 return (0);
2566         }
2567 
2568         /*
2569          * one of pagelock calls failed. The error type is in error variable.
2570          * Unlock what we've locked so far and retry with F_SOFTLOCK if error
2571          * type is either EFAULT or ENOTSUP. Otherwise just return the error
2572          * back to the caller.
2573          */
2574 
2575         eaddr = addr;
2576         seg = sv_seg;
2577 
2578         for (cnt = 0, addr = sv_addr; addr < eaddr; addr += ssize) {
2579                 if (addr >= seg->s_base + seg->s_size) {
2580                         seg = AS_SEGNEXT(as, seg);
2581                         ASSERT(seg != NULL && addr == seg->s_base);
2582                         cnt++;
2583                         ASSERT(cnt < segcnt);
2584                 }
2585                 if (eaddr > seg->s_base + seg->s_size) {
2586                         ssize = seg->s_base + seg->s_size - addr;
2587                 } else {
2588                         ssize = eaddr - addr;
2589                 }
2590                 pl = &plist[npages + cnt];
2591                 ASSERT(*pl != NULL);
2592                 (void) SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl,
2593                     L_PAGEUNLOCK, rw);
2594         }
2595 
2596         AS_LOCK_EXIT(as);
2597 
2598         kmem_free(plist, (npages + segcnt) * sizeof (page_t *));
2599 
2600         if (error != ENOTSUP && error != EFAULT) {
2601                 return (error);
2602         }
2603 
2604 slow:
2605         /*
2606          * If we are here because pagelock failed due to the need to cow fault
2607          * in the pages we want to lock F_SOFTLOCK will do this job and in
2608          * next as_pagelock() call for this address range pagelock will
2609          * hopefully succeed.
2610          */
2611         fault_err = as_fault(as->a_hat, as, sv_addr, sv_size, F_SOFTLOCK, rw);
2612         if (fault_err != 0) {
2613                 return (fc_decode(fault_err));
2614         }
2615         *ppp = NULL;
2616 
2617         return (0);
2618 }
2619 
2620 /*
2621  * lock pages in a given address space. Return shadow list. If
2622  * the list is NULL, the MMU mapping is also locked.
2623  */
2624 int
2625 as_pagelock(struct as *as, struct page ***ppp, caddr_t addr,
2626     size_t size, enum seg_rw rw)
2627 {
2628         size_t rsize;
2629         caddr_t raddr;
2630         faultcode_t fault_err;
2631         struct seg *seg;
2632         int err;
2633 
2634         TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_START,
2635             "as_pagelock_start: addr %p size %ld", addr, size);
2636 
2637         raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2638         rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2639             (size_t)raddr;
2640 
2641         /*
2642          * if the request crosses two segments let
2643          * as_fault handle it.
2644          */
2645         AS_LOCK_ENTER(as, RW_READER);
2646 
2647         seg = as_segat(as, raddr);
2648         if (seg == NULL) {
2649                 AS_LOCK_EXIT(as);
2650                 return (EFAULT);
2651         }
2652         ASSERT(raddr >= seg->s_base && raddr < seg->s_base + seg->s_size);
2653         if (raddr + rsize > seg->s_base + seg->s_size) {
2654                 return (as_pagelock_segs(as, seg, ppp, raddr, rsize, rw));
2655         }
2656         if (raddr + rsize <= raddr) {
2657                 AS_LOCK_EXIT(as);
2658                 return (EFAULT);
2659         }
2660 
2661         TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_START,
2662             "seg_lock_1_start: raddr %p rsize %ld", raddr, rsize);
2663 
2664         /*
2665          * try to lock pages and pass back shadow list
2666          */
2667         err = SEGOP_PAGELOCK(seg, raddr, rsize, ppp, L_PAGELOCK, rw);
2668 
2669         TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_END, "seg_lock_1_end");
2670 
2671         AS_LOCK_EXIT(as);
2672 
2673         if (err == 0 || (err != ENOTSUP && err != EFAULT)) {
2674                 return (err);
2675         }
2676 
2677         /*
2678          * Use F_SOFTLOCK to lock the pages because pagelock failed either due
2679          * to no pagelock support for this segment or pages need to be cow
2680          * faulted in. If fault is needed F_SOFTLOCK will do this job for
2681          * this as_pagelock() call and in the next as_pagelock() call for the
2682          * same address range pagelock call will hopefull succeed.
2683          */
2684         fault_err = as_fault(as->a_hat, as, addr, size, F_SOFTLOCK, rw);
2685         if (fault_err != 0) {
2686                 return (fc_decode(fault_err));
2687         }
2688         *ppp = NULL;
2689 
2690         TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_END, "as_pagelock_end");
2691         return (0);
2692 }
2693 
2694 /*
2695  * unlock pages locked by as_pagelock_segs().  Retrieve per segment shadow
2696  * lists from the end of plist and call pageunlock interface for each segment.
2697  * Drop as lock and free plist.
2698  */
2699 static void
2700 as_pageunlock_segs(struct as *as, struct seg *seg, caddr_t addr, size_t size,
2701     struct page **plist, enum seg_rw rw)
2702 {
2703         ulong_t cnt;
2704         caddr_t eaddr = addr + size;
2705         pgcnt_t npages = btop(size);
2706         size_t ssize;
2707         page_t **pl;
2708 
2709         ASSERT(AS_LOCK_HELD(as));
2710         ASSERT(seg != NULL);
2711         ASSERT(addr >= seg->s_base && addr < seg->s_base + seg->s_size);
2712         ASSERT(addr + size > seg->s_base + seg->s_size);
2713         ASSERT(IS_P2ALIGNED(size, PAGESIZE));
2714         ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
2715         ASSERT(plist != NULL);
2716 
2717         for (cnt = 0; addr < eaddr; addr += ssize) {
2718                 if (addr >= seg->s_base + seg->s_size) {
2719                         seg = AS_SEGNEXT(as, seg);
2720                         ASSERT(seg != NULL && addr == seg->s_base);
2721                         cnt++;
2722                 }
2723                 if (eaddr > seg->s_base + seg->s_size) {
2724                         ssize = seg->s_base + seg->s_size - addr;
2725                 } else {
2726                         ssize = eaddr - addr;
2727                 }
2728                 pl = &plist[npages + cnt];
2729                 ASSERT(*pl != NULL);
2730                 (void) SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl,
2731                     L_PAGEUNLOCK, rw);
2732         }
2733         ASSERT(cnt > 0);
2734         AS_LOCK_EXIT(as);
2735 
2736         cnt++;
2737         kmem_free(plist, (npages + cnt) * sizeof (page_t *));
2738 }
2739 
2740 /*
2741  * unlock pages in a given address range
2742  */
2743 void
2744 as_pageunlock(struct as *as, struct page **pp, caddr_t addr, size_t size,
2745     enum seg_rw rw)
2746 {
2747         struct seg *seg;
2748         size_t rsize;
2749         caddr_t raddr;
2750 
2751         TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_START,
2752             "as_pageunlock_start: addr %p size %ld", addr, size);
2753 
2754         /*
2755          * if the shadow list is NULL, as_pagelock was
2756          * falling back to as_fault
2757          */
2758         if (pp == NULL) {
2759                 (void) as_fault(as->a_hat, as, addr, size, F_SOFTUNLOCK, rw);
2760                 return;
2761         }
2762 
2763         raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2764         rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2765             (size_t)raddr;
2766 
2767         AS_LOCK_ENTER(as, RW_READER);
2768         seg = as_segat(as, raddr);
2769         ASSERT(seg != NULL);
2770 
2771         TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_UNLOCK_START,
2772             "seg_unlock_start: raddr %p rsize %ld", raddr, rsize);
2773 
2774         ASSERT(raddr >= seg->s_base && raddr < seg->s_base + seg->s_size);
2775         if (raddr + rsize <= seg->s_base + seg->s_size) {
2776                 SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGEUNLOCK, rw);
2777         } else {
2778                 as_pageunlock_segs(as, seg, raddr, rsize, pp, rw);
2779                 return;
2780         }
2781         AS_LOCK_EXIT(as);
2782         TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_END, "as_pageunlock_end");
2783 }
2784 
2785 int
2786 as_setpagesize(struct as *as, caddr_t addr, size_t size, uint_t szc,
2787     boolean_t wait)
2788 {
2789         struct seg *seg;
2790         size_t ssize;
2791         caddr_t raddr;                  /* rounded down addr */
2792         size_t rsize;                   /* rounded up size */
2793         int error = 0;
2794         size_t pgsz = page_get_pagesize(szc);
2795 
2796 setpgsz_top:
2797         if (!IS_P2ALIGNED(addr, pgsz) || !IS_P2ALIGNED(size, pgsz)) {
2798                 return (EINVAL);
2799         }
2800 
2801         raddr = addr;
2802         rsize = size;
2803 
2804         if (raddr + rsize < raddr)           /* check for wraparound */
2805                 return (ENOMEM);
2806 
2807         AS_LOCK_ENTER(as, RW_WRITER);
2808         as_clearwatchprot(as, raddr, rsize);
2809         seg = as_segat(as, raddr);
2810         if (seg == NULL) {
2811                 as_setwatch(as);
2812                 AS_LOCK_EXIT(as);
2813                 return (ENOMEM);
2814         }
2815 
2816         for (; rsize != 0; rsize -= ssize, raddr += ssize) {
2817                 if (raddr >= seg->s_base + seg->s_size) {
2818                         seg = AS_SEGNEXT(as, seg);
2819                         if (seg == NULL || raddr != seg->s_base) {
2820                                 error = ENOMEM;
2821                                 break;
2822                         }
2823                 }
2824                 if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
2825                         ssize = seg->s_base + seg->s_size - raddr;
2826                 } else {
2827                         ssize = rsize;
2828                 }
2829 
2830 retry:
2831                 error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc);
2832 
2833                 if (error == IE_NOMEM) {
2834                         error = EAGAIN;
2835                         break;
2836                 }
2837 
2838                 if (error == IE_RETRY) {
2839                         AS_LOCK_EXIT(as);
2840                         goto setpgsz_top;
2841                 }
2842 
2843                 if (error == ENOTSUP) {
2844                         error = EINVAL;
2845                         break;
2846                 }
2847 
2848                 if (wait && (error == EAGAIN)) {
2849                         /*
2850                          * Memory is currently locked.  It must be unlocked
2851                          * before this operation can succeed through a retry.
2852                          * The possible reasons for locked memory and
2853                          * corresponding strategies for unlocking are:
2854                          * (1) Normal I/O
2855                          *      wait for a signal that the I/O operation
2856                          *      has completed and the memory is unlocked.
2857                          * (2) Asynchronous I/O
2858                          *      The aio subsystem does not unlock pages when
2859                          *      the I/O is completed. Those pages are unlocked
2860                          *      when the application calls aiowait/aioerror.
2861                          *      So, to prevent blocking forever, cv_broadcast()
2862                          *      is done to wake up aio_cleanup_thread.
2863                          *      Subsequently, segvn_reclaim will be called, and
2864                          *      that will do AS_CLRUNMAPWAIT() and wake us up.
2865                          * (3) Long term page locking:
2866                          *      This is not relevant for as_setpagesize()
2867                          *      because we cannot change the page size for
2868                          *      driver memory. The attempt to do so will
2869                          *      fail with a different error than EAGAIN so
2870                          *      there's no need to trigger as callbacks like
2871                          *      as_unmap, as_setprot or as_free would do.
2872                          */
2873                         mutex_enter(&as->a_contents);
2874                         if (!AS_ISNOUNMAPWAIT(as)) {
2875                                 if (AS_ISUNMAPWAIT(as) == 0) {
2876                                         cv_broadcast(&as->a_cv);
2877                                 }
2878                                 AS_SETUNMAPWAIT(as);
2879                                 AS_LOCK_EXIT(as);
2880                                 while (AS_ISUNMAPWAIT(as)) {
2881                                         cv_wait(&as->a_cv, &as->a_contents);
2882                                 }
2883                         } else {
2884                                 /*
2885                                  * We may have raced with
2886                                  * segvn_reclaim()/segspt_reclaim(). In this
2887                                  * case clean nounmapwait flag and retry since
2888                                  * softlockcnt in this segment may be already
2889                                  * 0.  We don't drop as writer lock so our
2890                                  * number of retries without sleeping should
2891                                  * be very small. See segvn_reclaim() for
2892                                  * more comments.
2893                                  */
2894                                 AS_CLRNOUNMAPWAIT(as);
2895                                 mutex_exit(&as->a_contents);
2896                                 goto retry;
2897                         }
2898                         mutex_exit(&as->a_contents);
2899                         goto setpgsz_top;
2900                 } else if (error != 0) {
2901                         break;
2902                 }
2903         }
2904         as_setwatch(as);
2905         AS_LOCK_EXIT(as);
2906         return (error);
2907 }
2908 
2909 /*
2910  * as_iset3_default_lpsize() just calls SEGOP_SETPAGESIZE() on all segments
2911  * in its chunk where s_szc is less than the szc we want to set.
2912  */
2913 static int
2914 as_iset3_default_lpsize(struct as *as, caddr_t raddr, size_t rsize, uint_t szc,
2915     int *retry)
2916 {
2917         struct seg *seg;
2918         size_t ssize;
2919         int error;
2920 
2921         ASSERT(AS_WRITE_HELD(as));
2922 
2923         seg = as_segat(as, raddr);
2924         if (seg == NULL) {
2925                 panic("as_iset3_default_lpsize: no seg");
2926         }
2927 
2928         for (; rsize != 0; rsize -= ssize, raddr += ssize) {
2929                 if (raddr >= seg->s_base + seg->s_size) {
2930                         seg = AS_SEGNEXT(as, seg);
2931                         if (seg == NULL || raddr != seg->s_base) {
2932                                 panic("as_iset3_default_lpsize: as changed");
2933                         }
2934                 }
2935                 if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
2936                         ssize = seg->s_base + seg->s_size - raddr;
2937                 } else {
2938                         ssize = rsize;
2939                 }
2940 
2941                 if (szc > seg->s_szc) {
2942                         error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc);
2943                         /* Only retry on EINVAL segments that have no vnode. */
2944                         if (error == EINVAL) {
2945                                 vnode_t *vp = NULL;
2946                                 if ((SEGOP_GETTYPE(seg, raddr) & MAP_SHARED) &&
2947                                     (SEGOP_GETVP(seg, raddr, &vp) != 0 ||
2948                                     vp == NULL)) {
2949                                         *retry = 1;
2950                                 } else {
2951                                         *retry = 0;
2952                                 }
2953                         }
2954                         if (error) {
2955                                 return (error);
2956                         }
2957                 }
2958         }
2959         return (0);
2960 }
2961 
2962 /*
2963  * as_iset2_default_lpsize() calls as_iset3_default_lpsize() to set the
2964  * pagesize on each segment in its range, but if any fails with EINVAL,
2965  * then it reduces the pagesizes to the next size in the bitmap and
2966  * retries as_iset3_default_lpsize(). The reason why the code retries
2967  * smaller allowed sizes on EINVAL is because (a) the anon offset may not
2968  * match the bigger sizes, and (b) it's hard to get this offset (to begin
2969  * with) to pass to map_pgszcvec().
2970  */
2971 static int
2972 as_iset2_default_lpsize(struct as *as, caddr_t addr, size_t size, uint_t szc,
2973     uint_t szcvec)
2974 {
2975         int error;
2976         int retry;
2977 
2978         ASSERT(AS_WRITE_HELD(as));
2979 
2980         for (;;) {
2981                 error = as_iset3_default_lpsize(as, addr, size, szc, &retry);
2982                 if (error == EINVAL && retry) {
2983                         szcvec &= ~(1 << szc);
2984                         if (szcvec <= 1) {
2985                                 return (EINVAL);
2986                         }
2987                         szc = highbit(szcvec) - 1;
2988                 } else {
2989                         return (error);
2990                 }
2991         }
2992 }
2993 
2994 /*
2995  * as_iset1_default_lpsize() breaks its chunk into areas where existing
2996  * segments have a smaller szc than we want to set. For each such area,
2997  * it calls as_iset2_default_lpsize()
2998  */
2999 static int
3000 as_iset1_default_lpsize(struct as *as, caddr_t raddr, size_t rsize, uint_t szc,
3001     uint_t szcvec)
3002 {
3003         struct seg *seg;
3004         size_t ssize;
3005         caddr_t setaddr = raddr;
3006         size_t setsize = 0;
3007         int set;
3008         int error;
3009 
3010         ASSERT(AS_WRITE_HELD(as));
3011 
3012         seg = as_segat(as, raddr);
3013         if (seg == NULL) {
3014                 panic("as_iset1_default_lpsize: no seg");
3015         }
3016         if (seg->s_szc < szc) {
3017                 set = 1;
3018         } else {
3019                 set = 0;
3020         }
3021 
3022         for (; rsize != 0; rsize -= ssize, raddr += ssize, setsize += ssize) {
3023                 if (raddr >= seg->s_base + seg->s_size) {
3024                         seg = AS_SEGNEXT(as, seg);
3025                         if (seg == NULL || raddr != seg->s_base) {
3026                                 panic("as_iset1_default_lpsize: as changed");
3027                         }
3028                         if (seg->s_szc >= szc && set) {
3029                                 ASSERT(setsize != 0);
3030                                 error = as_iset2_default_lpsize(as,
3031                                     setaddr, setsize, szc, szcvec);
3032                                 if (error) {
3033                                         return (error);
3034                                 }
3035                                 set = 0;
3036                         } else if (seg->s_szc < szc && !set) {
3037                                 setaddr = raddr;
3038                                 setsize = 0;
3039                                 set = 1;
3040                         }
3041                 }
3042                 if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
3043                         ssize = seg->s_base + seg->s_size - raddr;
3044                 } else {
3045                         ssize = rsize;
3046                 }
3047         }
3048         error = 0;
3049         if (set) {
3050                 ASSERT(setsize != 0);
3051                 error = as_iset2_default_lpsize(as, setaddr, setsize,
3052                     szc, szcvec);
3053         }
3054         return (error);
3055 }
3056 
3057 /*
3058  * as_iset_default_lpsize() breaks its chunk according to the size code bitmap
3059  * returned by map_pgszcvec() (similar to as_map_segvn_segs()), and passes each
3060  * chunk to as_iset1_default_lpsize().
3061  */
3062 static int
3063 as_iset_default_lpsize(struct as *as, caddr_t addr, size_t size, int flags,
3064     int type)
3065 {
3066         int rtype = (type & MAP_SHARED) ? MAPPGSZC_SHM : MAPPGSZC_PRIVM;
3067         uint_t szcvec = map_pgszcvec(addr, size, (uintptr_t)addr,
3068             flags, rtype, 1);
3069         uint_t szc;
3070         uint_t nszc;
3071         int error;
3072         caddr_t a;
3073         caddr_t eaddr;
3074         size_t segsize;
3075         size_t pgsz;
3076         uint_t save_szcvec;
3077 
3078         ASSERT(AS_WRITE_HELD(as));
3079         ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
3080         ASSERT(IS_P2ALIGNED(size, PAGESIZE));
3081 
3082         szcvec &= ~1;
3083         if (szcvec <= 1) {   /* skip if base page size */
3084                 return (0);
3085         }
3086 
3087         /* Get the pagesize of the first larger page size. */
3088         szc = lowbit(szcvec) - 1;
3089         pgsz = page_get_pagesize(szc);
3090         eaddr = addr + size;
3091         addr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
3092         eaddr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
3093 
3094         save_szcvec = szcvec;
3095         szcvec >>= (szc + 1);
3096         nszc = szc;
3097         while (szcvec) {
3098                 if ((szcvec & 0x1) == 0) {
3099                         nszc++;
3100                         szcvec >>= 1;
3101                         continue;
3102                 }
3103                 nszc++;
3104                 pgsz = page_get_pagesize(nszc);
3105                 a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
3106                 if (a != addr) {
3107                         ASSERT(szc > 0);
3108                         ASSERT(a < eaddr);
3109                         segsize = a - addr;
3110                         error = as_iset1_default_lpsize(as, addr, segsize, szc,
3111                             save_szcvec);
3112                         if (error) {
3113                                 return (error);
3114                         }
3115                         addr = a;
3116                 }
3117                 szc = nszc;
3118                 szcvec >>= 1;
3119         }
3120 
3121         ASSERT(addr < eaddr);
3122         szcvec = save_szcvec;
3123         while (szcvec) {
3124                 a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
3125                 ASSERT(a >= addr);
3126                 if (a != addr) {
3127                         ASSERT(szc > 0);
3128                         segsize = a - addr;
3129                         error = as_iset1_default_lpsize(as, addr, segsize, szc,
3130                             save_szcvec);
3131                         if (error) {
3132                                 return (error);
3133                         }
3134                         addr = a;
3135                 }
3136                 szcvec &= ~(1 << szc);
3137                 if (szcvec) {
3138                         szc = highbit(szcvec) - 1;
3139                         pgsz = page_get_pagesize(szc);
3140                 }
3141         }
3142         ASSERT(addr == eaddr);
3143 
3144         return (0);
3145 }
3146 
3147 /*
3148  * Set the default large page size for the range. Called via memcntl with
3149  * page size set to 0. as_set_default_lpsize breaks the range down into
3150  * chunks with the same type/flags, ignores-non segvn segments, and passes
3151  * each chunk to as_iset_default_lpsize().
3152  */
3153 int
3154 as_set_default_lpsize(struct as *as, caddr_t addr, size_t size)
3155 {
3156         struct seg *seg;
3157         caddr_t raddr;
3158         size_t rsize;
3159         size_t ssize;
3160         int rtype, rflags;
3161         int stype, sflags;
3162         int error;
3163         caddr_t setaddr;
3164         size_t setsize;
3165         int segvn;
3166 
3167         if (size == 0)
3168                 return (0);
3169 
3170         AS_LOCK_ENTER(as, RW_WRITER);
3171 again:
3172         error = 0;
3173 
3174         raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
3175         rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
3176             (size_t)raddr;
3177 
3178         if (raddr + rsize < raddr) {         /* check for wraparound */
3179                 AS_LOCK_EXIT(as);
3180                 return (ENOMEM);
3181         }
3182         as_clearwatchprot(as, raddr, rsize);
3183         seg = as_segat(as, raddr);
3184         if (seg == NULL) {
3185                 as_setwatch(as);
3186                 AS_LOCK_EXIT(as);
3187                 return (ENOMEM);
3188         }
3189         if (seg->s_ops == &segvn_ops) {
3190                 rtype = SEGOP_GETTYPE(seg, addr);
3191                 rflags = rtype & (MAP_TEXT | MAP_INITDATA);
3192                 rtype = rtype & (MAP_SHARED | MAP_PRIVATE);
3193                 segvn = 1;
3194         } else {
3195                 segvn = 0;
3196         }
3197         setaddr = raddr;
3198         setsize = 0;
3199 
3200         for (; rsize != 0; rsize -= ssize, raddr += ssize, setsize += ssize) {
3201                 if (raddr >= (seg->s_base + seg->s_size)) {
3202                         seg = AS_SEGNEXT(as, seg);
3203                         if (seg == NULL || raddr != seg->s_base) {
3204                                 error = ENOMEM;
3205                                 break;
3206                         }
3207                         if (seg->s_ops == &segvn_ops) {
3208                                 stype = SEGOP_GETTYPE(seg, raddr);
3209                                 sflags = stype & (MAP_TEXT | MAP_INITDATA);
3210                                 stype &= (MAP_SHARED | MAP_PRIVATE);
3211                                 if (segvn && (rflags != sflags ||
3212                                     rtype != stype)) {
3213                                         /*
3214                                          * The next segment is also segvn but
3215                                          * has different flags and/or type.
3216                                          */
3217                                         ASSERT(setsize != 0);
3218                                         error = as_iset_default_lpsize(as,
3219                                             setaddr, setsize, rflags, rtype);
3220                                         if (error) {
3221                                                 break;
3222                                         }
3223                                         rflags = sflags;
3224                                         rtype = stype;
3225                                         setaddr = raddr;
3226                                         setsize = 0;
3227                                 } else if (!segvn) {
3228                                         rflags = sflags;
3229                                         rtype = stype;
3230                                         setaddr = raddr;
3231                                         setsize = 0;
3232                                         segvn = 1;
3233                                 }
3234                         } else if (segvn) {
3235                                 /* The next segment is not segvn. */
3236                                 ASSERT(setsize != 0);
3237                                 error = as_iset_default_lpsize(as,
3238                                     setaddr, setsize, rflags, rtype);
3239                                 if (error) {
3240                                         break;
3241                                 }
3242                                 segvn = 0;
3243                         }
3244                 }
3245                 if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
3246                         ssize = seg->s_base + seg->s_size - raddr;
3247                 } else {
3248                         ssize = rsize;
3249                 }
3250         }
3251         if (error == 0 && segvn) {
3252                 /* The last chunk when rsize == 0. */
3253                 ASSERT(setsize != 0);
3254                 error = as_iset_default_lpsize(as, setaddr, setsize,
3255                     rflags, rtype);
3256         }
3257 
3258         if (error == IE_RETRY) {
3259                 goto again;
3260         } else if (error == IE_NOMEM) {
3261                 error = EAGAIN;
3262         } else if (error == ENOTSUP) {
3263                 error = EINVAL;
3264         } else if (error == EAGAIN) {
3265                 mutex_enter(&as->a_contents);
3266                 if (!AS_ISNOUNMAPWAIT(as)) {
3267                         if (AS_ISUNMAPWAIT(as) == 0) {
3268                                 cv_broadcast(&as->a_cv);
3269                         }
3270                         AS_SETUNMAPWAIT(as);
3271                         AS_LOCK_EXIT(as);
3272                         while (AS_ISUNMAPWAIT(as)) {
3273                                 cv_wait(&as->a_cv, &as->a_contents);
3274                         }
3275                         mutex_exit(&as->a_contents);
3276                         AS_LOCK_ENTER(as, RW_WRITER);
3277                 } else {
3278                         /*
3279                          * We may have raced with
3280                          * segvn_reclaim()/segspt_reclaim(). In this case
3281                          * clean nounmapwait flag and retry since softlockcnt
3282                          * in this segment may be already 0.  We don't drop as
3283                          * writer lock so our number of retries without
3284                          * sleeping should be very small. See segvn_reclaim()
3285                          * for more comments.
3286                          */
3287                         AS_CLRNOUNMAPWAIT(as);
3288                         mutex_exit(&as->a_contents);
3289                 }
3290                 goto again;
3291         }
3292 
3293         as_setwatch(as);
3294         AS_LOCK_EXIT(as);
3295         return (error);
3296 }
3297 
3298 /*
3299  * Setup all of the uninitialized watched pages that we can.
3300  */
3301 void
3302 as_setwatch(struct as *as)
3303 {
3304         struct watched_page *pwp;
3305         struct seg *seg;
3306         caddr_t vaddr;
3307         uint_t prot;
3308         int  err, retrycnt;
3309 
3310         if (avl_numnodes(&as->a_wpage) == 0)
3311                 return;
3312 
3313         ASSERT(AS_WRITE_HELD(as));
3314 
3315         for (pwp = avl_first(&as->a_wpage); pwp != NULL;
3316             pwp = AVL_NEXT(&as->a_wpage, pwp)) {
3317                 retrycnt = 0;
3318         retry:
3319                 vaddr = pwp->wp_vaddr;
3320                 if (pwp->wp_oprot != 0 ||    /* already set up */
3321                     (seg = as_segat(as, vaddr)) == NULL ||
3322                     SEGOP_GETPROT(seg, vaddr, 0, &prot) != 0)
3323                         continue;
3324 
3325                 pwp->wp_oprot = prot;
3326                 if (pwp->wp_read)
3327                         prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
3328                 if (pwp->wp_write)
3329                         prot &= ~PROT_WRITE;
3330                 if (pwp->wp_exec)
3331                         prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
3332                 if (!(pwp->wp_flags & WP_NOWATCH) && prot != pwp->wp_oprot) {
3333                         err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot);
3334                         if (err == IE_RETRY) {
3335                                 pwp->wp_oprot = 0;
3336                                 ASSERT(retrycnt == 0);
3337                                 retrycnt++;
3338                                 goto retry;
3339                         }
3340                 }
3341                 pwp->wp_prot = prot;
3342         }
3343 }
3344 
3345 /*
3346  * Clear all of the watched pages in the address space.
3347  */
3348 void
3349 as_clearwatch(struct as *as)
3350 {
3351         struct watched_page *pwp;
3352         struct seg *seg;
3353         caddr_t vaddr;
3354         uint_t prot;
3355         int err, retrycnt;
3356 
3357         if (avl_numnodes(&as->a_wpage) == 0)
3358                 return;
3359 
3360         ASSERT(AS_WRITE_HELD(as));
3361 
3362         for (pwp = avl_first(&as->a_wpage); pwp != NULL;
3363             pwp = AVL_NEXT(&as->a_wpage, pwp)) {
3364                 retrycnt = 0;
3365         retry:
3366                 vaddr = pwp->wp_vaddr;
3367                 if (pwp->wp_oprot == 0 ||    /* not set up */
3368                     (seg = as_segat(as, vaddr)) == NULL)
3369                         continue;
3370 
3371                 if ((prot = pwp->wp_oprot) != pwp->wp_prot) {
3372                         err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot);
3373                         if (err == IE_RETRY) {
3374                                 ASSERT(retrycnt == 0);
3375                                 retrycnt++;
3376                                 goto retry;
3377                         }
3378                 }
3379                 pwp->wp_oprot = 0;
3380                 pwp->wp_prot = 0;
3381         }
3382 }
3383 
3384 /*
3385  * Force a new setup for all the watched pages in the range.
3386  */
3387 static void
3388 as_setwatchprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
3389 {
3390         struct watched_page *pwp;
3391         struct watched_page tpw;
3392         caddr_t eaddr = addr + size;
3393         caddr_t vaddr;
3394         struct seg *seg;
3395         int err, retrycnt;
3396         uint_t  wprot;
3397         avl_index_t where;
3398 
3399         if (avl_numnodes(&as->a_wpage) == 0)
3400                 return;
3401 
3402         ASSERT(AS_WRITE_HELD(as));
3403 
3404         tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
3405         if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL)
3406                 pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER);
3407 
3408         while (pwp != NULL && pwp->wp_vaddr < eaddr) {
3409                 retrycnt = 0;
3410                 vaddr = pwp->wp_vaddr;
3411 
3412                 wprot = prot;
3413                 if (pwp->wp_read)
3414                         wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
3415                 if (pwp->wp_write)
3416                         wprot &= ~PROT_WRITE;
3417                 if (pwp->wp_exec)
3418                         wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
3419                 if (!(pwp->wp_flags & WP_NOWATCH) && wprot != pwp->wp_oprot) {
3420                 retry:
3421                         seg = as_segat(as, vaddr);
3422                         if (seg == NULL) {
3423                                 panic("as_setwatchprot: no seg");
3424                                 /*NOTREACHED*/
3425                         }
3426                         err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, wprot);
3427                         if (err == IE_RETRY) {
3428                                 ASSERT(retrycnt == 0);
3429                                 retrycnt++;
3430                                 goto retry;
3431                         }
3432                 }
3433                 pwp->wp_oprot = prot;
3434                 pwp->wp_prot = wprot;
3435 
3436                 pwp = AVL_NEXT(&as->a_wpage, pwp);
3437         }
3438 }
3439 
3440 /*
3441  * Clear all of the watched pages in the range.
3442  */
3443 static void
3444 as_clearwatchprot(struct as *as, caddr_t addr, size_t size)
3445 {
3446         caddr_t eaddr = addr + size;
3447         struct watched_page *pwp;
3448         struct watched_page tpw;
3449         uint_t prot;
3450         struct seg *seg;
3451         int err, retrycnt;
3452         avl_index_t where;
3453 
3454         if (avl_numnodes(&as->a_wpage) == 0)
3455                 return;
3456 
3457         tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
3458         if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL)
3459                 pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER);
3460 
3461         ASSERT(AS_WRITE_HELD(as));
3462 
3463         while (pwp != NULL && pwp->wp_vaddr < eaddr) {
3464 
3465                 if ((prot = pwp->wp_oprot) != 0) {
3466                         retrycnt = 0;
3467 
3468                         if (prot != pwp->wp_prot) {
3469                         retry:
3470                                 seg = as_segat(as, pwp->wp_vaddr);
3471                                 if (seg == NULL)
3472                                         continue;
3473                                 err = SEGOP_SETPROT(seg, pwp->wp_vaddr,
3474                                     PAGESIZE, prot);
3475                                 if (err == IE_RETRY) {
3476                                         ASSERT(retrycnt == 0);
3477                                         retrycnt++;
3478                                         goto retry;
3479 
3480                                 }
3481                         }
3482                         pwp->wp_oprot = 0;
3483                         pwp->wp_prot = 0;
3484                 }
3485 
3486                 pwp = AVL_NEXT(&as->a_wpage, pwp);
3487         }
3488 }
3489 
3490 void
3491 as_signal_proc(struct as *as, k_siginfo_t *siginfo)
3492 {
3493         struct proc *p;
3494 
3495         mutex_enter(&pidlock);
3496         for (p = practive; p; p = p->p_next) {
3497                 if (p->p_as == as) {
3498                         mutex_enter(&p->p_lock);
3499                         if (p->p_as == as)
3500                                 sigaddq(p, NULL, siginfo, KM_NOSLEEP);
3501                         mutex_exit(&p->p_lock);
3502                 }
3503         }
3504         mutex_exit(&pidlock);
3505 }
3506 
3507 /*
3508  * return memory object ID
3509  */
3510 int
3511 as_getmemid(struct as *as, caddr_t addr, memid_t *memidp)
3512 {
3513         struct seg      *seg;
3514         int             sts;
3515 
3516         AS_LOCK_ENTER(as, RW_READER);
3517         seg = as_segat(as, addr);
3518         if (seg == NULL) {
3519                 AS_LOCK_EXIT(as);
3520                 return (EFAULT);
3521         }
3522         /*
3523          * catch old drivers which may not support getmemid
3524          */
3525         if (seg->s_ops->getmemid == NULL) {
3526                 AS_LOCK_EXIT(as);
3527                 return (ENODEV);
3528         }
3529 
3530         sts = SEGOP_GETMEMID(seg, addr, memidp);
3531 
3532         AS_LOCK_EXIT(as);
3533         return (sts);
3534 }