3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1986, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
26 /* All Rights Reserved */
27
28 /*
29 * University Copyright- Copyright (c) 1982, 1986, 1988
30 * The Regents of the University of California
31 * All Rights Reserved
32 *
33 * University Acknowledgment- Portions of this document are derived from
34 * software developed by the University of California, Berkeley, and its
35 * contributors.
36 */
37
38 /*
39 * VM - physical page management.
40 */
41
42 #include <sys/types.h>
250 uint_t page_create_overshoot;
251
252 uint_t page_reclaim_zero;
253 uint_t page_reclaim_zero_locked;
254
255 uint_t page_rename_exists;
256 uint_t page_rename_count;
257
258 uint_t page_lookup_cnt[20];
259 uint_t page_lookup_nowait_cnt[10];
260 uint_t page_find_cnt;
261 uint_t page_exists_cnt;
262 uint_t page_exists_forreal_cnt;
263 uint_t page_lookup_dev_cnt;
264 uint_t get_cachelist_cnt;
265 uint_t page_create_cnt[10];
266 uint_t alloc_pages[9];
267 uint_t page_exphcontg[19];
268 uint_t page_create_large_cnt[10];
269
270 /*
271 * Collects statistics.
272 */
273 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \
274 uint_t mylen = 0; \
275 \
276 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash, mylen++) { \
277 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
278 break; \
279 } \
280 if ((pp) != NULL) \
281 pagecnt.pc_find_hit++; \
282 else \
283 pagecnt.pc_find_miss++; \
284 if (mylen > PC_HASH_CNT) \
285 mylen = PC_HASH_CNT; \
286 pagecnt.pc_find_hashlen[mylen]++; \
287 }
288
289 #else /* VM_STATS */
290
291 /*
292 * Don't collect statistics
293 */
294 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \
295 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \
296 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
297 break; \
298 } \
299 }
300
301 #endif /* VM_STATS */
302
303
304
305 #ifdef DEBUG
306 #define MEMSEG_SEARCH_STATS
307 #endif
308
309 #ifdef MEMSEG_SEARCH_STATS
310 struct memseg_stats {
311 uint_t nsearch;
312 uint_t nlastwon;
313 uint_t nhashwon;
314 uint_t nnotfound;
315 } memseg_stats;
316
317 #define MEMSEG_STAT_INCR(v) \
318 atomic_inc_32(&memseg_stats.v)
319 #else
320 #define MEMSEG_STAT_INCR(x)
321 #endif
322
733 page_t *pp;
734 kmutex_t *phm;
735 ulong_t index;
736 uint_t hash_locked;
737 uint_t es;
738
739 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
740 VM_STAT_ADD(page_lookup_cnt[0]);
741 ASSERT(newpp ? PAGE_EXCL(newpp) : 1);
742
743 /*
744 * Acquire the appropriate page hash lock since
745 * we have to search the hash list. Pages that
746 * hash to this list can't change identity while
747 * this lock is held.
748 */
749 hash_locked = 0;
750 index = PAGE_HASH_FUNC(vp, off);
751 phm = NULL;
752 top:
753 PAGE_HASH_SEARCH(index, pp, vp, off);
754 if (pp != NULL) {
755 VM_STAT_ADD(page_lookup_cnt[1]);
756 es = (newpp != NULL) ? 1 : 0;
757 es |= flags;
758 if (!hash_locked) {
759 VM_STAT_ADD(page_lookup_cnt[2]);
760 if (!page_try_reclaim_lock(pp, se, es)) {
761 /*
762 * On a miss, acquire the phm. Then
763 * next time, page_lock() will be called,
764 * causing a wait if the page is busy.
765 * just looping with page_trylock() would
766 * get pretty boring.
767 */
768 VM_STAT_ADD(page_lookup_cnt[3]);
769 phm = PAGE_HASH_MUTEX(index);
770 mutex_enter(phm);
771 hash_locked = 1;
772 goto top;
773 }
774 } else {
775 VM_STAT_ADD(page_lookup_cnt[4]);
776 if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) {
777 VM_STAT_ADD(page_lookup_cnt[5]);
778 goto top;
779 }
780 }
781
782 /*
783 * Since `pp' is locked it can not change identity now.
784 * Reconfirm we locked the correct page.
785 *
786 * Both the p_vnode and p_offset *must* be cast volatile
787 * to force a reload of their values: The PAGE_HASH_SEARCH
788 * macro will have stuffed p_vnode and p_offset into
789 * registers before calling page_trylock(); another thread,
790 * actually holding the hash lock, could have changed the
791 * page's identity in memory, but our registers would not
792 * be changed, fooling the reconfirmation. If the hash
793 * lock was held during the search, the casting would
794 * not be needed.
795 */
796 VM_STAT_ADD(page_lookup_cnt[6]);
797 if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
798 ((volatile u_offset_t)(pp->p_offset) != off)) {
799 VM_STAT_ADD(page_lookup_cnt[7]);
800 if (hash_locked) {
801 panic("page_lookup_create: lost page %p",
802 (void *)pp);
803 /*NOTREACHED*/
804 }
805 page_unlock(pp);
806 phm = PAGE_HASH_MUTEX(index);
807 mutex_enter(phm);
808 hash_locked = 1;
931 }
932
933 /*
934 * Search the hash list for the page representing the
935 * specified [vp, offset] and return it locked. Skip
936 * free pages and pages that cannot be locked as requested.
937 * Used while attempting to kluster pages.
938 */
939 page_t *
940 page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se)
941 {
942 page_t *pp;
943 kmutex_t *phm;
944 ulong_t index;
945 uint_t locked;
946
947 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
948 VM_STAT_ADD(page_lookup_nowait_cnt[0]);
949
950 index = PAGE_HASH_FUNC(vp, off);
951 PAGE_HASH_SEARCH(index, pp, vp, off);
952 locked = 0;
953 if (pp == NULL) {
954 top:
955 VM_STAT_ADD(page_lookup_nowait_cnt[1]);
956 locked = 1;
957 phm = PAGE_HASH_MUTEX(index);
958 mutex_enter(phm);
959 PAGE_HASH_SEARCH(index, pp, vp, off);
960 }
961
962 if (pp == NULL || PP_ISFREE(pp)) {
963 VM_STAT_ADD(page_lookup_nowait_cnt[2]);
964 pp = NULL;
965 } else {
966 if (!page_trylock(pp, se)) {
967 VM_STAT_ADD(page_lookup_nowait_cnt[3]);
968 pp = NULL;
969 } else {
970 VM_STAT_ADD(page_lookup_nowait_cnt[4]);
971 /*
972 * See the comment in page_lookup()
973 */
974 if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
975 ((u_offset_t)(pp->p_offset) != off)) {
976 VM_STAT_ADD(page_lookup_nowait_cnt[5]);
977 if (locked) {
978 panic("page_lookup_nowait %p",
979 (void *)pp);
1001
1002 /*
1003 * Search the hash list for a page with the specified [vp, off]
1004 * that is known to exist and is already locked. This routine
1005 * is typically used by segment SOFTUNLOCK routines.
1006 */
1007 page_t *
1008 page_find(vnode_t *vp, u_offset_t off)
1009 {
1010 page_t *pp;
1011 kmutex_t *phm;
1012 ulong_t index;
1013
1014 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1015 VM_STAT_ADD(page_find_cnt);
1016
1017 index = PAGE_HASH_FUNC(vp, off);
1018 phm = PAGE_HASH_MUTEX(index);
1019
1020 mutex_enter(phm);
1021 PAGE_HASH_SEARCH(index, pp, vp, off);
1022 mutex_exit(phm);
1023
1024 ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr);
1025 return (pp);
1026 }
1027
1028 /*
1029 * Determine whether a page with the specified [vp, off]
1030 * currently exists in the system. Obviously this should
1031 * only be considered as a hint since nothing prevents the
1032 * page from disappearing or appearing immediately after
1033 * the return from this routine. Subsequently, we don't
1034 * even bother to lock the list.
1035 */
1036 page_t *
1037 page_exists(vnode_t *vp, u_offset_t off)
1038 {
1039 page_t *pp;
1040 ulong_t index;
1041
1042 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1043 VM_STAT_ADD(page_exists_cnt);
1044
1045 index = PAGE_HASH_FUNC(vp, off);
1046 PAGE_HASH_SEARCH(index, pp, vp, off);
1047
1048 return (pp);
1049 }
1050
1051 /*
1052 * Determine if physically contiguous pages exist for [vp, off] - [vp, off +
1053 * page_size(szc)) range. if they exist and ppa is not NULL fill ppa array
1054 * with these pages locked SHARED. If necessary reclaim pages from
1055 * freelist. Return 1 if contiguous pages exist and 0 otherwise.
1056 *
1057 * If we fail to lock pages still return 1 if pages exist and contiguous.
1058 * But in this case return value is just a hint. ppa array won't be filled.
1059 * Caller should initialize ppa[0] as NULL to distinguish return value.
1060 *
1061 * Returns 0 if pages don't exist or not physically contiguous.
1062 *
1063 * This routine doesn't work for anonymous(swapfs) pages.
1064 */
1065 int
1066 page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[])
1067 {
1068 pgcnt_t pages;
1075 kmutex_t *phm;
1076 page_t *pp;
1077 uint_t pszc;
1078 int loopcnt = 0;
1079
1080 ASSERT(szc != 0);
1081 ASSERT(vp != NULL);
1082 ASSERT(!IS_SWAPFSVP(vp));
1083 ASSERT(!VN_ISKAS(vp));
1084
1085 again:
1086 if (++loopcnt > 3) {
1087 VM_STAT_ADD(page_exphcontg[0]);
1088 return (0);
1089 }
1090
1091 index = PAGE_HASH_FUNC(vp, off);
1092 phm = PAGE_HASH_MUTEX(index);
1093
1094 mutex_enter(phm);
1095 PAGE_HASH_SEARCH(index, pp, vp, off);
1096 mutex_exit(phm);
1097
1098 VM_STAT_ADD(page_exphcontg[1]);
1099
1100 if (pp == NULL) {
1101 VM_STAT_ADD(page_exphcontg[2]);
1102 return (0);
1103 }
1104
1105 pages = page_get_pagecnt(szc);
1106 rootpp = pp;
1107 pfn = rootpp->p_pagenum;
1108
1109 if ((pszc = pp->p_szc) >= szc && ppa != NULL) {
1110 VM_STAT_ADD(page_exphcontg[3]);
1111 if (!page_trylock(pp, SE_SHARED)) {
1112 VM_STAT_ADD(page_exphcontg[4]);
1113 return (1);
1114 }
1115 /*
1302 * size code. Obviously this should only be considered as
1303 * a hint since nothing prevents the page from disappearing
1304 * or appearing immediately after the return from this routine.
1305 */
1306 int
1307 page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc)
1308 {
1309 page_t *pp;
1310 kmutex_t *phm;
1311 ulong_t index;
1312 int rc = 0;
1313
1314 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1315 ASSERT(szc != NULL);
1316 VM_STAT_ADD(page_exists_forreal_cnt);
1317
1318 index = PAGE_HASH_FUNC(vp, off);
1319 phm = PAGE_HASH_MUTEX(index);
1320
1321 mutex_enter(phm);
1322 PAGE_HASH_SEARCH(index, pp, vp, off);
1323 if (pp != NULL) {
1324 *szc = pp->p_szc;
1325 rc = 1;
1326 }
1327 mutex_exit(phm);
1328 return (rc);
1329 }
1330
1331 /* wakeup threads waiting for pages in page_create_get_something() */
1332 void
1333 wakeup_pcgs(void)
1334 {
1335 if (!CV_HAS_WAITERS(&pcgs_cv))
1336 return;
1337 cv_broadcast(&pcgs_cv);
1338 }
1339
1340 /*
1341 * 'freemem' is used all over the kernel as an indication of how many
1342 * pages are free (either on the cache list or on the free page list)
2430 }
2431 }
2432
2433 /*
2434 * We own this page!
2435 */
2436 ASSERT(PAGE_EXCL(npp));
2437 ASSERT(npp->p_vnode == NULL);
2438 ASSERT(!hat_page_is_mapped(npp));
2439 PP_CLRFREE(npp);
2440 PP_CLRAGED(npp);
2441
2442 /*
2443 * Here we have a page in our hot little mits and are
2444 * just waiting to stuff it on the appropriate lists.
2445 * Get the mutex and check to see if it really does
2446 * not exist.
2447 */
2448 phm = PAGE_HASH_MUTEX(index);
2449 mutex_enter(phm);
2450 PAGE_HASH_SEARCH(index, pp, vp, off);
2451 if (pp == NULL) {
2452 VM_STAT_ADD(page_create_new);
2453 pp = npp;
2454 npp = NULL;
2455 if (!page_hashin(pp, vp, off, phm)) {
2456 /*
2457 * Since we hold the page hash mutex and
2458 * just searched for this page, page_hashin
2459 * had better not fail. If it does, that
2460 * means somethread did not follow the
2461 * page hash mutex rules. Panic now and
2462 * get it over with. As usual, go down
2463 * holding all the locks.
2464 */
2465 ASSERT(MUTEX_HELD(phm));
2466 panic("page_create: "
2467 "hashin failed %p %p %llx %p",
2468 (void *)pp, (void *)vp, off, (void *)phm);
2469 /*NOTREACHED*/
2470 }
3260 page_hashout(opp, NULL);
3261 PP_CLRAGED(opp);
3262
3263 /*
3264 * Acquire the appropriate page hash lock, since
3265 * we're going to rename the page.
3266 */
3267 index = PAGE_HASH_FUNC(vp, off);
3268 phm = PAGE_HASH_MUTEX(index);
3269 mutex_enter(phm);
3270 top:
3271 /*
3272 * Look for an existing page with this name and destroy it if found.
3273 * By holding the page hash lock all the way to the page_hashin()
3274 * call, we are assured that no page can be created with this
3275 * identity. In the case when the phm lock is dropped to undo any
3276 * hat layer mappings, the existing page is held with an "exclusive"
3277 * lock, again preventing another page from being created with
3278 * this identity.
3279 */
3280 PAGE_HASH_SEARCH(index, pp, vp, off);
3281 if (pp != NULL) {
3282 VM_STAT_ADD(page_rename_exists);
3283
3284 /*
3285 * As it turns out, this is one of only two places where
3286 * page_lock() needs to hold the passed in lock in the
3287 * successful case. In all of the others, the lock could
3288 * be dropped as soon as the attempt is made to lock
3289 * the page. It is tempting to add yet another arguement,
3290 * PL_KEEP or PL_DROP, to let page_lock know what to do.
3291 */
3292 if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) {
3293 /*
3294 * Went to sleep because the page could not
3295 * be locked. We were woken up when the page
3296 * was unlocked, or when the page was destroyed.
3297 * In either case, `phm' was dropped while we
3298 * slept. Hence we should not just roar through
3299 * this loop.
3300 */
|
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1986, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2015, Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
24 */
25
26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
27 /* All Rights Reserved */
28
29 /*
30 * University Copyright- Copyright (c) 1982, 1986, 1988
31 * The Regents of the University of California
32 * All Rights Reserved
33 *
34 * University Acknowledgment- Portions of this document are derived from
35 * software developed by the University of California, Berkeley, and its
36 * contributors.
37 */
38
39 /*
40 * VM - physical page management.
41 */
42
43 #include <sys/types.h>
251 uint_t page_create_overshoot;
252
253 uint_t page_reclaim_zero;
254 uint_t page_reclaim_zero_locked;
255
256 uint_t page_rename_exists;
257 uint_t page_rename_count;
258
259 uint_t page_lookup_cnt[20];
260 uint_t page_lookup_nowait_cnt[10];
261 uint_t page_find_cnt;
262 uint_t page_exists_cnt;
263 uint_t page_exists_forreal_cnt;
264 uint_t page_lookup_dev_cnt;
265 uint_t get_cachelist_cnt;
266 uint_t page_create_cnt[10];
267 uint_t alloc_pages[9];
268 uint_t page_exphcontg[19];
269 uint_t page_create_large_cnt[10];
270
271 #endif
272
273 static inline page_t *
274 page_hash_search(ulong_t index, vnode_t *vnode, u_offset_t off)
275 {
276 uint_t mylen = 0;
277 page_t *page;
278
279 for (page = page_hash[index]; page; page = page->p_hash, mylen++)
280 if (page->p_vnode == vnode && page->p_offset == off)
281 break;
282
283 #ifdef VM_STATS
284 if (page != NULL)
285 pagecnt.pc_find_hit++;
286 else
287 pagecnt.pc_find_miss++;
288
289 pagecnt.pc_find_hashlen[MIN(mylen, PC_HASH_CNT)]++;
290 #endif
291
292 return (page);
293 }
294
295
296 #ifdef DEBUG
297 #define MEMSEG_SEARCH_STATS
298 #endif
299
300 #ifdef MEMSEG_SEARCH_STATS
301 struct memseg_stats {
302 uint_t nsearch;
303 uint_t nlastwon;
304 uint_t nhashwon;
305 uint_t nnotfound;
306 } memseg_stats;
307
308 #define MEMSEG_STAT_INCR(v) \
309 atomic_inc_32(&memseg_stats.v)
310 #else
311 #define MEMSEG_STAT_INCR(x)
312 #endif
313
724 page_t *pp;
725 kmutex_t *phm;
726 ulong_t index;
727 uint_t hash_locked;
728 uint_t es;
729
730 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
731 VM_STAT_ADD(page_lookup_cnt[0]);
732 ASSERT(newpp ? PAGE_EXCL(newpp) : 1);
733
734 /*
735 * Acquire the appropriate page hash lock since
736 * we have to search the hash list. Pages that
737 * hash to this list can't change identity while
738 * this lock is held.
739 */
740 hash_locked = 0;
741 index = PAGE_HASH_FUNC(vp, off);
742 phm = NULL;
743 top:
744 pp = page_hash_search(index, vp, off);
745 if (pp != NULL) {
746 VM_STAT_ADD(page_lookup_cnt[1]);
747 es = (newpp != NULL) ? 1 : 0;
748 es |= flags;
749 if (!hash_locked) {
750 VM_STAT_ADD(page_lookup_cnt[2]);
751 if (!page_try_reclaim_lock(pp, se, es)) {
752 /*
753 * On a miss, acquire the phm. Then
754 * next time, page_lock() will be called,
755 * causing a wait if the page is busy.
756 * just looping with page_trylock() would
757 * get pretty boring.
758 */
759 VM_STAT_ADD(page_lookup_cnt[3]);
760 phm = PAGE_HASH_MUTEX(index);
761 mutex_enter(phm);
762 hash_locked = 1;
763 goto top;
764 }
765 } else {
766 VM_STAT_ADD(page_lookup_cnt[4]);
767 if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) {
768 VM_STAT_ADD(page_lookup_cnt[5]);
769 goto top;
770 }
771 }
772
773 /*
774 * Since `pp' is locked it can not change identity now.
775 * Reconfirm we locked the correct page.
776 *
777 * Both the p_vnode and p_offset *must* be cast volatile
778 * to force a reload of their values: The page_hash_search
779 * function will have stuffed p_vnode and p_offset into
780 * registers before calling page_trylock(); another thread,
781 * actually holding the hash lock, could have changed the
782 * page's identity in memory, but our registers would not
783 * be changed, fooling the reconfirmation. If the hash
784 * lock was held during the search, the casting would
785 * not be needed.
786 */
787 VM_STAT_ADD(page_lookup_cnt[6]);
788 if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
789 ((volatile u_offset_t)(pp->p_offset) != off)) {
790 VM_STAT_ADD(page_lookup_cnt[7]);
791 if (hash_locked) {
792 panic("page_lookup_create: lost page %p",
793 (void *)pp);
794 /*NOTREACHED*/
795 }
796 page_unlock(pp);
797 phm = PAGE_HASH_MUTEX(index);
798 mutex_enter(phm);
799 hash_locked = 1;
922 }
923
924 /*
925 * Search the hash list for the page representing the
926 * specified [vp, offset] and return it locked. Skip
927 * free pages and pages that cannot be locked as requested.
928 * Used while attempting to kluster pages.
929 */
930 page_t *
931 page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se)
932 {
933 page_t *pp;
934 kmutex_t *phm;
935 ulong_t index;
936 uint_t locked;
937
938 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
939 VM_STAT_ADD(page_lookup_nowait_cnt[0]);
940
941 index = PAGE_HASH_FUNC(vp, off);
942 pp = page_hash_search(index, vp, off);
943 locked = 0;
944 if (pp == NULL) {
945 top:
946 VM_STAT_ADD(page_lookup_nowait_cnt[1]);
947 locked = 1;
948 phm = PAGE_HASH_MUTEX(index);
949 mutex_enter(phm);
950 pp = page_hash_search(index, vp, off);
951 }
952
953 if (pp == NULL || PP_ISFREE(pp)) {
954 VM_STAT_ADD(page_lookup_nowait_cnt[2]);
955 pp = NULL;
956 } else {
957 if (!page_trylock(pp, se)) {
958 VM_STAT_ADD(page_lookup_nowait_cnt[3]);
959 pp = NULL;
960 } else {
961 VM_STAT_ADD(page_lookup_nowait_cnt[4]);
962 /*
963 * See the comment in page_lookup()
964 */
965 if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
966 ((u_offset_t)(pp->p_offset) != off)) {
967 VM_STAT_ADD(page_lookup_nowait_cnt[5]);
968 if (locked) {
969 panic("page_lookup_nowait %p",
970 (void *)pp);
992
993 /*
994 * Search the hash list for a page with the specified [vp, off]
995 * that is known to exist and is already locked. This routine
996 * is typically used by segment SOFTUNLOCK routines.
997 */
998 page_t *
999 page_find(vnode_t *vp, u_offset_t off)
1000 {
1001 page_t *pp;
1002 kmutex_t *phm;
1003 ulong_t index;
1004
1005 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1006 VM_STAT_ADD(page_find_cnt);
1007
1008 index = PAGE_HASH_FUNC(vp, off);
1009 phm = PAGE_HASH_MUTEX(index);
1010
1011 mutex_enter(phm);
1012 pp = page_hash_search(index, vp, off);
1013 mutex_exit(phm);
1014
1015 ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr);
1016 return (pp);
1017 }
1018
1019 /*
1020 * Determine whether a page with the specified [vp, off]
1021 * currently exists in the system. Obviously this should
1022 * only be considered as a hint since nothing prevents the
1023 * page from disappearing or appearing immediately after
1024 * the return from this routine. Subsequently, we don't
1025 * even bother to lock the list.
1026 */
1027 page_t *
1028 page_exists(vnode_t *vp, u_offset_t off)
1029 {
1030 ulong_t index;
1031
1032 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1033 VM_STAT_ADD(page_exists_cnt);
1034
1035 index = PAGE_HASH_FUNC(vp, off);
1036
1037 return (page_hash_search(index, vp, off));
1038 }
1039
1040 /*
1041 * Determine if physically contiguous pages exist for [vp, off] - [vp, off +
1042 * page_size(szc)) range. if they exist and ppa is not NULL fill ppa array
1043 * with these pages locked SHARED. If necessary reclaim pages from
1044 * freelist. Return 1 if contiguous pages exist and 0 otherwise.
1045 *
1046 * If we fail to lock pages still return 1 if pages exist and contiguous.
1047 * But in this case return value is just a hint. ppa array won't be filled.
1048 * Caller should initialize ppa[0] as NULL to distinguish return value.
1049 *
1050 * Returns 0 if pages don't exist or not physically contiguous.
1051 *
1052 * This routine doesn't work for anonymous(swapfs) pages.
1053 */
1054 int
1055 page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[])
1056 {
1057 pgcnt_t pages;
1064 kmutex_t *phm;
1065 page_t *pp;
1066 uint_t pszc;
1067 int loopcnt = 0;
1068
1069 ASSERT(szc != 0);
1070 ASSERT(vp != NULL);
1071 ASSERT(!IS_SWAPFSVP(vp));
1072 ASSERT(!VN_ISKAS(vp));
1073
1074 again:
1075 if (++loopcnt > 3) {
1076 VM_STAT_ADD(page_exphcontg[0]);
1077 return (0);
1078 }
1079
1080 index = PAGE_HASH_FUNC(vp, off);
1081 phm = PAGE_HASH_MUTEX(index);
1082
1083 mutex_enter(phm);
1084 pp = page_hash_search(index, vp, off);
1085 mutex_exit(phm);
1086
1087 VM_STAT_ADD(page_exphcontg[1]);
1088
1089 if (pp == NULL) {
1090 VM_STAT_ADD(page_exphcontg[2]);
1091 return (0);
1092 }
1093
1094 pages = page_get_pagecnt(szc);
1095 rootpp = pp;
1096 pfn = rootpp->p_pagenum;
1097
1098 if ((pszc = pp->p_szc) >= szc && ppa != NULL) {
1099 VM_STAT_ADD(page_exphcontg[3]);
1100 if (!page_trylock(pp, SE_SHARED)) {
1101 VM_STAT_ADD(page_exphcontg[4]);
1102 return (1);
1103 }
1104 /*
1291 * size code. Obviously this should only be considered as
1292 * a hint since nothing prevents the page from disappearing
1293 * or appearing immediately after the return from this routine.
1294 */
1295 int
1296 page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc)
1297 {
1298 page_t *pp;
1299 kmutex_t *phm;
1300 ulong_t index;
1301 int rc = 0;
1302
1303 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1304 ASSERT(szc != NULL);
1305 VM_STAT_ADD(page_exists_forreal_cnt);
1306
1307 index = PAGE_HASH_FUNC(vp, off);
1308 phm = PAGE_HASH_MUTEX(index);
1309
1310 mutex_enter(phm);
1311 pp = page_hash_search(index, vp, off);
1312 if (pp != NULL) {
1313 *szc = pp->p_szc;
1314 rc = 1;
1315 }
1316 mutex_exit(phm);
1317 return (rc);
1318 }
1319
1320 /* wakeup threads waiting for pages in page_create_get_something() */
1321 void
1322 wakeup_pcgs(void)
1323 {
1324 if (!CV_HAS_WAITERS(&pcgs_cv))
1325 return;
1326 cv_broadcast(&pcgs_cv);
1327 }
1328
1329 /*
1330 * 'freemem' is used all over the kernel as an indication of how many
1331 * pages are free (either on the cache list or on the free page list)
2419 }
2420 }
2421
2422 /*
2423 * We own this page!
2424 */
2425 ASSERT(PAGE_EXCL(npp));
2426 ASSERT(npp->p_vnode == NULL);
2427 ASSERT(!hat_page_is_mapped(npp));
2428 PP_CLRFREE(npp);
2429 PP_CLRAGED(npp);
2430
2431 /*
2432 * Here we have a page in our hot little mits and are
2433 * just waiting to stuff it on the appropriate lists.
2434 * Get the mutex and check to see if it really does
2435 * not exist.
2436 */
2437 phm = PAGE_HASH_MUTEX(index);
2438 mutex_enter(phm);
2439 pp = page_hash_search(index, vp, off);
2440 if (pp == NULL) {
2441 VM_STAT_ADD(page_create_new);
2442 pp = npp;
2443 npp = NULL;
2444 if (!page_hashin(pp, vp, off, phm)) {
2445 /*
2446 * Since we hold the page hash mutex and
2447 * just searched for this page, page_hashin
2448 * had better not fail. If it does, that
2449 * means somethread did not follow the
2450 * page hash mutex rules. Panic now and
2451 * get it over with. As usual, go down
2452 * holding all the locks.
2453 */
2454 ASSERT(MUTEX_HELD(phm));
2455 panic("page_create: "
2456 "hashin failed %p %p %llx %p",
2457 (void *)pp, (void *)vp, off, (void *)phm);
2458 /*NOTREACHED*/
2459 }
3249 page_hashout(opp, NULL);
3250 PP_CLRAGED(opp);
3251
3252 /*
3253 * Acquire the appropriate page hash lock, since
3254 * we're going to rename the page.
3255 */
3256 index = PAGE_HASH_FUNC(vp, off);
3257 phm = PAGE_HASH_MUTEX(index);
3258 mutex_enter(phm);
3259 top:
3260 /*
3261 * Look for an existing page with this name and destroy it if found.
3262 * By holding the page hash lock all the way to the page_hashin()
3263 * call, we are assured that no page can be created with this
3264 * identity. In the case when the phm lock is dropped to undo any
3265 * hat layer mappings, the existing page is held with an "exclusive"
3266 * lock, again preventing another page from being created with
3267 * this identity.
3268 */
3269 pp = page_hash_search(index, vp, off);
3270 if (pp != NULL) {
3271 VM_STAT_ADD(page_rename_exists);
3272
3273 /*
3274 * As it turns out, this is one of only two places where
3275 * page_lock() needs to hold the passed in lock in the
3276 * successful case. In all of the others, the lock could
3277 * be dropped as soon as the attempt is made to lock
3278 * the page. It is tempting to add yet another arguement,
3279 * PL_KEEP or PL_DROP, to let page_lock know what to do.
3280 */
3281 if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) {
3282 /*
3283 * Went to sleep because the page could not
3284 * be locked. We were woken up when the page
3285 * was unlocked, or when the page was destroyed.
3286 * In either case, `phm' was dropped while we
3287 * slept. Hence we should not just roar through
3288 * this loop.
3289 */
|