642
643
644 /*
645 * The platform dependent hat structure.
646 * tte counts should be protected by cas.
647 * cpuset is protected by cas.
648 *
649 * ttecnt accounting for mappings which do not use shared hme is carried out
650 * during pagefault handling. In the shared hme case, only the first process
651 * to access a mapping generates a pagefault, subsequent processes simply
652 * find the shared hme entry during trap handling and therefore there is no
653 * corresponding event to initiate ttecnt accounting. Currently, as shared
654 * hmes are only used for text segments, when joining a region we assume the
655 * worst case and add the the number of ttes required to map the entire region
656 * to the ttecnt corresponding to the region pagesize. However, if the region
657 * has a 4M pagesize, and memory is low, the allocation of 4M pages may fail
658 * then 8K pages will be allocated instead and the first TSB which stores 8K
659 * mappings will potentially be undersized. To compensate for the potential
660 * underaccounting in this case we always add 1/4 of the region size to the 8K
661 * ttecnt.
662 *
663 * Note that sfmmu_xhat_provider MUST be the first element.
664 */
665
666 struct hat {
667 void *sfmmu_xhat_provider; /* NULL for CPU hat */
668 cpuset_t sfmmu_cpusran; /* cpu bit mask for efficient xcalls */
669 struct as *sfmmu_as; /* as this hat provides mapping for */
670 /* per pgsz private ttecnt + shme rgns ttecnt for rgns not in SCD */
671 ulong_t sfmmu_ttecnt[MMU_PAGE_SIZES];
672 /* shme rgns ttecnt for rgns in SCD */
673 ulong_t sfmmu_scdrttecnt[MMU_PAGE_SIZES];
674 /* est. ism ttes that are NOT in a SCD */
675 ulong_t sfmmu_ismttecnt[MMU_PAGE_SIZES];
676 /* ttecnt for isms that are in a SCD */
677 ulong_t sfmmu_scdismttecnt[MMU_PAGE_SIZES];
678 /* inflate tsb0 to allow for large page alloc failure in region */
679 ulong_t sfmmu_tsb0_4minflcnt;
680 union _h_un {
681 ism_blk_t *sfmmu_iblkp; /* maps to ismhat(s) */
682 ism_ment_t *sfmmu_imentp; /* ism hat's mapping list */
683 } h_un;
684 uint_t sfmmu_free:1; /* hat to be freed - set on as_free */
685 uint_t sfmmu_ismhat:1; /* hat is dummy ism hatid */
686 uint_t sfmmu_scdhat:1; /* hat is dummy scd hatid */
687 uchar_t sfmmu_rmstat; /* refmod stats refcnt */
1224 /*
1225 * Hment block structure.
1226 * The hme_blk is the node data structure which the hash structure
1227 * mantains. An hme_blk can have 2 different sizes depending on the
1228 * number of hments it implicitly contains. When dealing with 64K, 512K,
1229 * or 4M hments there is one hment per hme_blk. When dealing with
1230 * 8k hments we allocate an hme_blk plus an additional 7 hments to
1231 * give us a total of 8 (NHMENTS) hments that can be referenced through a
1232 * hme_blk.
1233 *
1234 * The hmeblk structure contains 2 tte reference counters used to determine if
1235 * it is ok to free up the hmeblk. Both counters have to be zero in order
1236 * to be able to free up hmeblk. They are protected by cas.
1237 * hblk_hmecnt is the number of hments present on pp mapping lists.
1238 * hblk_vcnt reflects number of valid ttes in hmeblk.
1239 *
1240 * The hmeblk now also has per tte lock cnts. This is required because
1241 * the counts can be high and there are not enough bits in the tte. When
1242 * physio is fixed to not lock the translations we should be able to move
1243 * the lock cnt back to the tte. See bug id 1198554.
1244 *
1245 * Note that xhat_hme_blk's layout follows this structure: hme_blk_misc
1246 * and sf_hment are at the same offsets in both structures. Whenever
1247 * hme_blk is changed, xhat_hme_blk may need to be updated as well.
1248 */
1249
1250 struct hme_blk_misc {
1251 uint_t notused:25;
1252 uint_t shared_bit:1; /* set for SRD shared hmeblk */
1253 uint_t xhat_bit:1; /* set for an xhat hme_blk */
1254 uint_t shadow_bit:1; /* set for a shadow hme_blk */
1255 uint_t nucleus_bit:1; /* set for a nucleus hme_blk */
1256 uint_t ttesize:3; /* contains ttesz of hmeblk */
1257 };
1258
1259 struct hme_blk {
1260 volatile uint64_t hblk_nextpa; /* physical address for hash list */
1261
1262 hmeblk_tag hblk_tag; /* tag used to obtain an hmeblk match */
1263
1264 struct hme_blk *hblk_next; /* on free list or on hash list */
1265 /* protected by hash lock */
1266
1267 struct hme_blk *hblk_shadow; /* pts to shadow hblk */
1268 /* protected by hash lock */
1269 uint_t hblk_span; /* span of memory hmeblk maps */
1270
1271 struct hme_blk_misc hblk_misc;
1272
1273 union {
1274 struct {
1275 ushort_t hblk_hmecount; /* hment on mlists counter */
1276 ushort_t hblk_validcnt; /* valid tte reference count */
1277 } hblk_counts;
1278 uint_t hblk_shadow_mask;
1279 } hblk_un;
1280
1281 uint_t hblk_lckcnt;
1282
1283 #ifdef HBLK_TRACE
1284 kmutex_t hblk_audit_lock; /* lock to protect index */
1285 uint_t hblk_audit_index; /* index into audit_cache */
1286 struct hblk_lockcnt_audit hblk_audit_cache[HBLK_AUDIT_CACHE_SIZE];
1287 #endif /* HBLK_AUDIT */
1288
1289 struct sf_hment hblk_hme[1]; /* hment array */
1290 };
1291
1292 #define hblk_shared hblk_misc.shared_bit
1293 #define hblk_xhat_bit hblk_misc.xhat_bit
1294 #define hblk_shw_bit hblk_misc.shadow_bit
1295 #define hblk_nuc_bit hblk_misc.nucleus_bit
1296 #define hblk_ttesz hblk_misc.ttesize
1297 #define hblk_hmecnt hblk_un.hblk_counts.hblk_hmecount
1298 #define hblk_vcnt hblk_un.hblk_counts.hblk_validcnt
1299 #define hblk_shw_mask hblk_un.hblk_shadow_mask
1300
1301 #define MAX_HBLK_LCKCNT 0xFFFFFFFF
1302 #define HMEBLK_ALIGN 0x8 /* hmeblk has to be double aligned */
1303
1304 #ifdef HBLK_TRACE
1305
1306 #define HBLK_STACK_TRACE(hmeblkp, lock) \
1307 { \
1308 int flag = lock; /* to pacify lint */ \
1309 int audit_index; \
1310 \
1311 mutex_enter(&hmeblkp->hblk_audit_lock); \
1312 audit_index = hmeblkp->hblk_audit_index; \
1313 hmeblkp->hblk_audit_index = ((hmeblkp->hblk_audit_index + 1) & \
2298 extern int sfmmu_getctx_sec(void);
2299 extern void sfmmu_setctx_sec(uint_t);
2300 extern void sfmmu_inv_tsb(caddr_t, uint_t);
2301 extern void sfmmu_init_ktsbinfo(void);
2302 extern int sfmmu_setup_4lp(void);
2303 extern void sfmmu_patch_mmu_asi(int);
2304 extern void sfmmu_init_nucleus_hblks(caddr_t, size_t, int, int);
2305 extern void sfmmu_cache_flushall(void);
2306 extern pgcnt_t sfmmu_tte_cnt(sfmmu_t *, uint_t);
2307 extern void *sfmmu_tsb_segkmem_alloc(vmem_t *, size_t, int);
2308 extern void sfmmu_tsb_segkmem_free(vmem_t *, void *, size_t);
2309 extern void sfmmu_reprog_pgsz_arr(sfmmu_t *, uint8_t *);
2310
2311 extern void hat_kern_setup(void);
2312 extern int hat_page_relocate(page_t **, page_t **, spgcnt_t *);
2313 extern int sfmmu_get_ppvcolor(struct page *);
2314 extern int sfmmu_get_addrvcolor(caddr_t);
2315 extern int sfmmu_hat_lock_held(sfmmu_t *);
2316 extern int sfmmu_alloc_ctx(sfmmu_t *, int, struct cpu *, int);
2317
2318 /*
2319 * Functions exported to xhat_sfmmu.c
2320 */
2321 extern kmutex_t *sfmmu_mlist_enter(page_t *);
2322 extern void sfmmu_mlist_exit(kmutex_t *);
2323 extern int sfmmu_mlist_held(struct page *);
2324 extern struct hme_blk *sfmmu_hmetohblk(struct sf_hment *);
2325
2326 /*
2327 * MMU-specific functions optionally imported from the CPU module
2328 */
2329 #pragma weak mmu_init_scd
2330 #pragma weak mmu_large_pages_disabled
2331 #pragma weak mmu_set_ctx_page_sizes
2332 #pragma weak mmu_check_page_sizes
2333
2334 extern void mmu_init_scd(sf_scd_t *);
2335 extern uint_t mmu_large_pages_disabled(uint_t);
2336 extern void mmu_set_ctx_page_sizes(sfmmu_t *);
2337 extern void mmu_check_page_sizes(sfmmu_t *, uint64_t *);
2338
2339 extern sfmmu_t *ksfmmup;
2340 extern caddr_t ktsb_base;
|
642
643
644 /*
645 * The platform dependent hat structure.
646 * tte counts should be protected by cas.
647 * cpuset is protected by cas.
648 *
649 * ttecnt accounting for mappings which do not use shared hme is carried out
650 * during pagefault handling. In the shared hme case, only the first process
651 * to access a mapping generates a pagefault, subsequent processes simply
652 * find the shared hme entry during trap handling and therefore there is no
653 * corresponding event to initiate ttecnt accounting. Currently, as shared
654 * hmes are only used for text segments, when joining a region we assume the
655 * worst case and add the the number of ttes required to map the entire region
656 * to the ttecnt corresponding to the region pagesize. However, if the region
657 * has a 4M pagesize, and memory is low, the allocation of 4M pages may fail
658 * then 8K pages will be allocated instead and the first TSB which stores 8K
659 * mappings will potentially be undersized. To compensate for the potential
660 * underaccounting in this case we always add 1/4 of the region size to the 8K
661 * ttecnt.
662 */
663
664 struct hat {
665 cpuset_t sfmmu_cpusran; /* cpu bit mask for efficient xcalls */
666 struct as *sfmmu_as; /* as this hat provides mapping for */
667 /* per pgsz private ttecnt + shme rgns ttecnt for rgns not in SCD */
668 ulong_t sfmmu_ttecnt[MMU_PAGE_SIZES];
669 /* shme rgns ttecnt for rgns in SCD */
670 ulong_t sfmmu_scdrttecnt[MMU_PAGE_SIZES];
671 /* est. ism ttes that are NOT in a SCD */
672 ulong_t sfmmu_ismttecnt[MMU_PAGE_SIZES];
673 /* ttecnt for isms that are in a SCD */
674 ulong_t sfmmu_scdismttecnt[MMU_PAGE_SIZES];
675 /* inflate tsb0 to allow for large page alloc failure in region */
676 ulong_t sfmmu_tsb0_4minflcnt;
677 union _h_un {
678 ism_blk_t *sfmmu_iblkp; /* maps to ismhat(s) */
679 ism_ment_t *sfmmu_imentp; /* ism hat's mapping list */
680 } h_un;
681 uint_t sfmmu_free:1; /* hat to be freed - set on as_free */
682 uint_t sfmmu_ismhat:1; /* hat is dummy ism hatid */
683 uint_t sfmmu_scdhat:1; /* hat is dummy scd hatid */
684 uchar_t sfmmu_rmstat; /* refmod stats refcnt */
1221 /*
1222 * Hment block structure.
1223 * The hme_blk is the node data structure which the hash structure
1224 * mantains. An hme_blk can have 2 different sizes depending on the
1225 * number of hments it implicitly contains. When dealing with 64K, 512K,
1226 * or 4M hments there is one hment per hme_blk. When dealing with
1227 * 8k hments we allocate an hme_blk plus an additional 7 hments to
1228 * give us a total of 8 (NHMENTS) hments that can be referenced through a
1229 * hme_blk.
1230 *
1231 * The hmeblk structure contains 2 tte reference counters used to determine if
1232 * it is ok to free up the hmeblk. Both counters have to be zero in order
1233 * to be able to free up hmeblk. They are protected by cas.
1234 * hblk_hmecnt is the number of hments present on pp mapping lists.
1235 * hblk_vcnt reflects number of valid ttes in hmeblk.
1236 *
1237 * The hmeblk now also has per tte lock cnts. This is required because
1238 * the counts can be high and there are not enough bits in the tte. When
1239 * physio is fixed to not lock the translations we should be able to move
1240 * the lock cnt back to the tte. See bug id 1198554.
1241 */
1242
1243 struct hme_blk_misc {
1244 uint_t notused:26;
1245 uint_t shared_bit:1; /* set for SRD shared hmeblk */
1246 uint_t shadow_bit:1; /* set for a shadow hme_blk */
1247 uint_t nucleus_bit:1; /* set for a nucleus hme_blk */
1248 uint_t ttesize:3; /* contains ttesz of hmeblk */
1249 };
1250
1251 struct hme_blk {
1252 volatile uint64_t hblk_nextpa; /* physical address for hash list */
1253
1254 hmeblk_tag hblk_tag; /* tag used to obtain an hmeblk match */
1255
1256 struct hme_blk *hblk_next; /* on free list or on hash list */
1257 /* protected by hash lock */
1258
1259 struct hme_blk *hblk_shadow; /* pts to shadow hblk */
1260 /* protected by hash lock */
1261 uint_t hblk_span; /* span of memory hmeblk maps */
1262
1263 struct hme_blk_misc hblk_misc;
1264
1265 union {
1266 struct {
1267 ushort_t hblk_hmecount; /* hment on mlists counter */
1268 ushort_t hblk_validcnt; /* valid tte reference count */
1269 } hblk_counts;
1270 uint_t hblk_shadow_mask;
1271 } hblk_un;
1272
1273 uint_t hblk_lckcnt;
1274
1275 #ifdef HBLK_TRACE
1276 kmutex_t hblk_audit_lock; /* lock to protect index */
1277 uint_t hblk_audit_index; /* index into audit_cache */
1278 struct hblk_lockcnt_audit hblk_audit_cache[HBLK_AUDIT_CACHE_SIZE];
1279 #endif /* HBLK_AUDIT */
1280
1281 struct sf_hment hblk_hme[1]; /* hment array */
1282 };
1283
1284 #define hblk_shared hblk_misc.shared_bit
1285 #define hblk_shw_bit hblk_misc.shadow_bit
1286 #define hblk_nuc_bit hblk_misc.nucleus_bit
1287 #define hblk_ttesz hblk_misc.ttesize
1288 #define hblk_hmecnt hblk_un.hblk_counts.hblk_hmecount
1289 #define hblk_vcnt hblk_un.hblk_counts.hblk_validcnt
1290 #define hblk_shw_mask hblk_un.hblk_shadow_mask
1291
1292 #define MAX_HBLK_LCKCNT 0xFFFFFFFF
1293 #define HMEBLK_ALIGN 0x8 /* hmeblk has to be double aligned */
1294
1295 #ifdef HBLK_TRACE
1296
1297 #define HBLK_STACK_TRACE(hmeblkp, lock) \
1298 { \
1299 int flag = lock; /* to pacify lint */ \
1300 int audit_index; \
1301 \
1302 mutex_enter(&hmeblkp->hblk_audit_lock); \
1303 audit_index = hmeblkp->hblk_audit_index; \
1304 hmeblkp->hblk_audit_index = ((hmeblkp->hblk_audit_index + 1) & \
2289 extern int sfmmu_getctx_sec(void);
2290 extern void sfmmu_setctx_sec(uint_t);
2291 extern void sfmmu_inv_tsb(caddr_t, uint_t);
2292 extern void sfmmu_init_ktsbinfo(void);
2293 extern int sfmmu_setup_4lp(void);
2294 extern void sfmmu_patch_mmu_asi(int);
2295 extern void sfmmu_init_nucleus_hblks(caddr_t, size_t, int, int);
2296 extern void sfmmu_cache_flushall(void);
2297 extern pgcnt_t sfmmu_tte_cnt(sfmmu_t *, uint_t);
2298 extern void *sfmmu_tsb_segkmem_alloc(vmem_t *, size_t, int);
2299 extern void sfmmu_tsb_segkmem_free(vmem_t *, void *, size_t);
2300 extern void sfmmu_reprog_pgsz_arr(sfmmu_t *, uint8_t *);
2301
2302 extern void hat_kern_setup(void);
2303 extern int hat_page_relocate(page_t **, page_t **, spgcnt_t *);
2304 extern int sfmmu_get_ppvcolor(struct page *);
2305 extern int sfmmu_get_addrvcolor(caddr_t);
2306 extern int sfmmu_hat_lock_held(sfmmu_t *);
2307 extern int sfmmu_alloc_ctx(sfmmu_t *, int, struct cpu *, int);
2308
2309 extern kmutex_t *sfmmu_mlist_enter(page_t *);
2310 extern void sfmmu_mlist_exit(kmutex_t *);
2311 extern int sfmmu_mlist_held(struct page *);
2312 extern struct hme_blk *sfmmu_hmetohblk(struct sf_hment *);
2313
2314 /*
2315 * MMU-specific functions optionally imported from the CPU module
2316 */
2317 #pragma weak mmu_init_scd
2318 #pragma weak mmu_large_pages_disabled
2319 #pragma weak mmu_set_ctx_page_sizes
2320 #pragma weak mmu_check_page_sizes
2321
2322 extern void mmu_init_scd(sf_scd_t *);
2323 extern uint_t mmu_large_pages_disabled(uint_t);
2324 extern void mmu_set_ctx_page_sizes(sfmmu_t *);
2325 extern void mmu_check_page_sizes(sfmmu_t *, uint64_t *);
2326
2327 extern sfmmu_t *ksfmmup;
2328 extern caddr_t ktsb_base;
|