Print this page
patch cstyle
patch sccs-keywords
patch SEGOP_SWAPOUT-delete
Split |
Close |
Expand all |
Collapse all |
--- old/usr/src/uts/common/vm/seg_kpm.c
+++ new/usr/src/uts/common/vm/seg_kpm.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License, Version 1.0 only
6 6 * (the "License"). You may not use this file except in compliance
7 7 * with the License.
8 8 *
9 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 10 * or http://www.opensolaris.org/os/licensing.
11 11 * See the License for the specific language governing permissions
12 12 * and limitations under the License.
13 13 *
14 14 * When distributing Covered Code, include this CDDL HEADER in each
15 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 16 * If applicable, add the following below this CDDL HEADER, with the
↓ open down ↓ |
16 lines elided |
↑ open up ↑ |
17 17 * fields enclosed by brackets "[]" replaced with your own identifying
18 18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 19 *
20 20 * CDDL HEADER END
21 21 */
22 22 /*
23 23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 */
26 26
27 -#pragma ident "%Z%%M% %I% %E% SMI"
28 -
29 27 /*
30 28 * Kernel Physical Mapping (kpm) segment driver (segkpm).
31 29 *
32 30 * This driver delivers along with the hat_kpm* interfaces an alternative
33 31 * mechanism for kernel mappings within the 64-bit Solaris operating system,
34 32 * which allows the mapping of all physical memory into the kernel address
35 33 * space at once. This is feasible in 64 bit kernels, e.g. for Ultrasparc II
36 34 * and beyond processors, since the available VA range is much larger than
37 35 * possible physical memory. Momentarily all physical memory is supported,
38 36 * that is represented by the list of memory segments (memsegs).
39 37 *
40 38 * Segkpm mappings have also very low overhead and large pages are used
41 39 * (when possible) to minimize the TLB and TSB footprint. It is also
42 40 * extentable for other than Sparc architectures (e.g. AMD64). Main
43 41 * advantage is the avoidance of the TLB-shootdown X-calls, which are
44 42 * normally needed when a kernel (global) mapping has to be removed.
45 43 *
46 44 * First example of a kernel facility that uses the segkpm mapping scheme
47 45 * is seg_map, where it is used as an alternative to hat_memload().
48 46 * See also hat layer for more information about the hat_kpm* routines.
49 47 * The kpm facilty can be turned off at boot time (e.g. /etc/system).
50 48 */
51 49
52 50 #include <sys/types.h>
53 51 #include <sys/param.h>
54 52 #include <sys/sysmacros.h>
55 53 #include <sys/systm.h>
56 54 #include <sys/vnode.h>
57 55 #include <sys/cmn_err.h>
58 56 #include <sys/debug.h>
59 57 #include <sys/thread.h>
60 58 #include <sys/cpuvar.h>
61 59 #include <sys/bitmap.h>
62 60 #include <sys/atomic.h>
63 61 #include <sys/lgrp.h>
64 62
65 63 #include <vm/seg_kmem.h>
66 64 #include <vm/seg_kpm.h>
67 65 #include <vm/hat.h>
68 66 #include <vm/as.h>
69 67 #include <vm/seg.h>
70 68 #include <vm/page.h>
71 69
72 70 /*
73 71 * Global kpm controls.
74 72 * See also platform and mmu specific controls.
75 73 *
76 74 * kpm_enable -- global on/off switch for segkpm.
77 75 * . Set by default on 64bit platforms that have kpm support.
78 76 * . Will be disabled from platform layer if not supported.
79 77 * . Can be disabled via /etc/system.
80 78 *
81 79 * kpm_smallpages -- use only regular/system pagesize for kpm mappings.
82 80 * . Can be useful for critical debugging of kpm clients.
83 81 * . Set to zero by default for platforms that support kpm large pages.
84 82 * The use of kpm large pages reduces the footprint of kpm meta data
85 83 * and has all the other advantages of using large pages (e.g TLB
86 84 * miss reduction).
87 85 * . Set by default for platforms that don't support kpm large pages or
88 86 * where large pages cannot be used for other reasons (e.g. there are
89 87 * only few full associative TLB entries available for large pages).
90 88 *
91 89 * segmap_kpm -- separate on/off switch for segmap using segkpm:
92 90 * . Set by default.
93 91 * . Will be disabled when kpm_enable is zero.
94 92 * . Will be disabled when MAXBSIZE != PAGESIZE.
95 93 * . Can be disabled via /etc/system.
96 94 *
97 95 */
98 96 int kpm_enable = 1;
99 97 int kpm_smallpages = 0;
100 98 int segmap_kpm = 1;
101 99
102 100 /*
103 101 * Private seg op routines.
104 102 */
105 103 faultcode_t segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr,
106 104 size_t len, enum fault_type type, enum seg_rw rw);
107 105 static void segkpm_dump(struct seg *);
108 106 static void segkpm_badop(void);
109 107 static int segkpm_notsup(void);
110 108 static int segkpm_capable(struct seg *, segcapability_t);
111 109
112 110 #define SEGKPM_BADOP(t) (t(*)())segkpm_badop
113 111 #define SEGKPM_NOTSUP (int(*)())segkpm_notsup
↓ open down ↓ |
75 lines elided |
↑ open up ↑ |
114 112
115 113 static struct seg_ops segkpm_ops = {
116 114 SEGKPM_BADOP(int), /* dup */
117 115 SEGKPM_BADOP(int), /* unmap */
118 116 SEGKPM_BADOP(void), /* free */
119 117 segkpm_fault,
120 118 SEGKPM_BADOP(int), /* faulta */
121 119 SEGKPM_BADOP(int), /* setprot */
122 120 SEGKPM_BADOP(int), /* checkprot */
123 121 SEGKPM_BADOP(int), /* kluster */
124 - SEGKPM_BADOP(size_t), /* swapout */
125 122 SEGKPM_BADOP(int), /* sync */
126 123 SEGKPM_BADOP(size_t), /* incore */
127 124 SEGKPM_BADOP(int), /* lockop */
128 125 SEGKPM_BADOP(int), /* getprot */
129 126 SEGKPM_BADOP(u_offset_t), /* getoffset */
130 127 SEGKPM_BADOP(int), /* gettype */
131 128 SEGKPM_BADOP(int), /* getvp */
132 129 SEGKPM_BADOP(int), /* advise */
133 130 segkpm_dump, /* dump */
134 131 SEGKPM_NOTSUP, /* pagelock */
135 132 SEGKPM_BADOP(int), /* setpgsz */
136 133 SEGKPM_BADOP(int), /* getmemid */
137 134 SEGKPM_BADOP(lgrp_mem_policy_info_t *), /* getpolicy */
138 135 segkpm_capable, /* capable */
139 136 };
140 137
141 138 /*
142 139 * kpm_pgsz and kpm_pgshft are set by platform layer.
143 140 */
144 141 size_t kpm_pgsz; /* kpm page size */
145 142 uint_t kpm_pgshft; /* kpm page shift */
146 143 u_offset_t kpm_pgoff; /* kpm page offset mask */
147 144 uint_t kpmp2pshft; /* kpm page to page shift */
148 145 pgcnt_t kpmpnpgs; /* how many pages per kpm page */
149 146
150 147
151 148 #ifdef SEGKPM_SUPPORT
152 149
↓ open down ↓ |
18 lines elided |
↑ open up ↑ |
153 150 int
154 151 segkpm_create(struct seg *seg, void *argsp)
155 152 {
156 153 struct segkpm_data *skd;
157 154 struct segkpm_crargs *b = (struct segkpm_crargs *)argsp;
158 155 ushort_t *p;
159 156 int i, j;
160 157
161 158 ASSERT(seg->s_as && RW_WRITE_HELD(&seg->s_as->a_lock));
162 159 ASSERT(btokpmp(seg->s_size) >= 1 &&
163 - kpmpageoff((uintptr_t)seg->s_base) == 0 &&
164 - kpmpageoff((uintptr_t)seg->s_base + seg->s_size) == 0);
160 + kpmpageoff((uintptr_t)seg->s_base) == 0 &&
161 + kpmpageoff((uintptr_t)seg->s_base + seg->s_size) == 0);
165 162
166 163 skd = kmem_zalloc(sizeof (struct segkpm_data), KM_SLEEP);
167 164
168 165 seg->s_data = (void *)skd;
169 166 seg->s_ops = &segkpm_ops;
170 167 skd->skd_prot = b->prot;
171 168
172 169 /*
173 170 * (1) Segkpm virtual addresses are based on physical adresses.
174 171 * From this and in opposite to other segment drivers it is
175 172 * often required to allocate a page first to be able to
176 173 * calculate the final segkpm virtual address.
177 174 * (2) Page allocation is done by calling page_create_va(),
178 175 * one important input argument is a virtual address (also
179 176 * expressed by the "va" in the function name). This function
180 177 * is highly optimized to select the right page for an optimal
181 178 * processor and platform support (e.g. virtual addressed
182 179 * caches (VAC), physical addressed caches, NUMA).
183 180 *
184 181 * Because of (1) the approach is to generate a faked virtual
185 182 * address for calling page_create_va(). In order to exploit
↓ open down ↓ |
11 lines elided |
↑ open up ↑ |
186 183 * the abilities of (2), especially to utilize the cache
187 184 * hierarchy (3) and to avoid VAC alias conflicts (4) the
188 185 * selection has to be done carefully. For each virtual color
189 186 * a separate counter is provided (4). The count values are
190 187 * used for the utilization of all cache lines (3) and are
191 188 * corresponding to the cache bins.
192 189 */
193 190 skd->skd_nvcolors = b->nvcolors;
194 191
195 192 p = skd->skd_va_select =
196 - kmem_zalloc(NCPU * b->nvcolors * sizeof (ushort_t), KM_SLEEP);
193 + kmem_zalloc(NCPU * b->nvcolors * sizeof (ushort_t), KM_SLEEP);
197 194
198 195 for (i = 0; i < NCPU; i++)
199 196 for (j = 0; j < b->nvcolors; j++, p++)
200 197 *p = j;
201 198
202 199 return (0);
203 200 }
204 201
205 202 /*
206 203 * This routine is called via a machine specific fault handling
207 204 * routine.
208 205 */
209 206 /* ARGSUSED */
210 207 faultcode_t
211 208 segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len,
212 209 enum fault_type type, enum seg_rw rw)
213 210 {
214 211 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
215 212
216 213 switch (type) {
217 214 case F_INVAL:
218 215 return (hat_kpm_fault(hat, addr));
219 216 case F_SOFTLOCK:
220 217 case F_SOFTUNLOCK:
221 218 return (0);
222 219 default:
223 220 return (FC_NOSUPPORT);
224 221 }
225 222 /*NOTREACHED*/
226 223 }
227 224
228 225 #define addr_to_vcolor(addr, vcolors) \
229 226 ((int)(((uintptr_t)(addr) & ((vcolors << PAGESHIFT) - 1)) >> PAGESHIFT))
230 227
231 228 /*
232 229 * Create a virtual address that can be used for invocations of
233 230 * page_create_va. Goal is to utilize the cache hierarchy (round
234 231 * robin bins) and to select the right color for virtual indexed
235 232 * caches. It isn't exact since we also increment the bin counter
236 233 * when the caller uses VOP_GETPAGE and gets a hit in the page
237 234 * cache, but we keep the bins turning for cache distribution
238 235 * (see also segkpm_create block comment).
239 236 */
240 237 caddr_t
241 238 segkpm_create_va(u_offset_t off)
242 239 {
243 240 int vcolor;
244 241 ushort_t *p;
245 242 struct segkpm_data *skd = (struct segkpm_data *)segkpm->s_data;
246 243 int nvcolors = skd->skd_nvcolors;
247 244 caddr_t va;
248 245
249 246 vcolor = (nvcolors > 1) ? addr_to_vcolor(off, nvcolors) : 0;
250 247 p = &skd->skd_va_select[(CPU->cpu_id * nvcolors) + vcolor];
251 248 va = (caddr_t)ptob(*p);
252 249
253 250 atomic_add_16(p, nvcolors);
254 251
255 252 return (va);
256 253 }
257 254
258 255 /*
259 256 * Unload mapping if the instance has an active kpm mapping.
260 257 */
261 258 void
262 259 segkpm_mapout_validkpme(struct kpme *kpme)
263 260 {
264 261 caddr_t vaddr;
265 262 page_t *pp;
266 263
267 264 retry:
268 265 if ((pp = kpme->kpe_page) == NULL) {
269 266 return;
270 267 }
271 268
272 269 if (page_lock(pp, SE_SHARED, (kmutex_t *)NULL, P_RECLAIM) == 0)
273 270 goto retry;
274 271
275 272 /*
276 273 * Check if segkpm mapping is not unloaded in the meantime
277 274 */
278 275 if (kpme->kpe_page == NULL) {
279 276 page_unlock(pp);
280 277 return;
281 278 }
282 279
283 280 vaddr = hat_kpm_page2va(pp, 1);
284 281 hat_kpm_mapout(pp, kpme, vaddr);
285 282 page_unlock(pp);
286 283 }
287 284
288 285 static void
289 286 segkpm_badop()
290 287 {
291 288 panic("segkpm_badop");
292 289 }
293 290
294 291 #else /* SEGKPM_SUPPORT */
295 292
296 293 /* segkpm stubs */
297 294
298 295 /*ARGSUSED*/
299 296 int segkpm_create(struct seg *seg, void *argsp) { return (0); }
300 297
301 298 /* ARGSUSED */
302 299 faultcode_t
303 300 segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len,
304 301 enum fault_type type, enum seg_rw rw)
305 302 {
306 303 return ((faultcode_t)0);
307 304 }
308 305
309 306 /* ARGSUSED */
310 307 caddr_t segkpm_create_va(u_offset_t off) { return (NULL); }
311 308
312 309 /* ARGSUSED */
313 310 void segkpm_mapout_validkpme(struct kpme *kpme) {}
314 311
315 312 static void
316 313 segkpm_badop() {}
317 314
318 315 #endif /* SEGKPM_SUPPORT */
319 316
320 317 static int
321 318 segkpm_notsup()
322 319 {
323 320 return (ENOTSUP);
324 321 }
325 322
326 323 /*
327 324 * segkpm pages are not dumped, so we just return
328 325 */
329 326 /*ARGSUSED*/
330 327 static void
331 328 segkpm_dump(struct seg *seg)
332 329 {}
333 330
334 331 /*
335 332 * We claim to have no special capabilities.
336 333 */
337 334 /*ARGSUSED*/
338 335 static int
339 336 segkpm_capable(struct seg *seg, segcapability_t capability)
340 337 {
341 338 return (0);
342 339 }
↓ open down ↓ |
136 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX