Print this page
5042 stop using deprecated atomic functions
Split |
Close |
Expand all |
Collapse all |
--- old/usr/src/uts/sun4u/io/zuluvm.c
+++ new/usr/src/uts/sun4u/io/zuluvm.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
↓ open down ↓ |
15 lines elided |
↑ open up ↑ |
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 -#pragma ident "%Z%%M% %I% %E% SMI"
27 -
28 26 /*
29 27 * zuluvm module
30 28 *
31 29 * Provides services required by the XVR-4000 graphics accelerator (zulu)
32 30 * that are not provided by the ddi. See PSARC 2002/231.
33 31 *
34 32 * Zulu has 2 dma engines with built in MMUs. zuluvm provides TLB miss
35 33 * interrupt support obtaining virtual to physical address translations
36 34 * using the XHAT interface PSARC/2003/517.
37 35 *
38 36 * The module has 3 components. This file, sun4u/vm/zulu_hat.c, and the
39 37 * assembly language routines in sun4u/ml/zulu_asm.s and
40 38 * sun4u/ml/zulu_hat_asm.s.
41 39 *
42 40 * The interrupt handler is a data bearing mondo interrupt handled at TL=1
43 41 * If no translation is found in the zulu hat's tsb, or if the tsb is locked by
44 42 * C code, the handler posts a soft interrupt which wakes up a parked
45 43 * thread belonging to zuludaemon(1M).
46 44 */
47 45
48 46 #include <sys/conf.h>
49 47 #include <sys/types.h>
50 48 #include <sys/kmem.h>
51 49 #include <sys/debug.h>
52 50 #include <sys/modctl.h>
53 51 #include <sys/autoconf.h>
54 52 #include <sys/ddi_impldefs.h>
55 53 #include <sys/ddi_subrdefs.h>
56 54 #include <sys/intr.h>
57 55 #include <sys/ddi.h>
58 56 #include <sys/sunndi.h>
59 57 #include <sys/proc.h>
60 58 #include <sys/thread.h>
61 59 #include <sys/machsystm.h>
62 60 #include <sys/ivintr.h>
63 61 #include <sys/tnf_probe.h>
64 62 #include <sys/intreg.h>
65 63 #include <sys/atomic.h>
66 64 #include <vm/as.h>
67 65 #include <vm/seg_enum.h>
68 66 #include <vm/faultcode.h>
69 67 #include <sys/dmv.h>
70 68 #include <sys/zulumod.h>
↓ open down ↓ |
33 lines elided |
↑ open up ↑ |
71 69 #include <sys/zulu_hat.h>
72 70
73 71 #define ZULUVM_GET_PAGE(val) \
74 72 (caddr_t)((uintptr_t)(val) & PAGEMASK)
75 73 #define ZULUVM_GET_AS curthread->t_procp->p_as
76 74
77 75 #define ZULUVM_LOCK mutex_enter(&(zdev->dev_lck))
78 76 #define ZULUVM_UNLOCK mutex_exit(&(zdev->dev_lck))
79 77
80 78 #define ZULUVM_SET_STATE(_z, b, c) \
81 - cas32((uint32_t *)&((_z)->zvm.state), c, b)
79 + atomic_cas_32((uint32_t *)&((_z)->zvm.state), c, b)
82 80 #define ZULUVM_GET_STATE(_z) \
83 81 (_z)->zvm.state
84 82 #define ZULUVM_SET_IDLE(_z) \
85 83 (_z)->zvm.state = ZULUVM_STATE_IDLE;
86 84
87 85 #define ZULUVM_INO_MASK ((1<<INO_SIZE)-1)
88 86 #define ZULUVM_IGN_MASK ((1<<IGN_SIZE)-1)
89 87 #define ZULUVM_MONDO(_zdev, _n) \
90 88 ((ZULUVM_IGN_MASK & _zdev->agentid) << INO_SIZE) | \
91 89 (ZULUVM_INO_MASK & (_n))
92 90
93 91 static void zuluvm_stop(zuluvm_state_t *, int, char *);
94 92 static zuluvm_proc_t *zuluvm_find_proc(zuluvm_state_t *, struct as *);
95 93 static int zuluvm_proc_release(zuluvm_state_t *zdev, zuluvm_proc_t *proc);
96 94 static int zuluvm_get_intr_props(zuluvm_state_t *zdev, dev_info_t *devi);
97 95 static int zuluvm_driver_attach(zuluvm_state_t *);
98 96 static int zuluvm_driver_detach(zuluvm_state_t *);
99 97 static void zuluvm_retarget_intr(void *arg);
100 98 static void zuluvm_do_retarget(zuluvm_state_t *zdev);
101 99
102 100 extern const unsigned int _mmu_pageshift;
103 101
104 102 extern int zuluvm_base_pgsize;
105 103 static int zuluvm_pagesizes[ZULUM_MAX_PG_SIZES + 1];
106 104
107 105 int zuluvm_fast_tlb = 1;
108 106
109 107 zuluvm_state_t *zuluvm_devtab[ZULUVM_MAX_DEV];
110 108 kmutex_t zuluvm_lck;
111 109
112 110 #ifdef DEBUG
113 111 int zuluvm_debug_state = 0;
114 112 #endif
115 113
116 114 unsigned long zuluvm_ctx_locked = 0;
117 115
118 116 /*
119 117 * Module linkage information for the kernel.
120 118 */
121 119 extern struct mod_ops mod_miscops;
122 120
123 121 static struct modlmisc modlmisc = {
124 122 &mod_miscops,
125 123 "sun4u support " ZULUVM_MOD_VERSION
126 124 };
127 125
128 126 static struct modlinkage modlinkage = {
129 127 MODREV_1,
130 128 (void *)&modlmisc,
131 129 NULL
132 130 };
133 131
134 132 int
135 133 _init(void)
136 134 {
137 135 zuluvm_base_pgsize = (_mmu_pageshift - 13) / 3;
138 136 if (zulu_hat_init() != 0) {
139 137 return (ZULUVM_ERROR);
140 138 }
141 139 mutex_init(&zuluvm_lck, NULL, MUTEX_DEFAULT, NULL);
142 140 return (mod_install(&modlinkage));
143 141 }
144 142
145 143 int
146 144 _fini(void)
147 145 {
148 146 mutex_destroy(&zuluvm_lck);
149 147 (void) zulu_hat_destroy();
150 148 return (mod_remove(&modlinkage));
151 149 }
152 150
153 151 int
154 152 _info(struct modinfo *modinfop)
155 153 {
156 154 return (mod_info(&modlinkage, modinfop));
157 155 }
158 156
159 157 /*
160 158 * currently the kernel driver makes the following assumptions:
161 159 * - there is only one TLB miss per zulu device handled at
162 160 * any given time
163 161 * ==> we only need local data storage per device, not per DMA
164 162 * ==> a page fault will block the DMA engine until the fault
165 163 * is resolved
166 164 * ==> a pagefault will not trigger a zulu DMA context switch
167 165 *
168 166 * If we want to implement asynnchronous zulu page fault, then we
169 167 * need to keep track of outstanding faults while zulu DMA runs
170 168 * in a different context.
171 169 */
172 170 static int
173 171 zuluvm_write_tte(zuluvm_state_t *zdev, void *arg, caddr_t addr,
174 172 int t_pfn, int t_perm, int t_size, uint64_t tag,
175 173 int tlbtype, int *size)
176 174 {
177 175 int error;
178 176
179 177 (void) addr;
180 178
181 179 ZULUVM_STATS_MISS(zdev, t_size);
182 180
183 181 if (tag == 0) { /* not coming from preload */
184 182 int state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_WRITE_TTE,
185 183 ZULUVM_STATE_INTR_PENDING);
186 184 if (state != ZULUVM_STATE_INTR_PENDING) {
187 185 zuluvm_stop(zdev, state, "zuluvm_write_tte");
188 186 return (ZULUVM_MISS_CANCELED);
189 187 }
190 188 }
191 189
192 190 if (!(tlbtype & ZULUVM_ITLB_FLAG) &&
193 191 t_size != zuluvm_base_pgsize &&
194 192 t_size != ZULU_TTE4M) {
195 193 t_size = zuluvm_base_pgsize;
196 194 TNF_PROBE_2(zuluvm_write_tte_new_pfn, "zuluvm", /* */,
197 195 tnf_opaque, t_pfn, t_pfn, tnf_int, pagesize, t_size);
198 196 }
199 197 TNF_PROBE_1(zuluvm_write_tte, "zuluvm", /* */,
200 198 tnf_opaque, t_pfn, t_pfn);
201 199 /*
202 200 * if the caller is zuluvm_preload, then we need to pass
203 201 * back the page size so it can add the right offset.
204 202 */
205 203 if (size)
206 204 *size = t_size;
207 205
208 206 error = zulud_write_tte(zdev, arg, t_size, tag, t_pfn,
209 207 t_perm, tlbtype);
210 208
211 209 return (error);
212 210 }
213 211
214 212 static void
215 213 zuluvm_stop(zuluvm_state_t *zdev, int state, char *tag)
216 214 {
217 215 int ostate = state;
218 216 while (state != ZULUVM_STATE_STOPPED) {
219 217 state = ZULUVM_SET_STATE(zdev,
220 218 ZULUVM_STATE_STOPPED, state);
221 219 #ifdef DEBUG
222 220 if (zuluvm_debug_state)
223 221 cmn_err(CE_NOTE, "zuluvm_stop(%s): (loop) state %d\n",
224 222 tag, state);
225 223 #endif
226 224 }
227 225 TNF_PROBE_2(zuluvm_stop, "zuluvm", /* */,
228 226 tnf_string, tag, tag,
229 227 tnf_int, state, ostate);
230 228 ZULUVM_STATS_CANCEL(zdev);
231 229 }
232 230
233 231 /*
234 232 * Executed with the context of the parked zulu deamon thread,
235 233 * uses zulu_hat_load to resolve the miss.
236 234 * The tte is loaded and miss done called by the function zuluvm_load_tte
237 235 * which is called from zulu_hat
238 236 *
239 237 * This function is synchronized with the zuluvm_as_free.
240 238 * zuluvm_as_free will block until miss servicing is complete.
241 239 *
242 240 * There is a race condition between as_free and the zulu tlb miss
243 241 * soft interrupt:
244 242 * - queue zulu interrupt
245 243 * - process dies, as_free runs
246 244 * - interrupt gets scheduled and runs as_fault on the
247 245 * already freed as.
248 246 * This is solved by keeping track of current zulu dma processes
249 247 * and invalidating them in zuluvm_as_free.
250 248 */
251 249 uint_t
252 250 zuluvm_tlb_handler(caddr_t data)
253 251 {
254 252 zuluvm_state_t *zdev = (zuluvm_state_t *)data;
255 253 int error;
256 254 int flag = 0;
257 255 int wait = 0;
258 256 zuluvm_proc_t *proc = NULL;
259 257 struct zulu_hat *zhat = NULL;
260 258 caddr_t addr;
261 259 int tlbtype;
262 260 void *arg;
263 261 int state, newstate;
264 262
265 263 TNF_PROBE_1(zuluvm_tlb_handler_lwp, "zuluvm", /* */,
266 264 tnf_opaque, lwp, ttolwp(curthread));
267 265
268 266 ZULUVM_LOCK;
269 267 error = ZULUVM_GET_TLB_ERRCODE(zdev);
270 268 addr = (caddr_t)ZULUVM_GET_TLB_ADDR(zdev);
271 269 tlbtype = ZULUVM_GET_TLB_TYPE(zdev);
272 270 arg = zdev->zvm.arg;
273 271
274 272 /*
275 273 * select the correct dma engine and remember the
276 274 * the as_free synchronization flags.
277 275 */
278 276 switch (tlbtype) {
279 277 case ZULUVM_ITLB1:
280 278 case ZULUVM_DMA1:
281 279 proc = zdev->zvm.proc1;
282 280 flag |= ZULUVM_DO_INTR1;
283 281 wait |= ZULUVM_WAIT_INTR1;
284 282 break;
285 283 case ZULUVM_ITLB2:
286 284 case ZULUVM_DMA2:
287 285 proc = zdev->zvm.proc2;
288 286 flag |= ZULUVM_DO_INTR2;
289 287 wait |= ZULUVM_WAIT_INTR2;
290 288 break;
291 289 }
292 290
293 291 state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_INTR_PENDING,
294 292 ZULUVM_STATE_INTR_QUEUED);
295 293 newstate = ZULUVM_GET_STATE(zdev);
296 294
297 295 TNF_PROBE_2(zuluvm_tlb_handler_state, "zuluvm", /* */,
298 296 tnf_int, oldstate, state,
299 297 tnf_int, newstate, newstate);
300 298 #ifdef DEBUG
301 299 if (zuluvm_debug_state)
302 300 cmn_err(CE_NOTE, "zuluvm_tlb_handler: state %d\n", state);
303 301 #endif
304 302 if (state != ZULUVM_STATE_INTR_PENDING &&
305 303 state != ZULUVM_STATE_INTR_QUEUED) {
306 304 ZULUVM_UNLOCK;
307 305
308 306 zuluvm_stop(zdev, state, "softintr1");
309 307 zulud_tlb_done(zdev, arg, tlbtype, ZULUVM_MISS_CANCELED);
310 308 return (1);
311 309 }
312 310
313 311 /*
314 312 * block the as_free callback in case it comes in
315 313 */
316 314 zdev->intr_flags |= flag;
317 315 ZULUVM_UNLOCK;
318 316
319 317 mutex_enter(&zdev->proc_lck);
320 318 /*
321 319 * check if this as is still valid
322 320 */
323 321 if (proc == NULL || proc->valid == 0 || proc->zhat == NULL) {
324 322 mutex_exit(&zdev->proc_lck);
325 323 /*
326 324 * we are on our way out, wake up the as_free
327 325 * callback if it is waiting for us
328 326 */
329 327 ZULUVM_LOCK;
330 328 zdev->intr_flags &= ~flag;
331 329 if (zdev->intr_flags | wait)
332 330 cv_broadcast(&zdev->intr_wait);
333 331 ZULUVM_UNLOCK;
334 332 state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_IDLE,
335 333 ZULUVM_STATE_INTR_PENDING);
336 334 if (state != ZULUVM_STATE_INTR_PENDING) {
337 335 zuluvm_stop(zdev, state, "softintr3");
338 336 }
339 337 zulud_tlb_done(zdev, arg, tlbtype, ZULUVM_NO_HAT);
340 338 return (1);
341 339 }
342 340 zhat = proc->zhat;
343 341 mutex_exit(&zdev->proc_lck);
344 342
345 343 TNF_PROBE_1(zuluvm_tlb_handler, "zuluvm", /* */,
346 344 tnf_opaque, addr, addr);
347 345
348 346 switch (error) {
349 347 case ZULUVM_CTX_LOCKED:
350 348 /*
351 349 * trap handler found that zulu_hat had the lock bit set
352 350 * rather than block in the fast trap handler, it punts
353 351 * in this rare instance
354 352 */
355 353 ++zuluvm_ctx_locked;
356 354 TNF_PROBE_1(zuluvm_ctx_locked, "zuluvm", /* CSTYLED */,
357 355 tnf_ulong, zuluvm_ctx_locked, zuluvm_ctx_locked);
358 356
359 357 /*FALLTHROUGH*/
360 358
361 359 case ZULUVM_TTE_DELAY:
362 360 /*
363 361 * fast tlb handler was skipped, see zuluvm_fast_tlb flag
364 362 */
365 363 /*FALLTHROUGH*/
366 364
367 365 case ZULUVM_NO_TTE:
368 366 /*
369 367 * no TSB entry and TTE in the hash
370 368 */
371 369 mutex_enter(&zdev->load_lck);
372 370 zdev->in_intr = 1;
373 371 error = zulu_hat_load(zhat, addr,
374 372 (tlbtype == ZULUVM_DMA2) ? S_WRITE : S_READ, NULL);
375 373 zdev->in_intr = 0;
376 374 mutex_exit(&zdev->load_lck);
377 375 if (error) {
378 376
379 377 error = ZULUVM_NO_MAP;
380 378 } else {
381 379 error = ZULUVM_SUCCESS;
382 380 TNF_PROBE_1(zuluvm_tlb_handler_done, "zuluvm", /* */,
383 381 tnf_int, error, error);
384 382 return (1);
385 383 }
386 384
387 385 default:
388 386 /*
389 387 * error case, fall through and tell zulu driver to abort DMA
390 388 */
391 389 break;
392 390 }
393 391
394 392 if (error != ZULUVM_MISS_CANCELED) {
395 393 state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_IDLE,
396 394 ZULUVM_STATE_WRITE_TTE);
397 395 newstate = ZULUVM_GET_STATE(zdev);
398 396 TNF_PROBE_2(zuluvm_tlb_handler_state_done, "zuluvm", /* */,
399 397 tnf_int, oldstate, state,
400 398 tnf_int, newstate, newstate);
401 399 if (state != ZULUVM_STATE_WRITE_TTE) {
402 400 zuluvm_stop(zdev, state, "softintr4");
403 401 }
404 402 }
405 403 /*
406 404 * synchronize with as_free callback
407 405 * It will set the wait flag, in that case we send
408 406 * a wake up.
409 407 */
410 408 ZULUVM_LOCK;
411 409 zdev->intr_flags &= ~flag;
412 410 if (zdev->intr_flags | wait)
413 411 cv_broadcast(&zdev->intr_wait);
414 412 ZULUVM_UNLOCK;
415 413
416 414 TNF_PROBE_1(zuluvm_tlb_handler_done, "zuluvm", /* */,
417 415 tnf_int, error, error);
418 416
419 417 zulud_tlb_done(zdev, arg, tlbtype, error);
420 418
421 419 return (1);
422 420 }
423 421
424 422
425 423 void
426 424 zuluvm_load_tte(struct zulu_hat *zhat, caddr_t addr, uint64_t pfn,
427 425 int perm, int size)
428 426 {
429 427 zuluvm_state_t *zdev = zhat->zdev;
430 428 int tlbtype = ZULUVM_GET_TLB_TYPE(zdev);
431 429
432 430 ASSERT(MUTEX_HELD(&zdev->load_lck));
433 431 ASSERT(pfn != 0);
434 432
435 433 if (zdev->in_intr) {
436 434 int error;
437 435 int flag = 0;
438 436 int wait = 0;
439 437
440 438 error = zuluvm_write_tte(zdev, zdev->zvm.arg, addr, pfn,
441 439 perm, size, 0, tlbtype, NULL);
442 440
443 441 if (error != ZULUVM_MISS_CANCELED) {
444 442 int state, newstate;
445 443
446 444 state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_IDLE,
447 445 ZULUVM_STATE_WRITE_TTE);
448 446 newstate = ZULUVM_GET_STATE(zdev);
449 447 TNF_PROBE_2(zuluvm_tlb_handler_state_done, "zuluvm",
450 448 /* */, tnf_int, oldstate, state,
451 449 tnf_int, newstate, newstate);
452 450 if (state != ZULUVM_STATE_WRITE_TTE) {
453 451 zuluvm_stop(zdev, state, "softintr4");
454 452 }
455 453 }
456 454 /*
457 455 * synchronize with as_free callback
458 456 * It will set the wait flag, in that case we send
459 457 * a wake up.
460 458 */
461 459 switch (tlbtype) {
462 460 case ZULUVM_ITLB1:
463 461 case ZULUVM_DMA1:
464 462 flag = ZULUVM_DO_INTR1;
465 463 wait = ZULUVM_WAIT_INTR1;
466 464 break;
467 465 case ZULUVM_ITLB2:
468 466 case ZULUVM_DMA2:
469 467 flag = ZULUVM_DO_INTR2;
470 468 wait = ZULUVM_WAIT_INTR2;
471 469 break;
472 470 }
473 471
474 472 ZULUVM_LOCK;
475 473 zdev->intr_flags &= ~flag;
476 474 if (zdev->intr_flags | wait)
477 475 cv_broadcast(&zdev->intr_wait);
478 476 ZULUVM_UNLOCK;
479 477
480 478 zulud_tlb_done(zdev, zdev->zvm.arg, tlbtype, error);
481 479 } else {
482 480 (void) zuluvm_write_tte(zdev, zdev->zvm.arg, addr, pfn,
483 481 perm, size, (uint64_t)addr |
484 482 zhat->zulu_ctx, tlbtype, NULL);
485 483 }
486 484 }
487 485
488 486
489 487
490 488
491 489 /*
492 490 * This function provides the faulting thread for zulu page faults
493 491 * It is call from the device driver in response to an ioctl issued
494 492 * by a zuludaemon thread.
495 493 * It sits in cv_wait_sig until it gets woken up by a signal or
496 494 * zulu tlb miss soft interrupt.
497 495 */
498 496 int
499 497 zuluvm_park(zuluvm_info_t devp)
500 498 {
501 499 int rval;
502 500 zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
503 501 mutex_enter(&zdev->park_lck);
504 502 zdev->parking = 1;
505 503 for (;;) {
506 504 rval = cv_wait_sig(&zdev->park_cv, &zdev->park_lck);
507 505 if (rval == 0)
508 506 break;
509 507 rval = zuluvm_tlb_handler(devp);
510 508 }
511 509 zdev->parking = 0;
512 510 mutex_exit(&zdev->park_lck);
513 511 return (rval);
514 512 }
515 513
516 514 /*
517 515 * zulu soft interrupt handler, just triggers the parked zulu fault
518 516 * thread
519 517 */
520 518 /*ARGSUSED*/
521 519 uint_t
522 520 zuluvm_softintr(caddr_t devp, caddr_t arg2)
523 521 {
524 522 int tlbtype;
525 523 void *arg;
526 524 zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
527 525 mutex_enter(&zdev->park_lck);
528 526 if (zdev->parking) {
529 527 cv_signal(&zdev->park_cv);
530 528 mutex_exit(&zdev->park_lck);
531 529 TNF_PROBE_1(zuluvm_fast_intr, "zuluvm", /* */,
532 530 tnf_opaque, devp, devp);
533 531 } else {
534 532 mutex_exit(&zdev->park_lck);
535 533 cmn_err(CE_NOTE, "zuluvm: no page fault thread\n");
536 534 ZULUVM_LOCK;
537 535 tlbtype = ZULUVM_GET_TLB_TYPE(zdev);
538 536 arg = zdev->zvm.arg;
539 537 ZULUVM_UNLOCK;
540 538 TNF_PROBE_0(zuluvm_fast_intr, "zuluvm", /* */);
541 539 zuluvm_stop(zdev, ZULUVM_STATE_INTR_QUEUED, "fast_intr");
542 540 zulud_tlb_done(zdev, arg, tlbtype, ZULUVM_NO_TTE);
543 541 }
544 542 return (1);
545 543 }
546 544
547 545 /* ***** public interface for process mapping events (hat layer) ***** */
548 546
549 547 /*
550 548 * If the page size matches the Zulu page sizes then just pass
551 549 * it thru. If not then emulate the page demap with demaps of
552 550 * smaller page size.
553 551 */
554 552 /* ARGSUSED */
555 553 void
556 554 zuluvm_demap_page(void *arg, struct hat *hat_ptr, short ctx,
557 555 caddr_t vaddr, uint_t size)
558 556 {
559 557 void *ddarg;
560 558 zuluvm_state_t *zdev = (zuluvm_state_t *)arg;
561 559
562 560 if (arg == NULL)
563 561 return;
564 562
565 563 ZULUVM_STATS_DEMAP_PAGE(zdev);
566 564
567 565 ddarg = zdev->zvm.arg;
568 566
569 567 TNF_PROBE_3(zuluvm_demap_page, "zuluvm", /* */,
570 568 tnf_opaque, addr, vaddr,
571 569 tnf_int, size, size,
572 570 tnf_int, ctx, ctx);
573 571
574 572 if (ddarg != NULL) {
575 573 if (size != zuluvm_base_pgsize &&
576 574 size != ZULU_TTE4M) {
577 575 int i;
578 576 int cnt = size - zuluvm_base_pgsize;
579 577 cnt = ZULU_HAT_SZ_SHIFT(cnt);
580 578 for (i = 0; i < cnt; i++) {
581 579 uintptr_t addr = (uintptr_t)vaddr |
582 580 i << ZULU_HAT_BP_SHIFT;
583 581 zulud_demap_page(zdev, ddarg,
584 582 (caddr_t)addr, ctx);
585 583 }
586 584 } else {
587 585 zulud_demap_page(zdev, ddarg, vaddr, ctx);
588 586 }
589 587 TNF_PROBE_0(zuluvm_demap_page_done, "zuluvm", /* */);
590 588 } else {
591 589 TNF_PROBE_0(zuluvm_demap_page_null_ddarg, "zuluvm", /* */);
592 590 }
593 591 }
594 592
595 593 /*
596 594 * An entire context has gone away, just pass it thru
597 595 */
598 596 void
599 597 zuluvm_demap_ctx(void *arg, short ctx)
600 598 {
601 599 void *ddarg;
602 600 zuluvm_state_t *zdev = (zuluvm_state_t *)arg;
603 601
604 602 if (arg == NULL)
605 603 return;
606 604
607 605 ZULUVM_STATS_DEMAP_CTX(zdev);
608 606
609 607 TNF_PROBE_1(zuluvm_demap_ctx, "zuluvm", /* */,
610 608 tnf_int, ctx, ctx);
611 609 ddarg = zdev->zvm.arg;
612 610
613 611 if (ddarg != NULL)
614 612 zulud_demap_ctx(zdev, ddarg, ctx);
615 613 }
616 614
617 615 static int
618 616 zuluvm_driver_attach(zuluvm_state_t *zdev)
619 617 {
620 618 int i;
621 619 mutex_enter(&zuluvm_lck);
622 620 for (i = 0; i < ZULUVM_MAX_DEV; i++) {
623 621 if (zuluvm_devtab[i] == NULL) {
624 622 zuluvm_devtab[i] = zdev;
625 623 ZULUVM_SET_IDLE(zdev);
626 624 break;
627 625 }
628 626 }
629 627 mutex_exit(&zuluvm_lck);
630 628 if (i >= ZULUVM_MAX_DEV)
631 629 return (ZULUVM_ERROR);
632 630
633 631 if (zulu_hat_attach((void *)zdev) != 0) {
634 632 return (ZULUVM_ERROR);
635 633 }
636 634
637 635 mutex_init(&zdev->dev_lck, NULL, MUTEX_DEFAULT, NULL);
638 636 mutex_init(&zdev->load_lck, NULL, MUTEX_DEFAULT, NULL);
639 637 mutex_init(&zdev->proc_lck, NULL, MUTEX_DEFAULT, NULL);
640 638 mutex_init(&zdev->park_lck, NULL, MUTEX_DEFAULT, NULL);
641 639 cv_init(&zdev->park_cv, NULL, CV_DEFAULT, NULL);
642 640 cv_init(&zdev->intr_wait, NULL, CV_DEFAULT, NULL);
643 641 zdev->parking = 0;
644 642
645 643 #ifdef ZULUVM_STATS
646 644 zdev->zvm.cancel = 0;
647 645 zdev->zvm.pagefault = 0;
648 646 zdev->zvm.no_mapping = 0;
649 647 zdev->zvm.preload = 0;
650 648 zdev->zvm.migrate = 0;
651 649 zdev->zvm.pagesize = 0;
652 650 zdev->zvm.tlb_miss[0] = 0;
653 651 zdev->zvm.tlb_miss[1] = 0;
654 652 zdev->zvm.tlb_miss[2] = 0;
655 653 zdev->zvm.tlb_miss[3] = 0;
656 654 zdev->zvm.itlb1miss = 0;
657 655 zdev->zvm.dtlb1miss = 0;
658 656 zdev->zvm.itlb2miss = 0;
659 657 zdev->zvm.dtlb2miss = 0;
660 658 #endif
661 659 zdev->zvm.pfncnt = 0;
662 660 for (i = 0; i < 50; i++)
663 661 zdev->zvm.pfnbuf[i] = 0;
664 662
665 663 zdev->zvm.mmu_pa = NULL;
666 664 zdev->zvm.proc1 = NULL;
667 665 zdev->zvm.proc2 = NULL;
668 666 zdev->procs = NULL;
669 667 return (ZULUVM_SUCCESS);
670 668 }
671 669
672 670 static int
673 671 zuluvm_driver_detach(zuluvm_state_t *zdev)
674 672 {
675 673 int i;
676 674 cv_destroy(&zdev->intr_wait);
677 675 cv_destroy(&zdev->park_cv);
678 676 mutex_destroy(&zdev->park_lck);
679 677 mutex_destroy(&zdev->proc_lck);
680 678 mutex_destroy(&zdev->dev_lck);
681 679 mutex_destroy(&zdev->load_lck);
682 680 zdev->dops = NULL;
683 681
684 682 mutex_enter(&zuluvm_lck);
685 683 for (i = 0; i < ZULUVM_MAX_DEV; i++) {
686 684 if (zuluvm_devtab[i] == zdev) {
687 685 zuluvm_devtab[i] = NULL;
688 686 break;
689 687 }
690 688 }
691 689 mutex_exit(&zuluvm_lck);
692 690
693 691 if (zulu_hat_detach((void *)zdev) == 0) {
694 692 return (ZULUVM_SUCCESS);
695 693 } else {
696 694 return (ZULUVM_ERROR);
697 695 }
698 696 }
699 697
700 698 zulud_ops_t *zuluvm_dops = NULL;
701 699
702 700 /*
703 701 * init the zulu kernel driver (variables, locks, etc)
704 702 */
705 703 int
706 704 zuluvm_init(zulud_ops_t *ops, int **pagesizes)
707 705 {
708 706 int error = ZULUVM_SUCCESS;
709 707 int i;
710 708 int size = zuluvm_base_pgsize; /* MMU_PAGESIZE; */
711 709
712 710 if (ops->version != ZULUVM_INTERFACE_VERSION)
713 711 return (ZULUVM_VERSION_MISMATCH);
714 712
715 713 zuluvm_dops = ops;
716 714 for (i = 0; i < ZULUM_MAX_PG_SIZES && size <= ZULU_TTE4M; i++) {
717 715 zuluvm_pagesizes[i] = size++;
718 716 }
719 717 zuluvm_pagesizes[i] = -1;
720 718 *pagesizes = zuluvm_pagesizes;
721 719
722 720 return (error);
723 721 }
724 722
725 723 /*
726 724 * cleanup afterwards
727 725 */
728 726 int
729 727 zuluvm_fini(void)
730 728 {
731 729 zuluvm_dops = NULL;
732 730 return (ZULUVM_SUCCESS);
733 731 }
734 732
735 733 /*
736 734 * allocate a zulu kernel driver instance for this zulu device
737 735 */
738 736 int
739 737 zuluvm_alloc_device(dev_info_t *devi, void *arg, zuluvm_info_t *devp,
740 738 caddr_t mmu, caddr_t imr)
741 739 {
742 740 uint64_t intr_num;
743 741 zuluvm_state_t *zdev;
744 742 int error = ZULUVM_SUCCESS;
745 743
746 744 TNF_PROBE_3(zuluvm_alloc_device, "zuluvm", /* */,
747 745 tnf_opaque, arg, arg,
748 746 tnf_opaque, mmu, mmu,
749 747 tnf_opaque, imr, imr);
750 748
751 749 zdev = kmem_zalloc(sizeof (zuluvm_state_t), KM_SLEEP);
752 750 zdev->dip = devi;
753 751 zdev->dops = zuluvm_dops;
754 752 error = zuluvm_driver_attach(zdev);
755 753 if (error != ZULUVM_SUCCESS) {
756 754 kmem_free(zdev, sizeof (zuluvm_state_t));
757 755 return (ZULUVM_NO_DEV);
758 756 }
759 757
760 758 ZULUVM_LOCK;
761 759 error = zuluvm_get_intr_props(zdev, devi);
762 760 if (error != ZULUVM_SUCCESS) {
763 761 ZULUVM_UNLOCK;
764 762 error = zuluvm_driver_detach(zdev);
765 763 if (error != ZULUVM_SUCCESS)
766 764 return (error);
767 765 kmem_free(zdev, sizeof (zuluvm_state_t));
768 766 return (ZULUVM_NO_DEV);
769 767 }
770 768 zdev->zvm.arg = arg;
771 769 zdev->zvm.mmu_pa = (uint64_t)va_to_pa((void *)mmu);
772 770 zdev->imr = (uint64_t *)imr;
773 771 zdev->zvm.dmv_intr = dmv_add_softintr(zuluvm_dmv_tlbmiss_tl1,
774 772 (void *)zdev);
775 773 zulud_set_itlb_pc(zdev, arg, DMV_MAKE_DMV(zdev->zvm.dmv_intr,
776 774 (void *)zdev));
777 775 zulud_set_dtlb_pc(zdev, arg, DMV_MAKE_DMV(zdev->zvm.dmv_intr,
778 776 (void *)zdev));
779 777 intr_dist_add(zuluvm_retarget_intr, (void *)zdev);
780 778 zuluvm_do_retarget(zdev);
781 779 intr_num = add_softintr(ZULUVM_PIL, zuluvm_softintr,
782 780 (caddr_t)zdev, SOFTINT_ST);
783 781 zdev->zvm.intr_num = intr_num;
784 782 *devp = (caddr_t)zdev;
785 783 ZULUVM_UNLOCK;
786 784 TNF_PROBE_1(zuluvm_alloc_device_done, "zuluvm", /* */,
787 785 tnf_opaque, devp, *devp);
788 786 return (ZULUVM_SUCCESS);
789 787 }
790 788
791 789 /*
792 790 * free a zulu kernel driver instance
793 791 */
794 792 int
795 793 zuluvm_free_device(zuluvm_info_t devp)
796 794 {
797 795 int error;
798 796 zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
799 797
800 798 TNF_PROBE_1(zuluvm_free_device, "zuluvm", /* */,
801 799 tnf_opaque, zdev, zdev);
802 800
803 801 if (zdev == NULL)
804 802 return (ZULUVM_NO_DEV);
805 803 ZULUVM_LOCK;
806 804 if (zdev->zvm.arg == NULL) {
807 805 ZULUVM_UNLOCK;
808 806 TNF_PROBE_1(zuluvm_free_device_done, "zuluvm", /* */,
809 807 tnf_int, error, ZULUVM_NO_DEV);
810 808 return (ZULUVM_NO_DEV);
811 809 }
812 810 (void) dmv_rem_intr(zdev->zvm.dmv_intr);
813 811 (void) rem_softintr(zdev->zvm.intr_num);
814 812 intr_dist_rem(zuluvm_retarget_intr, (void *)zdev);
815 813 zdev->zvm.arg = NULL;
816 814 ZULUVM_UNLOCK;
817 815 error = zuluvm_driver_detach(zdev);
818 816 if (error != ZULUVM_SUCCESS)
819 817 return (error);
820 818 zdev->dops = NULL;
821 819 kmem_free(zdev, sizeof (zuluvm_state_t));
822 820
823 821 TNF_PROBE_0(zuluvm_free_device_done, "zuluvm", /* */);
824 822 return (ZULUVM_SUCCESS);
825 823 }
826 824
827 825 /*
828 826 * find the as in the list of active zulu processes
829 827 * The caller has to hold zdev->proc_lck
830 828 */
831 829 static zuluvm_proc_t *
832 830 zuluvm_find_proc(zuluvm_state_t *zdev, struct as *asp)
833 831 {
834 832 zuluvm_proc_t *p;
835 833 TNF_PROBE_2(zuluvm_find_proc, "zuluvm", /* */,
836 834 tnf_opaque, zdev, zdev,
837 835 tnf_opaque, asp, asp);
838 836 for (p = zdev->procs; p != NULL; p = p->next) {
839 837 if (ZULU_HAT2AS(p->zhat) == asp) {
840 838 TNF_PROBE_1(zuluvm_find_proc_done,
841 839 "zuluvm", /* */, tnf_opaque, proc, p);
842 840 return (p);
843 841 }
844 842 }
845 843 TNF_PROBE_0(zuluvm_find_proc_fail, "zuluvm", /* */);
846 844 return (NULL);
847 845 }
848 846
849 847 void
850 848 zuluvm_as_free(struct as *as, void *arg, uint_t events)
851 849 {
852 850 zuluvm_proc_t *proc = (zuluvm_proc_t *)arg;
853 851 zuluvm_state_t *zdev = proc->zdev;
854 852 int wait = 0;
855 853 int flag = 0;
856 854 int valid;
857 855
858 856 (void) events;
859 857
860 858 TNF_PROBE_1(zuluvm_as_free, "zuluvm", /* */,
861 859 tnf_opaque, arg, arg);
862 860
863 861 (void) as_delete_callback(as, arg);
864 862 /*
865 863 * if this entry is still valid, then we need to sync
866 864 * with zuluvm_tlb_handler rountine.
867 865 */
868 866 mutex_enter(&zdev->proc_lck);
869 867 valid = proc->valid;
870 868 proc->valid = 0;
871 869 mutex_exit(&zdev->proc_lck);
872 870
873 871 if (valid) {
874 872 ZULUVM_LOCK;
875 873 if (proc == zdev->zvm.proc1) {
876 874 flag |= ZULUVM_WAIT_INTR1;
877 875 wait |= ZULUVM_DO_INTR1;
878 876 }
879 877 if (proc == zdev->zvm.proc2) {
880 878 flag |= ZULUVM_WAIT_INTR2;
881 879 wait |= ZULUVM_DO_INTR2;
882 880 }
883 881 if (flag) {
884 882 zdev->intr_flags |= flag;
885 883 /*
886 884 * wait until the tlb miss is resloved
887 885 */
888 886 while (zdev->intr_flags & wait) {
889 887 cv_wait(&zdev->intr_wait, &zdev->dev_lck);
890 888 }
891 889 zdev->intr_flags &= ~flag;
892 890 }
893 891 ZULUVM_UNLOCK;
894 892 }
895 893
896 894 if (proc->zhat != NULL) {
897 895 /*
898 896 * prevent any further tlb miss processing for this hat
899 897 */
900 898 zulu_hat_terminate(proc->zhat);
901 899 }
902 900
903 901 /*
904 902 * decrement the ref count and do the appropriate
905 903 * if it drops to zero.
906 904 */
907 905 mutex_enter(&zdev->proc_lck);
908 906 (void) zuluvm_proc_release(zdev, proc);
909 907 mutex_exit(&zdev->proc_lck);
910 908 }
911 909
912 910 /*
913 911 * notify zulu vm driver about a new process going to
914 912 * use zulu DMA. Create a zulu_hat.
915 913 */
916 914 int
917 915 zuluvm_dma_add_proc(zuluvm_info_t devp, uint64_t *cookie)
918 916 {
919 917 zuluvm_proc_t *proc;
920 918 int refcnt;
921 919 struct as *asp = ZULUVM_GET_AS;
922 920 zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
923 921
924 922 TNF_PROBE_1(zuluvm_dma_add_proc, "zuluvm", /* */,
925 923 tnf_opaque, zdev, zdev);
926 924 mutex_enter(&zdev->proc_lck);
927 925 proc = zuluvm_find_proc(zdev, asp);
928 926 if (proc == NULL) {
929 927 proc = kmem_zalloc(sizeof (zuluvm_proc_t), KM_SLEEP);
930 928 proc->zhat = zulu_hat_proc_attach(asp, zdev);
931 929 if (proc->zhat == NULL) {
932 930 mutex_exit(&zdev->proc_lck);
933 931 kmem_free(proc, sizeof (zuluvm_proc_t));
934 932 TNF_PROBE_2(zuluvm_dma_add_proc_done, "zuluvm", /* */,
935 933 tnf_int, valid, 0,
936 934 tnf_int, error, ZULUVM_ERROR);
937 935 return (ZULUVM_ERROR);
938 936 }
939 937 proc->zdev = zdev;
940 938 proc->valid = 1;
941 939 proc->refcnt = 1;
942 940 proc->next = zdev->procs;
943 941 if (zdev->procs)
944 942 zdev->procs->prev = proc;
945 943 proc->prev = NULL;
946 944 zdev->procs = proc;
947 945 proc->refcnt++;
948 946 (void) as_add_callback(asp, zuluvm_as_free, proc,
949 947 AS_FREE_EVENT, 0, -1, KM_SLEEP);
950 948 } else {
951 949 if (proc->valid == 0) {
952 950 mutex_exit(&zdev->proc_lck);
953 951 TNF_PROBE_2(zuluvm_dma_add_proc_done, "zuluvm", /* */,
954 952 tnf_int, valid, 0,
955 953 tnf_int, error, ZULUVM_ERROR);
956 954 return (ZULUVM_ERROR);
957 955 }
958 956 proc->refcnt++;
959 957 }
960 958 refcnt = proc->refcnt;
961 959 mutex_exit(&zdev->proc_lck);
962 960 *cookie = (uint64_t)proc;
963 961 TNF_PROBE_2(zuluvm_dma_add_proc_done, "zuluvm", /* */,
964 962 tnf_int, refcnt, refcnt,
965 963 tnf_int, error, ZULUVM_SUCCESS);
966 964 return (ZULUVM_SUCCESS);
967 965 }
968 966
969 967 void
970 968 zuluvm_proc_hold(zuluvm_state_t *zdev, zuluvm_proc_t *proc)
971 969 {
972 970 mutex_enter(&zdev->proc_lck);
973 971 proc->refcnt++;
974 972 mutex_exit(&zdev->proc_lck);
975 973 }
976 974
977 975 /*
978 976 * decrement ref count and free data if it drops to zero
979 977 */
980 978 static int
981 979 zuluvm_proc_release(zuluvm_state_t *zdev, zuluvm_proc_t *proc)
982 980 {
983 981 int refcnt;
984 982 ASSERT(MUTEX_HELD(&zdev->proc_lck));
985 983 refcnt = --proc->refcnt;
986 984 TNF_PROBE_3(zuluvm_proc_release, "zuluvm", /* */,
987 985 tnf_opaque, zdev, zdev,
988 986 tnf_opaque, proc, proc,
989 987 tnf_int, refcnt, refcnt);
990 988 if (refcnt == 0) {
991 989 if (proc->next)
992 990 proc->next->prev = proc->prev;
993 991 if (proc->prev)
994 992 proc->prev->next = proc->next;
995 993 else
996 994 zdev->procs = proc->next;
997 995 kmem_free(proc, sizeof (zuluvm_proc_t));
998 996 }
999 997 return (refcnt);
1000 998 }
1001 999
1002 1000 /*
1003 1001 * this process is not longer using DMA, all entries
1004 1002 * have been removed from the TLB.
1005 1003 */
1006 1004 int
1007 1005 zuluvm_dma_delete_proc(zuluvm_info_t devp, uint64_t cookie)
1008 1006 {
1009 1007 int refcnt;
1010 1008 zuluvm_proc_t *proc = (zuluvm_proc_t *)cookie;
1011 1009 zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
1012 1010
1013 1011 TNF_PROBE_2(zuluvm_dma_delete_proc, "zuluvm", /* */,
1014 1012 tnf_opaque, zdev, zdev,
1015 1013 tnf_opaque, cookie, cookie);
1016 1014 mutex_enter(&zdev->proc_lck);
1017 1015 if (proc != NULL) {
1018 1016 TNF_PROBE_1(zuluvm_dma_delete_proc, "zuluvm", /* */,
1019 1017 tnf_opaque, proc, proc);
1020 1018 if (proc->zhat != NULL) {
1021 1019 zulu_hat_proc_detach(proc->zhat);
1022 1020 proc->zhat = NULL;
1023 1021 }
1024 1022 refcnt = zuluvm_proc_release(zdev, proc);
1025 1023 }
1026 1024 mutex_exit(&zdev->proc_lck);
1027 1025
1028 1026 TNF_PROBE_2(zuluvm_dma_delete_proc_done, "zuluvm", /* */,
1029 1027 tnf_int, refcnt, refcnt,
1030 1028 tnf_int, error, ZULUVM_SUCCESS);
1031 1029 return (ZULUVM_SUCCESS);
1032 1030 }
1033 1031
1034 1032 /*
1035 1033 * barrier sync for device driver
1036 1034 * blocks until zuluvm_tlbmiss_tl1 function is done
1037 1035 */
1038 1036 void
1039 1037 zuluvm_fast_tlb_wait(caddr_t devp)
1040 1038 {
1041 1039 int state;
1042 1040 zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
1043 1041 int cnt = 0;
1044 1042
1045 1043 do {
1046 1044 state = ZULUVM_GET_STATE(zdev);
1047 1045 cnt++;
1048 1046 } while (state == ZULUVM_STATE_TLB_PENDING);
1049 1047 TNF_PROBE_1(zuluvm_fast_tlb_wait, "zuluvm", /* */,
1050 1048 tnf_int, loop_cnt, cnt);
1051 1049 }
1052 1050
1053 1051 /*
1054 1052 * setup DMA handling for this handle
1055 1053 */
1056 1054 int
1057 1055 zuluvm_dma_alloc_ctx(zuluvm_info_t devp, int dma, short *mmuctx,
1058 1056 uint64_t *tsbreg)
1059 1057 {
1060 1058 struct as *asp = ZULUVM_GET_AS;
1061 1059 int error = ZULUVM_NO_DEV;
1062 1060 zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
1063 1061 int state, newstate;
1064 1062
1065 1063 if (asp == NULL) {
1066 1064 TNF_PROBE_1(zuluvm_dma_alloc_ctx_done, "zuluvm", /* */,
1067 1065 tnf_int, error, ZULUVM_NO_HAT);
1068 1066 return (ZULUVM_NO_HAT);
1069 1067 }
1070 1068
1071 1069 *tsbreg = 0;
1072 1070 state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_IDLE,
1073 1071 ZULUVM_STATE_STOPPED);
1074 1072 newstate = ZULUVM_GET_STATE(zdev);
1075 1073 TNF_PROBE_4(zuluvm_dma_alloc_ctx, "zuluvm", /* */,
1076 1074 tnf_opaque, devp, devp,
1077 1075 tnf_int, dma, dma,
1078 1076 tnf_int, oldstate, state,
1079 1077 tnf_int, newstate, newstate);
1080 1078 #ifdef DEBUG
1081 1079 if (zuluvm_debug_state)
1082 1080 cmn_err(CE_NOTE, "zuluvm_dma_alloc_ctx: state %d\n", state);
1083 1081 #endif
1084 1082 if (state != ZULUVM_STATE_STOPPED && state != ZULUVM_STATE_IDLE) {
1085 1083 while (state != ZULUVM_STATE_IDLE) {
1086 1084 state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_IDLE,
1087 1085 ZULUVM_STATE_STOPPED);
1088 1086 #ifdef DEBUG
1089 1087 if (zuluvm_debug_state)
1090 1088 cmn_err(CE_NOTE, "zuluvm_dma_alloc_ctx: (loop)"
1091 1089 " state %d\n", state);
1092 1090 #endif
1093 1091 if (state != ZULUVM_STATE_IDLE)
1094 1092 delay(1);
1095 1093 }
1096 1094 }
1097 1095
1098 1096 if (zdev->zvm.arg != NULL) {
1099 1097 struct zulu_hat *zhat;
1100 1098 zuluvm_proc_t *proc;
1101 1099
1102 1100 mutex_enter(&zdev->proc_lck);
1103 1101 proc = zuluvm_find_proc(zdev, asp);
1104 1102 if (proc != NULL) {
1105 1103 zhat = proc->zhat;
1106 1104 proc->refcnt++;
1107 1105 }
1108 1106 mutex_exit(&zdev->proc_lck);
1109 1107
1110 1108 switch (dma) {
1111 1109 case ZULUVM_DMA1:
1112 1110 ZULUVM_LOCK;
1113 1111 zdev->zvm.proc1 = proc;
1114 1112 ZULUVM_UNLOCK;
1115 1113 error = ZULUVM_SUCCESS;
1116 1114 break;
1117 1115 case ZULUVM_DMA2:
1118 1116 ZULUVM_LOCK;
1119 1117 zdev->zvm.proc2 = proc;
1120 1118 ZULUVM_UNLOCK;
1121 1119 error = ZULUVM_SUCCESS;
1122 1120 break;
1123 1121 default:
1124 1122 mutex_enter(&zdev->proc_lck);
1125 1123 (void) zuluvm_proc_release(zdev, proc);
1126 1124 mutex_exit(&zdev->proc_lck);
1127 1125 }
1128 1126
1129 1127 if (error == ZULUVM_SUCCESS) {
1130 1128 zulu_hat_validate_ctx(zhat);
1131 1129 if (zhat->zulu_ctx >= 0) {
1132 1130 *mmuctx = zhat->zulu_ctx;
1133 1131 } else {
1134 1132 printf("invalid context value: %d\n",
1135 1133 zhat->zulu_ctx);
1136 1134
1137 1135 mutex_enter(&zdev->proc_lck);
1138 1136 (void) zuluvm_proc_release(zdev, proc);
1139 1137 mutex_exit(&zdev->proc_lck);
1140 1138
1141 1139 error = ZULUVM_ERROR;
1142 1140 }
1143 1141 } else {
1144 1142 error = ZULUVM_ERROR;
1145 1143 }
1146 1144 }
1147 1145 TNF_PROBE_1(zuluvm_dma_alloc_ctx_done, "zuluvm", /* */,
1148 1146 tnf_int, error, error);
1149 1147 return (error);
1150 1148 }
1151 1149
1152 1150 /*
1153 1151 * preload TLB
1154 1152 * this will try to pre-set the zulu tlb, mainly used for dma engine 2,
1155 1153 * video read-back.
1156 1154 */
1157 1155 int
1158 1156 zuluvm_dma_preload(zuluvm_info_t devp, int dma,
1159 1157 int num, zulud_preload_t *list)
1160 1158 {
1161 1159 int i;
1162 1160 int error = ZULUVM_SUCCESS;
1163 1161 struct zulu_hat *zhat;
1164 1162 zuluvm_proc_t *proc = NULL;
1165 1163
1166 1164 zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
1167 1165
1168 1166 TNF_PROBE_4(zuluvm_dma_preload, "zuluvm", /* */,
1169 1167 tnf_opaque, devp, devp,
1170 1168 tnf_int, dma, dma,
1171 1169 tnf_int, num, num,
1172 1170 tnf_opaque, list, list);
1173 1171 ZULUVM_LOCK;
1174 1172 switch (dma) {
1175 1173 case ZULUVM_DMA1:
1176 1174 proc = zdev->zvm.proc1;
1177 1175 break;
1178 1176 case ZULUVM_DMA2:
1179 1177 proc = zdev->zvm.proc2;
1180 1178 break;
1181 1179 }
1182 1180
1183 1181 mutex_enter(&zdev->proc_lck);
1184 1182 if (proc == NULL || proc->valid == 0 || proc->zhat == NULL) {
1185 1183 mutex_exit(&zdev->proc_lck);
1186 1184 ZULUVM_UNLOCK;
1187 1185 return (ZULUVM_NO_HAT);
1188 1186 }
1189 1187 mutex_exit(&zdev->proc_lck);
1190 1188
1191 1189 zhat = proc->zhat;
1192 1190 /*
1193 1191 * need to release this to avoid recursive enter in zuluvm_load_tte
1194 1192 * which gets called from zulu_hat_memload()
1195 1193 */
1196 1194 ZULUVM_UNLOCK;
1197 1195
1198 1196 mutex_enter(&zdev->load_lck);
1199 1197 for (i = 0; i < num; i++) {
1200 1198 int pg_size;
1201 1199 int res;
1202 1200 int first = 1;
1203 1201 caddr_t addr = ZULUVM_GET_PAGE(list[i].addr);
1204 1202 int64_t size = (int64_t)list[i].len;
1205 1203 while (size > 0) {
1206 1204 if (list[i].tlbtype & ~ZULUVM_DMA_MASK) {
1207 1205 error = ZULUVM_INVALID_MISS;
1208 1206 break;
1209 1207 }
1210 1208 res = zulu_hat_load(zhat, addr,
1211 1209 (list[i].tlbtype == ZULUVM_DMA2) ? S_WRITE : S_READ,
1212 1210 &pg_size);
1213 1211 if ((res != 0) || (pg_size < 0)) {
1214 1212 error = ZULUVM_NO_MAP;
1215 1213 break;
1216 1214 }
1217 1215 ZULUVM_STATS_PRELOAD(zdev);
1218 1216 TNF_PROBE_2(zuluvm_dma_preload_addr, "zuluvm", /* */,
1219 1217 tnf_opaque, addr, addr,
1220 1218 tnf_opaque, size, size);
1221 1219 if (first) {
1222 1220 first = 0;
1223 1221 size -= ZULU_HAT_PGDIFF(list[i].addr,
1224 1222 pg_size);
1225 1223 } else {
1226 1224 size -= ZULU_HAT_PGSZ(pg_size);
1227 1225 }
1228 1226 addr += ZULU_HAT_PGSZ(pg_size);
1229 1227 }
1230 1228 }
1231 1229 mutex_exit(&zdev->load_lck);
1232 1230 TNF_PROBE_1(zuluvm_dma_preload_done, "zuluvm", /* */,
1233 1231 tnf_int, error, error);
1234 1232 return (ZULUVM_SUCCESS);
1235 1233 }
1236 1234
1237 1235 /*
1238 1236 * destroy DMA handling for this handle
1239 1237 */
1240 1238 int
1241 1239 zuluvm_dma_free_ctx(zuluvm_info_t devp, int dma)
1242 1240 {
1243 1241 int error = ZULUVM_NO_DEV;
1244 1242 zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
1245 1243 int state, newstate;
1246 1244
1247 1245 state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_STOPPED,
1248 1246 ZULUVM_STATE_IDLE);
1249 1247 newstate = ZULUVM_GET_STATE(zdev);
1250 1248 TNF_PROBE_4(zuluvm_dma_free_ctx, "zuluvm", /* */,
1251 1249 tnf_opaque, devp, devp,
1252 1250 tnf_int, dma, dma,
1253 1251 tnf_int, oldstate, state,
1254 1252 tnf_int, newstate, newstate);
1255 1253 #ifdef DEBUG
1256 1254 if (zuluvm_debug_state)
1257 1255 cmn_err(CE_NOTE, "zuluvm_dma_free_ctx: state %d\n", state);
1258 1256 #endif
1259 1257 if (state != ZULUVM_STATE_IDLE && state != ZULUVM_STATE_STOPPED) {
1260 1258 int doit = 1;
1261 1259 while (doit) {
1262 1260 switch (state) {
1263 1261 case ZULUVM_STATE_CANCELED:
1264 1262 case ZULUVM_STATE_STOPPED:
1265 1263 doit = 0;
1266 1264 break;
1267 1265 case ZULUVM_STATE_IDLE:
1268 1266 state = ZULUVM_SET_STATE(zdev,
1269 1267 ZULUVM_STATE_STOPPED,
1270 1268 ZULUVM_STATE_IDLE);
1271 1269 break;
1272 1270 default:
1273 1271 state = ZULUVM_SET_STATE(zdev,
1274 1272 ZULUVM_STATE_CANCELED, state);
1275 1273 }
1276 1274 TNF_PROBE_1(zuluvm_dma_free_ctx, "zuluvm", /* */,
1277 1275 tnf_int, state, state);
1278 1276 #ifdef DEBUG
1279 1277 if (zuluvm_debug_state)
1280 1278 cmn_err(CE_NOTE, "zuluvm_dma_free_ctx: (loop1)"
1281 1279 " state %d\n", state);
1282 1280 #endif
1283 1281 }
1284 1282 }
1285 1283 TNF_PROBE_1(zuluvm_dma_free_ctx, "zuluvm", /* */,
1286 1284 tnf_int, state, state);
1287 1285
1288 1286 error = ZULUVM_SUCCESS;
1289 1287 while (state != ZULUVM_STATE_STOPPED) {
1290 1288 state = ZULUVM_GET_STATE(zdev);
1291 1289 #ifdef DEBUG
1292 1290 if (zuluvm_debug_state)
1293 1291 cmn_err(CE_NOTE, "zuluvm_dma_free: (loop2) state %d\n",
1294 1292 state);
1295 1293 #endif
1296 1294 if (state != ZULUVM_STATE_STOPPED)
1297 1295 delay(1);
1298 1296 }
1299 1297 ZULUVM_LOCK;
1300 1298 if (zdev->zvm.arg != NULL) {
1301 1299 zuluvm_proc_t *proc = NULL;
1302 1300 switch (dma) {
1303 1301 case ZULUVM_DMA1:
1304 1302 proc = zdev->zvm.proc1;
1305 1303 zdev->zvm.proc1 = NULL;
1306 1304 break;
1307 1305 case ZULUVM_DMA2:
1308 1306 proc = zdev->zvm.proc2;
1309 1307 zdev->zvm.proc2 = NULL;
1310 1308 break;
1311 1309 default:
1312 1310 error = ZULUVM_NO_DEV;
1313 1311 }
1314 1312 ZULUVM_UNLOCK;
1315 1313 if (proc) {
1316 1314 mutex_enter(&zdev->proc_lck);
1317 1315 (void) zuluvm_proc_release(zdev, proc);
1318 1316 mutex_exit(&zdev->proc_lck);
1319 1317 }
1320 1318 } else {
1321 1319 ZULUVM_UNLOCK;
1322 1320 error = ZULUVM_NO_DEV;
1323 1321 }
1324 1322 TNF_PROBE_1(zuluvm_dma_free_ctx_done, "zuluvm", /* */,
1325 1323 tnf_int, error, error);
1326 1324 return (error);
1327 1325 }
1328 1326
1329 1327 static void
1330 1328 zuluvm_do_retarget(zuluvm_state_t *zdev)
1331 1329 {
1332 1330 int i, idx;
1333 1331 uint_t cpu;
1334 1332 for (i = 0; i < ZULUVM_MAX_INTR; i++) {
1335 1333 if (zdev->interrupts[i].ino != -1) {
1336 1334 cpu = intr_dist_cpuid();
1337 1335 idx = zdev->interrupts[i].offset;
1338 1336 if (zdev->imr[idx] & ZULUVM_IMR_V_MASK)
1339 1337 zdev->imr[idx] = ZULUVM_IMR_V_MASK |
1340 1338 (cpu<<ZULUVM_IMR_TARGET_SHIFT);
1341 1339 else
1342 1340 zdev->imr[idx] =
1343 1341 cpu<<ZULUVM_IMR_TARGET_SHIFT;
1344 1342 }
1345 1343 }
1346 1344 }
1347 1345
1348 1346 static void
1349 1347 zuluvm_retarget_intr(void *arg)
1350 1348 {
1351 1349 zuluvm_state_t *zdev = (zuluvm_state_t *)arg;
1352 1350 ZULUVM_LOCK;
1353 1351 zuluvm_do_retarget(zdev);
1354 1352 ZULUVM_UNLOCK;
1355 1353 }
1356 1354
1357 1355 int
1358 1356 zuluvm_add_intr(zuluvm_info_t devp, int ino,
1359 1357 uint_t (*handler)(caddr_t), caddr_t arg)
1360 1358 {
1361 1359 zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
1362 1360 if (devp == NULL) {
1363 1361 TNF_PROBE_1(zuluvm_add_intr_done, "zuluvm", /* */,
1364 1362 tnf_int, error, ZULUVM_NO_DEV);
1365 1363 return (ZULUVM_NO_DEV);
1366 1364 }
1367 1365 if (ddi_add_intr(zdev->dip, ino, NULL, NULL, handler, arg)
1368 1366 != DDI_SUCCESS) {
1369 1367 TNF_PROBE_1(zuluvm_add_intr_done, "zuluvm", /* */,
1370 1368 tnf_int, error, ZULUVM_ERROR);
1371 1369 return (ZULUVM_ERROR);
1372 1370 }
1373 1371 return (ZULUVM_SUCCESS);
1374 1372 }
1375 1373
1376 1374 int
1377 1375 zuluvm_rem_intr(zuluvm_info_t devp, int ino)
1378 1376 {
1379 1377 zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
1380 1378 if (devp == NULL) {
1381 1379 TNF_PROBE_1(zuluvm_rem_intr_done, "zuluvm", /* */,
1382 1380 tnf_int, error, ZULUVM_NO_DEV);
1383 1381 return (ZULUVM_NO_DEV);
1384 1382 }
1385 1383 /* remove from distributin list */
1386 1384 ZULUVM_LOCK;
1387 1385 zdev->imr[zdev->interrupts[ino].offset] &= ~ZULUVM_IMR_V_MASK;
1388 1386 ZULUVM_UNLOCK;
1389 1387 ddi_remove_intr(zdev->dip, ino, NULL);
1390 1388 return (ZULUVM_SUCCESS);
1391 1389 }
1392 1390
1393 1391 int
1394 1392 zuluvm_enable_intr(zuluvm_info_t devp, int num)
1395 1393 {
1396 1394 zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
1397 1395
1398 1396 TNF_PROBE_2(zuluvm_enable_intr, "zuluvm_intr", /* */,
1399 1397 tnf_opaque, devp, devp,
1400 1398 tnf_int, num, num);
1401 1399 if (devp == NULL) {
1402 1400 TNF_PROBE_1(zuluvm_enable_intr_done, "zuluvm", /* */,
1403 1401 tnf_int, error, ZULUVM_NO_DEV);
1404 1402 return (ZULUVM_NO_DEV);
1405 1403 }
1406 1404 if (num < 0 || num > ZULUVM_IMR_MAX) {
1407 1405 TNF_PROBE_1(zuluvm_enable_intr_done, "zuluvm", /* */,
1408 1406 tnf_int, error, ZULUVM_BAD_IDX);
1409 1407 return (ZULUVM_BAD_IDX);
1410 1408 }
1411 1409 ZULUVM_LOCK;
1412 1410 zdev->imr[num] |= ZULUVM_IMR_V_MASK;
1413 1411 ZULUVM_UNLOCK;
1414 1412 TNF_PROBE_1(zuluvm_enable_intr_done, "zuluvm_intr", /* */,
1415 1413 tnf_int, error, ZULUVM_SUCCESS);
1416 1414 return (ZULUVM_SUCCESS);
1417 1415 }
1418 1416
1419 1417 int
1420 1418 zuluvm_disable_intr(zuluvm_info_t devp, int num)
1421 1419 {
1422 1420 zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
1423 1421
1424 1422 TNF_PROBE_2(zuluvm_disable_intr, "zuluvm_intr", /* */,
1425 1423 tnf_opaque, devp, devp,
1426 1424 tnf_int, num, num);
1427 1425 if (devp == NULL) {
1428 1426 TNF_PROBE_1(zuluvm_disable_intr_done, "zuluvm", /* */,
1429 1427 tnf_int, error, ZULUVM_NO_DEV);
1430 1428 return (ZULUVM_NO_DEV);
1431 1429 }
1432 1430 if (num < 0 || num > ZULUVM_IMR_MAX) {
1433 1431 TNF_PROBE_1(zuluvm_disable_intr_done, "zuluvm", /* */,
1434 1432 tnf_int, error, ZULUVM_BAD_IDX);
1435 1433 return (ZULUVM_BAD_IDX);
1436 1434 }
1437 1435 ZULUVM_LOCK;
1438 1436 zdev->imr[num] &= ~ZULUVM_IMR_V_MASK;
1439 1437 ZULUVM_UNLOCK;
1440 1438 TNF_PROBE_1(zuluvm_disable_intr_done, "zuluvm_intr", /* */,
1441 1439 tnf_int, error, ZULUVM_SUCCESS);
1442 1440 return (ZULUVM_SUCCESS);
1443 1441 }
1444 1442
1445 1443 static int
1446 1444 zuluvm_get_intr_props(zuluvm_state_t *zdev,
1447 1445 dev_info_t *devi)
1448 1446 {
1449 1447 int *intr;
1450 1448 int i;
1451 1449 uint_t nintr;
1452 1450
1453 1451 zdev->agentid = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
1454 1452 "portid", -1);
1455 1453 if (zdev->agentid == -1) {
1456 1454 cmn_err(CE_WARN, "%s%d: no portid property",
1457 1455 ddi_get_name(devi),
1458 1456 ddi_get_instance(devi));
1459 1457 return (ZULUVM_ERROR);
1460 1458 }
1461 1459
1462 1460 for (i = 0; i < ZULUVM_MAX_INTR; i++) {
1463 1461 zdev->interrupts[i].offset = 0;
1464 1462 zdev->interrupts[i].ino = -1;
1465 1463 }
1466 1464
1467 1465 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
1468 1466 "interrupts", &intr, &nintr) == DDI_PROP_SUCCESS) {
1469 1467
1470 1468 if (nintr == 0) {
1471 1469 cmn_err(CE_WARN, "%s%d: no interrupts in property",
1472 1470 ddi_get_name(devi),
1473 1471 ddi_get_instance(devi));
1474 1472 ddi_prop_free(intr);
1475 1473 return (ZULUVM_ERROR);
1476 1474 }
1477 1475 if (nintr >= ZULUVM_MAX_INTR) {
1478 1476 cmn_err(CE_WARN, "%s%d: to many interrupts (%d)",
1479 1477 ddi_get_name(devi),
1480 1478 ddi_get_instance(devi), nintr);
1481 1479 ddi_prop_free(intr);
1482 1480 return (ZULUVM_ERROR);
1483 1481 }
1484 1482 for (i = 0; i < nintr; i++) {
1485 1483 zdev->interrupts[i].offset = intr[i];
1486 1484 zdev->interrupts[i].ino = i;
1487 1485 }
1488 1486 ddi_prop_free(intr);
1489 1487 } else {
1490 1488 cmn_err(CE_WARN, "%s%d: no interrupts property",
1491 1489 ddi_get_name(devi),
1492 1490 ddi_get_instance(devi));
1493 1491 }
1494 1492 return (ZULUVM_SUCCESS);
1495 1493 }
1496 1494
1497 1495 /* *** enf of zulu *** */
↓ open down ↓ |
1406 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX