Print this page
5255 uts shouldn't open-code ISP2
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/dtrace/fasttrap.c
+++ new/usr/src/uts/common/dtrace/fasttrap.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 */
26 26
27 27 /*
28 28 * Copyright (c) 2013, Joyent, Inc. All rights reserved.
29 29 */
30 30
31 31 #include <sys/atomic.h>
32 32 #include <sys/errno.h>
33 33 #include <sys/stat.h>
34 34 #include <sys/modctl.h>
35 35 #include <sys/conf.h>
36 36 #include <sys/systm.h>
37 37 #include <sys/ddi.h>
38 38 #include <sys/sunddi.h>
39 39 #include <sys/cpuvar.h>
40 40 #include <sys/kmem.h>
41 41 #include <sys/strsubr.h>
42 42 #include <sys/fasttrap.h>
43 43 #include <sys/fasttrap_impl.h>
44 44 #include <sys/fasttrap_isa.h>
45 45 #include <sys/dtrace.h>
46 46 #include <sys/dtrace_impl.h>
47 47 #include <sys/sysmacros.h>
48 48 #include <sys/proc.h>
49 49 #include <sys/priv.h>
50 50 #include <sys/policy.h>
51 51 #include <util/qsort.h>
52 52
53 53 /*
54 54 * User-Land Trap-Based Tracing
55 55 * ----------------------------
56 56 *
57 57 * The fasttrap provider allows DTrace consumers to instrument any user-level
58 58 * instruction to gather data; this includes probes with semantic
59 59 * signifigance like entry and return as well as simple offsets into the
60 60 * function. While the specific techniques used are very ISA specific, the
61 61 * methodology is generalizable to any architecture.
62 62 *
63 63 *
64 64 * The General Methodology
65 65 * -----------------------
66 66 *
67 67 * With the primary goal of tracing every user-land instruction and the
68 68 * limitation that we can't trust user space so don't want to rely on much
69 69 * information there, we begin by replacing the instructions we want to trace
70 70 * with trap instructions. Each instruction we overwrite is saved into a hash
71 71 * table keyed by process ID and pc address. When we enter the kernel due to
72 72 * this trap instruction, we need the effects of the replaced instruction to
73 73 * appear to have occurred before we proceed with the user thread's
74 74 * execution.
75 75 *
76 76 * Each user level thread is represented by a ulwp_t structure which is
77 77 * always easily accessible through a register. The most basic way to produce
78 78 * the effects of the instruction we replaced is to copy that instruction out
79 79 * to a bit of scratch space reserved in the user thread's ulwp_t structure
80 80 * (a sort of kernel-private thread local storage), set the PC to that
81 81 * scratch space and single step. When we reenter the kernel after single
82 82 * stepping the instruction we must then adjust the PC to point to what would
83 83 * normally be the next instruction. Of course, special care must be taken
84 84 * for branches and jumps, but these represent such a small fraction of any
85 85 * instruction set that writing the code to emulate these in the kernel is
86 86 * not too difficult.
87 87 *
88 88 * Return probes may require several tracepoints to trace every return site,
89 89 * and, conversely, each tracepoint may activate several probes (the entry
90 90 * and offset 0 probes, for example). To solve this muliplexing problem,
91 91 * tracepoints contain lists of probes to activate and probes contain lists
92 92 * of tracepoints to enable. If a probe is activated, it adds its ID to
93 93 * existing tracepoints or creates new ones as necessary.
94 94 *
95 95 * Most probes are activated _before_ the instruction is executed, but return
96 96 * probes are activated _after_ the effects of the last instruction of the
97 97 * function are visible. Return probes must be fired _after_ we have
98 98 * single-stepped the instruction whereas all other probes are fired
99 99 * beforehand.
100 100 *
101 101 *
102 102 * Lock Ordering
103 103 * -------------
104 104 *
105 105 * The lock ordering below -- both internally and with respect to the DTrace
106 106 * framework -- is a little tricky and bears some explanation. Each provider
107 107 * has a lock (ftp_mtx) that protects its members including reference counts
108 108 * for enabled probes (ftp_rcount), consumers actively creating probes
109 109 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
110 110 * from being freed. A provider is looked up by taking the bucket lock for the
111 111 * provider hash table, and is returned with its lock held. The provider lock
112 112 * may be taken in functions invoked by the DTrace framework, but may not be
113 113 * held while calling functions in the DTrace framework.
114 114 *
115 115 * To ensure consistency over multiple calls to the DTrace framework, the
116 116 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
117 117 * not be taken when holding the provider lock as that would create a cyclic
118 118 * lock ordering. In situations where one would naturally take the provider
119 119 * lock and then the creation lock, we instead up a reference count to prevent
120 120 * the provider from disappearing, drop the provider lock, and acquire the
121 121 * creation lock.
122 122 *
123 123 * Briefly:
124 124 * bucket lock before provider lock
125 125 * DTrace before provider lock
126 126 * creation lock before DTrace
127 127 * never hold the provider lock and creation lock simultaneously
128 128 */
129 129
130 130 static dev_info_t *fasttrap_devi;
131 131 static dtrace_meta_provider_id_t fasttrap_meta_id;
132 132
133 133 static timeout_id_t fasttrap_timeout;
134 134 static kmutex_t fasttrap_cleanup_mtx;
135 135 static uint_t fasttrap_cleanup_work;
136 136
137 137 /*
138 138 * Generation count on modifications to the global tracepoint lookup table.
139 139 */
140 140 static volatile uint64_t fasttrap_mod_gen;
141 141
142 142 /*
143 143 * When the fasttrap provider is loaded, fasttrap_max is set to either
144 144 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
145 145 * fasttrap.conf file. Each time a probe is created, fasttrap_total is
146 146 * incremented by the number of tracepoints that may be associated with that
147 147 * probe; fasttrap_total is capped at fasttrap_max.
148 148 */
149 149 #define FASTTRAP_MAX_DEFAULT 250000
150 150 static uint32_t fasttrap_max;
151 151 static uint32_t fasttrap_total;
152 152
153 153
154 154 #define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000
155 155 #define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100
156 156 #define FASTTRAP_PROCS_DEFAULT_SIZE 0x100
157 157
158 158 #define FASTTRAP_PID_NAME "pid"
159 159
160 160 fasttrap_hash_t fasttrap_tpoints;
161 161 static fasttrap_hash_t fasttrap_provs;
162 162 static fasttrap_hash_t fasttrap_procs;
163 163
164 164 static uint64_t fasttrap_pid_count; /* pid ref count */
165 165 static kmutex_t fasttrap_count_mtx; /* lock on ref count */
166 166
167 167 #define FASTTRAP_ENABLE_FAIL 1
168 168 #define FASTTRAP_ENABLE_PARTIAL 2
169 169
170 170 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
171 171 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
172 172
173 173 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
174 174 const dtrace_pattr_t *);
175 175 static void fasttrap_provider_retire(pid_t, const char *, int);
176 176 static void fasttrap_provider_free(fasttrap_provider_t *);
177 177
178 178 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
179 179 static void fasttrap_proc_release(fasttrap_proc_t *);
180 180
181 181 #define FASTTRAP_PROVS_INDEX(pid, name) \
182 182 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
183 183
184 184 #define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
185 185
186 186 static int
187 187 fasttrap_highbit(ulong_t i)
188 188 {
189 189 int h = 1;
190 190
191 191 if (i == 0)
192 192 return (0);
193 193 #ifdef _LP64
194 194 if (i & 0xffffffff00000000ul) {
195 195 h += 32; i >>= 32;
196 196 }
197 197 #endif
198 198 if (i & 0xffff0000) {
199 199 h += 16; i >>= 16;
200 200 }
201 201 if (i & 0xff00) {
202 202 h += 8; i >>= 8;
203 203 }
204 204 if (i & 0xf0) {
205 205 h += 4; i >>= 4;
206 206 }
207 207 if (i & 0xc) {
208 208 h += 2; i >>= 2;
209 209 }
210 210 if (i & 0x2) {
211 211 h += 1;
212 212 }
213 213 return (h);
214 214 }
215 215
216 216 static uint_t
217 217 fasttrap_hash_str(const char *p)
218 218 {
219 219 unsigned int g;
220 220 uint_t hval = 0;
221 221
222 222 while (*p) {
223 223 hval = (hval << 4) + *p++;
224 224 if ((g = (hval & 0xf0000000)) != 0)
225 225 hval ^= g >> 24;
226 226 hval &= ~g;
227 227 }
228 228 return (hval);
229 229 }
230 230
231 231 void
232 232 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
233 233 {
234 234 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
235 235
236 236 sqp->sq_info.si_signo = SIGTRAP;
237 237 sqp->sq_info.si_code = TRAP_DTRACE;
238 238 sqp->sq_info.si_addr = (caddr_t)pc;
239 239
240 240 mutex_enter(&p->p_lock);
241 241 sigaddqa(p, t, sqp);
242 242 mutex_exit(&p->p_lock);
243 243
244 244 if (t != NULL)
245 245 aston(t);
246 246 }
247 247
248 248 /*
249 249 * This function ensures that no threads are actively using the memory
250 250 * associated with probes that were formerly live.
251 251 */
252 252 static void
253 253 fasttrap_mod_barrier(uint64_t gen)
254 254 {
255 255 int i;
256 256
257 257 if (gen < fasttrap_mod_gen)
258 258 return;
259 259
260 260 fasttrap_mod_gen++;
261 261
262 262 for (i = 0; i < NCPU; i++) {
263 263 mutex_enter(&cpu_core[i].cpuc_pid_lock);
264 264 mutex_exit(&cpu_core[i].cpuc_pid_lock);
265 265 }
266 266 }
267 267
268 268 /*
269 269 * This is the timeout's callback for cleaning up the providers and their
270 270 * probes.
271 271 */
272 272 /*ARGSUSED*/
273 273 static void
274 274 fasttrap_pid_cleanup_cb(void *data)
275 275 {
276 276 fasttrap_provider_t **fpp, *fp;
277 277 fasttrap_bucket_t *bucket;
278 278 dtrace_provider_id_t provid;
279 279 int i, later, rval;
280 280
281 281 static volatile int in = 0;
282 282 ASSERT(in == 0);
283 283 in = 1;
284 284
285 285 mutex_enter(&fasttrap_cleanup_mtx);
286 286 while (fasttrap_cleanup_work) {
287 287 fasttrap_cleanup_work = 0;
288 288 mutex_exit(&fasttrap_cleanup_mtx);
289 289
290 290 later = 0;
291 291
292 292 /*
293 293 * Iterate over all the providers trying to remove the marked
294 294 * ones. If a provider is marked but not retired, we just
295 295 * have to take a crack at removing it -- it's no big deal if
296 296 * we can't.
297 297 */
298 298 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
299 299 bucket = &fasttrap_provs.fth_table[i];
300 300 mutex_enter(&bucket->ftb_mtx);
301 301 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
302 302
303 303 while ((fp = *fpp) != NULL) {
304 304 if (!fp->ftp_marked) {
305 305 fpp = &fp->ftp_next;
306 306 continue;
307 307 }
308 308
309 309 mutex_enter(&fp->ftp_mtx);
310 310
311 311 /*
312 312 * If this provider has consumers actively
313 313 * creating probes (ftp_ccount) or is a USDT
314 314 * provider (ftp_mcount), we can't unregister
315 315 * or even condense.
316 316 */
317 317 if (fp->ftp_ccount != 0 ||
318 318 fp->ftp_mcount != 0) {
319 319 mutex_exit(&fp->ftp_mtx);
320 320 fp->ftp_marked = 0;
321 321 continue;
322 322 }
323 323
324 324 if (!fp->ftp_retired || fp->ftp_rcount != 0)
325 325 fp->ftp_marked = 0;
326 326
327 327 mutex_exit(&fp->ftp_mtx);
328 328
329 329 /*
330 330 * If we successfully unregister this
331 331 * provider we can remove it from the hash
332 332 * chain and free the memory. If our attempt
333 333 * to unregister fails and this is a retired
334 334 * provider, increment our flag to try again
335 335 * pretty soon. If we've consumed more than
336 336 * half of our total permitted number of
337 337 * probes call dtrace_condense() to try to
338 338 * clean out the unenabled probes.
339 339 */
340 340 provid = fp->ftp_provid;
341 341 if ((rval = dtrace_unregister(provid)) != 0) {
342 342 if (fasttrap_total > fasttrap_max / 2)
343 343 (void) dtrace_condense(provid);
344 344
345 345 if (rval == EAGAIN)
346 346 fp->ftp_marked = 1;
347 347
348 348 later += fp->ftp_marked;
349 349 fpp = &fp->ftp_next;
350 350 } else {
351 351 *fpp = fp->ftp_next;
352 352 fasttrap_provider_free(fp);
353 353 }
354 354 }
355 355 mutex_exit(&bucket->ftb_mtx);
356 356 }
357 357
358 358 mutex_enter(&fasttrap_cleanup_mtx);
359 359 }
360 360
361 361 ASSERT(fasttrap_timeout != 0);
362 362
363 363 /*
364 364 * If we were unable to remove a retired provider, try again after
365 365 * a second. This situation can occur in certain circumstances where
366 366 * providers cannot be unregistered even though they have no probes
367 367 * enabled because of an execution of dtrace -l or something similar.
368 368 * If the timeout has been disabled (set to 1 because we're trying
369 369 * to detach), we set fasttrap_cleanup_work to ensure that we'll
370 370 * get a chance to do that work if and when the timeout is reenabled
371 371 * (if detach fails).
372 372 */
373 373 if (later > 0) {
374 374 if (fasttrap_timeout != (timeout_id_t)1) {
375 375 fasttrap_timeout =
376 376 timeout(&fasttrap_pid_cleanup_cb, NULL, hz);
377 377 }
378 378
379 379 fasttrap_cleanup_work = 1;
380 380 } else {
381 381 fasttrap_timeout = 0;
382 382 }
383 383
384 384 mutex_exit(&fasttrap_cleanup_mtx);
385 385 in = 0;
386 386 }
387 387
388 388 /*
389 389 * Activates the asynchronous cleanup mechanism.
390 390 */
391 391 static void
392 392 fasttrap_pid_cleanup(void)
393 393 {
394 394 mutex_enter(&fasttrap_cleanup_mtx);
395 395 fasttrap_cleanup_work = 1;
396 396 if (fasttrap_timeout == 0)
397 397 fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1);
398 398 mutex_exit(&fasttrap_cleanup_mtx);
399 399 }
400 400
401 401 /*
402 402 * This is called from cfork() via dtrace_fasttrap_fork(). The child
403 403 * process's address space is (roughly) a copy of the parent process's so
404 404 * we have to remove all the instrumentation we had previously enabled in the
405 405 * parent.
406 406 */
407 407 static void
408 408 fasttrap_fork(proc_t *p, proc_t *cp)
409 409 {
410 410 pid_t ppid = p->p_pid;
411 411 int i;
412 412
413 413 ASSERT(curproc == p);
414 414 ASSERT(p->p_proc_flag & P_PR_LOCK);
415 415 ASSERT(p->p_dtrace_count > 0);
416 416 ASSERT(cp->p_dtrace_count == 0);
417 417
418 418 /*
419 419 * This would be simpler and faster if we maintained per-process
420 420 * hash tables of enabled tracepoints. It could, however, potentially
421 421 * slow down execution of a tracepoint since we'd need to go
422 422 * through two levels of indirection. In the future, we should
423 423 * consider either maintaining per-process ancillary lists of
424 424 * enabled tracepoints or hanging a pointer to a per-process hash
425 425 * table of enabled tracepoints off the proc structure.
426 426 */
427 427
428 428 /*
429 429 * We don't have to worry about the child process disappearing
430 430 * because we're in fork().
431 431 */
432 432 mutex_enter(&cp->p_lock);
433 433 sprlock_proc(cp);
434 434 mutex_exit(&cp->p_lock);
435 435
436 436 /*
437 437 * Iterate over every tracepoint looking for ones that belong to the
438 438 * parent process, and remove each from the child process.
439 439 */
440 440 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
441 441 fasttrap_tracepoint_t *tp;
442 442 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
443 443
444 444 mutex_enter(&bucket->ftb_mtx);
445 445 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
446 446 if (tp->ftt_pid == ppid &&
447 447 tp->ftt_proc->ftpc_acount != 0) {
448 448 int ret = fasttrap_tracepoint_remove(cp, tp);
449 449 ASSERT(ret == 0);
450 450
451 451 /*
452 452 * The count of active providers can only be
453 453 * decremented (i.e. to zero) during exec,
454 454 * exit, and removal of a meta provider so it
455 455 * should be impossible to drop the count
456 456 * mid-fork.
457 457 */
458 458 ASSERT(tp->ftt_proc->ftpc_acount != 0);
459 459 }
460 460 }
461 461 mutex_exit(&bucket->ftb_mtx);
462 462 }
463 463
464 464 mutex_enter(&cp->p_lock);
465 465 sprunlock(cp);
466 466 }
467 467
468 468 /*
469 469 * This is called from proc_exit() or from exec_common() if p_dtrace_probes
470 470 * is set on the proc structure to indicate that there is a pid provider
471 471 * associated with this process.
472 472 */
473 473 static void
474 474 fasttrap_exec_exit(proc_t *p)
475 475 {
476 476 ASSERT(p == curproc);
477 477 ASSERT(MUTEX_HELD(&p->p_lock));
478 478
479 479 mutex_exit(&p->p_lock);
480 480
481 481 /*
482 482 * We clean up the pid provider for this process here; user-land
483 483 * static probes are handled by the meta-provider remove entry point.
484 484 */
485 485 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
486 486
487 487 mutex_enter(&p->p_lock);
488 488 }
489 489
490 490
491 491 /*ARGSUSED*/
492 492 static void
493 493 fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc)
494 494 {
495 495 /*
496 496 * There are no "default" pid probes.
497 497 */
498 498 }
499 499
500 500 static int
501 501 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
502 502 {
503 503 fasttrap_tracepoint_t *tp, *new_tp = NULL;
504 504 fasttrap_bucket_t *bucket;
505 505 fasttrap_id_t *id;
506 506 pid_t pid;
507 507 uintptr_t pc;
508 508
509 509 ASSERT(index < probe->ftp_ntps);
510 510
511 511 pid = probe->ftp_pid;
512 512 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
513 513 id = &probe->ftp_tps[index].fit_id;
514 514
515 515 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
516 516
517 517 ASSERT(!(p->p_flag & SVFORK));
518 518
519 519 /*
520 520 * Before we make any modifications, make sure we've imposed a barrier
521 521 * on the generation in which this probe was last modified.
522 522 */
523 523 fasttrap_mod_barrier(probe->ftp_gen);
524 524
525 525 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
526 526
527 527 /*
528 528 * If the tracepoint has already been enabled, just add our id to the
529 529 * list of interested probes. This may be our second time through
530 530 * this path in which case we'll have constructed the tracepoint we'd
531 531 * like to install. If we can't find a match, and have an allocated
532 532 * tracepoint ready to go, enable that one now.
533 533 *
534 534 * A tracepoint whose process is defunct is also considered defunct.
535 535 */
536 536 again:
537 537 mutex_enter(&bucket->ftb_mtx);
538 538 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
539 539 /*
540 540 * Note that it's safe to access the active count on the
541 541 * associated proc structure because we know that at least one
542 542 * provider (this one) will still be around throughout this
543 543 * operation.
544 544 */
545 545 if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
546 546 tp->ftt_proc->ftpc_acount == 0)
547 547 continue;
548 548
549 549 /*
550 550 * Now that we've found a matching tracepoint, it would be
551 551 * a decent idea to confirm that the tracepoint is still
552 552 * enabled and the trap instruction hasn't been overwritten.
553 553 * Since this is a little hairy, we'll punt for now.
554 554 */
555 555
556 556 /*
557 557 * This can't be the first interested probe. We don't have
558 558 * to worry about another thread being in the midst of
559 559 * deleting this tracepoint (which would be the only valid
560 560 * reason for a tracepoint to have no interested probes)
561 561 * since we're holding P_PR_LOCK for this process.
562 562 */
563 563 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
564 564
565 565 switch (id->fti_ptype) {
566 566 case DTFTP_ENTRY:
567 567 case DTFTP_OFFSETS:
568 568 case DTFTP_IS_ENABLED:
569 569 id->fti_next = tp->ftt_ids;
570 570 membar_producer();
571 571 tp->ftt_ids = id;
572 572 membar_producer();
573 573 break;
574 574
575 575 case DTFTP_RETURN:
576 576 case DTFTP_POST_OFFSETS:
577 577 id->fti_next = tp->ftt_retids;
578 578 membar_producer();
579 579 tp->ftt_retids = id;
580 580 membar_producer();
581 581 break;
582 582
583 583 default:
584 584 ASSERT(0);
585 585 }
586 586
587 587 mutex_exit(&bucket->ftb_mtx);
588 588
589 589 if (new_tp != NULL) {
590 590 new_tp->ftt_ids = NULL;
591 591 new_tp->ftt_retids = NULL;
592 592 }
593 593
594 594 return (0);
595 595 }
596 596
597 597 /*
598 598 * If we have a good tracepoint ready to go, install it now while
599 599 * we have the lock held and no one can screw with us.
600 600 */
601 601 if (new_tp != NULL) {
602 602 int rc = 0;
603 603
604 604 new_tp->ftt_next = bucket->ftb_data;
605 605 membar_producer();
606 606 bucket->ftb_data = new_tp;
607 607 membar_producer();
608 608 mutex_exit(&bucket->ftb_mtx);
609 609
610 610 /*
611 611 * Activate the tracepoint in the ISA-specific manner.
612 612 * If this fails, we need to report the failure, but
613 613 * indicate that this tracepoint must still be disabled
614 614 * by calling fasttrap_tracepoint_disable().
615 615 */
616 616 if (fasttrap_tracepoint_install(p, new_tp) != 0)
617 617 rc = FASTTRAP_ENABLE_PARTIAL;
618 618
619 619 /*
620 620 * Increment the count of the number of tracepoints active in
621 621 * the victim process.
622 622 */
623 623 ASSERT(p->p_proc_flag & P_PR_LOCK);
624 624 p->p_dtrace_count++;
625 625
626 626 return (rc);
627 627 }
628 628
629 629 mutex_exit(&bucket->ftb_mtx);
630 630
631 631 /*
632 632 * Initialize the tracepoint that's been preallocated with the probe.
633 633 */
634 634 new_tp = probe->ftp_tps[index].fit_tp;
635 635
636 636 ASSERT(new_tp->ftt_pid == pid);
637 637 ASSERT(new_tp->ftt_pc == pc);
638 638 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
639 639 ASSERT(new_tp->ftt_ids == NULL);
640 640 ASSERT(new_tp->ftt_retids == NULL);
641 641
642 642 switch (id->fti_ptype) {
643 643 case DTFTP_ENTRY:
644 644 case DTFTP_OFFSETS:
645 645 case DTFTP_IS_ENABLED:
646 646 id->fti_next = NULL;
647 647 new_tp->ftt_ids = id;
648 648 break;
649 649
650 650 case DTFTP_RETURN:
651 651 case DTFTP_POST_OFFSETS:
652 652 id->fti_next = NULL;
653 653 new_tp->ftt_retids = id;
654 654 break;
655 655
656 656 default:
657 657 ASSERT(0);
658 658 }
659 659
660 660 /*
661 661 * If the ISA-dependent initialization goes to plan, go back to the
662 662 * beginning and try to install this freshly made tracepoint.
663 663 */
664 664 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
665 665 goto again;
666 666
667 667 new_tp->ftt_ids = NULL;
668 668 new_tp->ftt_retids = NULL;
669 669
670 670 return (FASTTRAP_ENABLE_FAIL);
671 671 }
672 672
673 673 static void
674 674 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
675 675 {
676 676 fasttrap_bucket_t *bucket;
677 677 fasttrap_provider_t *provider = probe->ftp_prov;
678 678 fasttrap_tracepoint_t **pp, *tp;
679 679 fasttrap_id_t *id, **idp;
680 680 pid_t pid;
681 681 uintptr_t pc;
682 682
683 683 ASSERT(index < probe->ftp_ntps);
684 684
685 685 pid = probe->ftp_pid;
686 686 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
687 687 id = &probe->ftp_tps[index].fit_id;
688 688
689 689 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
690 690
691 691 /*
692 692 * Find the tracepoint and make sure that our id is one of the
693 693 * ones registered with it.
694 694 */
695 695 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
696 696 mutex_enter(&bucket->ftb_mtx);
697 697 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
698 698 if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
699 699 tp->ftt_proc == provider->ftp_proc)
700 700 break;
701 701 }
702 702
703 703 /*
704 704 * If we somehow lost this tracepoint, we're in a world of hurt.
705 705 */
706 706 ASSERT(tp != NULL);
707 707
708 708 switch (id->fti_ptype) {
709 709 case DTFTP_ENTRY:
710 710 case DTFTP_OFFSETS:
711 711 case DTFTP_IS_ENABLED:
712 712 ASSERT(tp->ftt_ids != NULL);
713 713 idp = &tp->ftt_ids;
714 714 break;
715 715
716 716 case DTFTP_RETURN:
717 717 case DTFTP_POST_OFFSETS:
718 718 ASSERT(tp->ftt_retids != NULL);
719 719 idp = &tp->ftt_retids;
720 720 break;
721 721
722 722 default:
723 723 ASSERT(0);
724 724 }
725 725
726 726 while ((*idp)->fti_probe != probe) {
727 727 idp = &(*idp)->fti_next;
728 728 ASSERT(*idp != NULL);
729 729 }
730 730
731 731 id = *idp;
732 732 *idp = id->fti_next;
733 733 membar_producer();
734 734
735 735 ASSERT(id->fti_probe == probe);
736 736
737 737 /*
738 738 * If there are other registered enablings of this tracepoint, we're
739 739 * all done, but if this was the last probe assocated with this
740 740 * this tracepoint, we need to remove and free it.
741 741 */
742 742 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
743 743
744 744 /*
745 745 * If the current probe's tracepoint is in use, swap it
746 746 * for an unused tracepoint.
747 747 */
748 748 if (tp == probe->ftp_tps[index].fit_tp) {
749 749 fasttrap_probe_t *tmp_probe;
750 750 fasttrap_tracepoint_t **tmp_tp;
751 751 uint_t tmp_index;
752 752
753 753 if (tp->ftt_ids != NULL) {
754 754 tmp_probe = tp->ftt_ids->fti_probe;
755 755 /* LINTED - alignment */
756 756 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
757 757 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
758 758 } else {
759 759 tmp_probe = tp->ftt_retids->fti_probe;
760 760 /* LINTED - alignment */
761 761 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
762 762 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
763 763 }
764 764
765 765 ASSERT(*tmp_tp != NULL);
766 766 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
767 767 ASSERT((*tmp_tp)->ftt_ids == NULL);
768 768 ASSERT((*tmp_tp)->ftt_retids == NULL);
769 769
770 770 probe->ftp_tps[index].fit_tp = *tmp_tp;
771 771 *tmp_tp = tp;
772 772 }
773 773
774 774 mutex_exit(&bucket->ftb_mtx);
775 775
776 776 /*
777 777 * Tag the modified probe with the generation in which it was
778 778 * changed.
779 779 */
780 780 probe->ftp_gen = fasttrap_mod_gen;
781 781 return;
782 782 }
783 783
784 784 mutex_exit(&bucket->ftb_mtx);
785 785
786 786 /*
787 787 * We can't safely remove the tracepoint from the set of active
788 788 * tracepoints until we've actually removed the fasttrap instruction
789 789 * from the process's text. We can, however, operate on this
790 790 * tracepoint secure in the knowledge that no other thread is going to
791 791 * be looking at it since we hold P_PR_LOCK on the process if it's
792 792 * live or we hold the provider lock on the process if it's dead and
793 793 * gone.
794 794 */
795 795
796 796 /*
797 797 * We only need to remove the actual instruction if we're looking
798 798 * at an existing process
799 799 */
800 800 if (p != NULL) {
801 801 /*
802 802 * If we fail to restore the instruction we need to kill
803 803 * this process since it's in a completely unrecoverable
804 804 * state.
805 805 */
806 806 if (fasttrap_tracepoint_remove(p, tp) != 0)
807 807 fasttrap_sigtrap(p, NULL, pc);
808 808
809 809 /*
810 810 * Decrement the count of the number of tracepoints active
811 811 * in the victim process.
812 812 */
813 813 ASSERT(p->p_proc_flag & P_PR_LOCK);
814 814 p->p_dtrace_count--;
815 815 }
816 816
817 817 /*
818 818 * Remove the probe from the hash table of active tracepoints.
819 819 */
820 820 mutex_enter(&bucket->ftb_mtx);
821 821 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
822 822 ASSERT(*pp != NULL);
823 823 while (*pp != tp) {
824 824 pp = &(*pp)->ftt_next;
825 825 ASSERT(*pp != NULL);
826 826 }
827 827
828 828 *pp = tp->ftt_next;
829 829 membar_producer();
830 830
831 831 mutex_exit(&bucket->ftb_mtx);
832 832
833 833 /*
834 834 * Tag the modified probe with the generation in which it was changed.
835 835 */
836 836 probe->ftp_gen = fasttrap_mod_gen;
837 837 }
838 838
839 839 static void
840 840 fasttrap_enable_callbacks(void)
841 841 {
842 842 /*
843 843 * We don't have to play the rw lock game here because we're
844 844 * providing something rather than taking something away --
845 845 * we can be sure that no threads have tried to follow this
846 846 * function pointer yet.
847 847 */
848 848 mutex_enter(&fasttrap_count_mtx);
849 849 if (fasttrap_pid_count == 0) {
850 850 ASSERT(dtrace_pid_probe_ptr == NULL);
851 851 ASSERT(dtrace_return_probe_ptr == NULL);
852 852 dtrace_pid_probe_ptr = &fasttrap_pid_probe;
853 853 dtrace_return_probe_ptr = &fasttrap_return_probe;
854 854 }
855 855 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
856 856 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
857 857 fasttrap_pid_count++;
858 858 mutex_exit(&fasttrap_count_mtx);
859 859 }
860 860
861 861 static void
862 862 fasttrap_disable_callbacks(void)
863 863 {
864 864 ASSERT(MUTEX_HELD(&cpu_lock));
865 865
866 866 mutex_enter(&fasttrap_count_mtx);
867 867 ASSERT(fasttrap_pid_count > 0);
868 868 fasttrap_pid_count--;
869 869 if (fasttrap_pid_count == 0) {
870 870 cpu_t *cur, *cpu = CPU;
871 871
872 872 for (cur = cpu->cpu_next_onln; cur != cpu;
873 873 cur = cur->cpu_next_onln) {
874 874 rw_enter(&cur->cpu_ft_lock, RW_WRITER);
875 875 }
876 876
877 877 dtrace_pid_probe_ptr = NULL;
878 878 dtrace_return_probe_ptr = NULL;
879 879
880 880 for (cur = cpu->cpu_next_onln; cur != cpu;
881 881 cur = cur->cpu_next_onln) {
882 882 rw_exit(&cur->cpu_ft_lock);
883 883 }
884 884 }
885 885 mutex_exit(&fasttrap_count_mtx);
886 886 }
887 887
888 888 /*ARGSUSED*/
889 889 static int
890 890 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
891 891 {
892 892 fasttrap_probe_t *probe = parg;
893 893 proc_t *p;
894 894 int i, rc;
895 895
896 896 ASSERT(probe != NULL);
897 897 ASSERT(!probe->ftp_enabled);
898 898 ASSERT(id == probe->ftp_id);
899 899 ASSERT(MUTEX_HELD(&cpu_lock));
900 900
901 901 /*
902 902 * Increment the count of enabled probes on this probe's provider;
903 903 * the provider can't go away while the probe still exists. We
904 904 * must increment this even if we aren't able to properly enable
905 905 * this probe.
906 906 */
907 907 mutex_enter(&probe->ftp_prov->ftp_mtx);
908 908 probe->ftp_prov->ftp_rcount++;
909 909 mutex_exit(&probe->ftp_prov->ftp_mtx);
910 910
911 911 /*
912 912 * If this probe's provider is retired (meaning it was valid in a
913 913 * previously exec'ed incarnation of this address space), bail out. The
914 914 * provider can't go away while we're in this code path.
915 915 */
916 916 if (probe->ftp_prov->ftp_retired)
917 917 return (0);
918 918
919 919 /*
920 920 * If we can't find the process, it may be that we're in the context of
921 921 * a fork in which the traced process is being born and we're copying
922 922 * USDT probes. Otherwise, the process is gone so bail.
923 923 */
924 924 if ((p = sprlock(probe->ftp_pid)) == NULL) {
925 925 if ((curproc->p_flag & SFORKING) == 0)
926 926 return (0);
927 927
928 928 mutex_enter(&pidlock);
929 929 p = prfind(probe->ftp_pid);
930 930
931 931 /*
932 932 * Confirm that curproc is indeed forking the process in which
933 933 * we're trying to enable probes.
934 934 */
935 935 ASSERT(p != NULL);
936 936 ASSERT(p->p_parent == curproc);
937 937 ASSERT(p->p_stat == SIDL);
938 938
939 939 mutex_enter(&p->p_lock);
940 940 mutex_exit(&pidlock);
941 941
942 942 sprlock_proc(p);
943 943 }
944 944
945 945 ASSERT(!(p->p_flag & SVFORK));
946 946 mutex_exit(&p->p_lock);
947 947
948 948 /*
949 949 * We have to enable the trap entry point before any user threads have
950 950 * the chance to execute the trap instruction we're about to place
951 951 * in their process's text.
952 952 */
953 953 fasttrap_enable_callbacks();
954 954
955 955 /*
956 956 * Enable all the tracepoints and add this probe's id to each
957 957 * tracepoint's list of active probes.
958 958 */
959 959 for (i = 0; i < probe->ftp_ntps; i++) {
960 960 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
961 961 /*
962 962 * If enabling the tracepoint failed completely,
963 963 * we don't have to disable it; if the failure
964 964 * was only partial we must disable it.
965 965 */
966 966 if (rc == FASTTRAP_ENABLE_FAIL)
967 967 i--;
968 968 else
969 969 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
970 970
971 971 /*
972 972 * Back up and pull out all the tracepoints we've
973 973 * created so far for this probe.
974 974 */
975 975 while (i >= 0) {
976 976 fasttrap_tracepoint_disable(p, probe, i);
977 977 i--;
978 978 }
979 979
980 980 mutex_enter(&p->p_lock);
981 981 sprunlock(p);
982 982
983 983 /*
984 984 * Since we're not actually enabling this probe,
985 985 * drop our reference on the trap table entry.
986 986 */
987 987 fasttrap_disable_callbacks();
988 988 return (0);
989 989 }
990 990 }
991 991
992 992 mutex_enter(&p->p_lock);
993 993 sprunlock(p);
994 994
995 995 probe->ftp_enabled = 1;
996 996 return (0);
997 997 }
998 998
999 999 /*ARGSUSED*/
1000 1000 static void
1001 1001 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1002 1002 {
1003 1003 fasttrap_probe_t *probe = parg;
1004 1004 fasttrap_provider_t *provider = probe->ftp_prov;
1005 1005 proc_t *p;
1006 1006 int i, whack = 0;
1007 1007
1008 1008 ASSERT(id == probe->ftp_id);
1009 1009
1010 1010 /*
1011 1011 * We won't be able to acquire a /proc-esque lock on the process
1012 1012 * iff the process is dead and gone. In this case, we rely on the
1013 1013 * provider lock as a point of mutual exclusion to prevent other
1014 1014 * DTrace consumers from disabling this probe.
1015 1015 */
1016 1016 if ((p = sprlock(probe->ftp_pid)) != NULL) {
1017 1017 ASSERT(!(p->p_flag & SVFORK));
1018 1018 mutex_exit(&p->p_lock);
1019 1019 }
1020 1020
1021 1021 mutex_enter(&provider->ftp_mtx);
1022 1022
1023 1023 /*
1024 1024 * Disable all the associated tracepoints (for fully enabled probes).
1025 1025 */
1026 1026 if (probe->ftp_enabled) {
1027 1027 for (i = 0; i < probe->ftp_ntps; i++) {
1028 1028 fasttrap_tracepoint_disable(p, probe, i);
1029 1029 }
1030 1030 }
1031 1031
1032 1032 ASSERT(provider->ftp_rcount > 0);
1033 1033 provider->ftp_rcount--;
1034 1034
1035 1035 if (p != NULL) {
1036 1036 /*
1037 1037 * Even though we may not be able to remove it entirely, we
1038 1038 * mark this retired provider to get a chance to remove some
1039 1039 * of the associated probes.
1040 1040 */
1041 1041 if (provider->ftp_retired && !provider->ftp_marked)
1042 1042 whack = provider->ftp_marked = 1;
1043 1043 mutex_exit(&provider->ftp_mtx);
1044 1044
1045 1045 mutex_enter(&p->p_lock);
1046 1046 sprunlock(p);
1047 1047 } else {
1048 1048 /*
1049 1049 * If the process is dead, we're just waiting for the
1050 1050 * last probe to be disabled to be able to free it.
1051 1051 */
1052 1052 if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1053 1053 whack = provider->ftp_marked = 1;
1054 1054 mutex_exit(&provider->ftp_mtx);
1055 1055 }
1056 1056
1057 1057 if (whack)
1058 1058 fasttrap_pid_cleanup();
1059 1059
1060 1060 if (!probe->ftp_enabled)
1061 1061 return;
1062 1062
1063 1063 probe->ftp_enabled = 0;
1064 1064
1065 1065 ASSERT(MUTEX_HELD(&cpu_lock));
1066 1066 fasttrap_disable_callbacks();
1067 1067 }
1068 1068
1069 1069 /*ARGSUSED*/
1070 1070 static void
1071 1071 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1072 1072 dtrace_argdesc_t *desc)
1073 1073 {
1074 1074 fasttrap_probe_t *probe = parg;
1075 1075 char *str;
1076 1076 int i, ndx;
1077 1077
1078 1078 desc->dtargd_native[0] = '\0';
1079 1079 desc->dtargd_xlate[0] = '\0';
1080 1080
1081 1081 if (probe->ftp_prov->ftp_retired != 0 ||
1082 1082 desc->dtargd_ndx >= probe->ftp_nargs) {
1083 1083 desc->dtargd_ndx = DTRACE_ARGNONE;
1084 1084 return;
1085 1085 }
1086 1086
1087 1087 ndx = (probe->ftp_argmap != NULL) ?
1088 1088 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1089 1089
1090 1090 str = probe->ftp_ntypes;
1091 1091 for (i = 0; i < ndx; i++) {
1092 1092 str += strlen(str) + 1;
1093 1093 }
1094 1094
1095 1095 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1096 1096 (void) strcpy(desc->dtargd_native, str);
1097 1097
1098 1098 if (probe->ftp_xtypes == NULL)
1099 1099 return;
1100 1100
1101 1101 str = probe->ftp_xtypes;
1102 1102 for (i = 0; i < desc->dtargd_ndx; i++) {
1103 1103 str += strlen(str) + 1;
1104 1104 }
1105 1105
1106 1106 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1107 1107 (void) strcpy(desc->dtargd_xlate, str);
1108 1108 }
1109 1109
1110 1110 /*ARGSUSED*/
1111 1111 static void
1112 1112 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1113 1113 {
1114 1114 fasttrap_probe_t *probe = parg;
1115 1115 int i;
1116 1116 size_t size;
1117 1117
1118 1118 ASSERT(probe != NULL);
1119 1119 ASSERT(!probe->ftp_enabled);
1120 1120 ASSERT(fasttrap_total >= probe->ftp_ntps);
1121 1121
1122 1122 atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1123 1123 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1124 1124
1125 1125 if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1126 1126 fasttrap_mod_barrier(probe->ftp_gen);
1127 1127
1128 1128 for (i = 0; i < probe->ftp_ntps; i++) {
1129 1129 kmem_free(probe->ftp_tps[i].fit_tp,
1130 1130 sizeof (fasttrap_tracepoint_t));
1131 1131 }
1132 1132
1133 1133 kmem_free(probe, size);
1134 1134 }
1135 1135
1136 1136
1137 1137 static const dtrace_pattr_t pid_attr = {
1138 1138 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1139 1139 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1140 1140 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1141 1141 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1142 1142 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1143 1143 };
1144 1144
1145 1145 static dtrace_pops_t pid_pops = {
1146 1146 fasttrap_pid_provide,
1147 1147 NULL,
1148 1148 fasttrap_pid_enable,
1149 1149 fasttrap_pid_disable,
1150 1150 NULL,
1151 1151 NULL,
1152 1152 fasttrap_pid_getargdesc,
1153 1153 fasttrap_pid_getarg,
1154 1154 NULL,
1155 1155 fasttrap_pid_destroy
1156 1156 };
1157 1157
1158 1158 static dtrace_pops_t usdt_pops = {
1159 1159 fasttrap_pid_provide,
1160 1160 NULL,
1161 1161 fasttrap_pid_enable,
1162 1162 fasttrap_pid_disable,
1163 1163 NULL,
1164 1164 NULL,
1165 1165 fasttrap_pid_getargdesc,
1166 1166 fasttrap_usdt_getarg,
1167 1167 NULL,
1168 1168 fasttrap_pid_destroy
1169 1169 };
1170 1170
1171 1171 static fasttrap_proc_t *
1172 1172 fasttrap_proc_lookup(pid_t pid)
1173 1173 {
1174 1174 fasttrap_bucket_t *bucket;
1175 1175 fasttrap_proc_t *fprc, *new_fprc;
1176 1176
1177 1177 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1178 1178 mutex_enter(&bucket->ftb_mtx);
1179 1179
1180 1180 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1181 1181 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1182 1182 mutex_enter(&fprc->ftpc_mtx);
1183 1183 mutex_exit(&bucket->ftb_mtx);
1184 1184 fprc->ftpc_rcount++;
1185 1185 atomic_inc_64(&fprc->ftpc_acount);
1186 1186 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1187 1187 mutex_exit(&fprc->ftpc_mtx);
1188 1188
1189 1189 return (fprc);
1190 1190 }
1191 1191 }
1192 1192
1193 1193 /*
1194 1194 * Drop the bucket lock so we don't try to perform a sleeping
1195 1195 * allocation under it.
1196 1196 */
1197 1197 mutex_exit(&bucket->ftb_mtx);
1198 1198
1199 1199 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1200 1200 new_fprc->ftpc_pid = pid;
1201 1201 new_fprc->ftpc_rcount = 1;
1202 1202 new_fprc->ftpc_acount = 1;
1203 1203
1204 1204 mutex_enter(&bucket->ftb_mtx);
1205 1205
1206 1206 /*
1207 1207 * Take another lap through the list to make sure a proc hasn't
1208 1208 * been created for this pid while we weren't under the bucket lock.
1209 1209 */
1210 1210 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1211 1211 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1212 1212 mutex_enter(&fprc->ftpc_mtx);
1213 1213 mutex_exit(&bucket->ftb_mtx);
1214 1214 fprc->ftpc_rcount++;
1215 1215 atomic_inc_64(&fprc->ftpc_acount);
1216 1216 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1217 1217 mutex_exit(&fprc->ftpc_mtx);
1218 1218
1219 1219 kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1220 1220
1221 1221 return (fprc);
1222 1222 }
1223 1223 }
1224 1224
1225 1225 new_fprc->ftpc_next = bucket->ftb_data;
1226 1226 bucket->ftb_data = new_fprc;
1227 1227
1228 1228 mutex_exit(&bucket->ftb_mtx);
1229 1229
1230 1230 return (new_fprc);
1231 1231 }
1232 1232
1233 1233 static void
1234 1234 fasttrap_proc_release(fasttrap_proc_t *proc)
1235 1235 {
1236 1236 fasttrap_bucket_t *bucket;
1237 1237 fasttrap_proc_t *fprc, **fprcp;
1238 1238 pid_t pid = proc->ftpc_pid;
1239 1239
1240 1240 mutex_enter(&proc->ftpc_mtx);
1241 1241
1242 1242 ASSERT(proc->ftpc_rcount != 0);
1243 1243 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1244 1244
1245 1245 if (--proc->ftpc_rcount != 0) {
1246 1246 mutex_exit(&proc->ftpc_mtx);
1247 1247 return;
1248 1248 }
1249 1249
1250 1250 mutex_exit(&proc->ftpc_mtx);
1251 1251
1252 1252 /*
1253 1253 * There should definitely be no live providers associated with this
1254 1254 * process at this point.
1255 1255 */
1256 1256 ASSERT(proc->ftpc_acount == 0);
1257 1257
1258 1258 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1259 1259 mutex_enter(&bucket->ftb_mtx);
1260 1260
1261 1261 fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1262 1262 while ((fprc = *fprcp) != NULL) {
1263 1263 if (fprc == proc)
1264 1264 break;
1265 1265
1266 1266 fprcp = &fprc->ftpc_next;
1267 1267 }
1268 1268
1269 1269 /*
1270 1270 * Something strange has happened if we can't find the proc.
1271 1271 */
1272 1272 ASSERT(fprc != NULL);
1273 1273
1274 1274 *fprcp = fprc->ftpc_next;
1275 1275
1276 1276 mutex_exit(&bucket->ftb_mtx);
1277 1277
1278 1278 kmem_free(fprc, sizeof (fasttrap_proc_t));
1279 1279 }
1280 1280
1281 1281 /*
1282 1282 * Lookup a fasttrap-managed provider based on its name and associated pid.
1283 1283 * If the pattr argument is non-NULL, this function instantiates the provider
1284 1284 * if it doesn't exist otherwise it returns NULL. The provider is returned
1285 1285 * with its lock held.
1286 1286 */
1287 1287 static fasttrap_provider_t *
1288 1288 fasttrap_provider_lookup(pid_t pid, const char *name,
1289 1289 const dtrace_pattr_t *pattr)
1290 1290 {
1291 1291 fasttrap_provider_t *fp, *new_fp = NULL;
1292 1292 fasttrap_bucket_t *bucket;
1293 1293 char provname[DTRACE_PROVNAMELEN];
1294 1294 proc_t *p;
1295 1295 cred_t *cred;
1296 1296
1297 1297 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1298 1298 ASSERT(pattr != NULL);
1299 1299
1300 1300 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1301 1301 mutex_enter(&bucket->ftb_mtx);
1302 1302
1303 1303 /*
1304 1304 * Take a lap through the list and return the match if we find it.
1305 1305 */
1306 1306 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1307 1307 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1308 1308 !fp->ftp_retired) {
1309 1309 mutex_enter(&fp->ftp_mtx);
1310 1310 mutex_exit(&bucket->ftb_mtx);
1311 1311 return (fp);
1312 1312 }
1313 1313 }
1314 1314
1315 1315 /*
1316 1316 * Drop the bucket lock so we don't try to perform a sleeping
1317 1317 * allocation under it.
1318 1318 */
1319 1319 mutex_exit(&bucket->ftb_mtx);
1320 1320
1321 1321 /*
1322 1322 * Make sure the process exists, isn't a child created as the result
1323 1323 * of a vfork(2), and isn't a zombie (but may be in fork).
1324 1324 */
1325 1325 mutex_enter(&pidlock);
1326 1326 if ((p = prfind(pid)) == NULL) {
1327 1327 mutex_exit(&pidlock);
1328 1328 return (NULL);
1329 1329 }
1330 1330 mutex_enter(&p->p_lock);
1331 1331 mutex_exit(&pidlock);
1332 1332 if (p->p_flag & (SVFORK | SEXITING)) {
1333 1333 mutex_exit(&p->p_lock);
1334 1334 return (NULL);
1335 1335 }
1336 1336
1337 1337 /*
1338 1338 * Increment p_dtrace_probes so that the process knows to inform us
1339 1339 * when it exits or execs. fasttrap_provider_free() decrements this
1340 1340 * when we're done with this provider.
1341 1341 */
1342 1342 p->p_dtrace_probes++;
1343 1343
1344 1344 /*
1345 1345 * Grab the credentials for this process so we have
1346 1346 * something to pass to dtrace_register().
1347 1347 */
1348 1348 mutex_enter(&p->p_crlock);
1349 1349 crhold(p->p_cred);
1350 1350 cred = p->p_cred;
1351 1351 mutex_exit(&p->p_crlock);
1352 1352 mutex_exit(&p->p_lock);
1353 1353
1354 1354 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1355 1355 new_fp->ftp_pid = pid;
1356 1356 new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1357 1357
1358 1358 ASSERT(new_fp->ftp_proc != NULL);
1359 1359
1360 1360 mutex_enter(&bucket->ftb_mtx);
1361 1361
1362 1362 /*
1363 1363 * Take another lap through the list to make sure a provider hasn't
1364 1364 * been created for this pid while we weren't under the bucket lock.
1365 1365 */
1366 1366 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1367 1367 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1368 1368 !fp->ftp_retired) {
1369 1369 mutex_enter(&fp->ftp_mtx);
1370 1370 mutex_exit(&bucket->ftb_mtx);
1371 1371 fasttrap_provider_free(new_fp);
1372 1372 crfree(cred);
1373 1373 return (fp);
1374 1374 }
1375 1375 }
1376 1376
1377 1377 (void) strcpy(new_fp->ftp_name, name);
1378 1378
1379 1379 /*
1380 1380 * Fail and return NULL if either the provider name is too long
1381 1381 * or we fail to register this new provider with the DTrace
1382 1382 * framework. Note that this is the only place we ever construct
1383 1383 * the full provider name -- we keep it in pieces in the provider
1384 1384 * structure.
1385 1385 */
1386 1386 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1387 1387 sizeof (provname) ||
1388 1388 dtrace_register(provname, pattr,
1389 1389 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1390 1390 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1391 1391 &new_fp->ftp_provid) != 0) {
1392 1392 mutex_exit(&bucket->ftb_mtx);
1393 1393 fasttrap_provider_free(new_fp);
1394 1394 crfree(cred);
1395 1395 return (NULL);
1396 1396 }
1397 1397
1398 1398 new_fp->ftp_next = bucket->ftb_data;
1399 1399 bucket->ftb_data = new_fp;
1400 1400
1401 1401 mutex_enter(&new_fp->ftp_mtx);
1402 1402 mutex_exit(&bucket->ftb_mtx);
1403 1403
1404 1404 crfree(cred);
1405 1405 return (new_fp);
1406 1406 }
1407 1407
1408 1408 static void
1409 1409 fasttrap_provider_free(fasttrap_provider_t *provider)
1410 1410 {
1411 1411 pid_t pid = provider->ftp_pid;
1412 1412 proc_t *p;
1413 1413
1414 1414 /*
1415 1415 * There need to be no associated enabled probes, no consumers
1416 1416 * creating probes, and no meta providers referencing this provider.
1417 1417 */
1418 1418 ASSERT(provider->ftp_rcount == 0);
1419 1419 ASSERT(provider->ftp_ccount == 0);
1420 1420 ASSERT(provider->ftp_mcount == 0);
1421 1421
1422 1422 /*
1423 1423 * If this provider hasn't been retired, we need to explicitly drop the
1424 1424 * count of active providers on the associated process structure.
1425 1425 */
1426 1426 if (!provider->ftp_retired) {
1427 1427 atomic_dec_64(&provider->ftp_proc->ftpc_acount);
1428 1428 ASSERT(provider->ftp_proc->ftpc_acount <
1429 1429 provider->ftp_proc->ftpc_rcount);
1430 1430 }
1431 1431
1432 1432 fasttrap_proc_release(provider->ftp_proc);
1433 1433
1434 1434 kmem_free(provider, sizeof (fasttrap_provider_t));
1435 1435
1436 1436 /*
1437 1437 * Decrement p_dtrace_probes on the process whose provider we're
1438 1438 * freeing. We don't have to worry about clobbering somone else's
1439 1439 * modifications to it because we have locked the bucket that
1440 1440 * corresponds to this process's hash chain in the provider hash
1441 1441 * table. Don't sweat it if we can't find the process.
1442 1442 */
1443 1443 mutex_enter(&pidlock);
1444 1444 if ((p = prfind(pid)) == NULL) {
1445 1445 mutex_exit(&pidlock);
1446 1446 return;
1447 1447 }
1448 1448
1449 1449 mutex_enter(&p->p_lock);
1450 1450 mutex_exit(&pidlock);
1451 1451
1452 1452 p->p_dtrace_probes--;
1453 1453 mutex_exit(&p->p_lock);
1454 1454 }
1455 1455
1456 1456 static void
1457 1457 fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1458 1458 {
1459 1459 fasttrap_provider_t *fp;
1460 1460 fasttrap_bucket_t *bucket;
1461 1461 dtrace_provider_id_t provid;
1462 1462
1463 1463 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1464 1464
1465 1465 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1466 1466 mutex_enter(&bucket->ftb_mtx);
1467 1467
1468 1468 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1469 1469 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1470 1470 !fp->ftp_retired)
1471 1471 break;
1472 1472 }
1473 1473
1474 1474 if (fp == NULL) {
1475 1475 mutex_exit(&bucket->ftb_mtx);
1476 1476 return;
1477 1477 }
1478 1478
1479 1479 mutex_enter(&fp->ftp_mtx);
1480 1480 ASSERT(!mprov || fp->ftp_mcount > 0);
1481 1481 if (mprov && --fp->ftp_mcount != 0) {
1482 1482 mutex_exit(&fp->ftp_mtx);
1483 1483 mutex_exit(&bucket->ftb_mtx);
1484 1484 return;
1485 1485 }
1486 1486
1487 1487 /*
1488 1488 * Mark the provider to be removed in our post-processing step, mark it
1489 1489 * retired, and drop the active count on its proc. Marking it indicates
1490 1490 * that we should try to remove it; setting the retired flag indicates
1491 1491 * that we're done with this provider; dropping the active the proc
1492 1492 * releases our hold, and when this reaches zero (as it will during
1493 1493 * exit or exec) the proc and associated providers become defunct.
1494 1494 *
1495 1495 * We obviously need to take the bucket lock before the provider lock
1496 1496 * to perform the lookup, but we need to drop the provider lock
1497 1497 * before calling into the DTrace framework since we acquire the
1498 1498 * provider lock in callbacks invoked from the DTrace framework. The
1499 1499 * bucket lock therefore protects the integrity of the provider hash
1500 1500 * table.
1501 1501 */
1502 1502 atomic_dec_64(&fp->ftp_proc->ftpc_acount);
1503 1503 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1504 1504
1505 1505 fp->ftp_retired = 1;
1506 1506 fp->ftp_marked = 1;
1507 1507 provid = fp->ftp_provid;
1508 1508 mutex_exit(&fp->ftp_mtx);
1509 1509
1510 1510 /*
1511 1511 * We don't have to worry about invalidating the same provider twice
1512 1512 * since fasttrap_provider_lookup() will ignore provider that have
1513 1513 * been marked as retired.
1514 1514 */
1515 1515 dtrace_invalidate(provid);
1516 1516
1517 1517 mutex_exit(&bucket->ftb_mtx);
1518 1518
1519 1519 fasttrap_pid_cleanup();
1520 1520 }
1521 1521
1522 1522 static int
1523 1523 fasttrap_uint32_cmp(const void *ap, const void *bp)
1524 1524 {
1525 1525 return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1526 1526 }
1527 1527
1528 1528 static int
1529 1529 fasttrap_uint64_cmp(const void *ap, const void *bp)
1530 1530 {
1531 1531 return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1532 1532 }
1533 1533
1534 1534 static int
1535 1535 fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1536 1536 {
1537 1537 fasttrap_provider_t *provider;
1538 1538 fasttrap_probe_t *pp;
1539 1539 fasttrap_tracepoint_t *tp;
1540 1540 char *name;
1541 1541 int i, aframes, whack;
1542 1542
1543 1543 /*
1544 1544 * There needs to be at least one desired trace point.
1545 1545 */
1546 1546 if (pdata->ftps_noffs == 0)
1547 1547 return (EINVAL);
1548 1548
1549 1549 switch (pdata->ftps_type) {
1550 1550 case DTFTP_ENTRY:
1551 1551 name = "entry";
1552 1552 aframes = FASTTRAP_ENTRY_AFRAMES;
1553 1553 break;
1554 1554 case DTFTP_RETURN:
1555 1555 name = "return";
1556 1556 aframes = FASTTRAP_RETURN_AFRAMES;
1557 1557 break;
1558 1558 case DTFTP_OFFSETS:
1559 1559 name = NULL;
1560 1560 break;
1561 1561 default:
1562 1562 return (EINVAL);
1563 1563 }
1564 1564
1565 1565 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1566 1566 FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1567 1567 return (ESRCH);
1568 1568
1569 1569 /*
1570 1570 * Increment this reference count to indicate that a consumer is
1571 1571 * actively adding a new probe associated with this provider. This
1572 1572 * prevents the provider from being deleted -- we'll need to check
1573 1573 * for pending deletions when we drop this reference count.
1574 1574 */
1575 1575 provider->ftp_ccount++;
1576 1576 mutex_exit(&provider->ftp_mtx);
1577 1577
1578 1578 /*
1579 1579 * Grab the creation lock to ensure consistency between calls to
1580 1580 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1581 1581 * other threads creating probes. We must drop the provider lock
1582 1582 * before taking this lock to avoid a three-way deadlock with the
1583 1583 * DTrace framework.
1584 1584 */
1585 1585 mutex_enter(&provider->ftp_cmtx);
1586 1586
1587 1587 if (name == NULL) {
1588 1588 for (i = 0; i < pdata->ftps_noffs; i++) {
1589 1589 char name_str[17];
1590 1590
1591 1591 (void) sprintf(name_str, "%llx",
1592 1592 (unsigned long long)pdata->ftps_offs[i]);
1593 1593
1594 1594 if (dtrace_probe_lookup(provider->ftp_provid,
1595 1595 pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1596 1596 continue;
1597 1597
1598 1598 atomic_inc_32(&fasttrap_total);
1599 1599
1600 1600 if (fasttrap_total > fasttrap_max) {
1601 1601 atomic_dec_32(&fasttrap_total);
1602 1602 goto no_mem;
1603 1603 }
1604 1604
1605 1605 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1606 1606
1607 1607 pp->ftp_prov = provider;
1608 1608 pp->ftp_faddr = pdata->ftps_pc;
1609 1609 pp->ftp_fsize = pdata->ftps_size;
1610 1610 pp->ftp_pid = pdata->ftps_pid;
1611 1611 pp->ftp_ntps = 1;
1612 1612
1613 1613 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1614 1614 KM_SLEEP);
1615 1615
1616 1616 tp->ftt_proc = provider->ftp_proc;
1617 1617 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1618 1618 tp->ftt_pid = pdata->ftps_pid;
1619 1619
1620 1620 pp->ftp_tps[0].fit_tp = tp;
1621 1621 pp->ftp_tps[0].fit_id.fti_probe = pp;
1622 1622 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1623 1623
1624 1624 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1625 1625 pdata->ftps_mod, pdata->ftps_func, name_str,
1626 1626 FASTTRAP_OFFSET_AFRAMES, pp);
1627 1627 }
1628 1628
1629 1629 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1630 1630 pdata->ftps_func, name) == 0) {
1631 1631 atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1632 1632
1633 1633 if (fasttrap_total > fasttrap_max) {
1634 1634 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1635 1635 goto no_mem;
1636 1636 }
1637 1637
1638 1638 /*
1639 1639 * Make sure all tracepoint program counter values are unique.
1640 1640 * We later assume that each probe has exactly one tracepoint
1641 1641 * for a given pc.
1642 1642 */
1643 1643 qsort(pdata->ftps_offs, pdata->ftps_noffs,
1644 1644 sizeof (uint64_t), fasttrap_uint64_cmp);
1645 1645 for (i = 1; i < pdata->ftps_noffs; i++) {
1646 1646 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1647 1647 continue;
1648 1648
1649 1649 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1650 1650 goto no_mem;
1651 1651 }
1652 1652
1653 1653 ASSERT(pdata->ftps_noffs > 0);
1654 1654 pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1655 1655 ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1656 1656
1657 1657 pp->ftp_prov = provider;
1658 1658 pp->ftp_faddr = pdata->ftps_pc;
1659 1659 pp->ftp_fsize = pdata->ftps_size;
1660 1660 pp->ftp_pid = pdata->ftps_pid;
1661 1661 pp->ftp_ntps = pdata->ftps_noffs;
1662 1662
1663 1663 for (i = 0; i < pdata->ftps_noffs; i++) {
1664 1664 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1665 1665 KM_SLEEP);
1666 1666
1667 1667 tp->ftt_proc = provider->ftp_proc;
1668 1668 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1669 1669 tp->ftt_pid = pdata->ftps_pid;
1670 1670
1671 1671 pp->ftp_tps[i].fit_tp = tp;
1672 1672 pp->ftp_tps[i].fit_id.fti_probe = pp;
1673 1673 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
1674 1674 }
1675 1675
1676 1676 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1677 1677 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1678 1678 }
1679 1679
1680 1680 mutex_exit(&provider->ftp_cmtx);
1681 1681
1682 1682 /*
1683 1683 * We know that the provider is still valid since we incremented the
1684 1684 * creation reference count. If someone tried to clean up this provider
1685 1685 * while we were using it (e.g. because the process called exec(2) or
1686 1686 * exit(2)), take note of that and try to clean it up now.
1687 1687 */
1688 1688 mutex_enter(&provider->ftp_mtx);
1689 1689 provider->ftp_ccount--;
1690 1690 whack = provider->ftp_retired;
1691 1691 mutex_exit(&provider->ftp_mtx);
1692 1692
1693 1693 if (whack)
1694 1694 fasttrap_pid_cleanup();
1695 1695
1696 1696 return (0);
1697 1697
1698 1698 no_mem:
1699 1699 /*
1700 1700 * If we've exhausted the allowable resources, we'll try to remove
1701 1701 * this provider to free some up. This is to cover the case where
1702 1702 * the user has accidentally created many more probes than was
1703 1703 * intended (e.g. pid123:::).
1704 1704 */
1705 1705 mutex_exit(&provider->ftp_cmtx);
1706 1706 mutex_enter(&provider->ftp_mtx);
1707 1707 provider->ftp_ccount--;
1708 1708 provider->ftp_marked = 1;
1709 1709 mutex_exit(&provider->ftp_mtx);
1710 1710
1711 1711 fasttrap_pid_cleanup();
1712 1712
1713 1713 return (ENOMEM);
1714 1714 }
1715 1715
1716 1716 /*ARGSUSED*/
1717 1717 static void *
1718 1718 fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1719 1719 {
1720 1720 fasttrap_provider_t *provider;
1721 1721
1722 1722 /*
1723 1723 * A 32-bit unsigned integer (like a pid for example) can be
1724 1724 * expressed in 10 or fewer decimal digits. Make sure that we'll
1725 1725 * have enough space for the provider name.
1726 1726 */
1727 1727 if (strlen(dhpv->dthpv_provname) + 10 >=
1728 1728 sizeof (provider->ftp_name)) {
1729 1729 cmn_err(CE_WARN, "failed to instantiate provider %s: "
1730 1730 "name too long to accomodate pid", dhpv->dthpv_provname);
1731 1731 return (NULL);
1732 1732 }
1733 1733
1734 1734 /*
1735 1735 * Don't let folks spoof the true pid provider.
1736 1736 */
1737 1737 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
1738 1738 cmn_err(CE_WARN, "failed to instantiate provider %s: "
1739 1739 "%s is an invalid name", dhpv->dthpv_provname,
1740 1740 FASTTRAP_PID_NAME);
1741 1741 return (NULL);
1742 1742 }
1743 1743
1744 1744 /*
1745 1745 * The highest stability class that fasttrap supports is ISA; cap
1746 1746 * the stability of the new provider accordingly.
1747 1747 */
1748 1748 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
1749 1749 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
1750 1750 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
1751 1751 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
1752 1752 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
1753 1753 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
1754 1754 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
1755 1755 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
1756 1756 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
1757 1757 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
1758 1758
1759 1759 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
1760 1760 &dhpv->dthpv_pattr)) == NULL) {
1761 1761 cmn_err(CE_WARN, "failed to instantiate provider %s for "
1762 1762 "process %u", dhpv->dthpv_provname, (uint_t)pid);
1763 1763 return (NULL);
1764 1764 }
1765 1765
1766 1766 /*
1767 1767 * Up the meta provider count so this provider isn't removed until
1768 1768 * the meta provider has been told to remove it.
1769 1769 */
1770 1770 provider->ftp_mcount++;
1771 1771
1772 1772 mutex_exit(&provider->ftp_mtx);
1773 1773
1774 1774 return (provider);
1775 1775 }
1776 1776
1777 1777 /*ARGSUSED*/
1778 1778 static void
1779 1779 fasttrap_meta_create_probe(void *arg, void *parg,
1780 1780 dtrace_helper_probedesc_t *dhpb)
1781 1781 {
1782 1782 fasttrap_provider_t *provider = parg;
1783 1783 fasttrap_probe_t *pp;
1784 1784 fasttrap_tracepoint_t *tp;
1785 1785 int i, j;
1786 1786 uint32_t ntps;
1787 1787
1788 1788 /*
1789 1789 * Since the meta provider count is non-zero we don't have to worry
1790 1790 * about this provider disappearing.
1791 1791 */
1792 1792 ASSERT(provider->ftp_mcount > 0);
1793 1793
1794 1794 /*
1795 1795 * The offsets must be unique.
1796 1796 */
1797 1797 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
1798 1798 fasttrap_uint32_cmp);
1799 1799 for (i = 1; i < dhpb->dthpb_noffs; i++) {
1800 1800 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
1801 1801 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
1802 1802 return;
1803 1803 }
1804 1804
1805 1805 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
1806 1806 fasttrap_uint32_cmp);
1807 1807 for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
1808 1808 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
1809 1809 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
1810 1810 return;
1811 1811 }
1812 1812
1813 1813 /*
1814 1814 * Grab the creation lock to ensure consistency between calls to
1815 1815 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1816 1816 * other threads creating probes.
1817 1817 */
1818 1818 mutex_enter(&provider->ftp_cmtx);
1819 1819
1820 1820 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod,
1821 1821 dhpb->dthpb_func, dhpb->dthpb_name) != 0) {
1822 1822 mutex_exit(&provider->ftp_cmtx);
1823 1823 return;
1824 1824 }
1825 1825
1826 1826 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
1827 1827 ASSERT(ntps > 0);
1828 1828
1829 1829 atomic_add_32(&fasttrap_total, ntps);
1830 1830
1831 1831 if (fasttrap_total > fasttrap_max) {
1832 1832 atomic_add_32(&fasttrap_total, -ntps);
1833 1833 mutex_exit(&provider->ftp_cmtx);
1834 1834 return;
1835 1835 }
1836 1836
1837 1837 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
1838 1838
1839 1839 pp->ftp_prov = provider;
1840 1840 pp->ftp_pid = provider->ftp_pid;
1841 1841 pp->ftp_ntps = ntps;
1842 1842 pp->ftp_nargs = dhpb->dthpb_xargc;
1843 1843 pp->ftp_xtypes = dhpb->dthpb_xtypes;
1844 1844 pp->ftp_ntypes = dhpb->dthpb_ntypes;
1845 1845
1846 1846 /*
1847 1847 * First create a tracepoint for each actual point of interest.
1848 1848 */
1849 1849 for (i = 0; i < dhpb->dthpb_noffs; i++) {
1850 1850 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1851 1851
1852 1852 tp->ftt_proc = provider->ftp_proc;
1853 1853 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
1854 1854 tp->ftt_pid = provider->ftp_pid;
1855 1855
1856 1856 pp->ftp_tps[i].fit_tp = tp;
1857 1857 pp->ftp_tps[i].fit_id.fti_probe = pp;
1858 1858 #ifdef __sparc
1859 1859 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
1860 1860 #else
1861 1861 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
1862 1862 #endif
1863 1863 }
1864 1864
1865 1865 /*
1866 1866 * Then create a tracepoint for each is-enabled point.
1867 1867 */
1868 1868 for (j = 0; i < ntps; i++, j++) {
1869 1869 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1870 1870
1871 1871 tp->ftt_proc = provider->ftp_proc;
1872 1872 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
1873 1873 tp->ftt_pid = provider->ftp_pid;
1874 1874
1875 1875 pp->ftp_tps[i].fit_tp = tp;
1876 1876 pp->ftp_tps[i].fit_id.fti_probe = pp;
1877 1877 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
1878 1878 }
1879 1879
1880 1880 /*
1881 1881 * If the arguments are shuffled around we set the argument remapping
1882 1882 * table. Later, when the probe fires, we only remap the arguments
1883 1883 * if the table is non-NULL.
1884 1884 */
1885 1885 for (i = 0; i < dhpb->dthpb_xargc; i++) {
1886 1886 if (dhpb->dthpb_args[i] != i) {
1887 1887 pp->ftp_argmap = dhpb->dthpb_args;
1888 1888 break;
1889 1889 }
1890 1890 }
1891 1891
1892 1892 /*
1893 1893 * The probe is fully constructed -- register it with DTrace.
1894 1894 */
1895 1895 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
1896 1896 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
1897 1897
1898 1898 mutex_exit(&provider->ftp_cmtx);
1899 1899 }
1900 1900
1901 1901 /*ARGSUSED*/
1902 1902 static void
1903 1903 fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1904 1904 {
1905 1905 /*
1906 1906 * Clean up the USDT provider. There may be active consumers of the
1907 1907 * provider busy adding probes, no damage will actually befall the
1908 1908 * provider until that count has dropped to zero. This just puts
1909 1909 * the provider on death row.
1910 1910 */
1911 1911 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
1912 1912 }
1913 1913
1914 1914 static dtrace_mops_t fasttrap_mops = {
1915 1915 fasttrap_meta_create_probe,
1916 1916 fasttrap_meta_provide,
1917 1917 fasttrap_meta_remove
1918 1918 };
1919 1919
1920 1920 /*ARGSUSED*/
1921 1921 static int
1922 1922 fasttrap_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
1923 1923 {
1924 1924 return (0);
1925 1925 }
1926 1926
1927 1927 /*ARGSUSED*/
1928 1928 static int
1929 1929 fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
1930 1930 {
1931 1931 if (!dtrace_attached())
1932 1932 return (EAGAIN);
1933 1933
1934 1934 if (cmd == FASTTRAPIOC_MAKEPROBE) {
1935 1935 fasttrap_probe_spec_t *uprobe = (void *)arg;
1936 1936 fasttrap_probe_spec_t *probe;
1937 1937 uint64_t noffs;
1938 1938 size_t size;
1939 1939 int ret, err;
1940 1940
1941 1941 if (copyin(&uprobe->ftps_noffs, &noffs,
1942 1942 sizeof (uprobe->ftps_noffs)))
1943 1943 return (EFAULT);
1944 1944
1945 1945 /*
1946 1946 * Probes must have at least one tracepoint.
1947 1947 */
1948 1948 if (noffs == 0)
1949 1949 return (EINVAL);
1950 1950
1951 1951 size = sizeof (fasttrap_probe_spec_t) +
1952 1952 sizeof (probe->ftps_offs[0]) * (noffs - 1);
1953 1953
1954 1954 if (size > 1024 * 1024)
1955 1955 return (ENOMEM);
1956 1956
1957 1957 probe = kmem_alloc(size, KM_SLEEP);
1958 1958
1959 1959 if (copyin(uprobe, probe, size) != 0 ||
1960 1960 probe->ftps_noffs != noffs) {
1961 1961 kmem_free(probe, size);
1962 1962 return (EFAULT);
1963 1963 }
1964 1964
1965 1965 /*
1966 1966 * Verify that the function and module strings contain no
1967 1967 * funny characters.
1968 1968 */
1969 1969 if (u8_validate(probe->ftps_func, strlen(probe->ftps_func),
1970 1970 NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
1971 1971 ret = EINVAL;
1972 1972 goto err;
1973 1973 }
1974 1974
1975 1975 if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod),
1976 1976 NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
1977 1977 ret = EINVAL;
1978 1978 goto err;
1979 1979 }
1980 1980
1981 1981 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
1982 1982 proc_t *p;
1983 1983 pid_t pid = probe->ftps_pid;
1984 1984
1985 1985 mutex_enter(&pidlock);
1986 1986 /*
1987 1987 * Report an error if the process doesn't exist
1988 1988 * or is actively being birthed.
1989 1989 */
1990 1990 if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
1991 1991 mutex_exit(&pidlock);
1992 1992 return (ESRCH);
1993 1993 }
1994 1994 mutex_enter(&p->p_lock);
1995 1995 mutex_exit(&pidlock);
1996 1996
1997 1997 if ((ret = priv_proc_cred_perm(cr, p, NULL,
1998 1998 VREAD | VWRITE)) != 0) {
1999 1999 mutex_exit(&p->p_lock);
2000 2000 return (ret);
2001 2001 }
2002 2002
2003 2003 mutex_exit(&p->p_lock);
2004 2004 }
2005 2005
2006 2006 ret = fasttrap_add_probe(probe);
2007 2007 err:
2008 2008 kmem_free(probe, size);
2009 2009
2010 2010 return (ret);
2011 2011
2012 2012 } else if (cmd == FASTTRAPIOC_GETINSTR) {
2013 2013 fasttrap_instr_query_t instr;
2014 2014 fasttrap_tracepoint_t *tp;
2015 2015 uint_t index;
2016 2016 int ret;
2017 2017
2018 2018 if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2019 2019 return (EFAULT);
2020 2020
2021 2021 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2022 2022 proc_t *p;
2023 2023 pid_t pid = instr.ftiq_pid;
2024 2024
2025 2025 mutex_enter(&pidlock);
2026 2026 /*
2027 2027 * Report an error if the process doesn't exist
2028 2028 * or is actively being birthed.
2029 2029 */
2030 2030 if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
2031 2031 mutex_exit(&pidlock);
2032 2032 return (ESRCH);
2033 2033 }
2034 2034 mutex_enter(&p->p_lock);
2035 2035 mutex_exit(&pidlock);
2036 2036
2037 2037 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2038 2038 VREAD)) != 0) {
2039 2039 mutex_exit(&p->p_lock);
2040 2040 return (ret);
2041 2041 }
2042 2042
2043 2043 mutex_exit(&p->p_lock);
2044 2044 }
2045 2045
2046 2046 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2047 2047
2048 2048 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2049 2049 tp = fasttrap_tpoints.fth_table[index].ftb_data;
2050 2050 while (tp != NULL) {
2051 2051 if (instr.ftiq_pid == tp->ftt_pid &&
2052 2052 instr.ftiq_pc == tp->ftt_pc &&
2053 2053 tp->ftt_proc->ftpc_acount != 0)
2054 2054 break;
2055 2055
2056 2056 tp = tp->ftt_next;
2057 2057 }
2058 2058
2059 2059 if (tp == NULL) {
2060 2060 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2061 2061 return (ENOENT);
2062 2062 }
2063 2063
2064 2064 bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2065 2065 sizeof (instr.ftiq_instr));
2066 2066 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2067 2067
2068 2068 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2069 2069 return (EFAULT);
2070 2070
2071 2071 return (0);
2072 2072 }
2073 2073
2074 2074 return (EINVAL);
2075 2075 }
2076 2076
2077 2077 static struct cb_ops fasttrap_cb_ops = {
2078 2078 fasttrap_open, /* open */
2079 2079 nodev, /* close */
2080 2080 nulldev, /* strategy */
2081 2081 nulldev, /* print */
2082 2082 nodev, /* dump */
2083 2083 nodev, /* read */
2084 2084 nodev, /* write */
2085 2085 fasttrap_ioctl, /* ioctl */
2086 2086 nodev, /* devmap */
2087 2087 nodev, /* mmap */
2088 2088 nodev, /* segmap */
2089 2089 nochpoll, /* poll */
2090 2090 ddi_prop_op, /* cb_prop_op */
2091 2091 0, /* streamtab */
2092 2092 D_NEW | D_MP /* Driver compatibility flag */
2093 2093 };
2094 2094
2095 2095 /*ARGSUSED*/
2096 2096 static int
2097 2097 fasttrap_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
2098 2098 {
2099 2099 int error;
2100 2100
2101 2101 switch (infocmd) {
2102 2102 case DDI_INFO_DEVT2DEVINFO:
2103 2103 *result = (void *)fasttrap_devi;
2104 2104 error = DDI_SUCCESS;
2105 2105 break;
2106 2106 case DDI_INFO_DEVT2INSTANCE:
2107 2107 *result = (void *)0;
2108 2108 error = DDI_SUCCESS;
2109 2109 break;
2110 2110 default:
2111 2111 error = DDI_FAILURE;
2112 2112 }
2113 2113 return (error);
2114 2114 }
2115 2115
2116 2116 static int
2117 2117 fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
2118 2118 {
2119 2119 ulong_t nent;
2120 2120
2121 2121 switch (cmd) {
2122 2122 case DDI_ATTACH:
2123 2123 break;
2124 2124 case DDI_RESUME:
2125 2125 return (DDI_SUCCESS);
2126 2126 default:
2127 2127 return (DDI_FAILURE);
2128 2128 }
2129 2129
2130 2130 if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0,
2131 2131 DDI_PSEUDO, NULL) == DDI_FAILURE) {
2132 2132 ddi_remove_minor_node(devi, NULL);
2133 2133 return (DDI_FAILURE);
2134 2134 }
2135 2135
2136 2136 ddi_report_dev(devi);
2137 2137 fasttrap_devi = devi;
2138 2138
2139 2139 /*
2140 2140 * Install our hooks into fork(2), exec(2), and exit(2).
2141 2141 */
2142 2142 dtrace_fasttrap_fork_ptr = &fasttrap_fork;
2143 2143 dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit;
2144 2144 dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit;
2145 2145
2146 2146 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2147 2147 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2148 2148 fasttrap_total = 0;
|
↓ open down ↓ |
2148 lines elided |
↑ open up ↑ |
2149 2149
2150 2150 /*
2151 2151 * Conjure up the tracepoints hashtable...
2152 2152 */
2153 2153 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2154 2154 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2155 2155
2156 2156 if (nent == 0 || nent > 0x1000000)
2157 2157 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2158 2158
2159 - if ((nent & (nent - 1)) == 0)
2159 + if (ISP2(nent))
2160 2160 fasttrap_tpoints.fth_nent = nent;
2161 2161 else
2162 2162 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2163 2163 ASSERT(fasttrap_tpoints.fth_nent > 0);
2164 2164 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2165 2165 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2166 2166 sizeof (fasttrap_bucket_t), KM_SLEEP);
2167 2167
2168 2168 /*
2169 2169 * ... and the providers hash table...
2170 2170 */
2171 2171 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2172 - if ((nent & (nent - 1)) == 0)
2172 + if (ISP2(nent))
2173 2173 fasttrap_provs.fth_nent = nent;
2174 2174 else
2175 2175 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2176 2176 ASSERT(fasttrap_provs.fth_nent > 0);
2177 2177 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2178 2178 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2179 2179 sizeof (fasttrap_bucket_t), KM_SLEEP);
2180 2180
2181 2181 /*
2182 2182 * ... and the procs hash table.
2183 2183 */
2184 2184 nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2185 - if ((nent & (nent - 1)) == 0)
2185 + if (ISP2(nent))
2186 2186 fasttrap_procs.fth_nent = nent;
2187 2187 else
2188 2188 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2189 2189 ASSERT(fasttrap_procs.fth_nent > 0);
2190 2190 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2191 2191 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2192 2192 sizeof (fasttrap_bucket_t), KM_SLEEP);
2193 2193
2194 2194 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2195 2195 &fasttrap_meta_id);
2196 2196
2197 2197 return (DDI_SUCCESS);
2198 2198 }
2199 2199
2200 2200 static int
2201 2201 fasttrap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
2202 2202 {
2203 2203 int i, fail = 0;
2204 2204 timeout_id_t tmp;
2205 2205
2206 2206 switch (cmd) {
2207 2207 case DDI_DETACH:
2208 2208 break;
2209 2209 case DDI_SUSPEND:
2210 2210 return (DDI_SUCCESS);
2211 2211 default:
2212 2212 return (DDI_FAILURE);
2213 2213 }
2214 2214
2215 2215 /*
2216 2216 * Unregister the meta-provider to make sure no new fasttrap-
2217 2217 * managed providers come along while we're trying to close up
2218 2218 * shop. If we fail to detach, we'll need to re-register as a
2219 2219 * meta-provider. We can fail to unregister as a meta-provider
2220 2220 * if providers we manage still exist.
2221 2221 */
2222 2222 if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2223 2223 dtrace_meta_unregister(fasttrap_meta_id) != 0)
2224 2224 return (DDI_FAILURE);
2225 2225
2226 2226 /*
2227 2227 * Prevent any new timeouts from running by setting fasttrap_timeout
2228 2228 * to a non-zero value, and wait for the current timeout to complete.
2229 2229 */
2230 2230 mutex_enter(&fasttrap_cleanup_mtx);
2231 2231 fasttrap_cleanup_work = 0;
2232 2232
2233 2233 while (fasttrap_timeout != (timeout_id_t)1) {
2234 2234 tmp = fasttrap_timeout;
2235 2235 fasttrap_timeout = (timeout_id_t)1;
2236 2236
2237 2237 if (tmp != 0) {
2238 2238 mutex_exit(&fasttrap_cleanup_mtx);
2239 2239 (void) untimeout(tmp);
2240 2240 mutex_enter(&fasttrap_cleanup_mtx);
2241 2241 }
2242 2242 }
2243 2243
2244 2244 fasttrap_cleanup_work = 0;
2245 2245 mutex_exit(&fasttrap_cleanup_mtx);
2246 2246
2247 2247 /*
2248 2248 * Iterate over all of our providers. If there's still a process
2249 2249 * that corresponds to that pid, fail to detach.
2250 2250 */
2251 2251 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2252 2252 fasttrap_provider_t **fpp, *fp;
2253 2253 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2254 2254
2255 2255 mutex_enter(&bucket->ftb_mtx);
2256 2256 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2257 2257 while ((fp = *fpp) != NULL) {
2258 2258 /*
2259 2259 * Acquire and release the lock as a simple way of
2260 2260 * waiting for any other consumer to finish with
2261 2261 * this provider. A thread must first acquire the
2262 2262 * bucket lock so there's no chance of another thread
2263 2263 * blocking on the provider's lock.
2264 2264 */
2265 2265 mutex_enter(&fp->ftp_mtx);
2266 2266 mutex_exit(&fp->ftp_mtx);
2267 2267
2268 2268 if (dtrace_unregister(fp->ftp_provid) != 0) {
2269 2269 fail = 1;
2270 2270 fpp = &fp->ftp_next;
2271 2271 } else {
2272 2272 *fpp = fp->ftp_next;
2273 2273 fasttrap_provider_free(fp);
2274 2274 }
2275 2275 }
2276 2276
2277 2277 mutex_exit(&bucket->ftb_mtx);
2278 2278 }
2279 2279
2280 2280 if (fail) {
2281 2281 uint_t work;
2282 2282 /*
2283 2283 * If we're failing to detach, we need to unblock timeouts
2284 2284 * and start a new timeout if any work has accumulated while
2285 2285 * we've been unsuccessfully trying to detach.
2286 2286 */
2287 2287 mutex_enter(&fasttrap_cleanup_mtx);
2288 2288 fasttrap_timeout = 0;
2289 2289 work = fasttrap_cleanup_work;
2290 2290 mutex_exit(&fasttrap_cleanup_mtx);
2291 2291
2292 2292 if (work)
2293 2293 fasttrap_pid_cleanup();
2294 2294
2295 2295 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2296 2296 &fasttrap_meta_id);
2297 2297
2298 2298 return (DDI_FAILURE);
2299 2299 }
2300 2300
2301 2301 #ifdef DEBUG
2302 2302 mutex_enter(&fasttrap_count_mtx);
2303 2303 ASSERT(fasttrap_pid_count == 0);
2304 2304 mutex_exit(&fasttrap_count_mtx);
2305 2305 #endif
2306 2306
2307 2307 kmem_free(fasttrap_tpoints.fth_table,
2308 2308 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2309 2309 fasttrap_tpoints.fth_nent = 0;
2310 2310
2311 2311 kmem_free(fasttrap_provs.fth_table,
2312 2312 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2313 2313 fasttrap_provs.fth_nent = 0;
2314 2314
2315 2315 kmem_free(fasttrap_procs.fth_table,
2316 2316 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2317 2317 fasttrap_procs.fth_nent = 0;
2318 2318
2319 2319 /*
2320 2320 * We know there are no tracepoints in any process anywhere in
2321 2321 * the system so there is no process which has its p_dtrace_count
2322 2322 * greater than zero, therefore we know that no thread can actively
2323 2323 * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes
2324 2324 * and fasttrap_exec() and fasttrap_exit().
2325 2325 */
2326 2326 ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork);
2327 2327 dtrace_fasttrap_fork_ptr = NULL;
2328 2328
2329 2329 ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit);
2330 2330 dtrace_fasttrap_exec_ptr = NULL;
2331 2331
2332 2332 ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit);
2333 2333 dtrace_fasttrap_exit_ptr = NULL;
2334 2334
2335 2335 ddi_remove_minor_node(devi, NULL);
2336 2336
2337 2337 return (DDI_SUCCESS);
2338 2338 }
2339 2339
2340 2340 static struct dev_ops fasttrap_ops = {
2341 2341 DEVO_REV, /* devo_rev */
2342 2342 0, /* refcnt */
2343 2343 fasttrap_info, /* get_dev_info */
2344 2344 nulldev, /* identify */
2345 2345 nulldev, /* probe */
2346 2346 fasttrap_attach, /* attach */
2347 2347 fasttrap_detach, /* detach */
2348 2348 nodev, /* reset */
2349 2349 &fasttrap_cb_ops, /* driver operations */
2350 2350 NULL, /* bus operations */
2351 2351 nodev, /* dev power */
2352 2352 ddi_quiesce_not_needed, /* quiesce */
2353 2353 };
2354 2354
2355 2355 /*
2356 2356 * Module linkage information for the kernel.
2357 2357 */
2358 2358 static struct modldrv modldrv = {
2359 2359 &mod_driverops, /* module type (this is a pseudo driver) */
2360 2360 "Fasttrap Tracing", /* name of module */
2361 2361 &fasttrap_ops, /* driver ops */
2362 2362 };
2363 2363
2364 2364 static struct modlinkage modlinkage = {
2365 2365 MODREV_1,
2366 2366 (void *)&modldrv,
2367 2367 NULL
2368 2368 };
2369 2369
2370 2370 int
2371 2371 _init(void)
2372 2372 {
2373 2373 return (mod_install(&modlinkage));
2374 2374 }
2375 2375
2376 2376 int
2377 2377 _info(struct modinfo *modinfop)
2378 2378 {
2379 2379 return (mod_info(&modlinkage, modinfop));
2380 2380 }
2381 2381
2382 2382 int
2383 2383 _fini(void)
2384 2384 {
2385 2385 return (mod_remove(&modlinkage));
2386 2386 }
|
↓ open down ↓ |
191 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX