1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 /*
  28  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  29  */
  30 
  31 #include <sys/atomic.h>
  32 #include <sys/errno.h>
  33 #include <sys/stat.h>
  34 #include <sys/modctl.h>
  35 #include <sys/conf.h>
  36 #include <sys/systm.h>
  37 #include <sys/ddi.h>
  38 #include <sys/sunddi.h>
  39 #include <sys/cpuvar.h>
  40 #include <sys/kmem.h>
  41 #include <sys/strsubr.h>
  42 #include <sys/fasttrap.h>
  43 #include <sys/fasttrap_impl.h>
  44 #include <sys/fasttrap_isa.h>
  45 #include <sys/dtrace.h>
  46 #include <sys/dtrace_impl.h>
  47 #include <sys/sysmacros.h>
  48 #include <sys/proc.h>
  49 #include <sys/priv.h>
  50 #include <sys/policy.h>
  51 #include <util/qsort.h>
  52 
  53 /*
  54  * User-Land Trap-Based Tracing
  55  * ----------------------------
  56  *
  57  * The fasttrap provider allows DTrace consumers to instrument any user-level
  58  * instruction to gather data; this includes probes with semantic
  59  * signifigance like entry and return as well as simple offsets into the
  60  * function. While the specific techniques used are very ISA specific, the
  61  * methodology is generalizable to any architecture.
  62  *
  63  *
  64  * The General Methodology
  65  * -----------------------
  66  *
  67  * With the primary goal of tracing every user-land instruction and the
  68  * limitation that we can't trust user space so don't want to rely on much
  69  * information there, we begin by replacing the instructions we want to trace
  70  * with trap instructions. Each instruction we overwrite is saved into a hash
  71  * table keyed by process ID and pc address. When we enter the kernel due to
  72  * this trap instruction, we need the effects of the replaced instruction to
  73  * appear to have occurred before we proceed with the user thread's
  74  * execution.
  75  *
  76  * Each user level thread is represented by a ulwp_t structure which is
  77  * always easily accessible through a register. The most basic way to produce
  78  * the effects of the instruction we replaced is to copy that instruction out
  79  * to a bit of scratch space reserved in the user thread's ulwp_t structure
  80  * (a sort of kernel-private thread local storage), set the PC to that
  81  * scratch space and single step. When we reenter the kernel after single
  82  * stepping the instruction we must then adjust the PC to point to what would
  83  * normally be the next instruction. Of course, special care must be taken
  84  * for branches and jumps, but these represent such a small fraction of any
  85  * instruction set that writing the code to emulate these in the kernel is
  86  * not too difficult.
  87  *
  88  * Return probes may require several tracepoints to trace every return site,
  89  * and, conversely, each tracepoint may activate several probes (the entry
  90  * and offset 0 probes, for example). To solve this muliplexing problem,
  91  * tracepoints contain lists of probes to activate and probes contain lists
  92  * of tracepoints to enable. If a probe is activated, it adds its ID to
  93  * existing tracepoints or creates new ones as necessary.
  94  *
  95  * Most probes are activated _before_ the instruction is executed, but return
  96  * probes are activated _after_ the effects of the last instruction of the
  97  * function are visible. Return probes must be fired _after_ we have
  98  * single-stepped the instruction whereas all other probes are fired
  99  * beforehand.
 100  *
 101  *
 102  * Lock Ordering
 103  * -------------
 104  *
 105  * The lock ordering below -- both internally and with respect to the DTrace
 106  * framework -- is a little tricky and bears some explanation. Each provider
 107  * has a lock (ftp_mtx) that protects its members including reference counts
 108  * for enabled probes (ftp_rcount), consumers actively creating probes
 109  * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
 110  * from being freed. A provider is looked up by taking the bucket lock for the
 111  * provider hash table, and is returned with its lock held. The provider lock
 112  * may be taken in functions invoked by the DTrace framework, but may not be
 113  * held while calling functions in the DTrace framework.
 114  *
 115  * To ensure consistency over multiple calls to the DTrace framework, the
 116  * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
 117  * not be taken when holding the provider lock as that would create a cyclic
 118  * lock ordering. In situations where one would naturally take the provider
 119  * lock and then the creation lock, we instead up a reference count to prevent
 120  * the provider from disappearing, drop the provider lock, and acquire the
 121  * creation lock.
 122  *
 123  * Briefly:
 124  *      bucket lock before provider lock
 125  *      DTrace before provider lock
 126  *      creation lock before DTrace
 127  *      never hold the provider lock and creation lock simultaneously
 128  */
 129 
 130 static dev_info_t *fasttrap_devi;
 131 static dtrace_meta_provider_id_t fasttrap_meta_id;
 132 
 133 static timeout_id_t fasttrap_timeout;
 134 static kmutex_t fasttrap_cleanup_mtx;
 135 static uint_t fasttrap_cleanup_work;
 136 
 137 /*
 138  * Generation count on modifications to the global tracepoint lookup table.
 139  */
 140 static volatile uint64_t fasttrap_mod_gen;
 141 
 142 /*
 143  * When the fasttrap provider is loaded, fasttrap_max is set to either
 144  * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
 145  * fasttrap.conf file. Each time a probe is created, fasttrap_total is
 146  * incremented by the number of tracepoints that may be associated with that
 147  * probe; fasttrap_total is capped at fasttrap_max.
 148  */
 149 #define FASTTRAP_MAX_DEFAULT            250000
 150 static uint32_t fasttrap_max;
 151 static uint32_t fasttrap_total;
 152 
 153 
 154 #define FASTTRAP_TPOINTS_DEFAULT_SIZE   0x4000
 155 #define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100
 156 #define FASTTRAP_PROCS_DEFAULT_SIZE     0x100
 157 
 158 #define FASTTRAP_PID_NAME               "pid"
 159 
 160 fasttrap_hash_t                 fasttrap_tpoints;
 161 static fasttrap_hash_t          fasttrap_provs;
 162 static fasttrap_hash_t          fasttrap_procs;
 163 
 164 static uint64_t                 fasttrap_pid_count;     /* pid ref count */
 165 static kmutex_t                 fasttrap_count_mtx;     /* lock on ref count */
 166 
 167 #define FASTTRAP_ENABLE_FAIL    1
 168 #define FASTTRAP_ENABLE_PARTIAL 2
 169 
 170 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
 171 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
 172 
 173 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
 174     const dtrace_pattr_t *);
 175 static void fasttrap_provider_retire(pid_t, const char *, int);
 176 static void fasttrap_provider_free(fasttrap_provider_t *);
 177 
 178 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
 179 static void fasttrap_proc_release(fasttrap_proc_t *);
 180 
 181 #define FASTTRAP_PROVS_INDEX(pid, name) \
 182         ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
 183 
 184 #define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
 185 
 186 static int
 187 fasttrap_highbit(ulong_t i)
 188 {
 189         int h = 1;
 190 
 191         if (i == 0)
 192                 return (0);
 193 #ifdef _LP64
 194         if (i & 0xffffffff00000000ul) {
 195                 h += 32; i >>= 32;
 196         }
 197 #endif
 198         if (i & 0xffff0000) {
 199                 h += 16; i >>= 16;
 200         }
 201         if (i & 0xff00) {
 202                 h += 8; i >>= 8;
 203         }
 204         if (i & 0xf0) {
 205                 h += 4; i >>= 4;
 206         }
 207         if (i & 0xc) {
 208                 h += 2; i >>= 2;
 209         }
 210         if (i & 0x2) {
 211                 h += 1;
 212         }
 213         return (h);
 214 }
 215 
 216 static uint_t
 217 fasttrap_hash_str(const char *p)
 218 {
 219         unsigned int g;
 220         uint_t hval = 0;
 221 
 222         while (*p) {
 223                 hval = (hval << 4) + *p++;
 224                 if ((g = (hval & 0xf0000000)) != 0)
 225                         hval ^= g >> 24;
 226                 hval &= ~g;
 227         }
 228         return (hval);
 229 }
 230 
 231 void
 232 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
 233 {
 234         sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
 235 
 236         sqp->sq_info.si_signo = SIGTRAP;
 237         sqp->sq_info.si_code = TRAP_DTRACE;
 238         sqp->sq_info.si_addr = (caddr_t)pc;
 239 
 240         mutex_enter(&p->p_lock);
 241         sigaddqa(p, t, sqp);
 242         mutex_exit(&p->p_lock);
 243 
 244         if (t != NULL)
 245                 aston(t);
 246 }
 247 
 248 /*
 249  * This function ensures that no threads are actively using the memory
 250  * associated with probes that were formerly live.
 251  */
 252 static void
 253 fasttrap_mod_barrier(uint64_t gen)
 254 {
 255         int i;
 256 
 257         if (gen < fasttrap_mod_gen)
 258                 return;
 259 
 260         fasttrap_mod_gen++;
 261 
 262         for (i = 0; i < NCPU; i++) {
 263                 mutex_enter(&cpu_core[i].cpuc_pid_lock);
 264                 mutex_exit(&cpu_core[i].cpuc_pid_lock);
 265         }
 266 }
 267 
 268 /*
 269  * This is the timeout's callback for cleaning up the providers and their
 270  * probes.
 271  */
 272 /*ARGSUSED*/
 273 static void
 274 fasttrap_pid_cleanup_cb(void *data)
 275 {
 276         fasttrap_provider_t **fpp, *fp;
 277         fasttrap_bucket_t *bucket;
 278         dtrace_provider_id_t provid;
 279         int i, later, rval;
 280 
 281         static volatile int in = 0;
 282         ASSERT(in == 0);
 283         in = 1;
 284 
 285         mutex_enter(&fasttrap_cleanup_mtx);
 286         while (fasttrap_cleanup_work) {
 287                 fasttrap_cleanup_work = 0;
 288                 mutex_exit(&fasttrap_cleanup_mtx);
 289 
 290                 later = 0;
 291 
 292                 /*
 293                  * Iterate over all the providers trying to remove the marked
 294                  * ones. If a provider is marked but not retired, we just
 295                  * have to take a crack at removing it -- it's no big deal if
 296                  * we can't.
 297                  */
 298                 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
 299                         bucket = &fasttrap_provs.fth_table[i];
 300                         mutex_enter(&bucket->ftb_mtx);
 301                         fpp = (fasttrap_provider_t **)&bucket->ftb_data;
 302 
 303                         while ((fp = *fpp) != NULL) {
 304                                 if (!fp->ftp_marked) {
 305                                         fpp = &fp->ftp_next;
 306                                         continue;
 307                                 }
 308 
 309                                 mutex_enter(&fp->ftp_mtx);
 310 
 311                                 /*
 312                                  * If this provider has consumers actively
 313                                  * creating probes (ftp_ccount) or is a USDT
 314                                  * provider (ftp_mcount), we can't unregister
 315                                  * or even condense.
 316                                  */
 317                                 if (fp->ftp_ccount != 0 ||
 318                                     fp->ftp_mcount != 0) {
 319                                         mutex_exit(&fp->ftp_mtx);
 320                                         fp->ftp_marked = 0;
 321                                         continue;
 322                                 }
 323 
 324                                 if (!fp->ftp_retired || fp->ftp_rcount != 0)
 325                                         fp->ftp_marked = 0;
 326 
 327                                 mutex_exit(&fp->ftp_mtx);
 328 
 329                                 /*
 330                                  * If we successfully unregister this
 331                                  * provider we can remove it from the hash
 332                                  * chain and free the memory. If our attempt
 333                                  * to unregister fails and this is a retired
 334                                  * provider, increment our flag to try again
 335                                  * pretty soon. If we've consumed more than
 336                                  * half of our total permitted number of
 337                                  * probes call dtrace_condense() to try to
 338                                  * clean out the unenabled probes.
 339                                  */
 340                                 provid = fp->ftp_provid;
 341                                 if ((rval = dtrace_unregister(provid)) != 0) {
 342                                         if (fasttrap_total > fasttrap_max / 2)
 343                                                 (void) dtrace_condense(provid);
 344 
 345                                         if (rval == EAGAIN)
 346                                                 fp->ftp_marked = 1;
 347 
 348                                         later += fp->ftp_marked;
 349                                         fpp = &fp->ftp_next;
 350                                 } else {
 351                                         *fpp = fp->ftp_next;
 352                                         fasttrap_provider_free(fp);
 353                                 }
 354                         }
 355                         mutex_exit(&bucket->ftb_mtx);
 356                 }
 357 
 358                 mutex_enter(&fasttrap_cleanup_mtx);
 359         }
 360 
 361         ASSERT(fasttrap_timeout != 0);
 362 
 363         /*
 364          * If we were unable to remove a retired provider, try again after
 365          * a second. This situation can occur in certain circumstances where
 366          * providers cannot be unregistered even though they have no probes
 367          * enabled because of an execution of dtrace -l or something similar.
 368          * If the timeout has been disabled (set to 1 because we're trying
 369          * to detach), we set fasttrap_cleanup_work to ensure that we'll
 370          * get a chance to do that work if and when the timeout is reenabled
 371          * (if detach fails).
 372          */
 373         if (later > 0) {
 374                 if (fasttrap_timeout != (timeout_id_t)1) {
 375                         fasttrap_timeout =
 376                             timeout(&fasttrap_pid_cleanup_cb, NULL, hz);
 377                 }
 378 
 379                 fasttrap_cleanup_work = 1;
 380         } else {
 381                 fasttrap_timeout = 0;
 382         }
 383 
 384         mutex_exit(&fasttrap_cleanup_mtx);
 385         in = 0;
 386 }
 387 
 388 /*
 389  * Activates the asynchronous cleanup mechanism.
 390  */
 391 static void
 392 fasttrap_pid_cleanup(void)
 393 {
 394         mutex_enter(&fasttrap_cleanup_mtx);
 395         fasttrap_cleanup_work = 1;
 396         if (fasttrap_timeout == 0)
 397                 fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1);
 398         mutex_exit(&fasttrap_cleanup_mtx);
 399 }
 400 
 401 /*
 402  * This is called from cfork() via dtrace_fasttrap_fork(). The child
 403  * process's address space is (roughly) a copy of the parent process's so
 404  * we have to remove all the instrumentation we had previously enabled in the
 405  * parent.
 406  */
 407 static void
 408 fasttrap_fork(proc_t *p, proc_t *cp)
 409 {
 410         pid_t ppid = p->p_pid;
 411         int i;
 412 
 413         ASSERT(curproc == p);
 414         ASSERT(p->p_proc_flag & P_PR_LOCK);
 415         ASSERT(p->p_dtrace_count > 0);
 416         ASSERT(cp->p_dtrace_count == 0);
 417 
 418         /*
 419          * This would be simpler and faster if we maintained per-process
 420          * hash tables of enabled tracepoints. It could, however, potentially
 421          * slow down execution of a tracepoint since we'd need to go
 422          * through two levels of indirection. In the future, we should
 423          * consider either maintaining per-process ancillary lists of
 424          * enabled tracepoints or hanging a pointer to a per-process hash
 425          * table of enabled tracepoints off the proc structure.
 426          */
 427 
 428         /*
 429          * We don't have to worry about the child process disappearing
 430          * because we're in fork().
 431          */
 432         mutex_enter(&cp->p_lock);
 433         sprlock_proc(cp);
 434         mutex_exit(&cp->p_lock);
 435 
 436         /*
 437          * Iterate over every tracepoint looking for ones that belong to the
 438          * parent process, and remove each from the child process.
 439          */
 440         for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
 441                 fasttrap_tracepoint_t *tp;
 442                 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
 443 
 444                 mutex_enter(&bucket->ftb_mtx);
 445                 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
 446                         if (tp->ftt_pid == ppid &&
 447                             tp->ftt_proc->ftpc_acount != 0) {
 448                                 int ret = fasttrap_tracepoint_remove(cp, tp);
 449                                 ASSERT(ret == 0);
 450 
 451                                 /*
 452                                  * The count of active providers can only be
 453                                  * decremented (i.e. to zero) during exec,
 454                                  * exit, and removal of a meta provider so it
 455                                  * should be impossible to drop the count
 456                                  * mid-fork.
 457                                  */
 458                                 ASSERT(tp->ftt_proc->ftpc_acount != 0);
 459                         }
 460                 }
 461                 mutex_exit(&bucket->ftb_mtx);
 462         }
 463 
 464         mutex_enter(&cp->p_lock);
 465         sprunlock(cp);
 466 }
 467 
 468 /*
 469  * This is called from proc_exit() or from exec_common() if p_dtrace_probes
 470  * is set on the proc structure to indicate that there is a pid provider
 471  * associated with this process.
 472  */
 473 static void
 474 fasttrap_exec_exit(proc_t *p)
 475 {
 476         ASSERT(p == curproc);
 477         ASSERT(MUTEX_HELD(&p->p_lock));
 478 
 479         mutex_exit(&p->p_lock);
 480 
 481         /*
 482          * We clean up the pid provider for this process here; user-land
 483          * static probes are handled by the meta-provider remove entry point.
 484          */
 485         fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
 486 
 487         mutex_enter(&p->p_lock);
 488 }
 489 
 490 
 491 /*ARGSUSED*/
 492 static void
 493 fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc)
 494 {
 495         /*
 496          * There are no "default" pid probes.
 497          */
 498 }
 499 
 500 static int
 501 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
 502 {
 503         fasttrap_tracepoint_t *tp, *new_tp = NULL;
 504         fasttrap_bucket_t *bucket;
 505         fasttrap_id_t *id;
 506         pid_t pid;
 507         uintptr_t pc;
 508 
 509         ASSERT(index < probe->ftp_ntps);
 510 
 511         pid = probe->ftp_pid;
 512         pc = probe->ftp_tps[index].fit_tp->ftt_pc;
 513         id = &probe->ftp_tps[index].fit_id;
 514 
 515         ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
 516 
 517         ASSERT(!(p->p_flag & SVFORK));
 518 
 519         /*
 520          * Before we make any modifications, make sure we've imposed a barrier
 521          * on the generation in which this probe was last modified.
 522          */
 523         fasttrap_mod_barrier(probe->ftp_gen);
 524 
 525         bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
 526 
 527         /*
 528          * If the tracepoint has already been enabled, just add our id to the
 529          * list of interested probes. This may be our second time through
 530          * this path in which case we'll have constructed the tracepoint we'd
 531          * like to install. If we can't find a match, and have an allocated
 532          * tracepoint ready to go, enable that one now.
 533          *
 534          * A tracepoint whose process is defunct is also considered defunct.
 535          */
 536 again:
 537         mutex_enter(&bucket->ftb_mtx);
 538         for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
 539                 /*
 540                  * Note that it's safe to access the active count on the
 541                  * associated proc structure because we know that at least one
 542                  * provider (this one) will still be around throughout this
 543                  * operation.
 544                  */
 545                 if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
 546                     tp->ftt_proc->ftpc_acount == 0)
 547                         continue;
 548 
 549                 /*
 550                  * Now that we've found a matching tracepoint, it would be
 551                  * a decent idea to confirm that the tracepoint is still
 552                  * enabled and the trap instruction hasn't been overwritten.
 553                  * Since this is a little hairy, we'll punt for now.
 554                  */
 555 
 556                 /*
 557                  * This can't be the first interested probe. We don't have
 558                  * to worry about another thread being in the midst of
 559                  * deleting this tracepoint (which would be the only valid
 560                  * reason for a tracepoint to have no interested probes)
 561                  * since we're holding P_PR_LOCK for this process.
 562                  */
 563                 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
 564 
 565                 switch (id->fti_ptype) {
 566                 case DTFTP_ENTRY:
 567                 case DTFTP_OFFSETS:
 568                 case DTFTP_IS_ENABLED:
 569                         id->fti_next = tp->ftt_ids;
 570                         membar_producer();
 571                         tp->ftt_ids = id;
 572                         membar_producer();
 573                         break;
 574 
 575                 case DTFTP_RETURN:
 576                 case DTFTP_POST_OFFSETS:
 577                         id->fti_next = tp->ftt_retids;
 578                         membar_producer();
 579                         tp->ftt_retids = id;
 580                         membar_producer();
 581                         break;
 582 
 583                 default:
 584                         ASSERT(0);
 585                 }
 586 
 587                 mutex_exit(&bucket->ftb_mtx);
 588 
 589                 if (new_tp != NULL) {
 590                         new_tp->ftt_ids = NULL;
 591                         new_tp->ftt_retids = NULL;
 592                 }
 593 
 594                 return (0);
 595         }
 596 
 597         /*
 598          * If we have a good tracepoint ready to go, install it now while
 599          * we have the lock held and no one can screw with us.
 600          */
 601         if (new_tp != NULL) {
 602                 int rc = 0;
 603 
 604                 new_tp->ftt_next = bucket->ftb_data;
 605                 membar_producer();
 606                 bucket->ftb_data = new_tp;
 607                 membar_producer();
 608                 mutex_exit(&bucket->ftb_mtx);
 609 
 610                 /*
 611                  * Activate the tracepoint in the ISA-specific manner.
 612                  * If this fails, we need to report the failure, but
 613                  * indicate that this tracepoint must still be disabled
 614                  * by calling fasttrap_tracepoint_disable().
 615                  */
 616                 if (fasttrap_tracepoint_install(p, new_tp) != 0)
 617                         rc = FASTTRAP_ENABLE_PARTIAL;
 618 
 619                 /*
 620                  * Increment the count of the number of tracepoints active in
 621                  * the victim process.
 622                  */
 623                 ASSERT(p->p_proc_flag & P_PR_LOCK);
 624                 p->p_dtrace_count++;
 625 
 626                 return (rc);
 627         }
 628 
 629         mutex_exit(&bucket->ftb_mtx);
 630 
 631         /*
 632          * Initialize the tracepoint that's been preallocated with the probe.
 633          */
 634         new_tp = probe->ftp_tps[index].fit_tp;
 635 
 636         ASSERT(new_tp->ftt_pid == pid);
 637         ASSERT(new_tp->ftt_pc == pc);
 638         ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
 639         ASSERT(new_tp->ftt_ids == NULL);
 640         ASSERT(new_tp->ftt_retids == NULL);
 641 
 642         switch (id->fti_ptype) {
 643         case DTFTP_ENTRY:
 644         case DTFTP_OFFSETS:
 645         case DTFTP_IS_ENABLED:
 646                 id->fti_next = NULL;
 647                 new_tp->ftt_ids = id;
 648                 break;
 649 
 650         case DTFTP_RETURN:
 651         case DTFTP_POST_OFFSETS:
 652                 id->fti_next = NULL;
 653                 new_tp->ftt_retids = id;
 654                 break;
 655 
 656         default:
 657                 ASSERT(0);
 658         }
 659 
 660         /*
 661          * If the ISA-dependent initialization goes to plan, go back to the
 662          * beginning and try to install this freshly made tracepoint.
 663          */
 664         if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
 665                 goto again;
 666 
 667         new_tp->ftt_ids = NULL;
 668         new_tp->ftt_retids = NULL;
 669 
 670         return (FASTTRAP_ENABLE_FAIL);
 671 }
 672 
 673 static void
 674 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
 675 {
 676         fasttrap_bucket_t *bucket;
 677         fasttrap_provider_t *provider = probe->ftp_prov;
 678         fasttrap_tracepoint_t **pp, *tp;
 679         fasttrap_id_t *id, **idp;
 680         pid_t pid;
 681         uintptr_t pc;
 682 
 683         ASSERT(index < probe->ftp_ntps);
 684 
 685         pid = probe->ftp_pid;
 686         pc = probe->ftp_tps[index].fit_tp->ftt_pc;
 687         id = &probe->ftp_tps[index].fit_id;
 688 
 689         ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
 690 
 691         /*
 692          * Find the tracepoint and make sure that our id is one of the
 693          * ones registered with it.
 694          */
 695         bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
 696         mutex_enter(&bucket->ftb_mtx);
 697         for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
 698                 if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
 699                     tp->ftt_proc == provider->ftp_proc)
 700                         break;
 701         }
 702 
 703         /*
 704          * If we somehow lost this tracepoint, we're in a world of hurt.
 705          */
 706         ASSERT(tp != NULL);
 707 
 708         switch (id->fti_ptype) {
 709         case DTFTP_ENTRY:
 710         case DTFTP_OFFSETS:
 711         case DTFTP_IS_ENABLED:
 712                 ASSERT(tp->ftt_ids != NULL);
 713                 idp = &tp->ftt_ids;
 714                 break;
 715 
 716         case DTFTP_RETURN:
 717         case DTFTP_POST_OFFSETS:
 718                 ASSERT(tp->ftt_retids != NULL);
 719                 idp = &tp->ftt_retids;
 720                 break;
 721 
 722         default:
 723                 ASSERT(0);
 724         }
 725 
 726         while ((*idp)->fti_probe != probe) {
 727                 idp = &(*idp)->fti_next;
 728                 ASSERT(*idp != NULL);
 729         }
 730 
 731         id = *idp;
 732         *idp = id->fti_next;
 733         membar_producer();
 734 
 735         ASSERT(id->fti_probe == probe);
 736 
 737         /*
 738          * If there are other registered enablings of this tracepoint, we're
 739          * all done, but if this was the last probe assocated with this
 740          * this tracepoint, we need to remove and free it.
 741          */
 742         if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
 743 
 744                 /*
 745                  * If the current probe's tracepoint is in use, swap it
 746                  * for an unused tracepoint.
 747                  */
 748                 if (tp == probe->ftp_tps[index].fit_tp) {
 749                         fasttrap_probe_t *tmp_probe;
 750                         fasttrap_tracepoint_t **tmp_tp;
 751                         uint_t tmp_index;
 752 
 753                         if (tp->ftt_ids != NULL) {
 754                                 tmp_probe = tp->ftt_ids->fti_probe;
 755                                 /* LINTED - alignment */
 756                                 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
 757                                 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
 758                         } else {
 759                                 tmp_probe = tp->ftt_retids->fti_probe;
 760                                 /* LINTED - alignment */
 761                                 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
 762                                 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
 763                         }
 764 
 765                         ASSERT(*tmp_tp != NULL);
 766                         ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
 767                         ASSERT((*tmp_tp)->ftt_ids == NULL);
 768                         ASSERT((*tmp_tp)->ftt_retids == NULL);
 769 
 770                         probe->ftp_tps[index].fit_tp = *tmp_tp;
 771                         *tmp_tp = tp;
 772                 }
 773 
 774                 mutex_exit(&bucket->ftb_mtx);
 775 
 776                 /*
 777                  * Tag the modified probe with the generation in which it was
 778                  * changed.
 779                  */
 780                 probe->ftp_gen = fasttrap_mod_gen;
 781                 return;
 782         }
 783 
 784         mutex_exit(&bucket->ftb_mtx);
 785 
 786         /*
 787          * We can't safely remove the tracepoint from the set of active
 788          * tracepoints until we've actually removed the fasttrap instruction
 789          * from the process's text. We can, however, operate on this
 790          * tracepoint secure in the knowledge that no other thread is going to
 791          * be looking at it since we hold P_PR_LOCK on the process if it's
 792          * live or we hold the provider lock on the process if it's dead and
 793          * gone.
 794          */
 795 
 796         /*
 797          * We only need to remove the actual instruction if we're looking
 798          * at an existing process
 799          */
 800         if (p != NULL) {
 801                 /*
 802                  * If we fail to restore the instruction we need to kill
 803                  * this process since it's in a completely unrecoverable
 804                  * state.
 805                  */
 806                 if (fasttrap_tracepoint_remove(p, tp) != 0)
 807                         fasttrap_sigtrap(p, NULL, pc);
 808 
 809                 /*
 810                  * Decrement the count of the number of tracepoints active
 811                  * in the victim process.
 812                  */
 813                 ASSERT(p->p_proc_flag & P_PR_LOCK);
 814                 p->p_dtrace_count--;
 815         }
 816 
 817         /*
 818          * Remove the probe from the hash table of active tracepoints.
 819          */
 820         mutex_enter(&bucket->ftb_mtx);
 821         pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
 822         ASSERT(*pp != NULL);
 823         while (*pp != tp) {
 824                 pp = &(*pp)->ftt_next;
 825                 ASSERT(*pp != NULL);
 826         }
 827 
 828         *pp = tp->ftt_next;
 829         membar_producer();
 830 
 831         mutex_exit(&bucket->ftb_mtx);
 832 
 833         /*
 834          * Tag the modified probe with the generation in which it was changed.
 835          */
 836         probe->ftp_gen = fasttrap_mod_gen;
 837 }
 838 
 839 static void
 840 fasttrap_enable_callbacks(void)
 841 {
 842         /*
 843          * We don't have to play the rw lock game here because we're
 844          * providing something rather than taking something away --
 845          * we can be sure that no threads have tried to follow this
 846          * function pointer yet.
 847          */
 848         mutex_enter(&fasttrap_count_mtx);
 849         if (fasttrap_pid_count == 0) {
 850                 ASSERT(dtrace_pid_probe_ptr == NULL);
 851                 ASSERT(dtrace_return_probe_ptr == NULL);
 852                 dtrace_pid_probe_ptr = &fasttrap_pid_probe;
 853                 dtrace_return_probe_ptr = &fasttrap_return_probe;
 854         }
 855         ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
 856         ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
 857         fasttrap_pid_count++;
 858         mutex_exit(&fasttrap_count_mtx);
 859 }
 860 
 861 static void
 862 fasttrap_disable_callbacks(void)
 863 {
 864         ASSERT(MUTEX_HELD(&cpu_lock));
 865 
 866         mutex_enter(&fasttrap_count_mtx);
 867         ASSERT(fasttrap_pid_count > 0);
 868         fasttrap_pid_count--;
 869         if (fasttrap_pid_count == 0) {
 870                 cpu_t *cur, *cpu = CPU;
 871 
 872                 for (cur = cpu->cpu_next_onln; cur != cpu;
 873                     cur = cur->cpu_next_onln) {
 874                         rw_enter(&cur->cpu_ft_lock, RW_WRITER);
 875                 }
 876 
 877                 dtrace_pid_probe_ptr = NULL;
 878                 dtrace_return_probe_ptr = NULL;
 879 
 880                 for (cur = cpu->cpu_next_onln; cur != cpu;
 881                     cur = cur->cpu_next_onln) {
 882                         rw_exit(&cur->cpu_ft_lock);
 883                 }
 884         }
 885         mutex_exit(&fasttrap_count_mtx);
 886 }
 887 
 888 /*ARGSUSED*/
 889 static int
 890 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
 891 {
 892         fasttrap_probe_t *probe = parg;
 893         proc_t *p;
 894         int i, rc;
 895 
 896         ASSERT(probe != NULL);
 897         ASSERT(!probe->ftp_enabled);
 898         ASSERT(id == probe->ftp_id);
 899         ASSERT(MUTEX_HELD(&cpu_lock));
 900 
 901         /*
 902          * Increment the count of enabled probes on this probe's provider;
 903          * the provider can't go away while the probe still exists. We
 904          * must increment this even if we aren't able to properly enable
 905          * this probe.
 906          */
 907         mutex_enter(&probe->ftp_prov->ftp_mtx);
 908         probe->ftp_prov->ftp_rcount++;
 909         mutex_exit(&probe->ftp_prov->ftp_mtx);
 910 
 911         /*
 912          * If this probe's provider is retired (meaning it was valid in a
 913          * previously exec'ed incarnation of this address space), bail out. The
 914          * provider can't go away while we're in this code path.
 915          */
 916         if (probe->ftp_prov->ftp_retired)
 917                 return (0);
 918 
 919         /*
 920          * If we can't find the process, it may be that we're in the context of
 921          * a fork in which the traced process is being born and we're copying
 922          * USDT probes. Otherwise, the process is gone so bail.
 923          */
 924         if ((p = sprlock(probe->ftp_pid)) == NULL) {
 925                 if ((curproc->p_flag & SFORKING) == 0)
 926                         return (0);
 927 
 928                 mutex_enter(&pidlock);
 929                 p = prfind(probe->ftp_pid);
 930 
 931                 /*
 932                  * Confirm that curproc is indeed forking the process in which
 933                  * we're trying to enable probes.
 934                  */
 935                 ASSERT(p != NULL);
 936                 ASSERT(p->p_parent == curproc);
 937                 ASSERT(p->p_stat == SIDL);
 938 
 939                 mutex_enter(&p->p_lock);
 940                 mutex_exit(&pidlock);
 941 
 942                 sprlock_proc(p);
 943         }
 944 
 945         ASSERT(!(p->p_flag & SVFORK));
 946         mutex_exit(&p->p_lock);
 947 
 948         /*
 949          * We have to enable the trap entry point before any user threads have
 950          * the chance to execute the trap instruction we're about to place
 951          * in their process's text.
 952          */
 953         fasttrap_enable_callbacks();
 954 
 955         /*
 956          * Enable all the tracepoints and add this probe's id to each
 957          * tracepoint's list of active probes.
 958          */
 959         for (i = 0; i < probe->ftp_ntps; i++) {
 960                 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
 961                         /*
 962                          * If enabling the tracepoint failed completely,
 963                          * we don't have to disable it; if the failure
 964                          * was only partial we must disable it.
 965                          */
 966                         if (rc == FASTTRAP_ENABLE_FAIL)
 967                                 i--;
 968                         else
 969                                 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
 970 
 971                         /*
 972                          * Back up and pull out all the tracepoints we've
 973                          * created so far for this probe.
 974                          */
 975                         while (i >= 0) {
 976                                 fasttrap_tracepoint_disable(p, probe, i);
 977                                 i--;
 978                         }
 979 
 980                         mutex_enter(&p->p_lock);
 981                         sprunlock(p);
 982 
 983                         /*
 984                          * Since we're not actually enabling this probe,
 985                          * drop our reference on the trap table entry.
 986                          */
 987                         fasttrap_disable_callbacks();
 988                         return (0);
 989                 }
 990         }
 991 
 992         mutex_enter(&p->p_lock);
 993         sprunlock(p);
 994 
 995         probe->ftp_enabled = 1;
 996         return (0);
 997 }
 998 
 999 /*ARGSUSED*/
1000 static void
1001 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1002 {
1003         fasttrap_probe_t *probe = parg;
1004         fasttrap_provider_t *provider = probe->ftp_prov;
1005         proc_t *p;
1006         int i, whack = 0;
1007 
1008         ASSERT(id == probe->ftp_id);
1009 
1010         /*
1011          * We won't be able to acquire a /proc-esque lock on the process
1012          * iff the process is dead and gone. In this case, we rely on the
1013          * provider lock as a point of mutual exclusion to prevent other
1014          * DTrace consumers from disabling this probe.
1015          */
1016         if ((p = sprlock(probe->ftp_pid)) != NULL) {
1017                 ASSERT(!(p->p_flag & SVFORK));
1018                 mutex_exit(&p->p_lock);
1019         }
1020 
1021         mutex_enter(&provider->ftp_mtx);
1022 
1023         /*
1024          * Disable all the associated tracepoints (for fully enabled probes).
1025          */
1026         if (probe->ftp_enabled) {
1027                 for (i = 0; i < probe->ftp_ntps; i++) {
1028                         fasttrap_tracepoint_disable(p, probe, i);
1029                 }
1030         }
1031 
1032         ASSERT(provider->ftp_rcount > 0);
1033         provider->ftp_rcount--;
1034 
1035         if (p != NULL) {
1036                 /*
1037                  * Even though we may not be able to remove it entirely, we
1038                  * mark this retired provider to get a chance to remove some
1039                  * of the associated probes.
1040                  */
1041                 if (provider->ftp_retired && !provider->ftp_marked)
1042                         whack = provider->ftp_marked = 1;
1043                 mutex_exit(&provider->ftp_mtx);
1044 
1045                 mutex_enter(&p->p_lock);
1046                 sprunlock(p);
1047         } else {
1048                 /*
1049                  * If the process is dead, we're just waiting for the
1050                  * last probe to be disabled to be able to free it.
1051                  */
1052                 if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1053                         whack = provider->ftp_marked = 1;
1054                 mutex_exit(&provider->ftp_mtx);
1055         }
1056 
1057         if (whack)
1058                 fasttrap_pid_cleanup();
1059 
1060         if (!probe->ftp_enabled)
1061                 return;
1062 
1063         probe->ftp_enabled = 0;
1064 
1065         ASSERT(MUTEX_HELD(&cpu_lock));
1066         fasttrap_disable_callbacks();
1067 }
1068 
1069 /*ARGSUSED*/
1070 static void
1071 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1072     dtrace_argdesc_t *desc)
1073 {
1074         fasttrap_probe_t *probe = parg;
1075         char *str;
1076         int i, ndx;
1077 
1078         desc->dtargd_native[0] = '\0';
1079         desc->dtargd_xlate[0] = '\0';
1080 
1081         if (probe->ftp_prov->ftp_retired != 0 ||
1082             desc->dtargd_ndx >= probe->ftp_nargs) {
1083                 desc->dtargd_ndx = DTRACE_ARGNONE;
1084                 return;
1085         }
1086 
1087         ndx = (probe->ftp_argmap != NULL) ?
1088             probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1089 
1090         str = probe->ftp_ntypes;
1091         for (i = 0; i < ndx; i++) {
1092                 str += strlen(str) + 1;
1093         }
1094 
1095         ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1096         (void) strcpy(desc->dtargd_native, str);
1097 
1098         if (probe->ftp_xtypes == NULL)
1099                 return;
1100 
1101         str = probe->ftp_xtypes;
1102         for (i = 0; i < desc->dtargd_ndx; i++) {
1103                 str += strlen(str) + 1;
1104         }
1105 
1106         ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1107         (void) strcpy(desc->dtargd_xlate, str);
1108 }
1109 
1110 /*ARGSUSED*/
1111 static void
1112 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1113 {
1114         fasttrap_probe_t *probe = parg;
1115         int i;
1116         size_t size;
1117 
1118         ASSERT(probe != NULL);
1119         ASSERT(!probe->ftp_enabled);
1120         ASSERT(fasttrap_total >= probe->ftp_ntps);
1121 
1122         atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1123         size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1124 
1125         if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1126                 fasttrap_mod_barrier(probe->ftp_gen);
1127 
1128         for (i = 0; i < probe->ftp_ntps; i++) {
1129                 kmem_free(probe->ftp_tps[i].fit_tp,
1130                     sizeof (fasttrap_tracepoint_t));
1131         }
1132 
1133         kmem_free(probe, size);
1134 }
1135 
1136 
1137 static const dtrace_pattr_t pid_attr = {
1138 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1139 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1140 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1141 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1142 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1143 };
1144 
1145 static dtrace_pops_t pid_pops = {
1146         fasttrap_pid_provide,
1147         NULL,
1148         fasttrap_pid_enable,
1149         fasttrap_pid_disable,
1150         NULL,
1151         NULL,
1152         fasttrap_pid_getargdesc,
1153         fasttrap_pid_getarg,
1154         NULL,
1155         fasttrap_pid_destroy
1156 };
1157 
1158 static dtrace_pops_t usdt_pops = {
1159         fasttrap_pid_provide,
1160         NULL,
1161         fasttrap_pid_enable,
1162         fasttrap_pid_disable,
1163         NULL,
1164         NULL,
1165         fasttrap_pid_getargdesc,
1166         fasttrap_usdt_getarg,
1167         NULL,
1168         fasttrap_pid_destroy
1169 };
1170 
1171 static fasttrap_proc_t *
1172 fasttrap_proc_lookup(pid_t pid)
1173 {
1174         fasttrap_bucket_t *bucket;
1175         fasttrap_proc_t *fprc, *new_fprc;
1176 
1177         bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1178         mutex_enter(&bucket->ftb_mtx);
1179 
1180         for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1181                 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1182                         mutex_enter(&fprc->ftpc_mtx);
1183                         mutex_exit(&bucket->ftb_mtx);
1184                         fprc->ftpc_rcount++;
1185                         atomic_inc_64(&fprc->ftpc_acount);
1186                         ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1187                         mutex_exit(&fprc->ftpc_mtx);
1188 
1189                         return (fprc);
1190                 }
1191         }
1192 
1193         /*
1194          * Drop the bucket lock so we don't try to perform a sleeping
1195          * allocation under it.
1196          */
1197         mutex_exit(&bucket->ftb_mtx);
1198 
1199         new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1200         new_fprc->ftpc_pid = pid;
1201         new_fprc->ftpc_rcount = 1;
1202         new_fprc->ftpc_acount = 1;
1203 
1204         mutex_enter(&bucket->ftb_mtx);
1205 
1206         /*
1207          * Take another lap through the list to make sure a proc hasn't
1208          * been created for this pid while we weren't under the bucket lock.
1209          */
1210         for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1211                 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1212                         mutex_enter(&fprc->ftpc_mtx);
1213                         mutex_exit(&bucket->ftb_mtx);
1214                         fprc->ftpc_rcount++;
1215                         atomic_inc_64(&fprc->ftpc_acount);
1216                         ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1217                         mutex_exit(&fprc->ftpc_mtx);
1218 
1219                         kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1220 
1221                         return (fprc);
1222                 }
1223         }
1224 
1225         new_fprc->ftpc_next = bucket->ftb_data;
1226         bucket->ftb_data = new_fprc;
1227 
1228         mutex_exit(&bucket->ftb_mtx);
1229 
1230         return (new_fprc);
1231 }
1232 
1233 static void
1234 fasttrap_proc_release(fasttrap_proc_t *proc)
1235 {
1236         fasttrap_bucket_t *bucket;
1237         fasttrap_proc_t *fprc, **fprcp;
1238         pid_t pid = proc->ftpc_pid;
1239 
1240         mutex_enter(&proc->ftpc_mtx);
1241 
1242         ASSERT(proc->ftpc_rcount != 0);
1243         ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1244 
1245         if (--proc->ftpc_rcount != 0) {
1246                 mutex_exit(&proc->ftpc_mtx);
1247                 return;
1248         }
1249 
1250         mutex_exit(&proc->ftpc_mtx);
1251 
1252         /*
1253          * There should definitely be no live providers associated with this
1254          * process at this point.
1255          */
1256         ASSERT(proc->ftpc_acount == 0);
1257 
1258         bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1259         mutex_enter(&bucket->ftb_mtx);
1260 
1261         fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1262         while ((fprc = *fprcp) != NULL) {
1263                 if (fprc == proc)
1264                         break;
1265 
1266                 fprcp = &fprc->ftpc_next;
1267         }
1268 
1269         /*
1270          * Something strange has happened if we can't find the proc.
1271          */
1272         ASSERT(fprc != NULL);
1273 
1274         *fprcp = fprc->ftpc_next;
1275 
1276         mutex_exit(&bucket->ftb_mtx);
1277 
1278         kmem_free(fprc, sizeof (fasttrap_proc_t));
1279 }
1280 
1281 /*
1282  * Lookup a fasttrap-managed provider based on its name and associated pid.
1283  * If the pattr argument is non-NULL, this function instantiates the provider
1284  * if it doesn't exist otherwise it returns NULL. The provider is returned
1285  * with its lock held.
1286  */
1287 static fasttrap_provider_t *
1288 fasttrap_provider_lookup(pid_t pid, const char *name,
1289     const dtrace_pattr_t *pattr)
1290 {
1291         fasttrap_provider_t *fp, *new_fp = NULL;
1292         fasttrap_bucket_t *bucket;
1293         char provname[DTRACE_PROVNAMELEN];
1294         proc_t *p;
1295         cred_t *cred;
1296 
1297         ASSERT(strlen(name) < sizeof (fp->ftp_name));
1298         ASSERT(pattr != NULL);
1299 
1300         bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1301         mutex_enter(&bucket->ftb_mtx);
1302 
1303         /*
1304          * Take a lap through the list and return the match if we find it.
1305          */
1306         for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1307                 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1308                     !fp->ftp_retired) {
1309                         mutex_enter(&fp->ftp_mtx);
1310                         mutex_exit(&bucket->ftb_mtx);
1311                         return (fp);
1312                 }
1313         }
1314 
1315         /*
1316          * Drop the bucket lock so we don't try to perform a sleeping
1317          * allocation under it.
1318          */
1319         mutex_exit(&bucket->ftb_mtx);
1320 
1321         /*
1322          * Make sure the process exists, isn't a child created as the result
1323          * of a vfork(2), and isn't a zombie (but may be in fork).
1324          */
1325         mutex_enter(&pidlock);
1326         if ((p = prfind(pid)) == NULL) {
1327                 mutex_exit(&pidlock);
1328                 return (NULL);
1329         }
1330         mutex_enter(&p->p_lock);
1331         mutex_exit(&pidlock);
1332         if (p->p_flag & (SVFORK | SEXITING)) {
1333                 mutex_exit(&p->p_lock);
1334                 return (NULL);
1335         }
1336 
1337         /*
1338          * Increment p_dtrace_probes so that the process knows to inform us
1339          * when it exits or execs. fasttrap_provider_free() decrements this
1340          * when we're done with this provider.
1341          */
1342         p->p_dtrace_probes++;
1343 
1344         /*
1345          * Grab the credentials for this process so we have
1346          * something to pass to dtrace_register().
1347          */
1348         mutex_enter(&p->p_crlock);
1349         crhold(p->p_cred);
1350         cred = p->p_cred;
1351         mutex_exit(&p->p_crlock);
1352         mutex_exit(&p->p_lock);
1353 
1354         new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1355         new_fp->ftp_pid = pid;
1356         new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1357 
1358         ASSERT(new_fp->ftp_proc != NULL);
1359 
1360         mutex_enter(&bucket->ftb_mtx);
1361 
1362         /*
1363          * Take another lap through the list to make sure a provider hasn't
1364          * been created for this pid while we weren't under the bucket lock.
1365          */
1366         for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1367                 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1368                     !fp->ftp_retired) {
1369                         mutex_enter(&fp->ftp_mtx);
1370                         mutex_exit(&bucket->ftb_mtx);
1371                         fasttrap_provider_free(new_fp);
1372                         crfree(cred);
1373                         return (fp);
1374                 }
1375         }
1376 
1377         (void) strcpy(new_fp->ftp_name, name);
1378 
1379         /*
1380          * Fail and return NULL if either the provider name is too long
1381          * or we fail to register this new provider with the DTrace
1382          * framework. Note that this is the only place we ever construct
1383          * the full provider name -- we keep it in pieces in the provider
1384          * structure.
1385          */
1386         if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1387             sizeof (provname) ||
1388             dtrace_register(provname, pattr,
1389             DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1390             pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1391             &new_fp->ftp_provid) != 0) {
1392                 mutex_exit(&bucket->ftb_mtx);
1393                 fasttrap_provider_free(new_fp);
1394                 crfree(cred);
1395                 return (NULL);
1396         }
1397 
1398         new_fp->ftp_next = bucket->ftb_data;
1399         bucket->ftb_data = new_fp;
1400 
1401         mutex_enter(&new_fp->ftp_mtx);
1402         mutex_exit(&bucket->ftb_mtx);
1403 
1404         crfree(cred);
1405         return (new_fp);
1406 }
1407 
1408 static void
1409 fasttrap_provider_free(fasttrap_provider_t *provider)
1410 {
1411         pid_t pid = provider->ftp_pid;
1412         proc_t *p;
1413 
1414         /*
1415          * There need to be no associated enabled probes, no consumers
1416          * creating probes, and no meta providers referencing this provider.
1417          */
1418         ASSERT(provider->ftp_rcount == 0);
1419         ASSERT(provider->ftp_ccount == 0);
1420         ASSERT(provider->ftp_mcount == 0);
1421 
1422         /*
1423          * If this provider hasn't been retired, we need to explicitly drop the
1424          * count of active providers on the associated process structure.
1425          */
1426         if (!provider->ftp_retired) {
1427                 atomic_dec_64(&provider->ftp_proc->ftpc_acount);
1428                 ASSERT(provider->ftp_proc->ftpc_acount <
1429                     provider->ftp_proc->ftpc_rcount);
1430         }
1431 
1432         fasttrap_proc_release(provider->ftp_proc);
1433 
1434         kmem_free(provider, sizeof (fasttrap_provider_t));
1435 
1436         /*
1437          * Decrement p_dtrace_probes on the process whose provider we're
1438          * freeing. We don't have to worry about clobbering somone else's
1439          * modifications to it because we have locked the bucket that
1440          * corresponds to this process's hash chain in the provider hash
1441          * table. Don't sweat it if we can't find the process.
1442          */
1443         mutex_enter(&pidlock);
1444         if ((p = prfind(pid)) == NULL) {
1445                 mutex_exit(&pidlock);
1446                 return;
1447         }
1448 
1449         mutex_enter(&p->p_lock);
1450         mutex_exit(&pidlock);
1451 
1452         p->p_dtrace_probes--;
1453         mutex_exit(&p->p_lock);
1454 }
1455 
1456 static void
1457 fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1458 {
1459         fasttrap_provider_t *fp;
1460         fasttrap_bucket_t *bucket;
1461         dtrace_provider_id_t provid;
1462 
1463         ASSERT(strlen(name) < sizeof (fp->ftp_name));
1464 
1465         bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1466         mutex_enter(&bucket->ftb_mtx);
1467 
1468         for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1469                 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1470                     !fp->ftp_retired)
1471                         break;
1472         }
1473 
1474         if (fp == NULL) {
1475                 mutex_exit(&bucket->ftb_mtx);
1476                 return;
1477         }
1478 
1479         mutex_enter(&fp->ftp_mtx);
1480         ASSERT(!mprov || fp->ftp_mcount > 0);
1481         if (mprov && --fp->ftp_mcount != 0)  {
1482                 mutex_exit(&fp->ftp_mtx);
1483                 mutex_exit(&bucket->ftb_mtx);
1484                 return;
1485         }
1486 
1487         /*
1488          * Mark the provider to be removed in our post-processing step, mark it
1489          * retired, and drop the active count on its proc. Marking it indicates
1490          * that we should try to remove it; setting the retired flag indicates
1491          * that we're done with this provider; dropping the active the proc
1492          * releases our hold, and when this reaches zero (as it will during
1493          * exit or exec) the proc and associated providers become defunct.
1494          *
1495          * We obviously need to take the bucket lock before the provider lock
1496          * to perform the lookup, but we need to drop the provider lock
1497          * before calling into the DTrace framework since we acquire the
1498          * provider lock in callbacks invoked from the DTrace framework. The
1499          * bucket lock therefore protects the integrity of the provider hash
1500          * table.
1501          */
1502         atomic_dec_64(&fp->ftp_proc->ftpc_acount);
1503         ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1504 
1505         fp->ftp_retired = 1;
1506         fp->ftp_marked = 1;
1507         provid = fp->ftp_provid;
1508         mutex_exit(&fp->ftp_mtx);
1509 
1510         /*
1511          * We don't have to worry about invalidating the same provider twice
1512          * since fasttrap_provider_lookup() will ignore provider that have
1513          * been marked as retired.
1514          */
1515         dtrace_invalidate(provid);
1516 
1517         mutex_exit(&bucket->ftb_mtx);
1518 
1519         fasttrap_pid_cleanup();
1520 }
1521 
1522 static int
1523 fasttrap_uint32_cmp(const void *ap, const void *bp)
1524 {
1525         return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1526 }
1527 
1528 static int
1529 fasttrap_uint64_cmp(const void *ap, const void *bp)
1530 {
1531         return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1532 }
1533 
1534 static int
1535 fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1536 {
1537         fasttrap_provider_t *provider;
1538         fasttrap_probe_t *pp;
1539         fasttrap_tracepoint_t *tp;
1540         char *name;
1541         int i, aframes, whack;
1542 
1543         /*
1544          * There needs to be at least one desired trace point.
1545          */
1546         if (pdata->ftps_noffs == 0)
1547                 return (EINVAL);
1548 
1549         switch (pdata->ftps_type) {
1550         case DTFTP_ENTRY:
1551                 name = "entry";
1552                 aframes = FASTTRAP_ENTRY_AFRAMES;
1553                 break;
1554         case DTFTP_RETURN:
1555                 name = "return";
1556                 aframes = FASTTRAP_RETURN_AFRAMES;
1557                 break;
1558         case DTFTP_OFFSETS:
1559                 name = NULL;
1560                 break;
1561         default:
1562                 return (EINVAL);
1563         }
1564 
1565         if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1566             FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1567                 return (ESRCH);
1568 
1569         /*
1570          * Increment this reference count to indicate that a consumer is
1571          * actively adding a new probe associated with this provider. This
1572          * prevents the provider from being deleted -- we'll need to check
1573          * for pending deletions when we drop this reference count.
1574          */
1575         provider->ftp_ccount++;
1576         mutex_exit(&provider->ftp_mtx);
1577 
1578         /*
1579          * Grab the creation lock to ensure consistency between calls to
1580          * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1581          * other threads creating probes. We must drop the provider lock
1582          * before taking this lock to avoid a three-way deadlock with the
1583          * DTrace framework.
1584          */
1585         mutex_enter(&provider->ftp_cmtx);
1586 
1587         if (name == NULL) {
1588                 for (i = 0; i < pdata->ftps_noffs; i++) {
1589                         char name_str[17];
1590 
1591                         (void) sprintf(name_str, "%llx",
1592                             (unsigned long long)pdata->ftps_offs[i]);
1593 
1594                         if (dtrace_probe_lookup(provider->ftp_provid,
1595                             pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1596                                 continue;
1597 
1598                         atomic_inc_32(&fasttrap_total);
1599 
1600                         if (fasttrap_total > fasttrap_max) {
1601                                 atomic_dec_32(&fasttrap_total);
1602                                 goto no_mem;
1603                         }
1604 
1605                         pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1606 
1607                         pp->ftp_prov = provider;
1608                         pp->ftp_faddr = pdata->ftps_pc;
1609                         pp->ftp_fsize = pdata->ftps_size;
1610                         pp->ftp_pid = pdata->ftps_pid;
1611                         pp->ftp_ntps = 1;
1612 
1613                         tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1614                             KM_SLEEP);
1615 
1616                         tp->ftt_proc = provider->ftp_proc;
1617                         tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1618                         tp->ftt_pid = pdata->ftps_pid;
1619 
1620                         pp->ftp_tps[0].fit_tp = tp;
1621                         pp->ftp_tps[0].fit_id.fti_probe = pp;
1622                         pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1623 
1624                         pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1625                             pdata->ftps_mod, pdata->ftps_func, name_str,
1626                             FASTTRAP_OFFSET_AFRAMES, pp);
1627                 }
1628 
1629         } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1630             pdata->ftps_func, name) == 0) {
1631                 atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1632 
1633                 if (fasttrap_total > fasttrap_max) {
1634                         atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1635                         goto no_mem;
1636                 }
1637 
1638                 /*
1639                  * Make sure all tracepoint program counter values are unique.
1640                  * We later assume that each probe has exactly one tracepoint
1641                  * for a given pc.
1642                  */
1643                 qsort(pdata->ftps_offs, pdata->ftps_noffs,
1644                     sizeof (uint64_t), fasttrap_uint64_cmp);
1645                 for (i = 1; i < pdata->ftps_noffs; i++) {
1646                         if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1647                                 continue;
1648 
1649                         atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1650                         goto no_mem;
1651                 }
1652 
1653                 ASSERT(pdata->ftps_noffs > 0);
1654                 pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1655                     ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1656 
1657                 pp->ftp_prov = provider;
1658                 pp->ftp_faddr = pdata->ftps_pc;
1659                 pp->ftp_fsize = pdata->ftps_size;
1660                 pp->ftp_pid = pdata->ftps_pid;
1661                 pp->ftp_ntps = pdata->ftps_noffs;
1662 
1663                 for (i = 0; i < pdata->ftps_noffs; i++) {
1664                         tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1665                             KM_SLEEP);
1666 
1667                         tp->ftt_proc = provider->ftp_proc;
1668                         tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1669                         tp->ftt_pid = pdata->ftps_pid;
1670 
1671                         pp->ftp_tps[i].fit_tp = tp;
1672                         pp->ftp_tps[i].fit_id.fti_probe = pp;
1673                         pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
1674                 }
1675 
1676                 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1677                     pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1678         }
1679 
1680         mutex_exit(&provider->ftp_cmtx);
1681 
1682         /*
1683          * We know that the provider is still valid since we incremented the
1684          * creation reference count. If someone tried to clean up this provider
1685          * while we were using it (e.g. because the process called exec(2) or
1686          * exit(2)), take note of that and try to clean it up now.
1687          */
1688         mutex_enter(&provider->ftp_mtx);
1689         provider->ftp_ccount--;
1690         whack = provider->ftp_retired;
1691         mutex_exit(&provider->ftp_mtx);
1692 
1693         if (whack)
1694                 fasttrap_pid_cleanup();
1695 
1696         return (0);
1697 
1698 no_mem:
1699         /*
1700          * If we've exhausted the allowable resources, we'll try to remove
1701          * this provider to free some up. This is to cover the case where
1702          * the user has accidentally created many more probes than was
1703          * intended (e.g. pid123:::).
1704          */
1705         mutex_exit(&provider->ftp_cmtx);
1706         mutex_enter(&provider->ftp_mtx);
1707         provider->ftp_ccount--;
1708         provider->ftp_marked = 1;
1709         mutex_exit(&provider->ftp_mtx);
1710 
1711         fasttrap_pid_cleanup();
1712 
1713         return (ENOMEM);
1714 }
1715 
1716 /*ARGSUSED*/
1717 static void *
1718 fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1719 {
1720         fasttrap_provider_t *provider;
1721 
1722         /*
1723          * A 32-bit unsigned integer (like a pid for example) can be
1724          * expressed in 10 or fewer decimal digits. Make sure that we'll
1725          * have enough space for the provider name.
1726          */
1727         if (strlen(dhpv->dthpv_provname) + 10 >=
1728             sizeof (provider->ftp_name)) {
1729                 cmn_err(CE_WARN, "failed to instantiate provider %s: "
1730                     "name too long to accomodate pid", dhpv->dthpv_provname);
1731                 return (NULL);
1732         }
1733 
1734         /*
1735          * Don't let folks spoof the true pid provider.
1736          */
1737         if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
1738                 cmn_err(CE_WARN, "failed to instantiate provider %s: "
1739                     "%s is an invalid name", dhpv->dthpv_provname,
1740                     FASTTRAP_PID_NAME);
1741                 return (NULL);
1742         }
1743 
1744         /*
1745          * The highest stability class that fasttrap supports is ISA; cap
1746          * the stability of the new provider accordingly.
1747          */
1748         if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
1749                 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
1750         if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
1751                 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
1752         if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
1753                 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
1754         if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
1755                 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
1756         if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
1757                 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
1758 
1759         if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
1760             &dhpv->dthpv_pattr)) == NULL) {
1761                 cmn_err(CE_WARN, "failed to instantiate provider %s for "
1762                     "process %u",  dhpv->dthpv_provname, (uint_t)pid);
1763                 return (NULL);
1764         }
1765 
1766         /*
1767          * Up the meta provider count so this provider isn't removed until
1768          * the meta provider has been told to remove it.
1769          */
1770         provider->ftp_mcount++;
1771 
1772         mutex_exit(&provider->ftp_mtx);
1773 
1774         return (provider);
1775 }
1776 
1777 /*ARGSUSED*/
1778 static void
1779 fasttrap_meta_create_probe(void *arg, void *parg,
1780     dtrace_helper_probedesc_t *dhpb)
1781 {
1782         fasttrap_provider_t *provider = parg;
1783         fasttrap_probe_t *pp;
1784         fasttrap_tracepoint_t *tp;
1785         int i, j;
1786         uint32_t ntps;
1787 
1788         /*
1789          * Since the meta provider count is non-zero we don't have to worry
1790          * about this provider disappearing.
1791          */
1792         ASSERT(provider->ftp_mcount > 0);
1793 
1794         /*
1795          * The offsets must be unique.
1796          */
1797         qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
1798             fasttrap_uint32_cmp);
1799         for (i = 1; i < dhpb->dthpb_noffs; i++) {
1800                 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
1801                     dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
1802                         return;
1803         }
1804 
1805         qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
1806             fasttrap_uint32_cmp);
1807         for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
1808                 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
1809                     dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
1810                         return;
1811         }
1812 
1813         /*
1814          * Grab the creation lock to ensure consistency between calls to
1815          * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1816          * other threads creating probes.
1817          */
1818         mutex_enter(&provider->ftp_cmtx);
1819 
1820         if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod,
1821             dhpb->dthpb_func, dhpb->dthpb_name) != 0) {
1822                 mutex_exit(&provider->ftp_cmtx);
1823                 return;
1824         }
1825 
1826         ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
1827         ASSERT(ntps > 0);
1828 
1829         atomic_add_32(&fasttrap_total, ntps);
1830 
1831         if (fasttrap_total > fasttrap_max) {
1832                 atomic_add_32(&fasttrap_total, -ntps);
1833                 mutex_exit(&provider->ftp_cmtx);
1834                 return;
1835         }
1836 
1837         pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
1838 
1839         pp->ftp_prov = provider;
1840         pp->ftp_pid = provider->ftp_pid;
1841         pp->ftp_ntps = ntps;
1842         pp->ftp_nargs = dhpb->dthpb_xargc;
1843         pp->ftp_xtypes = dhpb->dthpb_xtypes;
1844         pp->ftp_ntypes = dhpb->dthpb_ntypes;
1845 
1846         /*
1847          * First create a tracepoint for each actual point of interest.
1848          */
1849         for (i = 0; i < dhpb->dthpb_noffs; i++) {
1850                 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1851 
1852                 tp->ftt_proc = provider->ftp_proc;
1853                 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
1854                 tp->ftt_pid = provider->ftp_pid;
1855 
1856                 pp->ftp_tps[i].fit_tp = tp;
1857                 pp->ftp_tps[i].fit_id.fti_probe = pp;
1858 #ifdef __sparc
1859                 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
1860 #else
1861                 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
1862 #endif
1863         }
1864 
1865         /*
1866          * Then create a tracepoint for each is-enabled point.
1867          */
1868         for (j = 0; i < ntps; i++, j++) {
1869                 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1870 
1871                 tp->ftt_proc = provider->ftp_proc;
1872                 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
1873                 tp->ftt_pid = provider->ftp_pid;
1874 
1875                 pp->ftp_tps[i].fit_tp = tp;
1876                 pp->ftp_tps[i].fit_id.fti_probe = pp;
1877                 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
1878         }
1879 
1880         /*
1881          * If the arguments are shuffled around we set the argument remapping
1882          * table. Later, when the probe fires, we only remap the arguments
1883          * if the table is non-NULL.
1884          */
1885         for (i = 0; i < dhpb->dthpb_xargc; i++) {
1886                 if (dhpb->dthpb_args[i] != i) {
1887                         pp->ftp_argmap = dhpb->dthpb_args;
1888                         break;
1889                 }
1890         }
1891 
1892         /*
1893          * The probe is fully constructed -- register it with DTrace.
1894          */
1895         pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
1896             dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
1897 
1898         mutex_exit(&provider->ftp_cmtx);
1899 }
1900 
1901 /*ARGSUSED*/
1902 static void
1903 fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1904 {
1905         /*
1906          * Clean up the USDT provider. There may be active consumers of the
1907          * provider busy adding probes, no damage will actually befall the
1908          * provider until that count has dropped to zero. This just puts
1909          * the provider on death row.
1910          */
1911         fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
1912 }
1913 
1914 static dtrace_mops_t fasttrap_mops = {
1915         fasttrap_meta_create_probe,
1916         fasttrap_meta_provide,
1917         fasttrap_meta_remove
1918 };
1919 
1920 /*ARGSUSED*/
1921 static int
1922 fasttrap_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
1923 {
1924         return (0);
1925 }
1926 
1927 /*ARGSUSED*/
1928 static int
1929 fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
1930 {
1931         if (!dtrace_attached())
1932                 return (EAGAIN);
1933 
1934         if (cmd == FASTTRAPIOC_MAKEPROBE) {
1935                 fasttrap_probe_spec_t *uprobe = (void *)arg;
1936                 fasttrap_probe_spec_t *probe;
1937                 uint64_t noffs;
1938                 size_t size;
1939                 int ret, err;
1940 
1941                 if (copyin(&uprobe->ftps_noffs, &noffs,
1942                     sizeof (uprobe->ftps_noffs)))
1943                         return (EFAULT);
1944 
1945                 /*
1946                  * Probes must have at least one tracepoint.
1947                  */
1948                 if (noffs == 0)
1949                         return (EINVAL);
1950 
1951                 size = sizeof (fasttrap_probe_spec_t) +
1952                     sizeof (probe->ftps_offs[0]) * (noffs - 1);
1953 
1954                 if (size > 1024 * 1024)
1955                         return (ENOMEM);
1956 
1957                 probe = kmem_alloc(size, KM_SLEEP);
1958 
1959                 if (copyin(uprobe, probe, size) != 0 ||
1960                     probe->ftps_noffs != noffs) {
1961                         kmem_free(probe, size);
1962                         return (EFAULT);
1963                 }
1964 
1965                 /*
1966                  * Verify that the function and module strings contain no
1967                  * funny characters.
1968                  */
1969                 if (u8_validate(probe->ftps_func, strlen(probe->ftps_func),
1970                     NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
1971                         ret = EINVAL;
1972                         goto err;
1973                 }
1974 
1975                 if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod),
1976                     NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
1977                         ret = EINVAL;
1978                         goto err;
1979                 }
1980 
1981                 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
1982                         proc_t *p;
1983                         pid_t pid = probe->ftps_pid;
1984 
1985                         mutex_enter(&pidlock);
1986                         /*
1987                          * Report an error if the process doesn't exist
1988                          * or is actively being birthed.
1989                          */
1990                         if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
1991                                 mutex_exit(&pidlock);
1992                                 return (ESRCH);
1993                         }
1994                         mutex_enter(&p->p_lock);
1995                         mutex_exit(&pidlock);
1996 
1997                         if ((ret = priv_proc_cred_perm(cr, p, NULL,
1998                             VREAD | VWRITE)) != 0) {
1999                                 mutex_exit(&p->p_lock);
2000                                 return (ret);
2001                         }
2002 
2003                         mutex_exit(&p->p_lock);
2004                 }
2005 
2006                 ret = fasttrap_add_probe(probe);
2007 err:
2008                 kmem_free(probe, size);
2009 
2010                 return (ret);
2011 
2012         } else if (cmd == FASTTRAPIOC_GETINSTR) {
2013                 fasttrap_instr_query_t instr;
2014                 fasttrap_tracepoint_t *tp;
2015                 uint_t index;
2016                 int ret;
2017 
2018                 if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2019                         return (EFAULT);
2020 
2021                 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2022                         proc_t *p;
2023                         pid_t pid = instr.ftiq_pid;
2024 
2025                         mutex_enter(&pidlock);
2026                         /*
2027                          * Report an error if the process doesn't exist
2028                          * or is actively being birthed.
2029                          */
2030                         if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
2031                                 mutex_exit(&pidlock);
2032                                 return (ESRCH);
2033                         }
2034                         mutex_enter(&p->p_lock);
2035                         mutex_exit(&pidlock);
2036 
2037                         if ((ret = priv_proc_cred_perm(cr, p, NULL,
2038                             VREAD)) != 0) {
2039                                 mutex_exit(&p->p_lock);
2040                                 return (ret);
2041                         }
2042 
2043                         mutex_exit(&p->p_lock);
2044                 }
2045 
2046                 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2047 
2048                 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2049                 tp = fasttrap_tpoints.fth_table[index].ftb_data;
2050                 while (tp != NULL) {
2051                         if (instr.ftiq_pid == tp->ftt_pid &&
2052                             instr.ftiq_pc == tp->ftt_pc &&
2053                             tp->ftt_proc->ftpc_acount != 0)
2054                                 break;
2055 
2056                         tp = tp->ftt_next;
2057                 }
2058 
2059                 if (tp == NULL) {
2060                         mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2061                         return (ENOENT);
2062                 }
2063 
2064                 bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2065                     sizeof (instr.ftiq_instr));
2066                 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2067 
2068                 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2069                         return (EFAULT);
2070 
2071                 return (0);
2072         }
2073 
2074         return (EINVAL);
2075 }
2076 
2077 static struct cb_ops fasttrap_cb_ops = {
2078         fasttrap_open,          /* open */
2079         nodev,                  /* close */
2080         nulldev,                /* strategy */
2081         nulldev,                /* print */
2082         nodev,                  /* dump */
2083         nodev,                  /* read */
2084         nodev,                  /* write */
2085         fasttrap_ioctl,         /* ioctl */
2086         nodev,                  /* devmap */
2087         nodev,                  /* mmap */
2088         nodev,                  /* segmap */
2089         nochpoll,               /* poll */
2090         ddi_prop_op,            /* cb_prop_op */
2091         0,                      /* streamtab  */
2092         D_NEW | D_MP            /* Driver compatibility flag */
2093 };
2094 
2095 /*ARGSUSED*/
2096 static int
2097 fasttrap_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
2098 {
2099         int error;
2100 
2101         switch (infocmd) {
2102         case DDI_INFO_DEVT2DEVINFO:
2103                 *result = (void *)fasttrap_devi;
2104                 error = DDI_SUCCESS;
2105                 break;
2106         case DDI_INFO_DEVT2INSTANCE:
2107                 *result = (void *)0;
2108                 error = DDI_SUCCESS;
2109                 break;
2110         default:
2111                 error = DDI_FAILURE;
2112         }
2113         return (error);
2114 }
2115 
2116 static int
2117 fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
2118 {
2119         ulong_t nent;
2120 
2121         switch (cmd) {
2122         case DDI_ATTACH:
2123                 break;
2124         case DDI_RESUME:
2125                 return (DDI_SUCCESS);
2126         default:
2127                 return (DDI_FAILURE);
2128         }
2129 
2130         if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0,
2131             DDI_PSEUDO, NULL) == DDI_FAILURE) {
2132                 ddi_remove_minor_node(devi, NULL);
2133                 return (DDI_FAILURE);
2134         }
2135 
2136         ddi_report_dev(devi);
2137         fasttrap_devi = devi;
2138 
2139         /*
2140          * Install our hooks into fork(2), exec(2), and exit(2).
2141          */
2142         dtrace_fasttrap_fork_ptr = &fasttrap_fork;
2143         dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit;
2144         dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit;
2145 
2146         fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2147             "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2148         fasttrap_total = 0;
2149 
2150         /*
2151          * Conjure up the tracepoints hashtable...
2152          */
2153         nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2154             "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2155 
2156         if (nent == 0 || nent > 0x1000000)
2157                 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2158 
2159         if (ISP2(nent))
2160                 fasttrap_tpoints.fth_nent = nent;
2161         else
2162                 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2163         ASSERT(fasttrap_tpoints.fth_nent > 0);
2164         fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2165         fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2166             sizeof (fasttrap_bucket_t), KM_SLEEP);
2167 
2168         /*
2169          * ... and the providers hash table...
2170          */
2171         nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2172         if (ISP2(nent))
2173                 fasttrap_provs.fth_nent = nent;
2174         else
2175                 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2176         ASSERT(fasttrap_provs.fth_nent > 0);
2177         fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2178         fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2179             sizeof (fasttrap_bucket_t), KM_SLEEP);
2180 
2181         /*
2182          * ... and the procs hash table.
2183          */
2184         nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2185         if (ISP2(nent))
2186                 fasttrap_procs.fth_nent = nent;
2187         else
2188                 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2189         ASSERT(fasttrap_procs.fth_nent > 0);
2190         fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2191         fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2192             sizeof (fasttrap_bucket_t), KM_SLEEP);
2193 
2194         (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2195             &fasttrap_meta_id);
2196 
2197         return (DDI_SUCCESS);
2198 }
2199 
2200 static int
2201 fasttrap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
2202 {
2203         int i, fail = 0;
2204         timeout_id_t tmp;
2205 
2206         switch (cmd) {
2207         case DDI_DETACH:
2208                 break;
2209         case DDI_SUSPEND:
2210                 return (DDI_SUCCESS);
2211         default:
2212                 return (DDI_FAILURE);
2213         }
2214 
2215         /*
2216          * Unregister the meta-provider to make sure no new fasttrap-
2217          * managed providers come along while we're trying to close up
2218          * shop. If we fail to detach, we'll need to re-register as a
2219          * meta-provider. We can fail to unregister as a meta-provider
2220          * if providers we manage still exist.
2221          */
2222         if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2223             dtrace_meta_unregister(fasttrap_meta_id) != 0)
2224                 return (DDI_FAILURE);
2225 
2226         /*
2227          * Prevent any new timeouts from running by setting fasttrap_timeout
2228          * to a non-zero value, and wait for the current timeout to complete.
2229          */
2230         mutex_enter(&fasttrap_cleanup_mtx);
2231         fasttrap_cleanup_work = 0;
2232 
2233         while (fasttrap_timeout != (timeout_id_t)1) {
2234                 tmp = fasttrap_timeout;
2235                 fasttrap_timeout = (timeout_id_t)1;
2236 
2237                 if (tmp != 0) {
2238                         mutex_exit(&fasttrap_cleanup_mtx);
2239                         (void) untimeout(tmp);
2240                         mutex_enter(&fasttrap_cleanup_mtx);
2241                 }
2242         }
2243 
2244         fasttrap_cleanup_work = 0;
2245         mutex_exit(&fasttrap_cleanup_mtx);
2246 
2247         /*
2248          * Iterate over all of our providers. If there's still a process
2249          * that corresponds to that pid, fail to detach.
2250          */
2251         for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2252                 fasttrap_provider_t **fpp, *fp;
2253                 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2254 
2255                 mutex_enter(&bucket->ftb_mtx);
2256                 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2257                 while ((fp = *fpp) != NULL) {
2258                         /*
2259                          * Acquire and release the lock as a simple way of
2260                          * waiting for any other consumer to finish with
2261                          * this provider. A thread must first acquire the
2262                          * bucket lock so there's no chance of another thread
2263                          * blocking on the provider's lock.
2264                          */
2265                         mutex_enter(&fp->ftp_mtx);
2266                         mutex_exit(&fp->ftp_mtx);
2267 
2268                         if (dtrace_unregister(fp->ftp_provid) != 0) {
2269                                 fail = 1;
2270                                 fpp = &fp->ftp_next;
2271                         } else {
2272                                 *fpp = fp->ftp_next;
2273                                 fasttrap_provider_free(fp);
2274                         }
2275                 }
2276 
2277                 mutex_exit(&bucket->ftb_mtx);
2278         }
2279 
2280         if (fail) {
2281                 uint_t work;
2282                 /*
2283                  * If we're failing to detach, we need to unblock timeouts
2284                  * and start a new timeout if any work has accumulated while
2285                  * we've been unsuccessfully trying to detach.
2286                  */
2287                 mutex_enter(&fasttrap_cleanup_mtx);
2288                 fasttrap_timeout = 0;
2289                 work = fasttrap_cleanup_work;
2290                 mutex_exit(&fasttrap_cleanup_mtx);
2291 
2292                 if (work)
2293                         fasttrap_pid_cleanup();
2294 
2295                 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2296                     &fasttrap_meta_id);
2297 
2298                 return (DDI_FAILURE);
2299         }
2300 
2301 #ifdef DEBUG
2302         mutex_enter(&fasttrap_count_mtx);
2303         ASSERT(fasttrap_pid_count == 0);
2304         mutex_exit(&fasttrap_count_mtx);
2305 #endif
2306 
2307         kmem_free(fasttrap_tpoints.fth_table,
2308             fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2309         fasttrap_tpoints.fth_nent = 0;
2310 
2311         kmem_free(fasttrap_provs.fth_table,
2312             fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2313         fasttrap_provs.fth_nent = 0;
2314 
2315         kmem_free(fasttrap_procs.fth_table,
2316             fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2317         fasttrap_procs.fth_nent = 0;
2318 
2319         /*
2320          * We know there are no tracepoints in any process anywhere in
2321          * the system so there is no process which has its p_dtrace_count
2322          * greater than zero, therefore we know that no thread can actively
2323          * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes
2324          * and fasttrap_exec() and fasttrap_exit().
2325          */
2326         ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork);
2327         dtrace_fasttrap_fork_ptr = NULL;
2328 
2329         ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit);
2330         dtrace_fasttrap_exec_ptr = NULL;
2331 
2332         ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit);
2333         dtrace_fasttrap_exit_ptr = NULL;
2334 
2335         ddi_remove_minor_node(devi, NULL);
2336 
2337         return (DDI_SUCCESS);
2338 }
2339 
2340 static struct dev_ops fasttrap_ops = {
2341         DEVO_REV,               /* devo_rev */
2342         0,                      /* refcnt */
2343         fasttrap_info,          /* get_dev_info */
2344         nulldev,                /* identify */
2345         nulldev,                /* probe */
2346         fasttrap_attach,        /* attach */
2347         fasttrap_detach,        /* detach */
2348         nodev,                  /* reset */
2349         &fasttrap_cb_ops,   /* driver operations */
2350         NULL,                   /* bus operations */
2351         nodev,                  /* dev power */
2352         ddi_quiesce_not_needed,         /* quiesce */
2353 };
2354 
2355 /*
2356  * Module linkage information for the kernel.
2357  */
2358 static struct modldrv modldrv = {
2359         &mod_driverops,             /* module type (this is a pseudo driver) */
2360         "Fasttrap Tracing",     /* name of module */
2361         &fasttrap_ops,              /* driver ops */
2362 };
2363 
2364 static struct modlinkage modlinkage = {
2365         MODREV_1,
2366         (void *)&modldrv,
2367         NULL
2368 };
2369 
2370 int
2371 _init(void)
2372 {
2373         return (mod_install(&modlinkage));
2374 }
2375 
2376 int
2377 _info(struct modinfo *modinfop)
2378 {
2379         return (mod_info(&modlinkage, modinfop));
2380 }
2381 
2382 int
2383 _fini(void)
2384 {
2385         return (mod_remove(&modlinkage));
2386 }