1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 #include <sys/dtrace.h>
  28 #include <sys/cmn_err.h>
  29 #include <sys/tnf.h>
  30 #include <sys/atomic.h>
  31 #include <sys/prsystm.h>
  32 #include <sys/modctl.h>
  33 #include <sys/aio_impl.h>
  34 
  35 #ifdef __sparc
  36 #include <sys/privregs.h>
  37 #endif
  38 
  39 void (*dtrace_cpu_init)(processorid_t);
  40 void (*dtrace_modload)(struct modctl *);
  41 void (*dtrace_modunload)(struct modctl *);
  42 void (*dtrace_helpers_cleanup)(void);
  43 void (*dtrace_helpers_fork)(proc_t *, proc_t *);
  44 void (*dtrace_cpustart_init)(void);
  45 void (*dtrace_cpustart_fini)(void);
  46 void (*dtrace_cpc_fire)(uint64_t);
  47 void (*dtrace_closef)(void);
  48 
  49 void (*dtrace_debugger_init)(void);
  50 void (*dtrace_debugger_fini)(void);
  51 
  52 dtrace_vtime_state_t dtrace_vtime_active = 0;
  53 dtrace_cacheid_t dtrace_predcache_id = DTRACE_CACHEIDNONE + 1;
  54 
  55 /*
  56  * dtrace_cpc_in_use usage statement: this global variable is used by the cpc
  57  * hardware overflow interrupt handler and the kernel cpc framework to check
  58  * whether or not the DTrace cpc provider is currently in use. The variable is
  59  * set before counters are enabled with the first enabling and cleared when
  60  * the last enabling is disabled. Its value at any given time indicates the
  61  * number of active dcpc based enablings. The global 'kcpc_cpuctx_lock' rwlock
  62  * is held during initial setting to protect races between kcpc_open() and the
  63  * first enabling. The locking provided by the DTrace subsystem, the kernel
  64  * cpc framework and the cpu management framework protect consumers from race
  65  * conditions on enabling and disabling probes.
  66  */
  67 uint32_t dtrace_cpc_in_use = 0;
  68 
  69 typedef struct dtrace_hrestime {
  70         lock_t          dthr_lock;              /* lock for this element */
  71         timestruc_t     dthr_hrestime;          /* hrestime value */
  72         int64_t         dthr_adj;               /* hrestime_adj value */
  73         hrtime_t        dthr_hrtime;            /* hrtime value */
  74 } dtrace_hrestime_t;
  75 
  76 static dtrace_hrestime_t dtrace_hrestime[2];
  77 
  78 /*
  79  * Making available adjustable high-resolution time in DTrace is regrettably
  80  * more complicated than one might think it should be.  The problem is that
  81  * the variables related to adjusted high-resolution time (hrestime,
  82  * hrestime_adj and friends) are adjusted under hres_lock -- and this lock may
  83  * be held when we enter probe context.  One might think that we could address
  84  * this by having a single snapshot copy that is stored under a different lock
  85  * from hres_tick(), using the snapshot iff hres_lock is locked in probe
  86  * context.  Unfortunately, this too won't work:  because hres_lock is grabbed
  87  * in more than just hres_tick() context, we could enter probe context
  88  * concurrently on two different CPUs with both locks (hres_lock and the
  89  * snapshot lock) held.  As this implies, the fundamental problem is that we
  90  * need to have access to a snapshot of these variables that we _know_ will
  91  * not be locked in probe context.  To effect this, we have two snapshots
  92  * protected by two different locks, and we mandate that these snapshots are
  93  * recorded in succession by a single thread calling dtrace_hres_tick().  (We
  94  * assure this by calling it out of the same CY_HIGH_LEVEL cyclic that calls
  95  * hres_tick().)  A single thread can't be in two places at once:  one of the
  96  * snapshot locks is guaranteed to be unheld at all times.  The
  97  * dtrace_gethrestime() algorithm is thus to check first one snapshot and then
  98  * the other to find the unlocked snapshot.
  99  */
 100 void
 101 dtrace_hres_tick(void)
 102 {
 103         int i;
 104         ushort_t spl;
 105 
 106         for (i = 0; i < 2; i++) {
 107                 dtrace_hrestime_t tmp;
 108 
 109                 spl = hr_clock_lock();
 110                 tmp.dthr_hrestime = hrestime;
 111                 tmp.dthr_adj = hrestime_adj;
 112                 tmp.dthr_hrtime = dtrace_gethrtime();
 113                 hr_clock_unlock(spl);
 114 
 115                 lock_set(&dtrace_hrestime[i].dthr_lock);
 116                 dtrace_hrestime[i].dthr_hrestime = tmp.dthr_hrestime;
 117                 dtrace_hrestime[i].dthr_adj = tmp.dthr_adj;
 118                 dtrace_hrestime[i].dthr_hrtime = tmp.dthr_hrtime;
 119                 dtrace_membar_producer();
 120 
 121                 /*
 122                  * To allow for lock-free examination of this lock, we use
 123                  * the same trick that is used hres_lock; for more details,
 124                  * see the description of this technique in sun4u/sys/clock.h.
 125                  */
 126                 dtrace_hrestime[i].dthr_lock++;
 127         }
 128 }
 129 
 130 hrtime_t
 131 dtrace_gethrestime(void)
 132 {
 133         dtrace_hrestime_t snap;
 134         hrtime_t now;
 135         int i = 0, adj, nslt;
 136 
 137         for (;;) {
 138                 snap.dthr_lock = dtrace_hrestime[i].dthr_lock;
 139                 dtrace_membar_consumer();
 140                 snap.dthr_hrestime = dtrace_hrestime[i].dthr_hrestime;
 141                 snap.dthr_hrtime = dtrace_hrestime[i].dthr_hrtime;
 142                 snap.dthr_adj = dtrace_hrestime[i].dthr_adj;
 143                 dtrace_membar_consumer();
 144 
 145                 if ((snap.dthr_lock & ~1) == dtrace_hrestime[i].dthr_lock)
 146                         break;
 147 
 148                 /*
 149                  * If we're here, the lock was either locked, or it
 150                  * transitioned while we were taking the snapshot.  Either
 151                  * way, we're going to try the other dtrace_hrestime element;
 152                  * we know that it isn't possible for both to be locked
 153                  * simultaneously, so we will ultimately get a good snapshot.
 154                  */
 155                 i ^= 1;
 156         }
 157 
 158         /*
 159          * We have a good snapshot.  Now perform any necessary adjustments.
 160          */
 161         nslt = dtrace_gethrtime() - snap.dthr_hrtime;
 162         ASSERT(nslt >= 0);
 163 
 164         now = ((hrtime_t)snap.dthr_hrestime.tv_sec * (hrtime_t)NANOSEC) +
 165             snap.dthr_hrestime.tv_nsec;
 166 
 167         if (snap.dthr_adj != 0) {
 168                 if (snap.dthr_adj > 0) {
 169                         adj = (nslt >> adj_shift);
 170                         if (adj > snap.dthr_adj)
 171                                 adj = (int)snap.dthr_adj;
 172                 } else {
 173                         adj = -(nslt >> adj_shift);
 174                         if (adj < snap.dthr_adj)
 175                                 adj = (int)snap.dthr_adj;
 176                 }
 177                 now += adj;
 178         }
 179 
 180         return (now);
 181 }
 182 
 183 void
 184 dtrace_vtime_enable(void)
 185 {
 186         dtrace_vtime_state_t state, nstate;
 187 
 188         do {
 189                 state = dtrace_vtime_active;
 190 
 191                 switch (state) {
 192                 case DTRACE_VTIME_INACTIVE:
 193                         nstate = DTRACE_VTIME_ACTIVE;
 194                         break;
 195 
 196                 case DTRACE_VTIME_INACTIVE_TNF:
 197                         nstate = DTRACE_VTIME_ACTIVE_TNF;
 198                         break;
 199 
 200                 case DTRACE_VTIME_ACTIVE:
 201                 case DTRACE_VTIME_ACTIVE_TNF:
 202                         panic("DTrace virtual time already enabled");
 203                         /*NOTREACHED*/
 204                 }
 205 
 206         } while (cas32((uint32_t *)&dtrace_vtime_active,
 207             state, nstate) != state);
 208 }
 209 
 210 void
 211 dtrace_vtime_disable(void)
 212 {
 213         dtrace_vtime_state_t state, nstate;
 214 
 215         do {
 216                 state = dtrace_vtime_active;
 217 
 218                 switch (state) {
 219                 case DTRACE_VTIME_ACTIVE:
 220                         nstate = DTRACE_VTIME_INACTIVE;
 221                         break;
 222 
 223                 case DTRACE_VTIME_ACTIVE_TNF:
 224                         nstate = DTRACE_VTIME_INACTIVE_TNF;
 225                         break;
 226 
 227                 case DTRACE_VTIME_INACTIVE:
 228                 case DTRACE_VTIME_INACTIVE_TNF:
 229                         panic("DTrace virtual time already disabled");
 230                         /*NOTREACHED*/
 231                 }
 232 
 233         } while (cas32((uint32_t *)&dtrace_vtime_active,
 234             state, nstate) != state);
 235 }
 236 
 237 void
 238 dtrace_vtime_enable_tnf(void)
 239 {
 240         dtrace_vtime_state_t state, nstate;
 241 
 242         do {
 243                 state = dtrace_vtime_active;
 244 
 245                 switch (state) {
 246                 case DTRACE_VTIME_ACTIVE:
 247                         nstate = DTRACE_VTIME_ACTIVE_TNF;
 248                         break;
 249 
 250                 case DTRACE_VTIME_INACTIVE:
 251                         nstate = DTRACE_VTIME_INACTIVE_TNF;
 252                         break;
 253 
 254                 case DTRACE_VTIME_ACTIVE_TNF:
 255                 case DTRACE_VTIME_INACTIVE_TNF:
 256                         panic("TNF already active");
 257                         /*NOTREACHED*/
 258                 }
 259 
 260         } while (cas32((uint32_t *)&dtrace_vtime_active,
 261             state, nstate) != state);
 262 }
 263 
 264 void
 265 dtrace_vtime_disable_tnf(void)
 266 {
 267         dtrace_vtime_state_t state, nstate;
 268 
 269         do {
 270                 state = dtrace_vtime_active;
 271 
 272                 switch (state) {
 273                 case DTRACE_VTIME_ACTIVE_TNF:
 274                         nstate = DTRACE_VTIME_ACTIVE;
 275                         break;
 276 
 277                 case DTRACE_VTIME_INACTIVE_TNF:
 278                         nstate = DTRACE_VTIME_INACTIVE;
 279                         break;
 280 
 281                 case DTRACE_VTIME_ACTIVE:
 282                 case DTRACE_VTIME_INACTIVE:
 283                         panic("TNF already inactive");
 284                         /*NOTREACHED*/
 285                 }
 286 
 287         } while (cas32((uint32_t *)&dtrace_vtime_active,
 288             state, nstate) != state);
 289 }
 290 
 291 void
 292 dtrace_vtime_switch(kthread_t *next)
 293 {
 294         dtrace_icookie_t cookie;
 295         hrtime_t ts;
 296 
 297         if (tnf_tracing_active) {
 298                 tnf_thread_switch(next);
 299 
 300                 if (dtrace_vtime_active == DTRACE_VTIME_INACTIVE_TNF)
 301                         return;
 302         }
 303 
 304         cookie = dtrace_interrupt_disable();
 305         ts = dtrace_gethrtime();
 306 
 307         if (curthread->t_dtrace_start != 0) {
 308                 curthread->t_dtrace_vtime += ts - curthread->t_dtrace_start;
 309                 curthread->t_dtrace_start = 0;
 310         }
 311 
 312         next->t_dtrace_start = ts;
 313 
 314         dtrace_interrupt_enable(cookie);
 315 }
 316 
 317 void (*dtrace_fasttrap_fork_ptr)(proc_t *, proc_t *);
 318 void (*dtrace_fasttrap_exec_ptr)(proc_t *);
 319 void (*dtrace_fasttrap_exit_ptr)(proc_t *);
 320 
 321 /*
 322  * This function is called by cfork() in the event that it appears that
 323  * there may be dtrace tracepoints active in the parent process's address
 324  * space. This first confirms the existence of dtrace tracepoints in the
 325  * parent process and calls into the fasttrap module to remove the
 326  * corresponding tracepoints from the child. By knowing that there are
 327  * existing tracepoints, and ensuring they can't be removed, we can rely
 328  * on the fasttrap module remaining loaded.
 329  */
 330 void
 331 dtrace_fasttrap_fork(proc_t *p, proc_t *cp)
 332 {
 333         ASSERT(p->p_proc_flag & P_PR_LOCK);
 334         ASSERT(p->p_dtrace_count > 0);
 335         ASSERT(dtrace_fasttrap_fork_ptr != NULL);
 336 
 337         dtrace_fasttrap_fork_ptr(p, cp);
 338 }