1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 /*
  28  * tavor_rsrc.c
  29  *    Tavor Resource Management Routines
  30  *
  31  *    Implements all the routines necessary for setup, teardown, and
  32  *    alloc/free of all Tavor resources, including those that are managed
  33  *    by Tavor hardware or which live in Tavor's direct attached DDR memory.
  34  */
  35 
  36 #include <sys/types.h>
  37 #include <sys/conf.h>
  38 #include <sys/ddi.h>
  39 #include <sys/sunddi.h>
  40 #include <sys/modctl.h>
  41 #include <sys/vmem.h>
  42 #include <sys/bitmap.h>
  43 
  44 #include <sys/ib/adapters/tavor/tavor.h>
  45 
  46 /*
  47  * The following routines are used for initializing and destroying
  48  * the resource pools used by the Tavor resource allocation routines.
  49  * They consist of four classes of object:
  50  *
  51  * Mailboxes:  The "In" and "Out" mailbox types are used by the Tavor
  52  *    command interface routines.  Mailboxes are used to pass information
  53  *    back and forth to the Tavor firmware.  Either type of mailbox may
  54  *    be allocated from Tavor's direct attached DDR memory or from system
  55  *    memory (although currently all "In" mailboxes are in DDR and all "out"
  56  *    mailboxes come from system memory.
  57  *
  58  * HW entry objects:  These objects represent resources required by the Tavor
  59  *    hardware.  These objects include things like Queue Pair contexts (QPC),
  60  *    Completion Queue contexts (CQC), Event Queue contexts (EQC), RDB (for
  61  *    supporting RDMA Read/Atomic), Multicast Group entries (MCG), Memory
  62  *    Protection Table entries (MPT), Memory Translation Table entries (MTT).
  63  *
  64  *    What these objects all have in common is that they are each required
  65  *    to come from DDR memory, they are always allocated from tables, and
  66  *    they are not to be directly accessed (read or written) by driver
  67  *    software.
  68  *    One notable exceptions to this rule are the Extended QP contexts (EQPC),
  69  *    and the UAR scratch area (UAR_SCR), both of which are not directly
  70  *    accessible through the Tavor resource allocation routines, but both
  71  *    of which are also required to reside in DDR memory and are not to be
  72  *    manipulated by driver software (they are privately managed by Tavor
  73  *    hardware).
  74  *    The other notable exceptions are the UAR pages (UAR_PG) which are
  75  *    allocated from the UAR address space rather than DDR, and the UD
  76  *    address vectors (UDAV) which are similar to the common object types
  77  *    with the major difference being that UDAVs _are_ directly read and
  78  *    written by driver software.
  79  *
  80  * SW handle objects: These objects represent resources required by Tavor
  81  *    driver software.  They are primarily software tracking structures,
  82  *    which are allocated from system memory (using kmem_cache).  Several of
  83  *    the objects have both a "constructor" and "destructor" method
  84  *    associated with them (see below).
  85  *
  86  * Protection Domain (PD) handle objects:  These objects are very much like
  87  *    a SW handle object with the notable difference that all PD handle
  88  *    objects have an actual Protection Domain number (PD) associated with
  89  *    them (and the PD number is allocated/managed through a separate
  90  *    vmem_arena specifically set aside for this purpose.
  91  */
  92 
  93 static int tavor_rsrc_mbox_init(tavor_state_t *state,
  94     tavor_rsrc_mbox_info_t *info);
  95 static void tavor_rsrc_mbox_fini(tavor_state_t *state,
  96     tavor_rsrc_mbox_info_t *info);
  97 
  98 static int tavor_rsrc_hw_entries_init(tavor_state_t *state,
  99     tavor_rsrc_hw_entry_info_t *info);
 100 static void tavor_rsrc_hw_entries_fini(tavor_state_t *state,
 101     tavor_rsrc_hw_entry_info_t *info);
 102 
 103 static int tavor_rsrc_sw_handles_init(tavor_state_t *state,
 104     tavor_rsrc_sw_hdl_info_t *info);
 105 static void tavor_rsrc_sw_handles_fini(tavor_state_t *state,
 106     tavor_rsrc_sw_hdl_info_t *info);
 107 
 108 static int tavor_rsrc_pd_handles_init(tavor_state_t *state,
 109     tavor_rsrc_sw_hdl_info_t *info);
 110 static void tavor_rsrc_pd_handles_fini(tavor_state_t *state,
 111     tavor_rsrc_sw_hdl_info_t *info);
 112 
 113 /*
 114  * The following routines are used for allocating and freeing the specific
 115  * types of objects described above from their associated resource pools.
 116  */
 117 static int tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info,
 118     uint_t num, tavor_rsrc_t *hdl);
 119 static void tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info,
 120     tavor_rsrc_t *hdl);
 121 
 122 static int tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info,
 123     uint_t num, uint_t num_align, ddi_acc_handle_t acc_handle,
 124     uint_t sleepflag, tavor_rsrc_t *hdl);
 125 static void tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info,
 126     tavor_rsrc_t *hdl);
 127 
 128 static int tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info,
 129     uint_t sleepflag, tavor_rsrc_t *hdl);
 130 static void tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info,
 131     tavor_rsrc_t *hdl);
 132 
 133 static int tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info,
 134     uint_t sleepflag, tavor_rsrc_t *hdl);
 135 static void tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info,
 136     tavor_rsrc_t *hdl);
 137 
 138 /*
 139  * The following routines are the constructors and destructors for several
 140  * of the SW handle type objects.  For certain types of SW handles objects
 141  * (all of which are implemented using kmem_cache), we need to do some
 142  * special field initialization (specifically, mutex_init/destroy).  These
 143  * routines enable that init and teardown.
 144  */
 145 static int tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags);
 146 static void tavor_rsrc_pdhdl_destructor(void *pd, void *state);
 147 static int tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags);
 148 static void tavor_rsrc_cqhdl_destructor(void *cq, void *state);
 149 static int tavor_rsrc_qphdl_constructor(void *cq, void *priv, int flags);
 150 static void tavor_rsrc_qphdl_destructor(void *cq, void *state);
 151 static int tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags);
 152 static void tavor_rsrc_srqhdl_destructor(void *srq, void *state);
 153 static int tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags);
 154 static void tavor_rsrc_refcnt_destructor(void *rc, void *state);
 155 static int tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags);
 156 static void tavor_rsrc_ahhdl_destructor(void *ah, void *state);
 157 static int tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags);
 158 static void tavor_rsrc_mrhdl_destructor(void *mr, void *state);
 159 
 160 /*
 161  * Special routine to calculate and return the size of a MCG object based
 162  * on current driver configuration (specifically, the number of QP per MCG
 163  * that has been configured.
 164  */
 165 static int tavor_rsrc_mcg_entry_get_size(tavor_state_t *state,
 166     uint_t *mcg_size_shift);
 167 
 168 
 169 /*
 170  * tavor_rsrc_alloc()
 171  *
 172  *    Context: Can be called from interrupt or base context.
 173  *    The "sleepflag" parameter is used by all object allocators to
 174  *    determine whether to SLEEP for resources or not.
 175  */
 176 int
 177 tavor_rsrc_alloc(tavor_state_t *state, tavor_rsrc_type_t rsrc, uint_t num,
 178     uint_t sleepflag, tavor_rsrc_t **hdl)
 179 {
 180         tavor_rsrc_pool_info_t  *rsrc_pool;
 181         tavor_rsrc_t            *tmp_rsrc_hdl;
 182         int                     flag, status = DDI_FAILURE;
 183 
 184         TAVOR_TNF_ENTER(tavor_rsrc_alloc);
 185 
 186         ASSERT(state != NULL);
 187         ASSERT(hdl != NULL);
 188 
 189         rsrc_pool = &state->ts_rsrc_hdl[rsrc];
 190         ASSERT(rsrc_pool != NULL);
 191 
 192         /*
 193          * Allocate space for the object used to track the resource handle
 194          */
 195         flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
 196         tmp_rsrc_hdl = (tavor_rsrc_t *)kmem_cache_alloc(state->ts_rsrc_cache,
 197             flag);
 198         if (tmp_rsrc_hdl == NULL) {
 199                 TNF_PROBE_0(tavor_rsrc_alloc_kmca_fail, TAVOR_TNF_ERROR, "");
 200                 TAVOR_TNF_EXIT(tavor_rsrc_alloc);
 201                 return (DDI_FAILURE);
 202         }
 203         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tmp_rsrc_hdl))
 204 
 205         /*
 206          * Set rsrc_hdl type.  This is later used by the tavor_rsrc_free call
 207          * to know what type of resource is being freed.
 208          */
 209         tmp_rsrc_hdl->rsrc_type = rsrc;
 210 
 211         /*
 212          * Depending on resource type, call the appropriate alloc routine
 213          */
 214         switch (rsrc_pool->rsrc_type) {
 215         case TAVOR_IN_MBOX:
 216         case TAVOR_OUT_MBOX:
 217         case TAVOR_INTR_IN_MBOX:
 218         case TAVOR_INTR_OUT_MBOX:
 219                 status = tavor_rsrc_mbox_alloc(rsrc_pool, num, tmp_rsrc_hdl);
 220                 break;
 221 
 222         case TAVOR_QPC:
 223         case TAVOR_CQC:
 224         case TAVOR_SRQC:
 225         case TAVOR_EQC:
 226         case TAVOR_RDB:
 227                 /*
 228                  * Because these objects are NOT accessed by Tavor driver
 229                  * software, we set the acc_handle parameter to zero.  But
 230                  * if they are allocated in multiples, we specify here that
 231                  * they must be aligned on a more restrictive boundary.
 232                  */
 233                 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num, 0,
 234                     sleepflag, tmp_rsrc_hdl);
 235                 break;
 236 
 237         case TAVOR_MPT:
 238                 /*
 239                  * Because these MPT objects are sometimes accessed by Tavor
 240                  * driver software (FMR), we set the acc_handle parameter.  But
 241                  * if they are allocated in multiples, we specify here that
 242                  * they must be aligned on a more restrictive boundary.
 243                  */
 244                 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num,
 245                     state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl);
 246                 break;
 247 
 248         case TAVOR_MCG:
 249                 /*
 250                  * Tavor MCG entries are also NOT accessed by Tavor driver
 251                  * software, but because MCG entries do not have the same
 252                  * alignnment restrictions we loosen the constraint here.
 253                  */
 254                 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 0,
 255                     sleepflag, tmp_rsrc_hdl);
 256                 break;
 257 
 258         case TAVOR_MTT:
 259         case TAVOR_UDAV:
 260                 /*
 261                  * Because MTT segments are among the few HW resources that
 262                  * may be allocated in odd numbers, we specify a less
 263                  * restrictive alignment than for the above resources.
 264                  *
 265                  * Also because both UDAV and MTT segment objects are read
 266                  * and/or written by Tavor driver software, we set the
 267                  * acc_handle parameter to point to the ddi_acc_handle_t for
 268                  * the Tavor DDR memory.
 269                  */
 270                 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1,
 271                     state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl);
 272                 break;
 273 
 274         case TAVOR_UARPG:
 275                 /*
 276                  * Because UAR pages are written by Tavor driver software (for
 277                  * doorbells), we set the acc_handle parameter to point to
 278                  * the ddi_acc_handle_t for the Tavor UAR memory.
 279                  */
 280                 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1,
 281                     state->ts_reg_uarhdl, sleepflag, tmp_rsrc_hdl);
 282                 break;
 283 
 284         case TAVOR_MRHDL:
 285         case TAVOR_EQHDL:
 286         case TAVOR_CQHDL:
 287         case TAVOR_SRQHDL:
 288         case TAVOR_AHHDL:
 289         case TAVOR_QPHDL:
 290         case TAVOR_REFCNT:
 291                 status = tavor_rsrc_swhdl_alloc(rsrc_pool, sleepflag,
 292                     tmp_rsrc_hdl);
 293                 break;
 294 
 295         case TAVOR_PDHDL:
 296                 status = tavor_rsrc_pdhdl_alloc(rsrc_pool, sleepflag,
 297                     tmp_rsrc_hdl);
 298                 break;
 299 
 300         default:
 301                 TAVOR_WARNING(state, "unexpected resource type in alloc");
 302                 TNF_PROBE_0(tavor_rsrc_alloc_inv_rsrctype_fail,
 303                     TAVOR_TNF_ERROR, "");
 304                 break;
 305         }
 306 
 307         /*
 308          * If the resource allocation failed, then free the special resource
 309          * tracking structure and return failure.  Otherwise return the
 310          * handle for the resource tracking structure.
 311          */
 312         if (status != DDI_SUCCESS) {
 313                 kmem_cache_free(state->ts_rsrc_cache, tmp_rsrc_hdl);
 314                 tmp_rsrc_hdl = NULL;
 315                 TNF_PROBE_1(tavor_rsrc_alloc_fail, TAVOR_TNF_ERROR, "",
 316                     tnf_uint, rsrc_type, rsrc_pool->rsrc_type);
 317                 TAVOR_TNF_EXIT(tavor_rsrc_alloc);
 318                 return (DDI_FAILURE);
 319         } else {
 320                 *hdl = tmp_rsrc_hdl;
 321                 TAVOR_TNF_EXIT(tavor_rsrc_alloc);
 322                 return (DDI_SUCCESS);
 323         }
 324 }
 325 
 326 
 327 /*
 328  * tavor_rsrc_free()
 329  *    Context: Can be called from interrupt or base context.
 330  */
 331 void
 332 tavor_rsrc_free(tavor_state_t *state, tavor_rsrc_t **hdl)
 333 {
 334         tavor_rsrc_pool_info_t  *rsrc_pool;
 335 
 336         TAVOR_TNF_ENTER(tavor_rsrc_free);
 337 
 338         ASSERT(state != NULL);
 339         ASSERT(hdl != NULL);
 340 
 341         rsrc_pool = &state->ts_rsrc_hdl[(*hdl)->rsrc_type];
 342         ASSERT(rsrc_pool != NULL);
 343 
 344         /*
 345          * Depending on resource type, call the appropriate free routine
 346          */
 347         switch (rsrc_pool->rsrc_type) {
 348         case TAVOR_IN_MBOX:
 349         case TAVOR_OUT_MBOX:
 350         case TAVOR_INTR_IN_MBOX:
 351         case TAVOR_INTR_OUT_MBOX:
 352                 tavor_rsrc_mbox_free(rsrc_pool, *hdl);
 353                 break;
 354 
 355         case TAVOR_QPC:
 356         case TAVOR_CQC:
 357         case TAVOR_SRQC:
 358         case TAVOR_EQC:
 359         case TAVOR_RDB:
 360         case TAVOR_MCG:
 361         case TAVOR_MPT:
 362         case TAVOR_MTT:
 363         case TAVOR_UDAV:
 364         case TAVOR_UARPG:
 365                 tavor_rsrc_hw_entry_free(rsrc_pool, *hdl);
 366                 break;
 367 
 368         case TAVOR_MRHDL:
 369         case TAVOR_EQHDL:
 370         case TAVOR_CQHDL:
 371         case TAVOR_SRQHDL:
 372         case TAVOR_AHHDL:
 373         case TAVOR_QPHDL:
 374         case TAVOR_REFCNT:
 375                 tavor_rsrc_swhdl_free(rsrc_pool, *hdl);
 376                 break;
 377 
 378         case TAVOR_PDHDL:
 379                 tavor_rsrc_pdhdl_free(rsrc_pool, *hdl);
 380                 break;
 381 
 382         default:
 383                 TAVOR_WARNING(state, "unexpected resource type in free");
 384                 TNF_PROBE_0(tavor_rsrc_free_inv_rsrctype_fail,
 385                     TAVOR_TNF_ERROR, "");
 386                 break;
 387         }
 388 
 389         /*
 390          * Free the special resource tracking structure, set the handle to
 391          * NULL, and return.
 392          */
 393         kmem_cache_free(state->ts_rsrc_cache, *hdl);
 394         *hdl = NULL;
 395 
 396         TAVOR_TNF_EXIT(tavor_rsrc_free);
 397 }
 398 
 399 
 400 /*
 401  * tavor_rsrc_init_phase1()
 402  *
 403  *    Completes the first phase of Tavor resource/configuration init.
 404  *    This involves creating the kmem_cache for the "tavor_rsrc_t"
 405  *    structs, allocating the space for the resource pool handles,
 406  *    and setting up the "Out" mailboxes.
 407  *
 408  *    When this function completes, the Tavor driver is ready to
 409  *    post the following commands which return information only in the
 410  *    "Out" mailbox: QUERY_DDR, QUERY_FW, QUERY_DEV_LIM, and QUERY_ADAPTER
 411  *    If any of these commands are to be posted at this time, they must be
 412  *    done so only when "spinning" (as the outstanding command list and
 413  *    EQ setup code has not yet run)
 414  *
 415  *    Context: Only called from attach() path context
 416  */
 417 int
 418 tavor_rsrc_init_phase1(tavor_state_t *state)
 419 {
 420         tavor_rsrc_pool_info_t          *rsrc_pool;
 421         tavor_rsrc_mbox_info_t          mbox_info;
 422         tavor_rsrc_cleanup_level_t      cleanup;
 423         tavor_cfg_profile_t             *cfgprof;
 424         uint64_t                        num, size;
 425         int                             status;
 426         char                            *errormsg, *rsrc_name;
 427 
 428         TAVOR_TNF_ENTER(tavor_rsrc_init_phase1);
 429 
 430         ASSERT(state != NULL);
 431 
 432         /* This is where Phase 1 of resource initialization begins */
 433         cleanup = TAVOR_RSRC_CLEANUP_LEVEL0;
 434 
 435         /* Build kmem cache name from Tavor instance */
 436         rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP);
 437         TAVOR_RSRC_NAME(rsrc_name, TAVOR_RSRC_CACHE);
 438 
 439         /*
 440          * Create the kmem_cache for "tavor_rsrc_t" structures
 441          * (kmem_cache_create will SLEEP until successful)
 442          */
 443         state->ts_rsrc_cache = kmem_cache_create(rsrc_name,
 444             sizeof (tavor_rsrc_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
 445 
 446         /*
 447          * Allocate an array of tavor_rsrc_pool_info_t's (used in all
 448          * subsequent resource allocations)
 449          */
 450         state->ts_rsrc_hdl = kmem_zalloc(TAVOR_NUM_RESOURCES *
 451             sizeof (tavor_rsrc_pool_info_t), KM_SLEEP);
 452 
 453         cfgprof = state->ts_cfg_profile;
 454 
 455         /*
 456          * Initialize the resource pool for "Out" mailboxes.  Notice that
 457          * the number of "Out" mailboxes, their size, and their location
 458          * (DDR or system memory) is configurable.  By default, however,
 459          * all "Out" mailboxes are located in system memory only (because
 460          * they are primarily read from and never written to)
 461          */
 462         num  =  ((uint64_t)1 << cfgprof->cp_log_num_outmbox);
 463         size =  ((uint64_t)1 << cfgprof->cp_log_outmbox_size);
 464         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX];
 465         rsrc_pool->rsrc_type   = TAVOR_OUT_MBOX;
 466         rsrc_pool->rsrc_loc    = TAVOR_IN_SYSMEM;
 467         rsrc_pool->rsrc_pool_size = (size * num);
 468         rsrc_pool->rsrc_shift          = cfgprof->cp_log_outmbox_size;
 469         rsrc_pool->rsrc_quantum        = size;
 470         rsrc_pool->rsrc_align          = TAVOR_MBOX_ALIGN;
 471         rsrc_pool->rsrc_state          = state;
 472         TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_OUTMBOX_VMEM);
 473         mbox_info.mbi_num         = num;
 474         mbox_info.mbi_size        = size;
 475         mbox_info.mbi_rsrcpool    = rsrc_pool;
 476         mbox_info.mbi_rsrcname    = rsrc_name;
 477         status = tavor_rsrc_mbox_init(state, &mbox_info);
 478         if (status != DDI_SUCCESS) {
 479                 tavor_rsrc_fini(state, cleanup);
 480                 /* Set "status" and "errormsg" and goto failure */
 481                 TAVOR_TNF_FAIL(DDI_FAILURE, "out mailboxes");
 482                 goto rsrcinitp1_fail;
 483         }
 484         cleanup = TAVOR_RSRC_CLEANUP_LEVEL1;
 485 
 486         /*
 487          * Initialize the Tavor "Out" mailbox list.  This step actually uses
 488          * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the
 489          * "Out" mailboxes, bind them for DMA access, and arrange them into
 490          * an easily accessed fast-allocation mechanism (see tavor_cmd.c
 491          * for more details)
 492          */
 493         status = tavor_outmbox_list_init(state);
 494         if (status != DDI_SUCCESS) {
 495                 tavor_rsrc_fini(state, cleanup);
 496                 /* Set "status" and "errormsg" and goto failure */
 497                 TAVOR_TNF_FAIL(DDI_FAILURE, "out mailbox list");
 498                 goto rsrcinitp1_fail;
 499         }
 500         cleanup = TAVOR_RSRC_CLEANUP_LEVEL2;
 501 
 502         /*
 503          * Initialize the resource pool for interrupt "Out" mailboxes.  Notice
 504          * that the number of interrupt "Out" mailboxes, their size, and their
 505          * location (DDR or system memory) is configurable.  By default,
 506          * however, all interrupt "Out" mailboxes are located in system memory
 507          * only (because they are primarily read from and never written to)
 508          */
 509         num  =  ((uint64_t)1 << cfgprof->cp_log_num_intr_outmbox);
 510         size =  ((uint64_t)1 << cfgprof->cp_log_outmbox_size);
 511         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_OUT_MBOX];
 512         rsrc_pool->rsrc_type   = TAVOR_INTR_OUT_MBOX;
 513         rsrc_pool->rsrc_loc    = TAVOR_IN_SYSMEM;
 514         rsrc_pool->rsrc_pool_size = (size * num);
 515         rsrc_pool->rsrc_shift          = cfgprof->cp_log_outmbox_size;
 516         rsrc_pool->rsrc_quantum        = size;
 517         rsrc_pool->rsrc_align          = TAVOR_MBOX_ALIGN;
 518         rsrc_pool->rsrc_state          = state;
 519         TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_OUTMBOX_VMEM);
 520         mbox_info.mbi_num         = num;
 521         mbox_info.mbi_size        = size;
 522         mbox_info.mbi_rsrcpool    = rsrc_pool;
 523         mbox_info.mbi_rsrcname    = rsrc_name;
 524         status = tavor_rsrc_mbox_init(state, &mbox_info);
 525         if (status != DDI_SUCCESS) {
 526                 tavor_rsrc_fini(state, cleanup);
 527                 /* Set "status" and "errormsg" and goto failure */
 528                 TAVOR_TNF_FAIL(DDI_FAILURE, "out intr mailboxes");
 529                 goto rsrcinitp1_fail;
 530         }
 531         cleanup = TAVOR_RSRC_CLEANUP_LEVEL3;
 532 
 533         /*
 534          * Initialize the Tavor "Out" mailbox list.  This step actually uses
 535          * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the
 536          * "Out" mailboxes, bind them for DMA access, and arrange them into
 537          * an easily accessed fast-allocation mechanism (see tavor_cmd.c
 538          * for more details)
 539          */
 540         status = tavor_intr_outmbox_list_init(state);
 541         if (status != DDI_SUCCESS) {
 542                 tavor_rsrc_fini(state, cleanup);
 543                 /* Set "status" and "errormsg" and goto failure */
 544                 TAVOR_TNF_FAIL(DDI_FAILURE, "out intr mailbox list");
 545                 goto rsrcinitp1_fail;
 546         }
 547         cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE;
 548 
 549         kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
 550         TAVOR_TNF_EXIT(tavor_rsrc_init_phase1);
 551         return (DDI_SUCCESS);
 552 
 553 rsrcinitp1_fail:
 554         kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
 555         TNF_PROBE_1(tavor_rsrc_init_phase1_fail, TAVOR_TNF_ERROR, "",
 556             tnf_string, msg, errormsg);
 557         TAVOR_TNF_EXIT(tavor_rsrc_init_phase1);
 558         return (status);
 559 }
 560 
 561 
 562 /*
 563  * tavor_rsrc_init_phase2()
 564  *    Context: Only called from attach() path context
 565  */
 566 int
 567 tavor_rsrc_init_phase2(tavor_state_t *state)
 568 {
 569         tavor_rsrc_sw_hdl_info_t        hdl_info;
 570         tavor_rsrc_hw_entry_info_t      entry_info;
 571         tavor_rsrc_mbox_info_t          mbox_info;
 572         tavor_rsrc_pool_info_t          *rsrc_pool;
 573         tavor_rsrc_cleanup_level_t      cleanup;
 574         tavor_cfg_profile_t             *cfgprof;
 575         uint64_t                        num, max, size, num_prealloc;
 576         uint64_t                        ddr_size, fw_size;
 577         uint_t                          mcg_size, mcg_size_shift;
 578         uint_t                          uarscr_size, mttsegment_sz;
 579         int                             status;
 580         char                            *errormsg, *rsrc_name;
 581 
 582         TAVOR_TNF_ENTER(tavor_rsrc_init_phase2);
 583 
 584         ASSERT(state != NULL);
 585 
 586         /* Phase 2 initialization begins where Phase 1 left off */
 587         cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE;
 588 
 589         /*
 590          * Calculate the extent of the DDR size and portion of which that
 591          * is already reserved for Tavor firmware.  (Note: this information
 592          * is available because the QUERY_DDR and QUERY_FW commands have
 593          * been posted to Tavor firmware prior to calling this routine)
 594          */
 595         ddr_size = state->ts_ddr.ddr_endaddr - state->ts_ddr.ddr_baseaddr + 1;
 596         fw_size  = state->ts_fw.fw_endaddr - state->ts_fw.fw_baseaddr + 1;
 597 
 598         /* Build the DDR vmem arena name from Tavor instance */
 599         rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP);
 600         TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_VMEM);
 601 
 602         /*
 603          * Do a vmem_create for the entire DDR range (not including the
 604          * portion consumed by Tavor firmware).  This creates the vmem arena
 605          * from which all other DDR objects (specifically, tables of HW
 606          * entries) will be allocated.
 607          */
 608         state->ts_ddrvmem = vmem_create(rsrc_name,
 609             (void *)(uintptr_t)state->ts_ddr.ddr_baseaddr, (ddr_size - fw_size),
 610             sizeof (uint64_t), NULL, NULL, NULL, 0, VM_SLEEP);
 611         if (state->ts_ddrvmem == NULL) {
 612                 tavor_rsrc_fini(state, cleanup);
 613                 /* Set "status" and "errormsg" and goto failure */
 614                 TAVOR_TNF_FAIL(DDI_FAILURE, "DDR vmem");
 615                 goto rsrcinitp2_fail;
 616         }
 617         cleanup = TAVOR_RSRC_CLEANUP_LEVEL5;
 618 
 619         /*
 620          * Initialize the resource pools for all objects that exist in
 621          * Tavor DDR memory.  This includes ("In") mailboxes, context tables
 622          * (QPC, CQC, EQC, etc...), and other miscellaneous HW objects.
 623          */
 624         cfgprof = state->ts_cfg_profile;
 625 
 626         /*
 627          * Initialize the resource pool for the MPT table entries.  Notice
 628          * that the number of MPTs is configurable.  The configured value must
 629          * be less that the maximum value (obtained from the QUERY_DEV_LIM
 630          * command) or the initialization will fail.  Note also that a certain
 631          * number of MPTs must be set aside for Tavor firmware use.
 632          */
 633         num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
 634         max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
 635         num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mpt);
 636         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT];
 637         rsrc_pool->rsrc_type   = TAVOR_MPT;
 638         rsrc_pool->rsrc_loc    = TAVOR_IN_DDR;
 639         rsrc_pool->rsrc_pool_size = (TAVOR_MPT_SIZE * num);
 640         rsrc_pool->rsrc_shift          = TAVOR_MPT_SIZE_SHIFT;
 641         rsrc_pool->rsrc_quantum        = TAVOR_MPT_SIZE;
 642         rsrc_pool->rsrc_align          = (TAVOR_MPT_SIZE * num);
 643         rsrc_pool->rsrc_state          = state;
 644         rsrc_pool->rsrc_start          = NULL;
 645         TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MPT_VMEM);
 646         entry_info.hwi_num        = num;
 647         entry_info.hwi_max        = max;
 648         entry_info.hwi_prealloc   = num_prealloc;
 649         entry_info.hwi_rsrcpool   = rsrc_pool;
 650         entry_info.hwi_rsrcname   = rsrc_name;
 651         status = tavor_rsrc_hw_entries_init(state, &entry_info);
 652         if (status != DDI_SUCCESS) {
 653                 tavor_rsrc_fini(state, cleanup);
 654                 /* Set "status" and "errormsg" and goto failure */
 655                 TAVOR_TNF_FAIL(DDI_FAILURE, "MPT table");
 656                 goto rsrcinitp2_fail;
 657         }
 658         cleanup = TAVOR_RSRC_CLEANUP_LEVEL6;
 659 
 660         /*
 661          * Initialize the resource pool for the MTT table entries.  Notice
 662          * that the number of MTTs is configurable.  The configured value must
 663          * be less that the maximum value (obtained from the QUERY_DEV_LIM
 664          * command) or the initialization will fail.  Note also that a certain
 665          * number of MTT segments must be set aside for Tavor firmware use.
 666          */
 667         mttsegment_sz = (TAVOR_MTTSEG_SIZE << TAVOR_MTT_SIZE_SHIFT);
 668         num = ((uint64_t)1 << cfgprof->cp_log_num_mttseg);
 669         max = ((uint64_t)1 << state->ts_devlim.log_max_mttseg);
 670         num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mttseg);
 671         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT];
 672         rsrc_pool->rsrc_type   = TAVOR_MTT;
 673         rsrc_pool->rsrc_loc    = TAVOR_IN_DDR;
 674         rsrc_pool->rsrc_pool_size = (TAVOR_MTT_SIZE * num);
 675         rsrc_pool->rsrc_shift          = TAVOR_MTT_SIZE_SHIFT;
 676         rsrc_pool->rsrc_quantum        = mttsegment_sz;
 677         rsrc_pool->rsrc_align          = (TAVOR_MTT_SIZE * num);
 678         rsrc_pool->rsrc_state          = state;
 679         rsrc_pool->rsrc_start          = NULL;
 680         TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MTT_VMEM);
 681         entry_info.hwi_num        = num;
 682         entry_info.hwi_max        = max;
 683         entry_info.hwi_prealloc   = num_prealloc;
 684         entry_info.hwi_rsrcpool   = rsrc_pool;
 685         entry_info.hwi_rsrcname   = rsrc_name;
 686         status = tavor_rsrc_hw_entries_init(state, &entry_info);
 687         if (status != DDI_SUCCESS) {
 688                 tavor_rsrc_fini(state, cleanup);
 689                 /* Set "status" and "errormsg" and goto failure */
 690                 TAVOR_TNF_FAIL(DDI_FAILURE, "MTT table");
 691                 goto rsrcinitp2_fail;
 692         }
 693         cleanup = TAVOR_RSRC_CLEANUP_LEVEL7;
 694 
 695         /*
 696          * Initialize the resource pool for the QPC table entries.  Notice
 697          * that the number of QPs is configurable.  The configured value must
 698          * be less that the maximum value (obtained from the QUERY_DEV_LIM
 699          * command) or the initialization will fail.  Note also that a certain
 700          * number of QP contexts must be set aside for Tavor firmware use.
 701          */
 702         num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
 703         max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
 704         num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_qp);
 705         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC];
 706         rsrc_pool->rsrc_type   = TAVOR_QPC;
 707         rsrc_pool->rsrc_loc    = TAVOR_IN_DDR;
 708         rsrc_pool->rsrc_pool_size = (TAVOR_QPC_SIZE * num);
 709         rsrc_pool->rsrc_shift          = TAVOR_QPC_SIZE_SHIFT;
 710         rsrc_pool->rsrc_quantum        = TAVOR_QPC_SIZE;
 711         rsrc_pool->rsrc_align          = (TAVOR_QPC_SIZE * num);
 712         rsrc_pool->rsrc_state          = state;
 713         rsrc_pool->rsrc_start          = NULL;
 714         TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_QPC_VMEM);
 715         entry_info.hwi_num        = num;
 716         entry_info.hwi_max        = max;
 717         entry_info.hwi_prealloc   = num_prealloc;
 718         entry_info.hwi_rsrcpool   = rsrc_pool;
 719         entry_info.hwi_rsrcname   = rsrc_name;
 720         status = tavor_rsrc_hw_entries_init(state, &entry_info);
 721         if (status != DDI_SUCCESS) {
 722                 tavor_rsrc_fini(state, cleanup);
 723                 /* Set "status" and "errormsg" and goto failure */
 724                 TAVOR_TNF_FAIL(DDI_FAILURE, "QPC table");
 725                 goto rsrcinitp2_fail;
 726         }
 727         cleanup = TAVOR_RSRC_CLEANUP_LEVEL8;
 728 
 729         /*
 730          * Initialize the resource pool for the RDB table entries.  Notice
 731          * that the number of RDBs is configurable.  The configured value must
 732          * be less that the maximum value (obtained from the QUERY_DEV_LIM
 733          * command) or the initialization will fail.
 734          */
 735         num = ((uint64_t)1 << cfgprof->cp_log_num_rdb);
 736         max = ((uint64_t)1 << state->ts_devlim.log_max_ra_glob);
 737         num_prealloc = 0;
 738         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB];
 739         rsrc_pool->rsrc_type   = TAVOR_RDB;
 740         rsrc_pool->rsrc_loc    = TAVOR_IN_DDR;
 741         rsrc_pool->rsrc_pool_size = (TAVOR_RDB_SIZE * num);
 742         rsrc_pool->rsrc_shift          = TAVOR_RDB_SIZE_SHIFT;
 743         rsrc_pool->rsrc_quantum        = TAVOR_RDB_SIZE;
 744         rsrc_pool->rsrc_align          = (TAVOR_RDB_SIZE * num);
 745         rsrc_pool->rsrc_state          = state;
 746         rsrc_pool->rsrc_start          = NULL;
 747         TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_RDB_VMEM);
 748         entry_info.hwi_num        = num;
 749         entry_info.hwi_max        = max;
 750         entry_info.hwi_prealloc   = num_prealloc;
 751         entry_info.hwi_rsrcpool   = rsrc_pool;
 752         entry_info.hwi_rsrcname   = rsrc_name;
 753         status = tavor_rsrc_hw_entries_init(state, &entry_info);
 754         if (status != DDI_SUCCESS) {
 755                 tavor_rsrc_fini(state, cleanup);
 756                 /* Set "status" and "errormsg" and goto failure */
 757                 TAVOR_TNF_FAIL(DDI_FAILURE, "RDB table");
 758                 goto rsrcinitp2_fail;
 759         }
 760         cleanup = TAVOR_RSRC_CLEANUP_LEVEL9;
 761 
 762         /*
 763          * Initialize the resource pool for the CQC table entries.  Notice
 764          * that the number of CQs is configurable.  The configured value must
 765          * be less that the maximum value (obtained from the QUERY_DEV_LIM
 766          * command) or the initialization will fail.  Note also that a certain
 767          * number of CQ contexts must be set aside for Tavor firmware use.
 768          */
 769         num = ((uint64_t)1 << cfgprof->cp_log_num_cq);
 770         max = ((uint64_t)1 << state->ts_devlim.log_max_cq);
 771         num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_cq);
 772         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC];
 773         rsrc_pool->rsrc_type   = TAVOR_CQC;
 774         rsrc_pool->rsrc_loc    = TAVOR_IN_DDR;
 775         rsrc_pool->rsrc_pool_size = (TAVOR_CQC_SIZE * num);
 776         rsrc_pool->rsrc_shift          = TAVOR_CQC_SIZE_SHIFT;
 777         rsrc_pool->rsrc_quantum        = TAVOR_CQC_SIZE;
 778         rsrc_pool->rsrc_align          = (TAVOR_CQC_SIZE * num);
 779         rsrc_pool->rsrc_state          = state;
 780         rsrc_pool->rsrc_start          = NULL;
 781         TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_CQC_VMEM);
 782         entry_info.hwi_num        = num;
 783         entry_info.hwi_max        = max;
 784         entry_info.hwi_prealloc   = num_prealloc;
 785         entry_info.hwi_rsrcpool   = rsrc_pool;
 786         entry_info.hwi_rsrcname   = rsrc_name;
 787         status = tavor_rsrc_hw_entries_init(state, &entry_info);
 788         if (status != DDI_SUCCESS) {
 789                 tavor_rsrc_fini(state, cleanup);
 790                 /* Set "status" and "errormsg" and goto failure */
 791                 TAVOR_TNF_FAIL(DDI_FAILURE, "CQC table");
 792                 goto rsrcinitp2_fail;
 793         }
 794         cleanup = TAVOR_RSRC_CLEANUP_LEVEL10;
 795 
 796         /*
 797          * Initialize the resource pool for the Extended QPC table entries.
 798          * Notice that the number of EQPCs must be the same as the number
 799          * of QP contexts.  So the initialization is constructed in a
 800          * similar way as above (for TAVOR_QPC).  One notable difference
 801          * here, however, is that by setting the rsrc_quantum field to
 802          * zero (indicating a zero-sized object) we indicate that the
 803          * object is not allocatable.  The EQPC table is, in fact, managed
 804          * internally by the hardware and it is, therefore, unnecessary to
 805          * initialize an additional vmem_arena for this type of object.
 806          */
 807         num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
 808         max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
 809         num_prealloc = 0;
 810         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC];
 811         rsrc_pool->rsrc_type   = TAVOR_EQPC;
 812         rsrc_pool->rsrc_loc    = TAVOR_IN_DDR;
 813         rsrc_pool->rsrc_pool_size = (TAVOR_EQPC_SIZE * num);
 814         rsrc_pool->rsrc_shift          = 0;
 815         rsrc_pool->rsrc_quantum        = 0;
 816         rsrc_pool->rsrc_align          = TAVOR_EQPC_SIZE;
 817         rsrc_pool->rsrc_state          = state;
 818         rsrc_pool->rsrc_start          = NULL;
 819         TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQPC_VMEM);
 820         entry_info.hwi_num        = num;
 821         entry_info.hwi_max        = max;
 822         entry_info.hwi_prealloc   = num_prealloc;
 823         entry_info.hwi_rsrcpool   = rsrc_pool;
 824         entry_info.hwi_rsrcname   = rsrc_name;
 825         status = tavor_rsrc_hw_entries_init(state, &entry_info);
 826         if (status != DDI_SUCCESS) {
 827                 tavor_rsrc_fini(state, cleanup);
 828                 /* Set "status" and "errormsg" and goto failure */
 829                 TAVOR_TNF_FAIL(DDI_FAILURE, "Extended QPC table");
 830                 goto rsrcinitp2_fail;
 831         }
 832         cleanup = TAVOR_RSRC_CLEANUP_LEVEL11;
 833 
 834         /*
 835          * Initialize the resource pool for the UD address vector table
 836          * entries.  Notice that the number of UDAVs is configurable.  The
 837          * configured value must be less that the maximum value (obtained
 838          * from the QUERY_DEV_LIM command) or the initialization will fail.
 839          */
 840         num = ((uint64_t)1 << cfgprof->cp_log_num_ah);
 841         max = ((uint64_t)1 << state->ts_devlim.log_max_av);
 842         num_prealloc = 0;
 843         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UDAV];
 844         rsrc_pool->rsrc_type   = TAVOR_UDAV;
 845         rsrc_pool->rsrc_loc    = TAVOR_IN_DDR;
 846         rsrc_pool->rsrc_pool_size = (TAVOR_UDAV_SIZE * num);
 847         rsrc_pool->rsrc_shift          = TAVOR_UDAV_SIZE_SHIFT;
 848         rsrc_pool->rsrc_quantum        = TAVOR_UDAV_SIZE;
 849         rsrc_pool->rsrc_align          = TAVOR_UDAV_SIZE;
 850         rsrc_pool->rsrc_state          = state;
 851         rsrc_pool->rsrc_start          = NULL;
 852         TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UDAV_VMEM);
 853         entry_info.hwi_num        = num;
 854         entry_info.hwi_max        = max;
 855         entry_info.hwi_prealloc   = num_prealloc;
 856         entry_info.hwi_rsrcpool   = rsrc_pool;
 857         entry_info.hwi_rsrcname   = rsrc_name;
 858         status = tavor_rsrc_hw_entries_init(state, &entry_info);
 859         if (status != DDI_SUCCESS) {
 860                 tavor_rsrc_fini(state, cleanup);
 861                 /* Set "status" and "errormsg" and goto failure */
 862                 TAVOR_TNF_FAIL(DDI_FAILURE, "UDAV table");
 863                 goto rsrcinitp2_fail;
 864         }
 865         cleanup = TAVOR_RSRC_CLEANUP_LEVEL12;
 866 
 867         /*
 868          * Initialize the resource pool for the UAR scratch table entries.
 869          * Notice that the number of UARSCRs is configurable.  The configured
 870          * value must be less that the maximum value (obtained from the
 871          * QUERY_DEV_LIM command) or the initialization will fail.
 872          * Like the EQPCs above, UARSCR objects are not allocatable.  The
 873          * UARSCR table is also managed internally by the hardware and it
 874          * is, therefore, unnecessary to initialize an additional vmem_arena
 875          * for this type of object.  We indicate this by setting the
 876          * rsrc_quantum field to zero (indicating a zero-sized object).
 877          */
 878         uarscr_size = state->ts_devlim.uarscr_entry_sz;
 879         num = ((uint64_t)1 << cfgprof->cp_log_num_uar);
 880         max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 -
 881             PAGESHIFT));
 882         num_prealloc = 0;
 883         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
 884         rsrc_pool->rsrc_type   = TAVOR_UAR_SCR;
 885         rsrc_pool->rsrc_loc    = TAVOR_IN_DDR;
 886         rsrc_pool->rsrc_pool_size = (uarscr_size * num);
 887         rsrc_pool->rsrc_shift          = 0;
 888         rsrc_pool->rsrc_quantum        = 0;
 889         rsrc_pool->rsrc_align          = uarscr_size;
 890         rsrc_pool->rsrc_state          = state;
 891         rsrc_pool->rsrc_start          = NULL;
 892         TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UARSCR_VMEM);
 893         entry_info.hwi_num        = num;
 894         entry_info.hwi_max        = max;
 895         entry_info.hwi_prealloc   = num_prealloc;
 896         entry_info.hwi_rsrcpool   = rsrc_pool;
 897         entry_info.hwi_rsrcname   = rsrc_name;
 898         status = tavor_rsrc_hw_entries_init(state, &entry_info);
 899         if (status != DDI_SUCCESS) {
 900                 tavor_rsrc_fini(state, cleanup);
 901                 /* Set "status" and "errormsg" and goto failure */
 902                 TAVOR_TNF_FAIL(DDI_FAILURE, "UAR scratch table");
 903                 goto rsrcinitp2_fail;
 904         }
 905         cleanup = TAVOR_RSRC_CLEANUP_LEVEL13;
 906 
 907         /*
 908          * Initialize the resource pool for the SRQC table entries.  Notice
 909          * that the number of SRQs is configurable.  The configured value must
 910          * be less that the maximum value (obtained from the QUERY_DEV_LIM
 911          * command) or the initialization will fail.  Note also that a certain
 912          * number of SRQ contexts must be set aside for Tavor firmware use.
 913          *
 914          * Note: We only allocate these resources if SRQ is enabled in the
 915          * config profile; see below.
 916          */
 917         num = ((uint64_t)1 << cfgprof->cp_log_num_srq);
 918         max = ((uint64_t)1 << state->ts_devlim.log_max_srq);
 919         num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_srq);
 920 
 921         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC];
 922         rsrc_pool->rsrc_type   = TAVOR_SRQC;
 923         rsrc_pool->rsrc_loc    = TAVOR_IN_DDR;
 924         rsrc_pool->rsrc_pool_size = (TAVOR_SRQC_SIZE * num);
 925         rsrc_pool->rsrc_shift          = TAVOR_SRQC_SIZE_SHIFT;
 926         rsrc_pool->rsrc_quantum        = TAVOR_SRQC_SIZE;
 927         rsrc_pool->rsrc_align          = (TAVOR_SRQC_SIZE * num);
 928         rsrc_pool->rsrc_state          = state;
 929         rsrc_pool->rsrc_start          = NULL;
 930         TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_SRQC_VMEM);
 931         entry_info.hwi_num        = num;
 932         entry_info.hwi_max        = max;
 933         entry_info.hwi_prealloc   = num_prealloc;
 934         entry_info.hwi_rsrcpool   = rsrc_pool;
 935         entry_info.hwi_rsrcname   = rsrc_name;
 936 
 937         /*
 938          * SRQ support is configurable.  Only if SRQ is enabled (the default)
 939          * do we actually try to configure these resources.  Otherwise, we
 940          * simply set the cleanup level and continue on to the next resource
 941          */
 942         if (state->ts_cfg_profile->cp_srq_enable != 0) {
 943                 status = tavor_rsrc_hw_entries_init(state, &entry_info);
 944                 if (status != DDI_SUCCESS) {
 945                         tavor_rsrc_fini(state, cleanup);
 946                         /* Set "status" and "errormsg" and goto failure */
 947                         TAVOR_TNF_FAIL(DDI_FAILURE, "SRQC table");
 948                         goto rsrcinitp2_fail;
 949                 }
 950         }
 951         cleanup = TAVOR_RSRC_CLEANUP_LEVEL14;
 952 
 953         /*
 954          * Initialize the resource pool for "In" mailboxes.  Notice that
 955          * the number of "In" mailboxes, their size, and their location
 956          * (DDR or system memory) is configurable.  By default, however,
 957          * all "In" mailboxes are located in system memory only (because
 958          * they are primarily written to and rarely read from)
 959          */
 960         num  =  ((uint64_t)1 << cfgprof->cp_log_num_inmbox);
 961         size =  ((uint64_t)1 << cfgprof->cp_log_inmbox_size);
 962         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX];
 963         rsrc_pool->rsrc_type   = TAVOR_IN_MBOX;
 964         rsrc_pool->rsrc_loc    = TAVOR_IN_DDR;
 965         rsrc_pool->rsrc_pool_size = (size * num);
 966         rsrc_pool->rsrc_shift          = cfgprof->cp_log_inmbox_size;
 967         rsrc_pool->rsrc_quantum        = size;
 968         rsrc_pool->rsrc_align          = TAVOR_MBOX_ALIGN;
 969         rsrc_pool->rsrc_state          = state;
 970         TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INMBOX_VMEM);
 971         mbox_info.mbi_num         = num;
 972         mbox_info.mbi_size        = size;
 973         mbox_info.mbi_rsrcpool    = rsrc_pool;
 974         mbox_info.mbi_rsrcname    = rsrc_name;
 975         status = tavor_rsrc_mbox_init(state, &mbox_info);
 976         if (status != DDI_SUCCESS) {
 977                 tavor_rsrc_fini(state, cleanup);
 978                 /* Set "status" and "errormsg" and goto failure */
 979                 TAVOR_TNF_FAIL(DDI_FAILURE, "in mailboxes");
 980                 goto rsrcinitp2_fail;
 981         }
 982         cleanup = TAVOR_RSRC_CLEANUP_LEVEL15;
 983 
 984         /*
 985          * Initialize the Tavor "In" mailbox list.  This step actually uses
 986          * the tavor_rsrc_alloc() for TAVOR_IN_MBOX to preallocate the
 987          * "In" mailboxes, bind them for DMA access, and arrange them into
 988          * an easily accessed fast-allocation mechanism (see tavor_cmd.c
 989          * for more details)
 990          */
 991         status = tavor_inmbox_list_init(state);
 992         if (status != DDI_SUCCESS) {
 993                 tavor_rsrc_fini(state, cleanup);
 994                 /* Set "status" and "errormsg" and goto failure */
 995                 TAVOR_TNF_FAIL(DDI_FAILURE, "in mailbox list");
 996                 goto rsrcinitp2_fail;
 997         }
 998         cleanup = TAVOR_RSRC_CLEANUP_LEVEL16;
 999 
1000         /*
1001          * Initialize the resource pool for interrupt "In" mailboxes.  Notice
1002          * that the number of interrupt "In" mailboxes, their size, and their
1003          * location (DDR or system memory) is configurable.  By default,
1004          * however, all interrupt "In" mailboxes are located in system memory
1005          * only (because they are primarily written to and rarely read from)
1006          */
1007         num  =  ((uint64_t)1 << cfgprof->cp_log_num_intr_inmbox);
1008         size =  ((uint64_t)1 << cfgprof->cp_log_inmbox_size);
1009         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_IN_MBOX];
1010         rsrc_pool->rsrc_type   = TAVOR_INTR_IN_MBOX;
1011         rsrc_pool->rsrc_loc    = TAVOR_IN_DDR;
1012         rsrc_pool->rsrc_pool_size = (size * num);
1013         rsrc_pool->rsrc_shift          = cfgprof->cp_log_inmbox_size;
1014         rsrc_pool->rsrc_quantum        = size;
1015         rsrc_pool->rsrc_align          = TAVOR_MBOX_ALIGN;
1016         rsrc_pool->rsrc_state          = state;
1017         TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_INMBOX_VMEM);
1018         mbox_info.mbi_num         = num;
1019         mbox_info.mbi_size        = size;
1020         mbox_info.mbi_rsrcpool    = rsrc_pool;
1021         mbox_info.mbi_rsrcname    = rsrc_name;
1022         status = tavor_rsrc_mbox_init(state, &mbox_info);
1023         if (status != DDI_SUCCESS) {
1024                 tavor_rsrc_fini(state, cleanup);
1025                 /* Set "status" and "errormsg" and goto failure */
1026                 TAVOR_TNF_FAIL(DDI_FAILURE, "in intr mailboxes");
1027                 goto rsrcinitp2_fail;
1028         }
1029         cleanup = TAVOR_RSRC_CLEANUP_LEVEL17;
1030 
1031         /*
1032          * Initialize the Tavor interrupt "In" mailbox list.  This step
1033          * actually uses the tavor_rsrc_alloc() for TAVOR_IN_MBOX to
1034          * preallocate the interrupt "In" mailboxes, bind them for DMA access,
1035          * and arrange them into an easily accessed fast-allocation mechanism
1036          * (see tavor_cmd.c for more details)
1037          */
1038         status = tavor_intr_inmbox_list_init(state);
1039         if (status != DDI_SUCCESS) {
1040                 tavor_rsrc_fini(state, cleanup);
1041                 /* Set "status" and "errormsg" and goto failure */
1042                 TAVOR_TNF_FAIL(DDI_FAILURE, "in intr mailbox list");
1043                 goto rsrcinitp2_fail;
1044         }
1045         cleanup = TAVOR_RSRC_CLEANUP_LEVEL18;
1046 
1047         /*
1048          * Initialize the Tavor command handling interfaces.  This step
1049          * sets up the outstanding command tracking mechanism for easy access
1050          * and fast allocation (see tavor_cmd.c for more details).
1051          */
1052         status = tavor_outstanding_cmdlist_init(state);
1053         if (status != DDI_SUCCESS) {
1054                 tavor_rsrc_fini(state, cleanup);
1055                 /* Set "status" and "errormsg" and goto failure */
1056                 TAVOR_TNF_FAIL(DDI_FAILURE, "outstanding cmd list");
1057                 goto rsrcinitp2_fail;
1058         }
1059         cleanup = TAVOR_RSRC_CLEANUP_LEVEL19;
1060 
1061         /*
1062          * Calculate (and validate) the size of Multicast Group (MCG) entries
1063          */
1064         status = tavor_rsrc_mcg_entry_get_size(state, &mcg_size_shift);
1065         if (status != DDI_SUCCESS) {
1066                 tavor_rsrc_fini(state, cleanup);
1067                 /* Set "status" and "errormsg" and goto failure */
1068                 TAVOR_TNF_FAIL(DDI_FAILURE, "failed get MCG size");
1069                 goto rsrcinitp2_fail;
1070         }
1071         mcg_size = TAVOR_MCGMEM_SZ(state);
1072 
1073         /*
1074          * Initialize the resource pool for the MCG table entries.  Notice
1075          * that the number of MCGs is configurable.  The configured value must
1076          * be less that the maximum value (obtained from the QUERY_DEV_LIM
1077          * command) or the initialization will fail.  Note also that a certain
1078          * number of MCGs must be set aside for Tavor firmware use (they
1079          * correspond to the number of MCGs used by the internal hash
1080          * function.
1081          */
1082         num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1083         max = ((uint64_t)1 << state->ts_devlim.log_max_mcg);
1084         num_prealloc = ((uint64_t)1 << cfgprof->cp_log_num_mcg_hash);
1085         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG];
1086         rsrc_pool->rsrc_type   = TAVOR_MCG;
1087         rsrc_pool->rsrc_loc    = TAVOR_IN_DDR;
1088         rsrc_pool->rsrc_pool_size = (mcg_size * num);
1089         rsrc_pool->rsrc_shift          = mcg_size_shift;
1090         rsrc_pool->rsrc_quantum        = mcg_size;
1091         rsrc_pool->rsrc_align          = mcg_size;
1092         rsrc_pool->rsrc_state          = state;
1093         rsrc_pool->rsrc_start          = NULL;
1094         TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MCG_VMEM);
1095         entry_info.hwi_num        = num;
1096         entry_info.hwi_max        = max;
1097         entry_info.hwi_prealloc   = num_prealloc;
1098         entry_info.hwi_rsrcpool   = rsrc_pool;
1099         entry_info.hwi_rsrcname   = rsrc_name;
1100         status = tavor_rsrc_hw_entries_init(state, &entry_info);
1101         if (status != DDI_SUCCESS) {
1102                 tavor_rsrc_fini(state, cleanup);
1103                 /* Set "status" and "errormsg" and goto failure */
1104                 TAVOR_TNF_FAIL(DDI_FAILURE, "MCG table");
1105                 goto rsrcinitp2_fail;
1106         }
1107         cleanup = TAVOR_RSRC_CLEANUP_LEVEL20;
1108 
1109         /*
1110          * Initialize the resource pool for the EQC table entries.  Notice
1111          * that the number of EQs is hardcoded.  The hardcoded value should
1112          * be less that the maximum value (obtained from the QUERY_DEV_LIM
1113          * command) or the initialization will fail.
1114          */
1115         num = TAVOR_NUM_EQ;
1116         max = ((uint64_t)1 << state->ts_devlim.log_max_eq);
1117         num_prealloc = 0;
1118         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC];
1119         rsrc_pool->rsrc_type   = TAVOR_EQC;
1120         rsrc_pool->rsrc_loc    = TAVOR_IN_DDR;
1121         rsrc_pool->rsrc_pool_size = (TAVOR_EQC_SIZE * num);
1122         rsrc_pool->rsrc_shift          = TAVOR_EQC_SIZE_SHIFT;
1123         rsrc_pool->rsrc_quantum        = TAVOR_EQC_SIZE;
1124         rsrc_pool->rsrc_align          = (TAVOR_EQC_SIZE * num);
1125         rsrc_pool->rsrc_state          = state;
1126         rsrc_pool->rsrc_start          = NULL;
1127         TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQC_VMEM);
1128         entry_info.hwi_num        = num;
1129         entry_info.hwi_max        = max;
1130         entry_info.hwi_prealloc   = num_prealloc;
1131         entry_info.hwi_rsrcpool   = rsrc_pool;
1132         entry_info.hwi_rsrcname   = rsrc_name;
1133         status = tavor_rsrc_hw_entries_init(state, &entry_info);
1134         if (status != DDI_SUCCESS) {
1135                 tavor_rsrc_fini(state, cleanup);
1136                 /* Set "status" and "errormsg" and goto failure */
1137                 TAVOR_TNF_FAIL(DDI_FAILURE, "EQC table");
1138                 goto rsrcinitp2_fail;
1139         }
1140         cleanup = TAVOR_RSRC_CLEANUP_LEVEL21;
1141 
1142         /*
1143          * Initialize the resource pools for all objects that exist in
1144          * system memory.  This includes PD handles, MR handle, EQ handles,
1145          * QP handles, etc.  These objects are almost entirely managed using
1146          * kmem_cache routines.  (See comment above for more detail)
1147          */
1148 
1149         /*
1150          * Initialize the resource pool for the PD handles.  Notice
1151          * that the number of PDHDLs is configurable.  The configured value
1152          * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1153          * command) or the initialization will fail.  Note also that the PD
1154          * handle has constructor and destructor methods associated with it.
1155          */
1156         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_PDHDL];
1157         rsrc_pool->rsrc_type  = TAVOR_PDHDL;
1158         rsrc_pool->rsrc_loc   = TAVOR_IN_SYSMEM;
1159         rsrc_pool->rsrc_quantum       = sizeof (struct tavor_sw_pd_s);
1160         rsrc_pool->rsrc_state         = state;
1161         TAVOR_RSRC_NAME(rsrc_name, TAVOR_PDHDL_CACHE);
1162         hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_pd);
1163         hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_pd);
1164         hdl_info.swi_rsrcpool    = rsrc_pool;
1165         hdl_info.swi_constructor = tavor_rsrc_pdhdl_constructor;
1166         hdl_info.swi_destructor  = tavor_rsrc_pdhdl_destructor;
1167         hdl_info.swi_rsrcname    = rsrc_name;
1168         hdl_info.swi_flags       = TAVOR_SWHDL_KMEMCACHE_INIT;
1169         status = tavor_rsrc_pd_handles_init(state, &hdl_info);
1170         if (status != DDI_SUCCESS) {
1171                 tavor_rsrc_fini(state, cleanup);
1172                 /* Set "status" and "errormsg" and goto failure */
1173                 TAVOR_TNF_FAIL(DDI_FAILURE, "PD handle");
1174                 goto rsrcinitp2_fail;
1175         }
1176         cleanup = TAVOR_RSRC_CLEANUP_LEVEL22;
1177 
1178         /*
1179          * Initialize the resource pool for the MR handles.  Notice
1180          * that the number of MRHDLs is configurable.  The configured value
1181          * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1182          * command) or the initialization will fail.
1183          */
1184         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MRHDL];
1185         rsrc_pool->rsrc_type  = TAVOR_MRHDL;
1186         rsrc_pool->rsrc_loc   = TAVOR_IN_SYSMEM;
1187         rsrc_pool->rsrc_quantum       = sizeof (struct tavor_sw_mr_s);
1188         rsrc_pool->rsrc_state         = state;
1189         TAVOR_RSRC_NAME(rsrc_name, TAVOR_MRHDL_CACHE);
1190         hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
1191         hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
1192         hdl_info.swi_rsrcpool    = rsrc_pool;
1193         hdl_info.swi_constructor = tavor_rsrc_mrhdl_constructor;
1194         hdl_info.swi_destructor  = tavor_rsrc_mrhdl_destructor;
1195         hdl_info.swi_rsrcname    = rsrc_name;
1196         hdl_info.swi_flags       = TAVOR_SWHDL_KMEMCACHE_INIT;
1197         status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1198         if (status != DDI_SUCCESS) {
1199                 tavor_rsrc_fini(state, cleanup);
1200                 /* Set "status" and "errormsg" and goto failure */
1201                 TAVOR_TNF_FAIL(DDI_FAILURE, "MR handle");
1202                 goto rsrcinitp2_fail;
1203         }
1204         cleanup = TAVOR_RSRC_CLEANUP_LEVEL23;
1205 
1206         /*
1207          * Initialize the resource pool for the EQ handles.  Notice
1208          * that the number of EQHDLs is hardcoded.  The hardcoded value
1209          * should be less that the maximum value (obtained from the
1210          * QUERY_DEV_LIM command) or the initialization will fail.
1211          */
1212         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQHDL];
1213         rsrc_pool->rsrc_type  = TAVOR_EQHDL;
1214         rsrc_pool->rsrc_loc   = TAVOR_IN_SYSMEM;
1215         rsrc_pool->rsrc_quantum       = sizeof (struct tavor_sw_eq_s);
1216         rsrc_pool->rsrc_state         = state;
1217         TAVOR_RSRC_NAME(rsrc_name, TAVOR_EQHDL_CACHE);
1218         hdl_info.swi_num = TAVOR_NUM_EQ;
1219         hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_eq);
1220         hdl_info.swi_rsrcpool    = rsrc_pool;
1221         hdl_info.swi_constructor = NULL;
1222         hdl_info.swi_destructor  = NULL;
1223         hdl_info.swi_rsrcname    = rsrc_name;
1224         hdl_info.swi_flags       = TAVOR_SWHDL_KMEMCACHE_INIT;
1225         status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1226         if (status != DDI_SUCCESS) {
1227                 tavor_rsrc_fini(state, cleanup);
1228                 /* Set "status" and "errormsg" and goto failure */
1229                 TAVOR_TNF_FAIL(DDI_FAILURE, "EQ handle");
1230                 goto rsrcinitp2_fail;
1231         }
1232         cleanup = TAVOR_RSRC_CLEANUP_LEVEL24;
1233 
1234         /*
1235          * Initialize the resource pool for the CQ handles.  Notice
1236          * that the number of CQHDLs is configurable.  The configured value
1237          * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1238          * command) or the initialization will fail.  Note also that the CQ
1239          * handle has constructor and destructor methods associated with it.
1240          */
1241         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQHDL];
1242         rsrc_pool->rsrc_type  = TAVOR_CQHDL;
1243         rsrc_pool->rsrc_loc   = TAVOR_IN_SYSMEM;
1244         rsrc_pool->rsrc_quantum       = sizeof (struct tavor_sw_cq_s);
1245         rsrc_pool->rsrc_state         = state;
1246         TAVOR_RSRC_NAME(rsrc_name, TAVOR_CQHDL_CACHE);
1247         hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_cq);
1248         hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_cq);
1249         hdl_info.swi_rsrcpool    = rsrc_pool;
1250         hdl_info.swi_constructor = tavor_rsrc_cqhdl_constructor;
1251         hdl_info.swi_destructor  = tavor_rsrc_cqhdl_destructor;
1252         hdl_info.swi_rsrcname    = rsrc_name;
1253         hdl_info.swi_flags       = (TAVOR_SWHDL_KMEMCACHE_INIT |
1254             TAVOR_SWHDL_TABLE_INIT);
1255         hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t);
1256         status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1257         if (status != DDI_SUCCESS) {
1258                 tavor_rsrc_fini(state, cleanup);
1259                 /* Set "status" and "errormsg" and goto failure */
1260                 TAVOR_TNF_FAIL(DDI_FAILURE, "CQ handle");
1261                 goto rsrcinitp2_fail;
1262         }
1263 
1264         /*
1265          * Save away the pointer to the central list of CQ handle pointers
1266          * This this is used as a mechanism to enable fast CQnumber-to-CQhandle
1267          * lookup during EQ event processing.  The table is a list of
1268          * tavor_cqhdl_t allocated by the above routine because of the
1269          * TAVOR_SWHDL_TABLE_INIT flag.  The table has as many tavor_cqhdl_t
1270          * as the number of CQs.
1271          */
1272         state->ts_cqhdl = hdl_info.swi_table_ptr;
1273         cleanup = TAVOR_RSRC_CLEANUP_LEVEL25;
1274 
1275         /*
1276          * Initialize the resource pool for the SRQ handles.  Notice
1277          * that the number of SRQHDLs is configurable.  The configured value
1278          * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1279          * command) or the initialization will fail.  Note also that the SRQ
1280          * handle has constructor and destructor methods associated with it.
1281          *
1282          * Note: We only allocate these resources if SRQ is enabled in the
1283          * config profile; see below.
1284          */
1285         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQHDL];
1286         rsrc_pool->rsrc_type  = TAVOR_SRQHDL;
1287         rsrc_pool->rsrc_loc   = TAVOR_IN_SYSMEM;
1288         rsrc_pool->rsrc_quantum       = sizeof (struct tavor_sw_srq_s);
1289         rsrc_pool->rsrc_state         = state;
1290         TAVOR_RSRC_NAME(rsrc_name, TAVOR_SRQHDL_CACHE);
1291         hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_srq);
1292         hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_srq);
1293         hdl_info.swi_rsrcpool    = rsrc_pool;
1294         hdl_info.swi_constructor = tavor_rsrc_srqhdl_constructor;
1295         hdl_info.swi_destructor  = tavor_rsrc_srqhdl_destructor;
1296         hdl_info.swi_rsrcname    = rsrc_name;
1297         hdl_info.swi_flags       = (TAVOR_SWHDL_KMEMCACHE_INIT |
1298             TAVOR_SWHDL_TABLE_INIT);
1299         hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t);
1300 
1301         /*
1302          * SRQ support is configurable.  Only if SRQ is enabled (the default)
1303          * do we actually try to configure these resources.  Otherwise, we
1304          * simply set the cleanup level and continue on to the next resource
1305          */
1306         if (state->ts_cfg_profile->cp_srq_enable != 0) {
1307                 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1308                 if (status != DDI_SUCCESS) {
1309                         tavor_rsrc_fini(state, cleanup);
1310                         /* Set "status" and "errormsg" and goto failure */
1311                         TAVOR_TNF_FAIL(DDI_FAILURE, "SRQ handle");
1312                         goto rsrcinitp2_fail;
1313                 }
1314 
1315                 /*
1316                  * Save away the pointer to the central list of SRQ handle
1317                  * pointers This this is used as a mechanism to enable fast
1318                  * SRQnumber-to-SRQhandle lookup.  The table is a list of
1319                  * tavor_srqhdl_t allocated by the above routine because of the
1320                  * TAVOR_SWHDL_TABLE_INIT flag.  The table has as many
1321                  * tavor_srqhdl_t as the number of SRQs.
1322                  */
1323                 state->ts_srqhdl = hdl_info.swi_table_ptr;
1324         }
1325         cleanup = TAVOR_RSRC_CLEANUP_LEVEL26;
1326 
1327         /*
1328          * Initialize the resource pool for the address handles.  Notice
1329          * that the number of AHHDLs is configurable.  The configured value
1330          * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1331          * command) or the initialization will fail.
1332          */
1333         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_AHHDL];
1334         rsrc_pool->rsrc_type  = TAVOR_AHHDL;
1335         rsrc_pool->rsrc_loc   = TAVOR_IN_SYSMEM;
1336         rsrc_pool->rsrc_quantum       = sizeof (struct tavor_sw_ah_s);
1337         rsrc_pool->rsrc_state         = state;
1338         TAVOR_RSRC_NAME(rsrc_name, TAVOR_AHHDL_CACHE);
1339         hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_ah);
1340         hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_av);
1341         hdl_info.swi_rsrcpool    = rsrc_pool;
1342         hdl_info.swi_constructor = tavor_rsrc_ahhdl_constructor;
1343         hdl_info.swi_destructor  = tavor_rsrc_ahhdl_destructor;
1344         hdl_info.swi_rsrcname    = rsrc_name;
1345         hdl_info.swi_flags       = TAVOR_SWHDL_KMEMCACHE_INIT;
1346         status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1347         if (status != DDI_SUCCESS) {
1348                 tavor_rsrc_fini(state, cleanup);
1349                 /* Set "status" and "errormsg" and goto failure */
1350                 TAVOR_TNF_FAIL(DDI_FAILURE, "AH handle");
1351                 goto rsrcinitp2_fail;
1352         }
1353         cleanup = TAVOR_RSRC_CLEANUP_LEVEL27;
1354 
1355         /*
1356          * Initialize the resource pool for the QP handles.  Notice
1357          * that the number of QPHDLs is configurable.  The configured value
1358          * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1359          * command) or the initialization will fail.  Note also that the QP
1360          * handle has constructor and destructor methods associated with it.
1361          */
1362         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPHDL];
1363         rsrc_pool->rsrc_type  = TAVOR_QPHDL;
1364         rsrc_pool->rsrc_loc   = TAVOR_IN_SYSMEM;
1365         rsrc_pool->rsrc_quantum       = sizeof (struct tavor_sw_qp_s);
1366         rsrc_pool->rsrc_state         = state;
1367         TAVOR_RSRC_NAME(rsrc_name, TAVOR_QPHDL_CACHE);
1368         hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
1369         hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
1370         hdl_info.swi_rsrcpool    = rsrc_pool;
1371         hdl_info.swi_constructor = tavor_rsrc_qphdl_constructor;
1372         hdl_info.swi_destructor  = tavor_rsrc_qphdl_destructor;
1373         hdl_info.swi_rsrcname    = rsrc_name;
1374         hdl_info.swi_flags       = (TAVOR_SWHDL_KMEMCACHE_INIT |
1375             TAVOR_SWHDL_TABLE_INIT);
1376         hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t);
1377         status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1378         if (status != DDI_SUCCESS) {
1379                 tavor_rsrc_fini(state, cleanup);
1380                 /* Set "status" and "errormsg" and goto failure */
1381                 TAVOR_TNF_FAIL(DDI_FAILURE, "QP handle");
1382                 goto rsrcinitp2_fail;
1383         }
1384 
1385         /*
1386          * Save away the pointer to the central list of QP handle pointers
1387          * This this is used as a mechanism to enable fast QPnumber-to-QPhandle
1388          * lookup during CQ event processing.  The table is a list of
1389          * tavor_qphdl_t allocated by the above routine because of the
1390          * TAVOR_SWHDL_TABLE_INIT flag.  The table has as many tavor_qphdl_t
1391          * as the number of QPs.
1392          */
1393         state->ts_qphdl = hdl_info.swi_table_ptr;
1394         cleanup = TAVOR_RSRC_CLEANUP_LEVEL28;
1395 
1396         /*
1397          * Initialize the resource pool for the reference count handles.
1398          * Notice that the number of REFCNTs is configurable, but it's value
1399          * is set to the number of MPTs.  Since REFCNTs are used to support
1400          * shared memory regions, it is possible that we might require as
1401          * one REFCNT for every MPT.
1402          */
1403         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_REFCNT];
1404         rsrc_pool->rsrc_type  = TAVOR_REFCNT;
1405         rsrc_pool->rsrc_loc   = TAVOR_IN_SYSMEM;
1406         rsrc_pool->rsrc_quantum       = sizeof (tavor_sw_refcnt_t);
1407         rsrc_pool->rsrc_state         = state;
1408         TAVOR_RSRC_NAME(rsrc_name, TAVOR_REFCNT_CACHE);
1409         hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
1410         hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
1411         hdl_info.swi_rsrcpool    = rsrc_pool;
1412         hdl_info.swi_constructor = tavor_rsrc_refcnt_constructor;
1413         hdl_info.swi_destructor  = tavor_rsrc_refcnt_destructor;
1414         hdl_info.swi_rsrcname    = rsrc_name;
1415         hdl_info.swi_flags       = TAVOR_SWHDL_KMEMCACHE_INIT;
1416         status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1417         if (status != DDI_SUCCESS) {
1418                 tavor_rsrc_fini(state, cleanup);
1419                 /* Set "status" and "errormsg" and goto failure */
1420                 TAVOR_TNF_FAIL(DDI_FAILURE, "reference count handle");
1421                 goto rsrcinitp2_fail;
1422         }
1423         cleanup = TAVOR_RSRC_CLEANUP_LEVEL29;
1424 
1425         /*
1426          * Initialize the resource pool for the MCG handles.  Notice that for
1427          * these MCG handles, we are allocating a table of structures (used to
1428          * keep track of the MCG entries that are being written to hardware
1429          * and to speed up multicast attach/detach operations).
1430          */
1431         hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1432         hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mcg);
1433         hdl_info.swi_flags = TAVOR_SWHDL_TABLE_INIT;
1434         hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s);
1435         status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1436         if (status != DDI_SUCCESS) {
1437                 tavor_rsrc_fini(state, cleanup);
1438                 /* Set "status" and "errormsg" and goto failure */
1439                 TAVOR_TNF_FAIL(DDI_FAILURE, "MCG handle");
1440                 goto rsrcinitp2_fail;
1441         }
1442         state->ts_mcghdl = hdl_info.swi_table_ptr;
1443         cleanup = TAVOR_RSRC_CLEANUP_LEVEL30;
1444 
1445         /*
1446          * Initialize the resource pools for all objects that exist in
1447          * UAR memory.  The only objects that are allocated from UAR memory
1448          * are the UAR pages which are used for holding Tavor hardware's
1449          * doorbell registers.
1450          */
1451 
1452         /*
1453          * Initialize the resource pool for the UAR pages.  Notice
1454          * that the number of UARPGs is configurable.  The configured value
1455          * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1456          * command) or the initialization will fail.  Note also that by
1457          * specifying the rsrc_start parameter in advance, we direct the
1458          * initialization routine not to attempt to allocated space from the
1459          * Tavor DDR vmem_arena.
1460          */
1461         num  = ((uint64_t)1 << cfgprof->cp_log_num_uar);
1462         max  = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 -
1463             PAGESHIFT));
1464         num_prealloc = 0;
1465         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UARPG];
1466         rsrc_pool->rsrc_type   = TAVOR_UARPG;
1467         rsrc_pool->rsrc_loc    = TAVOR_IN_UAR;
1468         rsrc_pool->rsrc_pool_size = (num << PAGESHIFT);
1469         rsrc_pool->rsrc_shift          = PAGESHIFT;
1470         rsrc_pool->rsrc_quantum        = PAGESIZE;
1471         rsrc_pool->rsrc_align          = PAGESIZE;
1472         rsrc_pool->rsrc_state          = state;
1473         rsrc_pool->rsrc_start          = (void *)state->ts_reg_uar_baseaddr;
1474         TAVOR_RSRC_NAME(rsrc_name, TAVOR_UAR_VMEM);
1475         entry_info.hwi_num        = num;
1476         entry_info.hwi_max        = max;
1477         entry_info.hwi_prealloc   = num_prealloc;
1478         entry_info.hwi_rsrcpool   = rsrc_pool;
1479         entry_info.hwi_rsrcname   = rsrc_name;
1480         status = tavor_rsrc_hw_entries_init(state, &entry_info);
1481         if (status != DDI_SUCCESS) {
1482                 tavor_rsrc_fini(state, cleanup);
1483                 /* Set "status" and "errormsg" and goto failure */
1484                 TAVOR_TNF_FAIL(DDI_FAILURE, "UAR page table");
1485                 goto rsrcinitp2_fail;
1486         }
1487         cleanup = TAVOR_RSRC_CLEANUP_ALL;
1488 
1489         kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
1490         TAVOR_TNF_EXIT(tavor_rsrc_init_phase2);
1491         return (DDI_SUCCESS);
1492 
1493 rsrcinitp2_fail:
1494         kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
1495         TNF_PROBE_1(tavor_rsrc_init_phase2_fail, TAVOR_TNF_ERROR, "",
1496             tnf_string, msg, errormsg);
1497         TAVOR_TNF_EXIT(tavor_rsrc_init_phase2);
1498         return (status);
1499 }
1500 
1501 
1502 /*
1503  * tavor_rsrc_fini()
1504  *    Context: Only called from attach() and/or detach() path contexts
1505  */
1506 void
1507 tavor_rsrc_fini(tavor_state_t *state, tavor_rsrc_cleanup_level_t clean)
1508 {
1509         tavor_rsrc_sw_hdl_info_t        hdl_info;
1510         tavor_rsrc_hw_entry_info_t      entry_info;
1511         tavor_rsrc_mbox_info_t          mbox_info;
1512         tavor_cfg_profile_t             *cfgprof;
1513 
1514         TAVOR_TNF_ENTER(tavor_rsrc_fini);
1515 
1516         ASSERT(state != NULL);
1517 
1518         cfgprof = state->ts_cfg_profile;
1519 
1520         switch (clean) {
1521         /*
1522          * If we add more resources that need to be cleaned up here, we should
1523          * ensure that TAVOR_RSRC_CLEANUP_ALL is still the first entry (i.e.
1524          * corresponds to the last resource allocated).
1525          */
1526         case TAVOR_RSRC_CLEANUP_ALL:
1527                 /* Cleanup the UAR page resource pool */
1528                 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UARPG];
1529                 tavor_rsrc_hw_entries_fini(state, &entry_info);
1530                 /* FALLTHROUGH */
1531 
1532         case TAVOR_RSRC_CLEANUP_LEVEL30:
1533                 /* Cleanup the central MCG handle pointers list */
1534                 hdl_info.swi_rsrcpool  = NULL;
1535                 hdl_info.swi_table_ptr = state->ts_mcghdl;
1536                 hdl_info.swi_num =
1537                     ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1538                 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s);
1539                 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1540                 /* FALLTHROUGH */
1541 
1542         case TAVOR_RSRC_CLEANUP_LEVEL29:
1543                 /* Cleanup the reference count resource pool */
1544                 hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_REFCNT];
1545                 hdl_info.swi_table_ptr = NULL;
1546                 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1547                 /* FALLTHROUGH */
1548 
1549         case TAVOR_RSRC_CLEANUP_LEVEL28:
1550                 /* Cleanup the QP handle resource pool */
1551                 hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_QPHDL];
1552                 hdl_info.swi_table_ptr = state->ts_qphdl;
1553                 hdl_info.swi_num =
1554                     ((uint64_t)1 << cfgprof->cp_log_num_qp);
1555                 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t);
1556                 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1557                 /* FALLTHROUGH */
1558 
1559         case TAVOR_RSRC_CLEANUP_LEVEL27:
1560                 /* Cleanup the address handle resource pool */
1561                 hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_AHHDL];
1562                 hdl_info.swi_table_ptr = NULL;
1563                 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1564                 /* FALLTHROUGH */
1565 
1566         case TAVOR_RSRC_CLEANUP_LEVEL26:
1567                 /*
1568                  * Cleanup the SRQ handle resource pool.
1569                  *
1570                  * Note: We only clean up if SRQ is enabled.  Otherwise we
1571                  * simply fallthrough to the next resource cleanup.
1572                  */
1573                 if (state->ts_cfg_profile->cp_srq_enable != 0) {
1574                         hdl_info.swi_rsrcpool  =
1575                             &state->ts_rsrc_hdl[TAVOR_SRQHDL];
1576                         hdl_info.swi_table_ptr = state->ts_srqhdl;
1577                         hdl_info.swi_num =
1578                             ((uint64_t)1 << cfgprof->cp_log_num_srq);
1579                         hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t);
1580                         tavor_rsrc_sw_handles_fini(state, &hdl_info);
1581                 }
1582                 /* FALLTHROUGH */
1583 
1584         case TAVOR_RSRC_CLEANUP_LEVEL25:
1585                 /* Cleanup the CQ handle resource pool */
1586                 hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_CQHDL];
1587                 hdl_info.swi_table_ptr = state->ts_cqhdl;
1588                 hdl_info.swi_num =
1589                     ((uint64_t)1 << cfgprof->cp_log_num_cq);
1590                 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t);
1591                 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1592                 /* FALLTHROUGH */
1593 
1594         case TAVOR_RSRC_CLEANUP_LEVEL24:
1595                 /* Cleanup the EQ handle resource pool */
1596                 hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_EQHDL];
1597                 hdl_info.swi_table_ptr = NULL;
1598                 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1599                 /* FALLTHROUGH */
1600 
1601         case TAVOR_RSRC_CLEANUP_LEVEL23:
1602                 /* Cleanup the MR handle resource pool */
1603                 hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_MRHDL];
1604                 hdl_info.swi_table_ptr = NULL;
1605                 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1606                 /* FALLTHROUGH */
1607 
1608         case TAVOR_RSRC_CLEANUP_LEVEL22:
1609                 /* Cleanup the PD handle resource pool */
1610                 hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_PDHDL];
1611                 hdl_info.swi_table_ptr = NULL;
1612                 tavor_rsrc_pd_handles_fini(state, &hdl_info);
1613                 /* FALLTHROUGH */
1614 
1615         case TAVOR_RSRC_CLEANUP_LEVEL21:
1616                 /* Cleanup the EQC table resource pool */
1617                 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQC];
1618                 tavor_rsrc_hw_entries_fini(state, &entry_info);
1619                 /* FALLTHROUGH */
1620 
1621         case TAVOR_RSRC_CLEANUP_LEVEL20:
1622                 /* Cleanup the MCG table resource pool */
1623                 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MCG];
1624                 tavor_rsrc_hw_entries_fini(state, &entry_info);
1625                 /* FALLTHROUGH */
1626 
1627         case TAVOR_RSRC_CLEANUP_LEVEL19:
1628                 /* Cleanup the outstanding command list  */
1629                 tavor_outstanding_cmdlist_fini(state);
1630                 /* FALLTHROUGH */
1631 
1632         case TAVOR_RSRC_CLEANUP_LEVEL18:
1633                 /* Cleanup the "In" mailbox list  */
1634                 tavor_intr_inmbox_list_fini(state);
1635                 /* FALLTHROUGH */
1636 
1637         case TAVOR_RSRC_CLEANUP_LEVEL17:
1638                 /* Cleanup the interrupt "In" mailbox resource pool */
1639                 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[
1640                     TAVOR_INTR_IN_MBOX];
1641                 tavor_rsrc_mbox_fini(state, &mbox_info);
1642                 /* FALLTHROUGH */
1643 
1644         case TAVOR_RSRC_CLEANUP_LEVEL16:
1645                 /* Cleanup the "In" mailbox list  */
1646                 tavor_inmbox_list_fini(state);
1647                 /* FALLTHROUGH */
1648 
1649         case TAVOR_RSRC_CLEANUP_LEVEL15:
1650                 /* Cleanup the "In" mailbox resource pool */
1651                 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX];
1652                 tavor_rsrc_mbox_fini(state, &mbox_info);
1653                 /* FALLTHROUGH */
1654 
1655         case TAVOR_RSRC_CLEANUP_LEVEL14:
1656                 /*
1657                  * Cleanup the SRQC table resource pool.
1658                  *
1659                  * Note: We only clean up if SRQ is enabled.  Otherwise we
1660                  * simply fallthrough to the next resource cleanup.
1661                  */
1662                 if (state->ts_cfg_profile->cp_srq_enable != 0) {
1663                         entry_info.hwi_rsrcpool =
1664                             &state->ts_rsrc_hdl[TAVOR_SRQC];
1665                         tavor_rsrc_hw_entries_fini(state, &entry_info);
1666                 }
1667                 /* FALLTHROUGH */
1668 
1669         case TAVOR_RSRC_CLEANUP_LEVEL13:
1670                 /* Cleanup the UAR scratch table resource pool */
1671                 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
1672                 tavor_rsrc_hw_entries_fini(state, &entry_info);
1673                 /* FALLTHROUGH */
1674 
1675         case TAVOR_RSRC_CLEANUP_LEVEL12:
1676                 /* Cleanup the UDAV table resource pool */
1677                 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UDAV];
1678                 tavor_rsrc_hw_entries_fini(state, &entry_info);
1679                 /* FALLTHROUGH */
1680 
1681         case TAVOR_RSRC_CLEANUP_LEVEL11:
1682                 /* Cleanup the EQPC table resource pool */
1683                 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQPC];
1684                 tavor_rsrc_hw_entries_fini(state, &entry_info);
1685                 /* FALLTHROUGH */
1686 
1687         case TAVOR_RSRC_CLEANUP_LEVEL10:
1688                 /* Cleanup the CQC table resource pool */
1689                 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQC];
1690                 tavor_rsrc_hw_entries_fini(state, &entry_info);
1691                 /* FALLTHROUGH */
1692 
1693         case TAVOR_RSRC_CLEANUP_LEVEL9:
1694                 /* Cleanup the RDB table resource pool */
1695                 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_RDB];
1696                 tavor_rsrc_hw_entries_fini(state, &entry_info);
1697                 /* FALLTHROUGH */
1698 
1699         case TAVOR_RSRC_CLEANUP_LEVEL8:
1700                 /* Cleanup the QPC table resource pool */
1701                 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPC];
1702                 tavor_rsrc_hw_entries_fini(state, &entry_info);
1703                 /* FALLTHROUGH */
1704 
1705         case TAVOR_RSRC_CLEANUP_LEVEL7:
1706                 /* Cleanup the MTT table resource pool */
1707                 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MTT];
1708                 tavor_rsrc_hw_entries_fini(state, &entry_info);
1709                 /* FALLTHROUGH */
1710 
1711         case TAVOR_RSRC_CLEANUP_LEVEL6:
1712                 /* Cleanup the MPT table resource pool */
1713                 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MPT];
1714                 tavor_rsrc_hw_entries_fini(state, &entry_info);
1715                 /* FALLTHROUGH */
1716 
1717         case TAVOR_RSRC_CLEANUP_LEVEL5:
1718                 /* Destroy the vmem arena for DDR memory */
1719                 vmem_destroy(state->ts_ddrvmem);
1720                 break;
1721 
1722         /*
1723          * The cleanup below comes from the "Phase 1" initialization step.
1724          * (see tavor_rsrc_init_phase1() above)
1725          */
1726         case TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE:
1727                 /* Cleanup the interrupt "Out" mailbox list  */
1728                 tavor_intr_outmbox_list_fini(state);
1729                 /* FALLTHROUGH */
1730 
1731         case TAVOR_RSRC_CLEANUP_LEVEL3:
1732                 /* Cleanup the "Out" mailbox resource pool */
1733                 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[
1734                     TAVOR_INTR_OUT_MBOX];
1735                 tavor_rsrc_mbox_fini(state, &mbox_info);
1736                 /* FALLTHROUGH */
1737 
1738         case TAVOR_RSRC_CLEANUP_LEVEL2:
1739                 /* Cleanup the "Out" mailbox list  */
1740                 tavor_outmbox_list_fini(state);
1741                 /* FALLTHROUGH */
1742 
1743         case TAVOR_RSRC_CLEANUP_LEVEL1:
1744                 /* Cleanup the "Out" mailbox resource pool */
1745                 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX];
1746                 tavor_rsrc_mbox_fini(state, &mbox_info);
1747                 /* FALLTHROUGH */
1748 
1749         case TAVOR_RSRC_CLEANUP_LEVEL0:
1750                 /* Free the array of tavor_rsrc_pool_info_t's */
1751                 kmem_free(state->ts_rsrc_hdl, TAVOR_NUM_RESOURCES *
1752                     sizeof (tavor_rsrc_pool_info_t));
1753                 kmem_cache_destroy(state->ts_rsrc_cache);
1754                 break;
1755 
1756         default:
1757                 TAVOR_WARNING(state, "unexpected resource cleanup level");
1758                 TNF_PROBE_0(tavor_rsrc_fini_default_fail, TAVOR_TNF_ERROR, "");
1759                 TAVOR_TNF_EXIT(tavor_rsrc_fini);
1760                 return;
1761         }
1762 
1763         TAVOR_TNF_EXIT(tavor_rsrc_fini);
1764 }
1765 
1766 
1767 /*
1768  * tavor_rsrc_mbox_init()
1769  *    Context: Only called from attach() path context
1770  */
1771 static int
1772 tavor_rsrc_mbox_init(tavor_state_t *state, tavor_rsrc_mbox_info_t *info)
1773 {
1774         tavor_rsrc_pool_info_t  *rsrc_pool;
1775         tavor_rsrc_priv_mbox_t  *priv;
1776         vmem_t                  *vmp;
1777         uint64_t                offset;
1778         uint_t                  dma_xfer_mode;
1779 
1780         TAVOR_TNF_ENTER(tavor_rsrc_mbox_init);
1781 
1782         ASSERT(state != NULL);
1783         ASSERT(info != NULL);
1784 
1785         rsrc_pool = info->mbi_rsrcpool;
1786         ASSERT(rsrc_pool != NULL);
1787 
1788         dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;
1789 
1790         /* Allocate and initialize mailbox private structure */
1791         priv = kmem_zalloc(sizeof (tavor_rsrc_priv_mbox_t), KM_SLEEP);
1792         priv->pmb_dip                = state->ts_dip;
1793         priv->pmb_acchdl     = state->ts_reg_ddrhdl;
1794         priv->pmb_devaccattr = state->ts_reg_accattr;
1795         priv->pmb_xfer_mode  = dma_xfer_mode;
1796 
1797         /*
1798          * Initialize many of the default DMA attributes.  Then set alignment
1799          * and scatter-gather restrictions specific for mailbox memory.
1800          */
1801         tavor_dma_attr_init(&priv->pmb_dmaattr);
1802         priv->pmb_dmaattr.dma_attr_align  = TAVOR_MBOX_ALIGN;
1803         priv->pmb_dmaattr.dma_attr_sgllen = 1;
1804 
1805         rsrc_pool->rsrc_private = priv;
1806 
1807         /* Is object in DDR memory or system memory? */
1808         if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) {
1809                 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem,
1810                     rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align,
1811                     0, 0, NULL, NULL, VM_SLEEP);
1812                 if (rsrc_pool->rsrc_ddr_offset == NULL) {
1813                         /* Unable to alloc space for mailboxes */
1814                         kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t));
1815                         TNF_PROBE_0(tavor_rsrc_mbox_init_vma_fail,
1816                             TAVOR_TNF_ERROR, "");
1817                         TAVOR_TNF_EXIT(tavor_rsrc_mbox_init);
1818                         return (DDI_FAILURE);
1819                 }
1820 
1821                 /* Calculate offset and starting point (in DDR) */
1822                 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset -
1823                     state->ts_ddr.ddr_baseaddr);
1824                 rsrc_pool->rsrc_start =
1825                     (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr +
1826                     offset);
1827 
1828                 /* Create new vmem arena for the mailboxes */
1829                 vmp = vmem_create(info->mbi_rsrcname,
1830                     rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size,
1831                     rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1832                 if (vmp == NULL) {
1833                         /* Unable to create vmem arena */
1834                         vmem_xfree(state->ts_ddrvmem,
1835                             rsrc_pool->rsrc_ddr_offset,
1836                             rsrc_pool->rsrc_pool_size);
1837                         kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t));
1838                         TNF_PROBE_0(tavor_rsrc_mbox_init_vmem_create_fail,
1839                             TAVOR_TNF_ERROR, "");
1840                         TAVOR_TNF_EXIT(tavor_rsrc_mbox_init);
1841                         return (DDI_FAILURE);
1842                 }
1843                 rsrc_pool->rsrc_vmp = vmp;
1844         } else {
1845                 rsrc_pool->rsrc_ddr_offset = NULL;
1846                 rsrc_pool->rsrc_start = NULL;
1847                 rsrc_pool->rsrc_vmp = NULL;
1848         }
1849 
1850         TAVOR_TNF_EXIT(tavor_rsrc_mbox_init);
1851         return (DDI_SUCCESS);
1852 }
1853 
1854 
1855 /*
1856  * tavor_rsrc_mbox_fini()
1857  *    Context: Only called from attach() and/or detach() path contexts
1858  */
1859 static void
1860 tavor_rsrc_mbox_fini(tavor_state_t *state, tavor_rsrc_mbox_info_t *info)
1861 {
1862         tavor_rsrc_pool_info_t  *rsrc_pool;
1863 
1864         TAVOR_TNF_ENTER(tavor_rsrc_mbox_fini);
1865 
1866         ASSERT(state != NULL);
1867         ASSERT(info != NULL);
1868 
1869         rsrc_pool = info->mbi_rsrcpool;
1870         ASSERT(rsrc_pool != NULL);
1871 
1872         /* If mailboxes are DDR memory, then destroy and free up vmem */
1873         if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) {
1874 
1875                 /* Destroy the specially created mbox vmem arena */
1876                 vmem_destroy(rsrc_pool->rsrc_vmp);
1877 
1878                 /* Free up the region from the ddr_vmem arena */
1879                 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset,
1880                     rsrc_pool->rsrc_pool_size);
1881         }
1882 
1883         /* Free up the private struct */
1884         kmem_free(rsrc_pool->rsrc_private, sizeof (tavor_rsrc_priv_mbox_t));
1885 
1886         TAVOR_TNF_EXIT(tavor_rsrc_mbox_fini);
1887 }
1888 
1889 
1890 /*
1891  * tavor_rsrc_hw_entries_init()
1892  *    Context: Only called from attach() path context
1893  */
1894 static int
1895 tavor_rsrc_hw_entries_init(tavor_state_t *state,
1896     tavor_rsrc_hw_entry_info_t *info)
1897 {
1898         tavor_rsrc_pool_info_t  *rsrc_pool;
1899         tavor_rsrc_t            *rsvd_rsrc = NULL;
1900         vmem_t                  *vmp;
1901         uint64_t                num_hwentry, max_hwentry, num_prealloc;
1902         uint64_t                offset;
1903         int                     status;
1904 
1905         TAVOR_TNF_ENTER(tavor_rsrc_hw_entries_init);
1906 
1907         ASSERT(state != NULL);
1908         ASSERT(info != NULL);
1909 
1910         rsrc_pool       = info->hwi_rsrcpool;
1911         ASSERT(rsrc_pool != NULL);
1912         num_hwentry     = info->hwi_num;
1913         max_hwentry     = info->hwi_max;
1914         num_prealloc    = info->hwi_prealloc;
1915 
1916         /* Make sure number of HW entries makes sense */
1917         if (num_hwentry > max_hwentry) {
1918                 TNF_PROBE_2(tavor_rsrc_hw_entries_init_toomany_fail,
1919                     TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of HW "
1920                     "entries exceeds device maximum", tnf_uint, maxhw,
1921                     max_hwentry);
1922                 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
1923                 return (DDI_FAILURE);
1924         }
1925 
1926         /*
1927          * Determine if we need to allocate DDR space to set up the
1928          * "rsrc_start" pointer.  Not necessary if "rsrc_start" has already
1929          * been initialized (as is the case for the UAR page init).
1930          */
1931         if (rsrc_pool->rsrc_start == NULL) {
1932                 /* Make sure HW entries table is aligned as specified */
1933                 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem,
1934                     rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align,
1935                     0, 0, NULL, NULL, VM_NOSLEEP | VM_FIRSTFIT);
1936                 if (rsrc_pool->rsrc_ddr_offset == NULL) {
1937                         /* Unable to alloc space for aligned HW table */
1938                         TNF_PROBE_0(tavor_rsrc_hw_entry_table_vmxalloc_fail,
1939                             TAVOR_TNF_ERROR, "");
1940                         TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
1941                         return (DDI_FAILURE);
1942                 }
1943 
1944                 /* Calculate offset and starting point (in DDR) */
1945                 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset -
1946                     state->ts_ddr.ddr_baseaddr);
1947                 rsrc_pool->rsrc_start =
1948                     (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr +
1949                     offset);
1950         } else {
1951                 rsrc_pool->rsrc_ddr_offset = rsrc_pool->rsrc_start;
1952         }
1953 
1954         /*
1955          * Create new vmem arena for the HW entries table (if rsrc_quantum
1956          * is non-zero).  Otherwise if rsrc_quantum is zero, then these HW
1957          * entries are not going to be dynamically allocatable (i.e. they
1958          * won't be allocated/freed through tavor_rsrc_alloc/free).  This
1959          * latter option is used for EQPC and UARSCR resource which are, in
1960          * fact, managed by the Tavor hardware.
1961          */
1962         if (rsrc_pool->rsrc_quantum != 0) {
1963                 vmp = vmem_create(info->hwi_rsrcname,
1964                     rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size,
1965                     rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1966                 if (vmp == NULL) {
1967                         /* Unable to create vmem arena */
1968                         if (rsrc_pool->rsrc_ddr_offset !=
1969                             rsrc_pool->rsrc_start) {
1970                                 vmem_xfree(state->ts_ddrvmem,
1971                                     rsrc_pool->rsrc_ddr_offset,
1972                                     rsrc_pool->rsrc_pool_size);
1973                         }
1974                         TNF_PROBE_0(tavor_rsrc_hw_entries_init_vmemcreate_fail,
1975                             TAVOR_TNF_ERROR, "");
1976                         TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
1977                         return (DDI_FAILURE);
1978                 }
1979                 rsrc_pool->rsrc_vmp = vmp;
1980         } else {
1981                 rsrc_pool->rsrc_vmp = NULL;
1982         }
1983 
1984         /* The first HW entries may be reserved by Tavor firmware */
1985         if (num_prealloc != 0) {
1986                 status = tavor_rsrc_alloc(state, rsrc_pool->rsrc_type,
1987                     num_prealloc, TAVOR_SLEEP, &rsvd_rsrc);
1988                 if (status != DDI_SUCCESS) {
1989                         /* Unable to preallocate the reserved HW entries */
1990                         if (rsrc_pool->rsrc_vmp != NULL) {
1991                                 vmem_destroy(rsrc_pool->rsrc_vmp);
1992                         }
1993                         if (rsrc_pool->rsrc_ddr_offset !=
1994                             rsrc_pool->rsrc_start) {
1995                                 vmem_xfree(state->ts_ddrvmem,
1996                                     rsrc_pool->rsrc_ddr_offset,
1997                                     rsrc_pool->rsrc_pool_size);
1998                         }
1999                         TNF_PROBE_0(tavor_rsrc_hw_entries_init_pre_fail,
2000                             TAVOR_TNF_ERROR, "");
2001                         TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
2002                         return (DDI_FAILURE);
2003                 }
2004         }
2005         rsrc_pool->rsrc_private = rsvd_rsrc;
2006 
2007         TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
2008         return (DDI_SUCCESS);
2009 }
2010 
2011 
2012 /*
2013  * tavor_rsrc_hw_entries_fini()
2014  *    Context: Only called from attach() and/or detach() path contexts
2015  */
2016 static void
2017 tavor_rsrc_hw_entries_fini(tavor_state_t *state,
2018     tavor_rsrc_hw_entry_info_t *info)
2019 {
2020         tavor_rsrc_pool_info_t  *rsrc_pool;
2021         tavor_rsrc_t            *rsvd_rsrc;
2022 
2023         TAVOR_TNF_ENTER(tavor_rsrc_hw_entries_fini);
2024 
2025         ASSERT(state != NULL);
2026         ASSERT(info != NULL);
2027 
2028         rsrc_pool = info->hwi_rsrcpool;
2029         ASSERT(rsrc_pool != NULL);
2030 
2031         /* Free up any "reserved" (i.e. preallocated) HW entries */
2032         rsvd_rsrc = (tavor_rsrc_t *)rsrc_pool->rsrc_private;
2033         if (rsvd_rsrc != NULL) {
2034                 tavor_rsrc_free(state, &rsvd_rsrc);
2035         }
2036 
2037         /*
2038          * If we've actually setup a vmem arena for the HW entries, then
2039          * destroy it now
2040          */
2041         if (rsrc_pool->rsrc_vmp != NULL) {
2042                 vmem_destroy(rsrc_pool->rsrc_vmp);
2043         }
2044 
2045         /*
2046          * Determine if a region was allocated from the tavor_ddr_vmem
2047          * arena (and free it up if necessary)
2048          */
2049         if (rsrc_pool->rsrc_ddr_offset != rsrc_pool->rsrc_start) {
2050                 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset,
2051                     rsrc_pool->rsrc_pool_size);
2052         }
2053 
2054         TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_fini);
2055 }
2056 
2057 
2058 /*
2059  * tavor_rsrc_sw_handles_init()
2060  *    Context: Only called from attach() path context
2061  */
2062 /* ARGSUSED */
2063 static int
2064 tavor_rsrc_sw_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2065 {
2066         tavor_rsrc_pool_info_t  *rsrc_pool;
2067         uint64_t                num_swhdl, max_swhdl, prealloc_sz;
2068 
2069         TAVOR_TNF_ENTER(tavor_rsrc_sw_handles_init);
2070 
2071         ASSERT(state != NULL);
2072         ASSERT(info != NULL);
2073 
2074         rsrc_pool       = info->swi_rsrcpool;
2075         ASSERT(rsrc_pool != NULL);
2076         num_swhdl       = info->swi_num;
2077         max_swhdl       = info->swi_max;
2078         prealloc_sz     = info->swi_prealloc_sz;
2079 
2080         /* Make sure number of SW handles makes sense */
2081         if (num_swhdl > max_swhdl) {
2082                 TNF_PROBE_2(tavor_rsrc_sw_handles_init_toomany_fail,
2083                     TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of SW "
2084                     "handles exceeds maximum", tnf_uint, maxsw, max_swhdl);
2085                 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_init);
2086                 return (DDI_FAILURE);
2087         }
2088 
2089         /*
2090          * Depending on the flags parameter, create a kmem_cache for some
2091          * number of software handle structures.  Note: kmem_cache_create()
2092          * will SLEEP until successful.
2093          */
2094         if (info->swi_flags & TAVOR_SWHDL_KMEMCACHE_INIT) {
2095                 rsrc_pool->rsrc_private = kmem_cache_create(
2096                     info->swi_rsrcname, rsrc_pool->rsrc_quantum, 0,
2097                     info->swi_constructor, info->swi_destructor, NULL,
2098                     rsrc_pool->rsrc_state, NULL, 0);
2099         }
2100 
2101         /* Allocate the central list of SW handle pointers */
2102         if (info->swi_flags & TAVOR_SWHDL_TABLE_INIT) {
2103                 info->swi_table_ptr = kmem_zalloc(num_swhdl * prealloc_sz,
2104                     KM_SLEEP);
2105         }
2106 
2107         TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_init);
2108         return (DDI_SUCCESS);
2109 }
2110 
2111 
2112 /*
2113  * tavor_rsrc_sw_handles_fini()
2114  *    Context: Only called from attach() and/or detach() path contexts
2115  */
2116 /* ARGSUSED */
2117 static void
2118 tavor_rsrc_sw_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2119 {
2120         tavor_rsrc_pool_info_t  *rsrc_pool;
2121         uint64_t                num_swhdl, prealloc_sz;
2122 
2123         TAVOR_TNF_ENTER(tavor_rsrc_sw_handles_fini);
2124 
2125         ASSERT(state != NULL);
2126         ASSERT(info != NULL);
2127 
2128         rsrc_pool       = info->swi_rsrcpool;
2129         num_swhdl       = info->swi_num;
2130         prealloc_sz     = info->swi_prealloc_sz;
2131 
2132         /*
2133          * If a "software handle" kmem_cache exists for this resource, then
2134          * destroy it now
2135          */
2136         if (rsrc_pool != NULL) {
2137                 kmem_cache_destroy(rsrc_pool->rsrc_private);
2138         }
2139 
2140         /* Free up this central list of SW handle pointers */
2141         if (info->swi_table_ptr != NULL) {
2142                 kmem_free(info->swi_table_ptr, num_swhdl * prealloc_sz);
2143         }
2144 
2145         TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_fini);
2146 }
2147 
2148 
2149 /*
2150  * tavor_rsrc_pd_handles_init()
2151  *    Context: Only called from attach() path context
2152  */
2153 static int
2154 tavor_rsrc_pd_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2155 {
2156         tavor_rsrc_pool_info_t  *rsrc_pool;
2157         vmem_t                  *vmp;
2158         char                    vmem_name[TAVOR_RSRC_NAME_MAXLEN];
2159         int                     status;
2160 
2161         TAVOR_TNF_ENTER(tavor_rsrc_pd_handles_init);
2162 
2163         ASSERT(state != NULL);
2164         ASSERT(info != NULL);
2165 
2166         rsrc_pool = info->swi_rsrcpool;
2167         ASSERT(rsrc_pool != NULL);
2168 
2169         /* Initialize the resource pool for software handle table */
2170         status = tavor_rsrc_sw_handles_init(state, info);
2171         if (status != DDI_SUCCESS) {
2172                 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_fail, TAVOR_TNF_ERROR, "");
2173                 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc);
2174                 return (DDI_FAILURE);
2175         }
2176 
2177         /* Build vmem arena name from Tavor instance */
2178         TAVOR_RSRC_NAME(vmem_name, TAVOR_PDHDL_VMEM);
2179 
2180         /* Create new vmem arena for PD numbers */
2181         vmp = vmem_create(vmem_name, (caddr_t)1, info->swi_num, 1, NULL,
2182             NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
2183         if (vmp == NULL) {
2184                 /* Unable to create vmem arena */
2185                 info->swi_table_ptr = NULL;
2186                 tavor_rsrc_sw_handles_fini(state, info);
2187                 TNF_PROBE_0(tavor_rsrc_pd_handles_init_vmem_create_fail,
2188                     TAVOR_TNF_ERROR, "");
2189                 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_init);
2190                 return (DDI_FAILURE);
2191         }
2192         rsrc_pool->rsrc_vmp = vmp;
2193 
2194         TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_init);
2195         return (DDI_SUCCESS);
2196 }
2197 
2198 
2199 /*
2200  * tavor_rsrc_pd_handles_fini()
2201  *    Context: Only called from attach() and/or detach() path contexts
2202  */
2203 static void
2204 tavor_rsrc_pd_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2205 {
2206         tavor_rsrc_pool_info_t  *rsrc_pool;
2207 
2208         TAVOR_TNF_ENTER(tavor_rsrc_pd_handles_fini);
2209 
2210         ASSERT(state != NULL);
2211         ASSERT(info != NULL);
2212 
2213         rsrc_pool = info->swi_rsrcpool;
2214 
2215         /* Destroy the specially created UAR scratch table vmem arena */
2216         vmem_destroy(rsrc_pool->rsrc_vmp);
2217 
2218         /* Destroy the "tavor_sw_pd_t" kmem_cache */
2219         tavor_rsrc_sw_handles_fini(state, info);
2220 
2221         TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_fini);
2222 }
2223 
2224 
2225 /*
2226  * tavor_rsrc_mbox_alloc()
2227  *    Context: Only called from attach() path context
2228  */
2229 static int
2230 tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num,
2231     tavor_rsrc_t *hdl)
2232 {
2233         tavor_rsrc_priv_mbox_t  *priv;
2234         void                    *addr;
2235         caddr_t                 kaddr;
2236         uint64_t                offset;
2237         size_t                  real_len, temp_len;
2238         int                     status;
2239 
2240         TAVOR_TNF_ENTER(tavor_rsrc_mbox_alloc);
2241 
2242         ASSERT(pool_info != NULL);
2243         ASSERT(hdl != NULL);
2244 
2245         /* Get the private pointer for the mailboxes */
2246         priv = pool_info->rsrc_private;
2247         ASSERT(priv != NULL);
2248 
2249         /*
2250          * Allocate a DMA handle for the mailbox.  This will be used for
2251          * two purposes (potentially).  First, it could be used below in
2252          * the call to ddi_dma_mem_alloc() - if the mailbox is to come from
2253          * system memory.  Second, it is definitely used later to bind
2254          * the mailbox for DMA access from/by the hardware.
2255          */
2256         status = ddi_dma_alloc_handle(priv->pmb_dip, &priv->pmb_dmaattr,
2257             DDI_DMA_SLEEP, NULL, &hdl->tr_dmahdl);
2258         if (status != DDI_SUCCESS) {
2259                 TNF_PROBE_1(tavor_rsrc_mbox_alloc_dmahdl_fail, TAVOR_TNF_ERROR,
2260                     "", tnf_uint, status, status);
2261                 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc);
2262                 return (DDI_FAILURE);
2263         }
2264 
2265         /* Is mailbox in DDR memory or system memory? */
2266         if (pool_info->rsrc_loc == TAVOR_IN_DDR) {
2267                 /* Use vmem_alloc() to get DDR address of mbox */
2268                 hdl->tr_len = (num * pool_info->rsrc_quantum);
2269                 addr = vmem_alloc(pool_info->rsrc_vmp, hdl->tr_len,
2270                     VM_SLEEP);
2271                 if (addr == NULL) {
2272                         /* No more DDR available for mailbox entries */
2273                         ddi_dma_free_handle(&hdl->tr_dmahdl);
2274                         TNF_PROBE_0(tavor_rsrc_mbox_alloc_vma_fail,
2275                             TAVOR_TNF_ERROR, "");
2276                         TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc);
2277                         return (DDI_FAILURE);
2278                 }
2279                 hdl->tr_acchdl = priv->pmb_acchdl;
2280 
2281                 /* Calculate kernel virtual address (from the DDR offset) */
2282                 offset = ((uintptr_t)addr -
2283                     (uintptr_t)pool_info->rsrc_ddr_offset);
2284                 hdl->tr_addr = (void *)(uintptr_t)(offset +
2285                     (uintptr_t)pool_info->rsrc_start);
2286 
2287         } else { /* TAVOR_IN_SYSMEM */
2288 
2289                 /* Use ddi_dma_mem_alloc() to get memory for mailbox */
2290                 temp_len = (num * pool_info->rsrc_quantum);
2291                 status = ddi_dma_mem_alloc(hdl->tr_dmahdl, temp_len,
2292                     &priv->pmb_devaccattr, priv->pmb_xfer_mode, DDI_DMA_SLEEP,
2293                     NULL, &kaddr, &real_len, &hdl->tr_acchdl);
2294                 if (status != DDI_SUCCESS) {
2295                         /* No more sys memory available for mailbox entries */
2296                         ddi_dma_free_handle(&hdl->tr_dmahdl);
2297                         TNF_PROBE_0(tavor_rsrc_mbox_alloc_dma_memalloc_fail,
2298                             TAVOR_TNF_ERROR, "");
2299                         TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc);
2300                         return (DDI_FAILURE);
2301                 }
2302                 hdl->tr_addr = (void *)kaddr;
2303                 hdl->tr_len  = real_len;
2304         }
2305 
2306         TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc);
2307         return (DDI_SUCCESS);
2308 }
2309 
2310 
2311 /*
2312  * tavor_rsrc_mbox_free()
2313  *    Context: Can be called from interrupt or base context.
2314  */
2315 static void
2316 tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2317 {
2318         void            *addr;
2319         uint64_t        offset;
2320 
2321         TAVOR_TNF_ENTER(tavor_rsrc_mbox_free);
2322 
2323         ASSERT(pool_info != NULL);
2324         ASSERT(hdl != NULL);
2325 
2326         /* Is mailbox in DDR memory or system memory? */
2327         if (pool_info->rsrc_loc == TAVOR_IN_DDR) {
2328 
2329                 /* Calculate the allocated address (the mbox's DDR offset) */
2330                 offset = ((uintptr_t)hdl->tr_addr -
2331                     (uintptr_t)pool_info->rsrc_start);
2332                 addr = (void *)(uintptr_t)(offset +
2333                     (uintptr_t)pool_info->rsrc_ddr_offset);
2334 
2335                 /* Use vmem_free() to free up DDR memory for mailbox */
2336                 vmem_free(pool_info->rsrc_vmp, addr, hdl->tr_len);
2337 
2338         } else { /* TAVOR_IN_SYSMEM */
2339 
2340                 /* Use ddi_dma_mem_free() to free up sys memory for mailbox */
2341                 ddi_dma_mem_free(&hdl->tr_acchdl);
2342         }
2343 
2344         /* Free the DMA handle for the mailbox */
2345         ddi_dma_free_handle(&hdl->tr_dmahdl);
2346 
2347         TAVOR_TNF_EXIT(tavor_rsrc_mbox_free);
2348 }
2349 
2350 
2351 /*
2352  * tavor_rsrc_hw_entry_alloc()
2353  *    Context: Can be called from interrupt or base context.
2354  */
2355 static int
2356 tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num,
2357     uint_t num_align, ddi_acc_handle_t acc_handle, uint_t sleepflag,
2358     tavor_rsrc_t *hdl)
2359 {
2360         void            *addr;
2361         uint64_t        offset;
2362         uint32_t        align;
2363         int             flag;
2364 
2365         TAVOR_TNF_ENTER(tavor_rsrc_hw_entry_alloc);
2366 
2367         ASSERT(pool_info != NULL);
2368         ASSERT(hdl != NULL);
2369 
2370         /*
2371          * Tavor hardware entries (QPC, CQC, EQC, MPT, MTT, etc.) do not
2372          * use dma_handle (because they are in Tavor locally attached DDR
2373          * memory) and, generally, don't use the acc_handle (because the
2374          * entries are not directly accessed by software).  The exceptions
2375          * to this rule are the UARPG and UDAV entries.
2376          */
2377 
2378         /*
2379          * Use vmem_xalloc() to get a properly aligned pointer (based on
2380          * the number requested) to the HW entry(ies).  This handles the
2381          * cases (for special QPCs and for RDB entries) where we need more
2382          * than one and need to ensure that they are properly aligned.
2383          */
2384         flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
2385         hdl->tr_len = (num * pool_info->rsrc_quantum);
2386         align       = (num_align * pool_info->rsrc_quantum);
2387         addr = vmem_xalloc(pool_info->rsrc_vmp, hdl->tr_len,
2388             align, 0, 0, NULL, NULL, flag | VM_FIRSTFIT);
2389         if (addr == NULL) {
2390                 /* No more HW entries available */
2391                 TNF_PROBE_0(tavor_rsrc_hw_entry_alloc_vmxa_fail,
2392                     TAVOR_TNF_ERROR, "");
2393                 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_alloc);
2394                 return (DDI_FAILURE);
2395         }
2396 
2397         /* If an access handle was provided, fill it in */
2398         if (acc_handle != 0) {
2399                 hdl->tr_acchdl = acc_handle;
2400         }
2401 
2402         /* Calculate vaddr and HW table index (from the DDR offset) */
2403         offset = ((uintptr_t)addr - (uintptr_t)pool_info->rsrc_ddr_offset);
2404         hdl->tr_addr = (void *)(uintptr_t)(offset +
2405             (uintptr_t)pool_info->rsrc_start);
2406         hdl->tr_indx = (offset >> pool_info->rsrc_shift);
2407 
2408         TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_alloc);
2409         return (DDI_SUCCESS);
2410 }
2411 
2412 
2413 /*
2414  * tavor_rsrc_hw_entry_free()
2415  *    Context: Can be called from interrupt or base context.
2416  */
2417 static void
2418 tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2419 {
2420         void            *addr;
2421         uint64_t        offset;
2422 
2423         TAVOR_TNF_ENTER(tavor_rsrc_hw_entry_free);
2424 
2425         ASSERT(pool_info != NULL);
2426         ASSERT(hdl != NULL);
2427 
2428         /* Calculate the allocated address (the entry's DDR offset) */
2429         offset = ((uintptr_t)hdl->tr_addr - (uintptr_t)pool_info->rsrc_start);
2430         addr   = (void *)(uintptr_t)(offset +
2431             (uintptr_t)pool_info->rsrc_ddr_offset);
2432 
2433         /* Use vmem_xfree() to free up the HW table entry */
2434         vmem_xfree(pool_info->rsrc_vmp, addr, hdl->tr_len);
2435 
2436         TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_free);
2437 }
2438 
2439 
2440 /*
2441  * tavor_rsrc_swhdl_alloc()
2442  *    Context: Can be called from interrupt or base context.
2443  */
2444 static int
2445 tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag,
2446     tavor_rsrc_t *hdl)
2447 {
2448         void    *addr;
2449         int     flag;
2450 
2451         TAVOR_TNF_ENTER(tavor_rsrc_swhdl_alloc);
2452 
2453         ASSERT(pool_info != NULL);
2454         ASSERT(hdl != NULL);
2455 
2456         /* Allocate the software handle structure */
2457         flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
2458         addr = kmem_cache_alloc(pool_info->rsrc_private, flag);
2459         if (addr == NULL) {
2460                 TNF_PROBE_0(tavor_rsrc_swhdl_alloc_kmca_fail, TAVOR_TNF_ERROR,
2461                     "");
2462                 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_alloc);
2463                 return (DDI_FAILURE);
2464         }
2465         hdl->tr_len  = pool_info->rsrc_quantum;
2466         hdl->tr_addr = addr;
2467 
2468         TAVOR_TNF_EXIT(tavor_rsrc_swhdl_alloc);
2469         return (DDI_SUCCESS);
2470 }
2471 
2472 
2473 /*
2474  * tavor_rsrc_swhdl_free()
2475  *    Context: Can be called from interrupt or base context.
2476  */
2477 static void
2478 tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2479 {
2480         TAVOR_TNF_ENTER(tavor_rsrc_swhdl_free);
2481 
2482         ASSERT(pool_info != NULL);
2483         ASSERT(hdl != NULL);
2484 
2485         /* Free the software handle structure */
2486         kmem_cache_free(pool_info->rsrc_private, hdl->tr_addr);
2487 
2488         TAVOR_TNF_EXIT(tavor_rsrc_swhdl_free);
2489 }
2490 
2491 
2492 /*
2493  * tavor_rsrc_pdhdl_alloc()
2494  *    Context: Can be called from interrupt or base context.
2495  */
2496 static int
2497 tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag,
2498     tavor_rsrc_t *hdl)
2499 {
2500         tavor_pdhdl_t   addr;
2501         void            *tmpaddr;
2502         int             flag, status;
2503 
2504         TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_alloc);
2505 
2506         ASSERT(pool_info != NULL);
2507         ASSERT(hdl != NULL);
2508 
2509         /* Allocate the software handle */
2510         status = tavor_rsrc_swhdl_alloc(pool_info, sleepflag, hdl);
2511         if (status != DDI_SUCCESS) {
2512                 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_fail, TAVOR_TNF_ERROR, "");
2513                 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc);
2514                 return (DDI_FAILURE);
2515         }
2516         addr = (tavor_pdhdl_t)hdl->tr_addr;
2517         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*addr))
2518 
2519         /* Allocate a PD number for the handle */
2520         flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
2521         tmpaddr = vmem_alloc(pool_info->rsrc_vmp, 1, flag);
2522         if (tmpaddr == NULL) {
2523                 /* No more PD number entries available */
2524                 tavor_rsrc_swhdl_free(pool_info, hdl);
2525                 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_vma_fail,
2526                     TAVOR_TNF_ERROR, "");
2527                 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc);
2528                 return (DDI_FAILURE);
2529         }
2530         addr->pd_pdnum = (uint32_t)(uintptr_t)tmpaddr;
2531         addr->pd_rsrcp = hdl;
2532         hdl->tr_indx   = addr->pd_pdnum;
2533 
2534         TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc);
2535         return (DDI_SUCCESS);
2536 }
2537 
2538 
2539 /*
2540  * tavor_rsrc_pdhdl_free()
2541  *    Context: Can be called from interrupt or base context.
2542  */
2543 static void
2544 tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2545 {
2546         TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_free);
2547 
2548         ASSERT(pool_info != NULL);
2549         ASSERT(hdl != NULL);
2550 
2551         /* Use vmem_free() to free up the PD number */
2552         vmem_free(pool_info->rsrc_vmp, (void *)(uintptr_t)hdl->tr_indx, 1);
2553 
2554         /* Free the software handle structure */
2555         tavor_rsrc_swhdl_free(pool_info, hdl);
2556 
2557         TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_free);
2558 }
2559 
2560 
2561 /*
2562  * tavor_rsrc_pdhdl_constructor()
2563  *    Context: Can be called from interrupt or base context.
2564  */
2565 /* ARGSUSED */
2566 static int
2567 tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags)
2568 {
2569         tavor_pdhdl_t   pdhdl;
2570         tavor_state_t   *state;
2571 
2572         TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_constructor);
2573 
2574         pdhdl = (tavor_pdhdl_t)pd;
2575         state = (tavor_state_t *)priv;
2576 
2577         mutex_init(&pdhdl->pd_lock, NULL, MUTEX_DRIVER,
2578             DDI_INTR_PRI(state->ts_intrmsi_pri));
2579 
2580         TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_constructor);
2581         return (DDI_SUCCESS);
2582 }
2583 
2584 
2585 /*
2586  * tavor_rsrc_pdhdl_destructor()
2587  *    Context: Can be called from interrupt or base context.
2588  */
2589 /* ARGSUSED */
2590 static void
2591 tavor_rsrc_pdhdl_destructor(void *pd, void *priv)
2592 {
2593         tavor_pdhdl_t   pdhdl;
2594 
2595         TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_destructor);
2596 
2597         pdhdl = (tavor_pdhdl_t)pd;
2598 
2599         mutex_destroy(&pdhdl->pd_lock);
2600 
2601         TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_destructor);
2602 }
2603 
2604 
2605 /*
2606  * tavor_rsrc_cqhdl_constructor()
2607  *    Context: Can be called from interrupt or base context.
2608  */
2609 /* ARGSUSED */
2610 static int
2611 tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags)
2612 {
2613         tavor_cqhdl_t   cqhdl;
2614         tavor_state_t   *state;
2615 
2616         TAVOR_TNF_ENTER(tavor_rsrc_cqhdl_constructor);
2617 
2618         cqhdl = (tavor_cqhdl_t)cq;
2619         state = (tavor_state_t *)priv;
2620 
2621         mutex_init(&cqhdl->cq_lock, NULL, MUTEX_DRIVER,
2622             DDI_INTR_PRI(state->ts_intrmsi_pri));
2623         mutex_init(&cqhdl->cq_wrid_wqhdr_lock, NULL, MUTEX_DRIVER,
2624             DDI_INTR_PRI(state->ts_intrmsi_pri));
2625 
2626         TAVOR_TNF_EXIT(tavor_rsrc_cqhdl_constructor);
2627         return (DDI_SUCCESS);
2628 }
2629 
2630 
2631 /*
2632  * tavor_rsrc_cqhdl_destructor()
2633  *    Context: Can be called from interrupt or base context.
2634  */
2635 /* ARGSUSED */
2636 static void
2637 tavor_rsrc_cqhdl_destructor(void *cq, void *priv)
2638 {
2639         tavor_cqhdl_t   cqhdl;
2640 
2641         TAVOR_TNF_ENTER(tavor_rsrc_cqhdl_destructor);
2642 
2643         cqhdl = (tavor_cqhdl_t)cq;
2644 
2645         mutex_destroy(&cqhdl->cq_wrid_wqhdr_lock);
2646         mutex_destroy(&cqhdl->cq_lock);
2647 
2648         TAVOR_TNF_EXIT(tavor_rsrc_cqhdl_destructor);
2649 }
2650 
2651 
2652 /*
2653  * tavor_rsrc_qphdl_constructor()
2654  *    Context: Can be called from interrupt or base context.
2655  */
2656 /* ARGSUSED */
2657 static int
2658 tavor_rsrc_qphdl_constructor(void *qp, void *priv, int flags)
2659 {
2660         tavor_qphdl_t   qphdl;
2661         tavor_state_t   *state;
2662 
2663         TAVOR_TNF_ENTER(tavor_rsrc_qphdl_constructor);
2664 
2665         qphdl = (tavor_qphdl_t)qp;
2666         state = (tavor_state_t *)priv;
2667 
2668         mutex_init(&qphdl->qp_lock, NULL, MUTEX_DRIVER,
2669             DDI_INTR_PRI(state->ts_intrmsi_pri));
2670 
2671         TAVOR_TNF_EXIT(tavor_rsrc_qphdl_constructor);
2672         return (DDI_SUCCESS);
2673 }
2674 
2675 
2676 /*
2677  * tavor_rsrc_qphdl_destructor()
2678  *    Context: Can be called from interrupt or base context.
2679  */
2680 /* ARGSUSED */
2681 static void
2682 tavor_rsrc_qphdl_destructor(void *qp, void *priv)
2683 {
2684         tavor_qphdl_t   qphdl;
2685 
2686         TAVOR_TNF_ENTER(tavor_rsrc_qphdl_destructor);
2687 
2688         qphdl = (tavor_qphdl_t)qp;
2689 
2690         mutex_destroy(&qphdl->qp_lock);
2691 
2692         TAVOR_TNF_ENTER(tavor_rsrc_qphdl_destructor);
2693 }
2694 
2695 
2696 /*
2697  * tavor_rsrc_srqhdl_constructor()
2698  *    Context: Can be called from interrupt or base context.
2699  */
2700 /* ARGSUSED */
2701 static int
2702 tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags)
2703 {
2704         tavor_srqhdl_t  srqhdl;
2705         tavor_state_t   *state;
2706 
2707         TAVOR_TNF_ENTER(tavor_rsrc_srqhdl_constructor);
2708 
2709         srqhdl = (tavor_srqhdl_t)srq;
2710         state = (tavor_state_t *)priv;
2711 
2712         mutex_init(&srqhdl->srq_lock, NULL, MUTEX_DRIVER,
2713             DDI_INTR_PRI(state->ts_intrmsi_pri));
2714 
2715         TAVOR_TNF_EXIT(tavor_rsrc_srqhdl_constructor);
2716         return (DDI_SUCCESS);
2717 }
2718 
2719 
2720 /*
2721  * tavor_rsrc_srqhdl_destructor()
2722  *    Context: Can be called from interrupt or base context.
2723  */
2724 /* ARGSUSED */
2725 static void
2726 tavor_rsrc_srqhdl_destructor(void *srq, void *priv)
2727 {
2728         tavor_srqhdl_t  srqhdl;
2729 
2730         TAVOR_TNF_ENTER(tavor_rsrc_srqhdl_destructor);
2731 
2732         srqhdl = (tavor_srqhdl_t)srq;
2733 
2734         mutex_destroy(&srqhdl->srq_lock);
2735 
2736         TAVOR_TNF_EXIT(tavor_rsrc_srqhdl_destructor);
2737 }
2738 
2739 
2740 /*
2741  * tavor_rsrc_refcnt_constructor()
2742  *    Context: Can be called from interrupt or base context.
2743  */
2744 /* ARGSUSED */
2745 static int
2746 tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags)
2747 {
2748         tavor_sw_refcnt_t       *refcnt;
2749         tavor_state_t           *state;
2750 
2751         TAVOR_TNF_ENTER(tavor_rsrc_refcnt_constructor);
2752 
2753         refcnt = (tavor_sw_refcnt_t *)rc;
2754         state  = (tavor_state_t *)priv;
2755 
2756         mutex_init(&refcnt->swrc_lock, NULL, MUTEX_DRIVER,
2757             DDI_INTR_PRI(state->ts_intrmsi_pri));
2758 
2759         TAVOR_TNF_EXIT(tavor_rsrc_refcnt_constructor);
2760         return (DDI_SUCCESS);
2761 }
2762 
2763 
2764 /*
2765  * tavor_rsrc_refcnt_destructor()
2766  *    Context: Can be called from interrupt or base context.
2767  */
2768 /* ARGSUSED */
2769 static void
2770 tavor_rsrc_refcnt_destructor(void *rc, void *priv)
2771 {
2772         tavor_sw_refcnt_t       *refcnt;
2773 
2774         TAVOR_TNF_ENTER(tavor_rsrc_refcnt_destructor);
2775 
2776         refcnt = (tavor_sw_refcnt_t *)rc;
2777 
2778         mutex_destroy(&refcnt->swrc_lock);
2779 
2780         TAVOR_TNF_ENTER(tavor_rsrc_refcnt_destructor);
2781 }
2782 
2783 
2784 /*
2785  * tavor_rsrc_ahhdl_constructor()
2786  *    Context: Can be called from interrupt or base context.
2787  */
2788 /* ARGSUSED */
2789 static int
2790 tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags)
2791 {
2792         tavor_ahhdl_t   ahhdl;
2793         tavor_state_t   *state;
2794 
2795         TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_constructor);
2796 
2797         ahhdl = (tavor_ahhdl_t)ah;
2798         state = (tavor_state_t *)priv;
2799 
2800         mutex_init(&ahhdl->ah_lock, NULL, MUTEX_DRIVER,
2801             DDI_INTR_PRI(state->ts_intrmsi_pri));
2802 
2803         TAVOR_TNF_EXIT(tavor_rsrc_ahhdl_constructor);
2804         return (DDI_SUCCESS);
2805 }
2806 
2807 
2808 /*
2809  * tavor_rsrc_ahhdl_destructor()
2810  *    Context: Can be called from interrupt or base context.
2811  */
2812 /* ARGSUSED */
2813 static void
2814 tavor_rsrc_ahhdl_destructor(void *ah, void *priv)
2815 {
2816         tavor_ahhdl_t   ahhdl;
2817 
2818         TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_destructor);
2819 
2820         ahhdl = (tavor_ahhdl_t)ah;
2821 
2822         mutex_destroy(&ahhdl->ah_lock);
2823 
2824         TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_destructor);
2825 }
2826 
2827 
2828 /*
2829  * tavor_rsrc_mrhdl_constructor()
2830  *    Context: Can be called from interrupt or base context.
2831  */
2832 /* ARGSUSED */
2833 static int
2834 tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags)
2835 {
2836         tavor_mrhdl_t   mrhdl;
2837         tavor_state_t   *state;
2838 
2839         TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_constructor);
2840 
2841         mrhdl = (tavor_mrhdl_t)mr;
2842         state = (tavor_state_t *)priv;
2843 
2844         mutex_init(&mrhdl->mr_lock, NULL, MUTEX_DRIVER,
2845             DDI_INTR_PRI(state->ts_intrmsi_pri));
2846 
2847         TAVOR_TNF_EXIT(tavor_rsrc_mrhdl_constructor);
2848         return (DDI_SUCCESS);
2849 }
2850 
2851 
2852 /*
2853  * tavor_rsrc_mrhdl_destructor()
2854  *    Context: Can be called from interrupt or base context.
2855  */
2856 /* ARGSUSED */
2857 static void
2858 tavor_rsrc_mrhdl_destructor(void *mr, void *priv)
2859 {
2860         tavor_mrhdl_t   mrhdl;
2861 
2862         TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_destructor);
2863 
2864         mrhdl = (tavor_mrhdl_t)mr;
2865 
2866         mutex_destroy(&mrhdl->mr_lock);
2867 
2868         TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_destructor);
2869 }
2870 
2871 
2872 /*
2873  * tavor_rsrc_mcg_entry_get_size()
2874  */
2875 static int
2876 tavor_rsrc_mcg_entry_get_size(tavor_state_t *state, uint_t *mcg_size_shift)
2877 {
2878         uint_t  num_qp_per_mcg, max_qp_per_mcg, log2;
2879 
2880         TAVOR_TNF_ENTER(tavor_rsrc_mcg_entry_get_size);
2881 
2882         /*
2883          * Round the configured number of QP per MCG to next larger
2884          * power-of-2 size and update.
2885          */
2886         num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg + 8;
2887         log2 = highbit(num_qp_per_mcg);
2888         if ((num_qp_per_mcg & (num_qp_per_mcg - 1)) == 0) {
2889                 log2 = log2 - 1;
2890         }
2891         state->ts_cfg_profile->cp_num_qp_per_mcg = (1 << log2) - 8;
2892 
2893         /* Now make sure number of QP per MCG makes sense */
2894         num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg;
2895         max_qp_per_mcg = (1 << state->ts_devlim.log_max_qp_mcg);
2896         if (num_qp_per_mcg > max_qp_per_mcg) {
2897                 TNF_PROBE_1(tavor_rsrc_mcg_getsz_toomany_qppermcg_fail,
2898                     TAVOR_TNF_ERROR, "", tnf_uint, maxqpmcg, max_qp_per_mcg);
2899                 TAVOR_TNF_EXIT(tavor_rsrc_mcg_entry_get_size);
2900                 return (DDI_FAILURE);
2901         }
2902 
2903         /* Return the (shift) size of an individual MCG HW entry */
2904         *mcg_size_shift = log2 + 2;
2905 
2906         TAVOR_TNF_EXIT(tavor_rsrc_mcg_entry_get_size);
2907         return (DDI_SUCCESS);
2908 }