1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /*
27 * hermon_rsrc.c
28 * Hermon Resource Management Routines
29 *
30 * Implements all the routines necessary for setup, teardown, and
31 * alloc/free of all Hermon resources, including those that are managed
32 * by Hermon hardware or which live in Hermon's direct attached DDR memory.
33 */
34
35 #include <sys/sysmacros.h>
36 #include <sys/types.h>
37 #include <sys/conf.h>
38 #include <sys/ddi.h>
39 #include <sys/sunddi.h>
40 #include <sys/modctl.h>
41 #include <sys/vmem.h>
42 #include <sys/bitmap.h>
43
44 #include <sys/ib/adapters/hermon/hermon.h>
45
46 int hermon_rsrc_verbose = 0;
47
48 /*
49 * The following routines are used for initializing and destroying
50 * the resource pools used by the Hermon resource allocation routines.
51 * They consist of four classes of object:
52 *
53 * Mailboxes: The "In" and "Out" mailbox types are used by the Hermon
54 * command interface routines. Mailboxes are used to pass information
55 * back and forth to the Hermon firmware. Either type of mailbox may
56 * be allocated from Hermon's direct attached DDR memory or from system
57 * memory (although currently all "In" mailboxes are in DDR and all "out"
58 * mailboxes come from system memory.
59 *
60 * HW entry objects: These objects represent resources required by the Hermon
61 * hardware. These objects include things like Queue Pair contexts (QPC),
62 * Completion Queue contexts (CQC), Event Queue contexts (EQC), RDB (for
63 * supporting RDMA Read/Atomic), Multicast Group entries (MCG), Memory
64 * Protection Table entries (MPT), Memory Translation Table entries (MTT).
65 *
66 * What these objects all have in common is that they are each required
67 * to come from ICM memory, they are always allocated from tables, and
68 * they are not to be directly accessed (read or written) by driver
69 * software (Mellanox FMR access to MPT is an exception).
70 * The other notable exceptions are the UAR pages (UAR_PG) which are
71 * allocated from the UAR address space rather than DDR, and the UD
72 * address vectors (UDAV) which are similar to the common object types
73 * with the major difference being that UDAVs _are_ directly read and
74 * written by driver software.
75 *
76 * SW handle objects: These objects represent resources required by Hermon
77 * driver software. They are primarily software tracking structures,
78 * which are allocated from system memory (using kmem_cache). Several of
79 * the objects have both a "constructor" and "destructor" method
80 * associated with them (see below).
81 *
82 * Protection Domain (PD) handle objects: These objects are very much like
83 * a SW handle object with the notable difference that all PD handle
84 * objects have an actual Protection Domain number (PD) associated with
85 * them (and the PD number is allocated/managed through a separate
86 * vmem_arena specifically set aside for this purpose.
87 */
88
89 static int hermon_rsrc_mbox_init(hermon_state_t *state,
90 hermon_rsrc_mbox_info_t *info);
91 static void hermon_rsrc_mbox_fini(hermon_state_t *state,
92 hermon_rsrc_mbox_info_t *info);
93
94 static int hermon_rsrc_sw_handles_init(hermon_state_t *state,
95 hermon_rsrc_sw_hdl_info_t *info);
96 static void hermon_rsrc_sw_handles_fini(hermon_state_t *state,
97 hermon_rsrc_sw_hdl_info_t *info);
98
99 static int hermon_rsrc_pd_handles_init(hermon_state_t *state,
100 hermon_rsrc_sw_hdl_info_t *info);
101 static void hermon_rsrc_pd_handles_fini(hermon_state_t *state,
102 hermon_rsrc_sw_hdl_info_t *info);
103
104 /*
105 * The following routines are used for allocating and freeing the specific
106 * types of objects described above from their associated resource pools.
107 */
108 static int hermon_rsrc_mbox_alloc(hermon_rsrc_pool_info_t *pool_info,
109 uint_t num, hermon_rsrc_t *hdl);
110 static void hermon_rsrc_mbox_free(hermon_rsrc_t *hdl);
111
112 static int hermon_rsrc_hw_entry_alloc(hermon_rsrc_pool_info_t *pool_info,
113 uint_t num, uint_t num_align, uint_t sleepflag, hermon_rsrc_t *hdl);
114 static void hermon_rsrc_hw_entry_free(hermon_rsrc_pool_info_t *pool_info,
115 hermon_rsrc_t *hdl);
116 static int hermon_rsrc_hw_entry_reserve(hermon_rsrc_pool_info_t *pool_info,
117 uint_t num, uint_t num_align, uint_t sleepflag, hermon_rsrc_t *hdl);
118
119 static int hermon_rsrc_hw_entry_icm_confirm(hermon_rsrc_pool_info_t *pool_info,
120 uint_t num, hermon_rsrc_t *hdl, int num_to_hdl);
121 static int hermon_rsrc_hw_entry_icm_free(hermon_rsrc_pool_info_t *pool_info,
122 hermon_rsrc_t *hdl, int num_to_hdl);
123
124 static int hermon_rsrc_swhdl_alloc(hermon_rsrc_pool_info_t *pool_info,
125 uint_t sleepflag, hermon_rsrc_t *hdl);
126 static void hermon_rsrc_swhdl_free(hermon_rsrc_pool_info_t *pool_info,
127 hermon_rsrc_t *hdl);
128
129 static int hermon_rsrc_pdhdl_alloc(hermon_rsrc_pool_info_t *pool_info,
130 uint_t sleepflag, hermon_rsrc_t *hdl);
131 static void hermon_rsrc_pdhdl_free(hermon_rsrc_pool_info_t *pool_info,
132 hermon_rsrc_t *hdl);
133
134 static int hermon_rsrc_fexch_alloc(hermon_state_t *state,
135 hermon_rsrc_type_t rsrc, uint_t num, uint_t sleepflag, hermon_rsrc_t *hdl);
136 static void hermon_rsrc_fexch_free(hermon_state_t *state, hermon_rsrc_t *hdl);
137 static int hermon_rsrc_rfci_alloc(hermon_state_t *state,
138 hermon_rsrc_type_t rsrc, uint_t num, uint_t sleepflag, hermon_rsrc_t *hdl);
139 static void hermon_rsrc_rfci_free(hermon_state_t *state, hermon_rsrc_t *hdl);
140
141 /*
142 * The following routines are the constructors and destructors for several
143 * of the SW handle type objects. For certain types of SW handles objects
144 * (all of which are implemented using kmem_cache), we need to do some
145 * special field initialization (specifically, mutex_init/destroy). These
146 * routines enable that init and teardown.
147 */
148 static int hermon_rsrc_pdhdl_constructor(void *pd, void *priv, int flags);
149 static void hermon_rsrc_pdhdl_destructor(void *pd, void *state);
150 static int hermon_rsrc_cqhdl_constructor(void *cq, void *priv, int flags);
151 static void hermon_rsrc_cqhdl_destructor(void *cq, void *state);
152 static int hermon_rsrc_qphdl_constructor(void *cq, void *priv, int flags);
153 static void hermon_rsrc_qphdl_destructor(void *cq, void *state);
154 static int hermon_rsrc_srqhdl_constructor(void *srq, void *priv, int flags);
155 static void hermon_rsrc_srqhdl_destructor(void *srq, void *state);
156 static int hermon_rsrc_refcnt_constructor(void *rc, void *priv, int flags);
157 static void hermon_rsrc_refcnt_destructor(void *rc, void *state);
158 static int hermon_rsrc_ahhdl_constructor(void *ah, void *priv, int flags);
159 static void hermon_rsrc_ahhdl_destructor(void *ah, void *state);
160 static int hermon_rsrc_mrhdl_constructor(void *mr, void *priv, int flags);
161 static void hermon_rsrc_mrhdl_destructor(void *mr, void *state);
162
163 /*
164 * Special routine to calculate and return the size of a MCG object based
165 * on current driver configuration (specifically, the number of QP per MCG
166 * that has been configured.
167 */
168 static int hermon_rsrc_mcg_entry_get_size(hermon_state_t *state,
169 uint_t *mcg_size_shift);
170
171
172 /*
173 * hermon_rsrc_alloc()
174 *
175 * Context: Can be called from interrupt or base context.
176 * The "sleepflag" parameter is used by all object allocators to
177 * determine whether to SLEEP for resources or not.
178 */
179 int
180 hermon_rsrc_alloc(hermon_state_t *state, hermon_rsrc_type_t rsrc, uint_t num,
181 uint_t sleepflag, hermon_rsrc_t **hdl)
182 {
183 hermon_rsrc_pool_info_t *rsrc_pool;
184 hermon_rsrc_t *tmp_rsrc_hdl;
185 int flag, status = DDI_FAILURE;
186
187 ASSERT(state != NULL);
188 ASSERT(hdl != NULL);
189
190 rsrc_pool = &state->hs_rsrc_hdl[rsrc];
191 ASSERT(rsrc_pool != NULL);
192
193 /*
194 * Allocate space for the object used to track the resource handle
195 */
196 flag = (sleepflag == HERMON_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
197 tmp_rsrc_hdl = kmem_cache_alloc(state->hs_rsrc_cache, flag);
198 if (tmp_rsrc_hdl == NULL) {
199 return (DDI_FAILURE);
200 }
201 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tmp_rsrc_hdl))
202
203 /*
204 * Set rsrc_hdl type. This is later used by the hermon_rsrc_free call
205 * to know what type of resource is being freed.
206 */
207 tmp_rsrc_hdl->rsrc_type = rsrc;
208
209 /*
210 * Depending on resource type, call the appropriate alloc routine
211 */
212 switch (rsrc) {
213 case HERMON_IN_MBOX:
214 case HERMON_OUT_MBOX:
215 case HERMON_INTR_IN_MBOX:
216 case HERMON_INTR_OUT_MBOX:
217 status = hermon_rsrc_mbox_alloc(rsrc_pool, num, tmp_rsrc_hdl);
218 break;
219
220 case HERMON_DMPT:
221 /* Allocate "num" (contiguous/aligned for FEXCH) DMPTs */
222 case HERMON_QPC:
223 /* Allocate "num" (contiguous/aligned for RSS) QPCs */
224 status = hermon_rsrc_hw_entry_alloc(rsrc_pool, num, num,
225 sleepflag, tmp_rsrc_hdl);
226 break;
227
228 case HERMON_QPC_FEXCH_PORT1:
229 case HERMON_QPC_FEXCH_PORT2:
230 /* Allocate "num" contiguous/aligned QPCs for FEXCH */
231 status = hermon_rsrc_fexch_alloc(state, rsrc, num,
232 sleepflag, tmp_rsrc_hdl);
233 break;
234
235 case HERMON_QPC_RFCI_PORT1:
236 case HERMON_QPC_RFCI_PORT2:
237 /* Allocate "num" contiguous/aligned QPCs for RFCI */
238 status = hermon_rsrc_rfci_alloc(state, rsrc, num,
239 sleepflag, tmp_rsrc_hdl);
240 break;
241
242 case HERMON_MTT:
243 case HERMON_CQC:
244 case HERMON_SRQC:
245 case HERMON_EQC:
246 case HERMON_MCG:
247 case HERMON_UARPG:
248 /* Allocate "num" unaligned resources */
249 status = hermon_rsrc_hw_entry_alloc(rsrc_pool, num, 1,
250 sleepflag, tmp_rsrc_hdl);
251 break;
252
253 case HERMON_MRHDL:
254 case HERMON_EQHDL:
255 case HERMON_CQHDL:
256 case HERMON_SRQHDL:
257 case HERMON_AHHDL:
258 case HERMON_QPHDL:
259 case HERMON_REFCNT:
260 status = hermon_rsrc_swhdl_alloc(rsrc_pool, sleepflag,
261 tmp_rsrc_hdl);
262 break;
263
264 case HERMON_PDHDL:
265 status = hermon_rsrc_pdhdl_alloc(rsrc_pool, sleepflag,
266 tmp_rsrc_hdl);
267 break;
268
269 case HERMON_RDB: /* handled during HERMON_QPC */
270 case HERMON_ALTC: /* handled during HERMON_QPC */
271 case HERMON_AUXC: /* handled during HERMON_QPC */
272 case HERMON_CMPT_QPC: /* handled during HERMON_QPC */
273 case HERMON_CMPT_SRQC: /* handled during HERMON_SRQC */
274 case HERMON_CMPT_CQC: /* handled during HERMON_CPC */
275 case HERMON_CMPT_EQC: /* handled during HERMON_EPC */
276 default:
277 HERMON_WARNING(state, "unexpected resource type in alloc ");
278 cmn_err(CE_WARN, "Resource type %x \n", rsrc_pool->rsrc_type);
279 break;
280 }
281
282 /*
283 * If the resource allocation failed, then free the special resource
284 * tracking structure and return failure. Otherwise return the
285 * handle for the resource tracking structure.
286 */
287 if (status != DDI_SUCCESS) {
288 kmem_cache_free(state->hs_rsrc_cache, tmp_rsrc_hdl);
289 return (DDI_FAILURE);
290 } else {
291 *hdl = tmp_rsrc_hdl;
292 return (DDI_SUCCESS);
293 }
294 }
295
296
297 /*
298 * hermon_rsrc_reserve()
299 *
300 * Context: Can only be called from attach.
301 * The "sleepflag" parameter is used by all object allocators to
302 * determine whether to SLEEP for resources or not.
303 */
304 int
305 hermon_rsrc_reserve(hermon_state_t *state, hermon_rsrc_type_t rsrc, uint_t num,
306 uint_t sleepflag, hermon_rsrc_t **hdl)
307 {
308 hermon_rsrc_pool_info_t *rsrc_pool;
309 hermon_rsrc_t *tmp_rsrc_hdl;
310 int flag, status = DDI_FAILURE;
311
312 ASSERT(state != NULL);
313 ASSERT(hdl != NULL);
314
315 rsrc_pool = &state->hs_rsrc_hdl[rsrc];
316 ASSERT(rsrc_pool != NULL);
317
318 /*
319 * Allocate space for the object used to track the resource handle
320 */
321 flag = (sleepflag == HERMON_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
322 tmp_rsrc_hdl = kmem_cache_alloc(state->hs_rsrc_cache, flag);
323 if (tmp_rsrc_hdl == NULL) {
324 return (DDI_FAILURE);
325 }
326 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tmp_rsrc_hdl))
327
328 /*
329 * Set rsrc_hdl type. This is later used by the hermon_rsrc_free call
330 * to know what type of resource is being freed.
331 */
332 tmp_rsrc_hdl->rsrc_type = rsrc;
333
334 switch (rsrc) {
335 case HERMON_QPC:
336 case HERMON_DMPT:
337 case HERMON_MTT:
338 /*
339 * Reserve num resources, naturally aligned (N * num).
340 */
341 status = hermon_rsrc_hw_entry_reserve(rsrc_pool, num, num,
342 sleepflag, tmp_rsrc_hdl);
343 break;
344
345 default:
346 HERMON_WARNING(state, "unexpected resource type in reserve ");
347 cmn_err(CE_WARN, "Resource type %x \n", rsrc);
348 break;
349 }
350
351 /*
352 * If the resource allocation failed, then free the special resource
353 * tracking structure and return failure. Otherwise return the
354 * handle for the resource tracking structure.
355 */
356 if (status != DDI_SUCCESS) {
357 kmem_cache_free(state->hs_rsrc_cache, tmp_rsrc_hdl);
358 return (DDI_FAILURE);
359 } else {
360 *hdl = tmp_rsrc_hdl;
361 return (DDI_SUCCESS);
362 }
363 }
364
365
366 /*
367 * hermon_rsrc_fexch_alloc()
368 *
369 * Context: Can only be called from base context.
370 * The "sleepflag" parameter is used by all object allocators to
371 * determine whether to SLEEP for resources or not.
372 */
373 static int
374 hermon_rsrc_fexch_alloc(hermon_state_t *state, hermon_rsrc_type_t rsrc,
375 uint_t num, uint_t sleepflag, hermon_rsrc_t *hdl)
376 {
377 hermon_fcoib_t *fcoib;
378 void *addr;
379 uint32_t fexch_qpn_base;
380 hermon_rsrc_pool_info_t *qpc_pool, *mpt_pool, *mtt_pool;
381 int flag, status;
382 hermon_rsrc_t mpt_hdl; /* temporary, just for icm_confirm */
383 hermon_rsrc_t mtt_hdl; /* temporary, just for icm_confirm */
384 uint_t portm1; /* hca_port_number - 1 */
385 uint_t nummtt;
386 vmem_t *vmp;
387
388 ASSERT(state != NULL);
389 ASSERT(hdl != NULL);
390
391 if ((state->hs_ibtfinfo.hca_attr->hca_flags2 & IBT_HCA2_FC) == 0)
392 return (DDI_FAILURE);
393
394 portm1 = rsrc - HERMON_QPC_FEXCH_PORT1;
395 fcoib = &state->hs_fcoib;
396 flag = (sleepflag == HERMON_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
397
398 /* Allocate from the FEXCH QP range */
399 vmp = fcoib->hfc_fexch_vmemp[portm1];
400 addr = vmem_xalloc(vmp, num, num, 0, 0, NULL, NULL, flag | VM_FIRSTFIT);
401 if (addr == NULL) {
402 return (DDI_FAILURE);
403 }
404 fexch_qpn_base = (uint32_t)((uintptr_t)addr -
405 fcoib->hfc_vmemstart + fcoib->hfc_fexch_base[portm1]);
406
407 /* ICM confirm for the FEXCH QP range */
408 qpc_pool = &state->hs_rsrc_hdl[HERMON_QPC];
409 hdl->hr_len = num << qpc_pool->rsrc_shift;
410 hdl->hr_addr = addr; /* used only for vmem_xfree */
411 hdl->hr_indx = fexch_qpn_base;
412
413 status = hermon_rsrc_hw_entry_icm_confirm(qpc_pool, num, hdl, 1);
414 if (status != DDI_SUCCESS) {
415 vmem_xfree(vmp, addr, num);
416 return (DDI_FAILURE);
417 }
418
419 /* ICM confirm for the Primary MKEYs (client side only) */
420 mpt_pool = &state->hs_rsrc_hdl[HERMON_DMPT];
421 mpt_hdl.hr_len = num << mpt_pool->rsrc_shift;
422 mpt_hdl.hr_addr = NULL;
423 mpt_hdl.hr_indx = fcoib->hfc_mpt_base[portm1] +
424 (fexch_qpn_base - fcoib->hfc_fexch_base[portm1]);
425
426 status = hermon_rsrc_hw_entry_icm_confirm(mpt_pool, num, &mpt_hdl, 0);
427 if (status != DDI_SUCCESS) {
428 status = hermon_rsrc_hw_entry_icm_free(qpc_pool, hdl, 1);
429 vmem_xfree(vmp, addr, num);
430 return (DDI_FAILURE);
431 }
432
433 /* ICM confirm for the MTTs of the Primary MKEYs (client side only) */
434 nummtt = fcoib->hfc_mtts_per_mpt;
435 num *= nummtt;
436 mtt_pool = &state->hs_rsrc_hdl[HERMON_MTT];
437 mtt_hdl.hr_len = num << mtt_pool->rsrc_shift;
438 mtt_hdl.hr_addr = NULL;
439 mtt_hdl.hr_indx = fcoib->hfc_mtt_base[portm1] +
440 (fexch_qpn_base - fcoib->hfc_fexch_base[portm1]) *
441 nummtt;
442
443 status = hermon_rsrc_hw_entry_icm_confirm(mtt_pool, num, &mtt_hdl, 0);
444 if (status != DDI_SUCCESS) {
445 vmem_xfree(vmp, addr, num);
446 return (DDI_FAILURE);
447 }
448 return (DDI_SUCCESS);
449 }
450
451 static void
452 hermon_rsrc_fexch_free(hermon_state_t *state, hermon_rsrc_t *hdl)
453 {
454 hermon_fcoib_t *fcoib;
455 uint_t portm1; /* hca_port_number - 1 */
456
457 ASSERT(state != NULL);
458 ASSERT(hdl != NULL);
459
460 portm1 = hdl->rsrc_type - HERMON_QPC_FEXCH_PORT1;
461 fcoib = &state->hs_fcoib;
462 vmem_xfree(fcoib->hfc_fexch_vmemp[portm1], hdl->hr_addr,
463 hdl->hr_len >> state->hs_rsrc_hdl[HERMON_QPC].rsrc_shift);
464 }
465
466 /*
467 * hermon_rsrc_rfci_alloc()
468 *
469 * Context: Can only be called from base context.
470 * The "sleepflag" parameter is used by all object allocators to
471 * determine whether to SLEEP for resources or not.
472 */
473 static int
474 hermon_rsrc_rfci_alloc(hermon_state_t *state, hermon_rsrc_type_t rsrc,
475 uint_t num, uint_t sleepflag, hermon_rsrc_t *hdl)
476 {
477 hermon_fcoib_t *fcoib;
478 void *addr;
479 uint32_t rfci_qpn_base;
480 hermon_rsrc_pool_info_t *qpc_pool;
481 int flag, status;
482 uint_t portm1; /* hca_port_number - 1 */
483 vmem_t *vmp;
484
485 ASSERT(state != NULL);
486 ASSERT(hdl != NULL);
487
488 if ((state->hs_ibtfinfo.hca_attr->hca_flags2 & IBT_HCA2_FC) == 0)
489 return (DDI_FAILURE);
490
491 portm1 = rsrc - HERMON_QPC_RFCI_PORT1;
492 fcoib = &state->hs_fcoib;
493 flag = (sleepflag == HERMON_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
494
495 /* Allocate from the RFCI QP range */
496 vmp = fcoib->hfc_rfci_vmemp[portm1];
497 addr = vmem_xalloc(vmp, num, num, 0, 0, NULL, NULL, flag | VM_FIRSTFIT);
498 if (addr == NULL) {
499 return (DDI_FAILURE);
500 }
501 rfci_qpn_base = (uint32_t)((uintptr_t)addr -
502 fcoib->hfc_vmemstart + fcoib->hfc_rfci_base[portm1]);
503
504 /* ICM confirm for the RFCI QP */
505 qpc_pool = &state->hs_rsrc_hdl[HERMON_QPC];
506 hdl->hr_len = num << qpc_pool->rsrc_shift;
507 hdl->hr_addr = addr; /* used only for vmem_xfree */
508 hdl->hr_indx = rfci_qpn_base;
509
510 status = hermon_rsrc_hw_entry_icm_confirm(qpc_pool, num, hdl, 1);
511 if (status != DDI_SUCCESS) {
512 vmem_xfree(vmp, addr, num);
513 return (DDI_FAILURE);
514 }
515 return (DDI_SUCCESS);
516 }
517
518 static void
519 hermon_rsrc_rfci_free(hermon_state_t *state, hermon_rsrc_t *hdl)
520 {
521 hermon_fcoib_t *fcoib;
522 uint_t portm1; /* hca_port_number - 1 */
523
524 ASSERT(state != NULL);
525 ASSERT(hdl != NULL);
526
527 portm1 = hdl->rsrc_type - HERMON_QPC_RFCI_PORT1;
528 fcoib = &state->hs_fcoib;
529 vmem_xfree(fcoib->hfc_rfci_vmemp[portm1], hdl->hr_addr,
530 hdl->hr_len >> state->hs_rsrc_hdl[HERMON_QPC].rsrc_shift);
531 }
532
533
534 /*
535 * hermon_rsrc_free()
536 * Context: Can be called from interrupt or base context.
537 */
538 void
539 hermon_rsrc_free(hermon_state_t *state, hermon_rsrc_t **hdl)
540 {
541 hermon_rsrc_pool_info_t *rsrc_pool;
542
543 ASSERT(state != NULL);
544 ASSERT(hdl != NULL);
545
546 rsrc_pool = &state->hs_rsrc_hdl[(*hdl)->rsrc_type];
547 ASSERT(rsrc_pool != NULL);
548
549 /*
550 * Depending on resource type, call the appropriate free routine
551 */
552 switch (rsrc_pool->rsrc_type) {
553 case HERMON_IN_MBOX:
554 case HERMON_OUT_MBOX:
555 case HERMON_INTR_IN_MBOX:
556 case HERMON_INTR_OUT_MBOX:
557 hermon_rsrc_mbox_free(*hdl);
558 break;
559
560 case HERMON_QPC_FEXCH_PORT1:
561 case HERMON_QPC_FEXCH_PORT2:
562 hermon_rsrc_fexch_free(state, *hdl);
563 break;
564
565 case HERMON_QPC_RFCI_PORT1:
566 case HERMON_QPC_RFCI_PORT2:
567 hermon_rsrc_rfci_free(state, *hdl);
568 break;
569
570 case HERMON_QPC:
571 case HERMON_CQC:
572 case HERMON_SRQC:
573 case HERMON_EQC:
574 case HERMON_DMPT:
575 case HERMON_MCG:
576 case HERMON_MTT:
577 case HERMON_UARPG:
578 hermon_rsrc_hw_entry_free(rsrc_pool, *hdl);
579 break;
580
581 case HERMON_MRHDL:
582 case HERMON_EQHDL:
583 case HERMON_CQHDL:
584 case HERMON_SRQHDL:
585 case HERMON_AHHDL:
586 case HERMON_QPHDL:
587 case HERMON_REFCNT:
588 hermon_rsrc_swhdl_free(rsrc_pool, *hdl);
589 break;
590
591 case HERMON_PDHDL:
592 hermon_rsrc_pdhdl_free(rsrc_pool, *hdl);
593 break;
594
595 case HERMON_RDB:
596 case HERMON_ALTC:
597 case HERMON_AUXC:
598 case HERMON_CMPT_QPC:
599 case HERMON_CMPT_SRQC:
600 case HERMON_CMPT_CQC:
601 case HERMON_CMPT_EQC:
602 default:
603 cmn_err(CE_CONT, "!rsrc_type = 0x%x\n", rsrc_pool->rsrc_type);
604 break;
605 }
606
607 /*
608 * Free the special resource tracking structure, set the handle to
609 * NULL, and return.
610 */
611 kmem_cache_free(state->hs_rsrc_cache, *hdl);
612 *hdl = NULL;
613 }
614
615
616 /*
617 * hermon_rsrc_init_phase1()
618 *
619 * Completes the first phase of Hermon resource/configuration init.
620 * This involves creating the kmem_cache for the "hermon_rsrc_t"
621 * structs, allocating the space for the resource pool handles,
622 * and setting up the "Out" mailboxes.
623 *
624 * When this function completes, the Hermon driver is ready to
625 * post the following commands which return information only in the
626 * "Out" mailbox: QUERY_DDR, QUERY_FW, QUERY_DEV_LIM, and QUERY_ADAPTER
627 * If any of these commands are to be posted at this time, they must be
628 * done so only when "spinning" (as the outstanding command list and
629 * EQ setup code has not yet run)
630 *
631 * Context: Only called from attach() path context
632 */
633 int
634 hermon_rsrc_init_phase1(hermon_state_t *state)
635 {
636 hermon_rsrc_pool_info_t *rsrc_pool;
637 hermon_rsrc_mbox_info_t mbox_info;
638 hermon_rsrc_cleanup_level_t cleanup;
639 hermon_cfg_profile_t *cfgprof;
640 uint64_t num, size;
641 int status;
642 char *rsrc_name;
643
644 ASSERT(state != NULL);
645
646 /* This is where Phase 1 of resource initialization begins */
647 cleanup = HERMON_RSRC_CLEANUP_LEVEL0;
648
649 /* Build kmem cache name from Hermon instance */
650 rsrc_name = kmem_zalloc(HERMON_RSRC_NAME_MAXLEN, KM_SLEEP);
651 HERMON_RSRC_NAME(rsrc_name, HERMON_RSRC_CACHE);
652
653 /*
654 * Create the kmem_cache for "hermon_rsrc_t" structures
655 * (kmem_cache_create will SLEEP until successful)
656 */
657 state->hs_rsrc_cache = kmem_cache_create(rsrc_name,
658 sizeof (hermon_rsrc_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
659
660 /*
661 * Allocate an array of hermon_rsrc_pool_info_t's (used in all
662 * subsequent resource allocations)
663 */
664 state->hs_rsrc_hdl = kmem_zalloc(HERMON_NUM_RESOURCES *
665 sizeof (hermon_rsrc_pool_info_t), KM_SLEEP);
666
667 /* Pull in the configuration profile */
668 cfgprof = state->hs_cfg_profile;
669
670 /* Initialize the resource pool for "out" mailboxes */
671 num = ((uint64_t)1 << cfgprof->cp_log_num_outmbox);
672 size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size);
673 rsrc_pool = &state->hs_rsrc_hdl[HERMON_OUT_MBOX];
674 rsrc_pool->rsrc_loc = HERMON_IN_SYSMEM;
675 rsrc_pool->rsrc_pool_size = (size * num);
676 rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size;
677 rsrc_pool->rsrc_quantum = (uint_t)size;
678 rsrc_pool->rsrc_align = HERMON_MBOX_ALIGN;
679 rsrc_pool->rsrc_state = state;
680 mbox_info.mbi_num = num;
681 mbox_info.mbi_size = size;
682 mbox_info.mbi_rsrcpool = rsrc_pool;
683 status = hermon_rsrc_mbox_init(state, &mbox_info);
684 if (status != DDI_SUCCESS) {
685 hermon_rsrc_fini(state, cleanup);
686 status = DDI_FAILURE;
687 goto rsrcinitp1_fail;
688 }
689 cleanup = HERMON_RSRC_CLEANUP_LEVEL1;
690
691 /* Initialize the mailbox list */
692 status = hermon_outmbox_list_init(state);
693 if (status != DDI_SUCCESS) {
694 hermon_rsrc_fini(state, cleanup);
695 status = DDI_FAILURE;
696 goto rsrcinitp1_fail;
697 }
698 cleanup = HERMON_RSRC_CLEANUP_LEVEL2;
699
700 /* Initialize the resource pool for "interrupt out" mailboxes */
701 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_outmbox);
702 size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size);
703 rsrc_pool = &state->hs_rsrc_hdl[HERMON_INTR_OUT_MBOX];
704 rsrc_pool->rsrc_loc = HERMON_IN_SYSMEM;
705 rsrc_pool->rsrc_pool_size = (size * num);
706 rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size;
707 rsrc_pool->rsrc_quantum = (uint_t)size;
708 rsrc_pool->rsrc_align = HERMON_MBOX_ALIGN;
709 rsrc_pool->rsrc_state = state;
710 mbox_info.mbi_num = num;
711 mbox_info.mbi_size = size;
712 mbox_info.mbi_rsrcpool = rsrc_pool;
713 status = hermon_rsrc_mbox_init(state, &mbox_info);
714 if (status != DDI_SUCCESS) {
715 hermon_rsrc_fini(state, cleanup);
716 status = DDI_FAILURE;
717 goto rsrcinitp1_fail;
718 }
719 cleanup = HERMON_RSRC_CLEANUP_LEVEL3;
720
721 /* Initialize the mailbox list */
722 status = hermon_intr_outmbox_list_init(state);
723 if (status != DDI_SUCCESS) {
724 hermon_rsrc_fini(state, cleanup);
725 status = DDI_FAILURE;
726 goto rsrcinitp1_fail;
727 }
728 cleanup = HERMON_RSRC_CLEANUP_LEVEL4;
729
730 /* Initialize the resource pool for "in" mailboxes */
731 num = ((uint64_t)1 << cfgprof->cp_log_num_inmbox);
732 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size);
733 rsrc_pool = &state->hs_rsrc_hdl[HERMON_IN_MBOX];
734 rsrc_pool->rsrc_loc = HERMON_IN_SYSMEM;
735 rsrc_pool->rsrc_pool_size = (size * num);
736 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size;
737 rsrc_pool->rsrc_quantum = (uint_t)size;
738 rsrc_pool->rsrc_align = HERMON_MBOX_ALIGN;
739 rsrc_pool->rsrc_state = state;
740 mbox_info.mbi_num = num;
741 mbox_info.mbi_size = size;
742 mbox_info.mbi_rsrcpool = rsrc_pool;
743 status = hermon_rsrc_mbox_init(state, &mbox_info);
744 if (status != DDI_SUCCESS) {
745 hermon_rsrc_fini(state, cleanup);
746 status = DDI_FAILURE;
747 goto rsrcinitp1_fail;
748 }
749 cleanup = HERMON_RSRC_CLEANUP_LEVEL5;
750
751 /* Initialize the mailbox list */
752 status = hermon_inmbox_list_init(state);
753 if (status != DDI_SUCCESS) {
754 hermon_rsrc_fini(state, cleanup);
755 status = DDI_FAILURE;
756 goto rsrcinitp1_fail;
757 }
758 cleanup = HERMON_RSRC_CLEANUP_LEVEL6;
759
760 /* Initialize the resource pool for "interrupt in" mailboxes */
761 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_inmbox);
762 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size);
763 rsrc_pool = &state->hs_rsrc_hdl[HERMON_INTR_IN_MBOX];
764 rsrc_pool->rsrc_loc = HERMON_IN_SYSMEM;
765 rsrc_pool->rsrc_pool_size = (size * num);
766 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size;
767 rsrc_pool->rsrc_quantum = (uint_t)size;
768 rsrc_pool->rsrc_align = HERMON_MBOX_ALIGN;
769 rsrc_pool->rsrc_state = state;
770 mbox_info.mbi_num = num;
771 mbox_info.mbi_size = size;
772 mbox_info.mbi_rsrcpool = rsrc_pool;
773 status = hermon_rsrc_mbox_init(state, &mbox_info);
774 if (status != DDI_SUCCESS) {
775 hermon_rsrc_fini(state, cleanup);
776 status = DDI_FAILURE;
777 goto rsrcinitp1_fail;
778 }
779 cleanup = HERMON_RSRC_CLEANUP_LEVEL7;
780
781 /* Initialize the mailbox list */
782 status = hermon_intr_inmbox_list_init(state);
783 if (status != DDI_SUCCESS) {
784 hermon_rsrc_fini(state, cleanup);
785 status = DDI_FAILURE;
786 goto rsrcinitp1_fail;
787 }
788 cleanup = HERMON_RSRC_CLEANUP_PHASE1_COMPLETE;
789 kmem_free(rsrc_name, HERMON_RSRC_NAME_MAXLEN);
790 return (DDI_SUCCESS);
791
792 rsrcinitp1_fail:
793 kmem_free(rsrc_name, HERMON_RSRC_NAME_MAXLEN);
794 return (status);
795 }
796
797
798 /*
799 * hermon_rsrc_init_phase2()
800 * Context: Only called from attach() path context
801 */
802 int
803 hermon_rsrc_init_phase2(hermon_state_t *state)
804 {
805 hermon_rsrc_sw_hdl_info_t hdl_info;
806 hermon_rsrc_hw_entry_info_t entry_info;
807 hermon_rsrc_pool_info_t *rsrc_pool;
808 hermon_rsrc_cleanup_level_t cleanup, ncleanup;
809 hermon_cfg_profile_t *cfgprof;
810 hermon_hw_querydevlim_t *devlim;
811 uint64_t num, max, num_prealloc;
812 uint_t mcg_size, mcg_size_shift;
813 int i, status;
814 char *rsrc_name;
815
816 ASSERT(state != NULL);
817
818 /* Phase 2 initialization begins where Phase 1 left off */
819 cleanup = HERMON_RSRC_CLEANUP_PHASE1_COMPLETE;
820
821 /* Allocate the ICM resource name space */
822
823 /* Build the ICM vmem arena names from Hermon instance */
824 rsrc_name = kmem_zalloc(HERMON_RSRC_NAME_MAXLEN, KM_SLEEP);
825
826 /*
827 * Initialize the resource pools for all objects that exist in
828 * context memory (ICM). The ICM consists of context tables, each
829 * type of resource (QP, CQ, EQ, etc) having it's own context table
830 * (QPC, CQC, EQC, etc...).
831 */
832 cfgprof = state->hs_cfg_profile;
833 devlim = &state->hs_devlim;
834
835 /*
836 * Initialize the resource pools for each of the driver resources.
837 * With a few exceptions, these resources fall into the two cateogories
838 * of either hw_entries or sw_entries.
839 */
840
841 /*
842 * Initialize the resource pools for ICM (hardware) types first.
843 * These resources are managed through vmem arenas, which are
844 * created via the rsrc pool initialization routine. Note that,
845 * due to further calculations, the MCG resource pool is
846 * initialized seperately.
847 */
848 for (i = 0; i < HERMON_NUM_ICM_RESOURCES; i++) {
849
850 rsrc_pool = &state->hs_rsrc_hdl[i];
851 rsrc_pool->rsrc_type = i;
852 rsrc_pool->rsrc_state = state;
853
854 /* Set the resource-specific attributes */
855 switch (i) {
856 case HERMON_MTT:
857 max = ((uint64_t)1 << devlim->log_max_mtt);
858 num_prealloc = ((uint64_t)1 << devlim->log_rsvd_mtt);
859 HERMON_RSRC_NAME(rsrc_name, HERMON_MTT_VMEM);
860 ncleanup = HERMON_RSRC_CLEANUP_LEVEL9;
861 break;
862
863 case HERMON_DMPT:
864 max = ((uint64_t)1 << devlim->log_max_dmpt);
865 num_prealloc = ((uint64_t)1 << devlim->log_rsvd_dmpt);
866 HERMON_RSRC_NAME(rsrc_name, HERMON_DMPT_VMEM);
867 ncleanup = HERMON_RSRC_CLEANUP_LEVEL10;
868 break;
869
870 case HERMON_QPC:
871 max = ((uint64_t)1 << devlim->log_max_qp);
872 num_prealloc = ((uint64_t)1 << devlim->log_rsvd_qp);
873 HERMON_RSRC_NAME(rsrc_name, HERMON_QPC_VMEM);
874 ncleanup = HERMON_RSRC_CLEANUP_LEVEL11;
875 break;
876
877 case HERMON_CQC:
878 max = ((uint64_t)1 << devlim->log_max_cq);
879 num_prealloc = ((uint64_t)1 << devlim->log_rsvd_cq);
880 HERMON_RSRC_NAME(rsrc_name, HERMON_CQC_VMEM);
881 ncleanup = HERMON_RSRC_CLEANUP_LEVEL13;
882 break;
883
884 case HERMON_SRQC:
885 max = ((uint64_t)1 << devlim->log_max_srq);
886 num_prealloc = ((uint64_t)1 << devlim->log_rsvd_srq);
887 HERMON_RSRC_NAME(rsrc_name, HERMON_SRQC_VMEM);
888 ncleanup = HERMON_RSRC_CLEANUP_LEVEL16;
889 break;
890
891 case HERMON_EQC:
892 max = ((uint64_t)1 << devlim->log_max_eq);
893 num_prealloc = state->hs_rsvd_eqs;
894 HERMON_RSRC_NAME(rsrc_name, HERMON_EQC_VMEM);
895 ncleanup = HERMON_RSRC_CLEANUP_LEVEL18;
896 break;
897
898 case HERMON_MCG: /* handled below */
899 case HERMON_AUXC:
900 case HERMON_ALTC:
901 case HERMON_RDB:
902 case HERMON_CMPT_QPC:
903 case HERMON_CMPT_SRQC:
904 case HERMON_CMPT_CQC:
905 case HERMON_CMPT_EQC:
906 default:
907 /* We don't need to initialize this rsrc here. */
908 continue;
909 }
910
911 /* Set the common values for all resource pools */
912 rsrc_pool->rsrc_state = state;
913 rsrc_pool->rsrc_loc = HERMON_IN_ICM;
914 rsrc_pool->rsrc_pool_size = state->hs_icm[i].table_size;
915 rsrc_pool->rsrc_align = state->hs_icm[i].table_size;
916 rsrc_pool->rsrc_shift = state->hs_icm[i].log_object_size;
917 rsrc_pool->rsrc_quantum = state->hs_icm[i].object_size;
918
919 /* Now, initialize the entry_info and call the init routine */
920 entry_info.hwi_num = state->hs_icm[i].num_entries;
921 entry_info.hwi_max = max;
922 entry_info.hwi_prealloc = num_prealloc;
923 entry_info.hwi_rsrcpool = rsrc_pool;
924 entry_info.hwi_rsrcname = rsrc_name;
925 status = hermon_rsrc_hw_entries_init(state, &entry_info);
926 if (status != DDI_SUCCESS) {
927 hermon_rsrc_fini(state, cleanup);
928 status = DDI_FAILURE;
929 goto rsrcinitp2_fail;
930 }
931 cleanup = ncleanup;
932 }
933
934 /*
935 * Initialize the Multicast Group (MCG) entries. First, calculate
936 * (and validate) the size of the MCGs.
937 */
938 status = hermon_rsrc_mcg_entry_get_size(state, &mcg_size_shift);
939 if (status != DDI_SUCCESS) {
940 hermon_rsrc_fini(state, cleanup);
941 status = DDI_FAILURE;
942 goto rsrcinitp2_fail;
943 }
944 mcg_size = HERMON_MCGMEM_SZ(state);
945
946 /*
947 * Initialize the resource pool for the MCG table entries. Notice
948 * that the number of MCGs is configurable. Note also that a certain
949 * number of MCGs must be set aside for Hermon firmware use (they
950 * correspond to the number of MCGs used by the internal hash
951 * function).
952 */
953 num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
954 max = ((uint64_t)1 << devlim->log_max_mcg);
955 num_prealloc = ((uint64_t)1 << cfgprof->cp_log_num_mcg_hash);
956 rsrc_pool = &state->hs_rsrc_hdl[HERMON_MCG];
957 rsrc_pool->rsrc_loc = HERMON_IN_ICM;
958 rsrc_pool->rsrc_pool_size = (mcg_size * num);
959 rsrc_pool->rsrc_shift = mcg_size_shift;
960 rsrc_pool->rsrc_quantum = mcg_size;
961 rsrc_pool->rsrc_align = (mcg_size * num);
962 rsrc_pool->rsrc_state = state;
963 HERMON_RSRC_NAME(rsrc_name, HERMON_MCG_VMEM);
964 entry_info.hwi_num = num;
965 entry_info.hwi_max = max;
966 entry_info.hwi_prealloc = num_prealloc;
967 entry_info.hwi_rsrcpool = rsrc_pool;
968 entry_info.hwi_rsrcname = rsrc_name;
969 status = hermon_rsrc_hw_entries_init(state, &entry_info);
970 if (status != DDI_SUCCESS) {
971 hermon_rsrc_fini(state, cleanup);
972 status = DDI_FAILURE;
973 goto rsrcinitp2_fail;
974 }
975 cleanup = HERMON_RSRC_CLEANUP_LEVEL19;
976
977 /*
978 * Initialize the full range of ICM for the AUXC resource.
979 * This is done because its size is so small, about 1 byte per QP.
980 */
981
982 /*
983 * Initialize the Hermon command handling interfaces. This step
984 * sets up the outstanding command tracking mechanism for easy access
985 * and fast allocation (see hermon_cmd.c for more details).
986 */
987 status = hermon_outstanding_cmdlist_init(state);
988 if (status != DDI_SUCCESS) {
989 hermon_rsrc_fini(state, cleanup);
990 status = DDI_FAILURE;
991 goto rsrcinitp2_fail;
992 }
993 cleanup = HERMON_RSRC_CLEANUP_LEVEL20;
994
995 /* Initialize the resource pool and vmem arena for the PD handles */
996 rsrc_pool = &state->hs_rsrc_hdl[HERMON_PDHDL];
997 rsrc_pool->rsrc_loc = HERMON_IN_SYSMEM;
998 rsrc_pool->rsrc_quantum = sizeof (struct hermon_sw_pd_s);
999 rsrc_pool->rsrc_state = state;
1000 HERMON_RSRC_NAME(rsrc_name, HERMON_PDHDL_CACHE);
1001 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_pd);
1002 hdl_info.swi_max = ((uint64_t)1 << devlim->log_max_pd);
1003 hdl_info.swi_rsrcpool = rsrc_pool;
1004 hdl_info.swi_constructor = hermon_rsrc_pdhdl_constructor;
1005 hdl_info.swi_destructor = hermon_rsrc_pdhdl_destructor;
1006 hdl_info.swi_rsrcname = rsrc_name;
1007 hdl_info.swi_flags = HERMON_SWHDL_KMEMCACHE_INIT;
1008 status = hermon_rsrc_pd_handles_init(state, &hdl_info);
1009 if (status != DDI_SUCCESS) {
1010 hermon_rsrc_fini(state, cleanup);
1011 status = DDI_FAILURE;
1012 goto rsrcinitp2_fail;
1013 }
1014 cleanup = HERMON_RSRC_CLEANUP_LEVEL21;
1015
1016 /*
1017 * Initialize the resource pools for the rest of the software handles.
1018 * This includes MR handles, EQ handles, QP handles, etc. These
1019 * objects are almost entirely managed using kmem_cache routines,
1020 * and do not utilize a vmem arena.
1021 */
1022 for (i = HERMON_NUM_ICM_RESOURCES; i < HERMON_NUM_RESOURCES; i++) {
1023 rsrc_pool = &state->hs_rsrc_hdl[i];
1024 rsrc_pool->rsrc_type = i;
1025
1026 /* Set the resource-specific attributes */
1027 switch (i) {
1028 case HERMON_MRHDL:
1029 rsrc_pool->rsrc_quantum =
1030 sizeof (struct hermon_sw_mr_s);
1031 HERMON_RSRC_NAME(rsrc_name, HERMON_MRHDL_CACHE);
1032 hdl_info.swi_num =
1033 ((uint64_t)1 << cfgprof->cp_log_num_dmpt) +
1034 ((uint64_t)1 << cfgprof->cp_log_num_cmpt);
1035 hdl_info.swi_max =
1036 ((uint64_t)1 << cfgprof->cp_log_num_dmpt) +
1037 ((uint64_t)1 << cfgprof->cp_log_num_cmpt);
1038 hdl_info.swi_constructor =
1039 hermon_rsrc_mrhdl_constructor;
1040 hdl_info.swi_destructor = hermon_rsrc_mrhdl_destructor;
1041 hdl_info.swi_flags = HERMON_SWHDL_KMEMCACHE_INIT;
1042 ncleanup = HERMON_RSRC_CLEANUP_LEVEL22;
1043 break;
1044
1045 case HERMON_EQHDL:
1046 rsrc_pool->rsrc_quantum =
1047 sizeof (struct hermon_sw_eq_s);
1048 HERMON_RSRC_NAME(rsrc_name, HERMON_EQHDL_CACHE);
1049 hdl_info.swi_num = HERMON_NUM_EQ;
1050 hdl_info.swi_max = ((uint64_t)1 << devlim->log_max_eq);
1051 hdl_info.swi_constructor = NULL;
1052 hdl_info.swi_destructor = NULL;
1053 hdl_info.swi_flags = HERMON_SWHDL_KMEMCACHE_INIT;
1054 ncleanup = HERMON_RSRC_CLEANUP_LEVEL23;
1055 break;
1056
1057 case HERMON_CQHDL:
1058 rsrc_pool->rsrc_quantum =
1059 sizeof (struct hermon_sw_cq_s);
1060 HERMON_RSRC_NAME(rsrc_name, HERMON_CQHDL_CACHE);
1061 hdl_info.swi_num =
1062 (uint64_t)1 << cfgprof->cp_log_num_cq;
1063 hdl_info.swi_max = (uint64_t)1 << devlim->log_max_cq;
1064 hdl_info.swi_constructor =
1065 hermon_rsrc_cqhdl_constructor;
1066 hdl_info.swi_destructor = hermon_rsrc_cqhdl_destructor;
1067 hdl_info.swi_flags = HERMON_SWHDL_KMEMCACHE_INIT;
1068 hdl_info.swi_prealloc_sz = sizeof (hermon_cqhdl_t);
1069 ncleanup = HERMON_RSRC_CLEANUP_LEVEL24;
1070 break;
1071
1072 case HERMON_SRQHDL:
1073 rsrc_pool->rsrc_quantum =
1074 sizeof (struct hermon_sw_srq_s);
1075 HERMON_RSRC_NAME(rsrc_name, HERMON_SRQHDL_CACHE);
1076 hdl_info.swi_num =
1077 (uint64_t)1 << cfgprof->cp_log_num_srq;
1078 hdl_info.swi_max = (uint64_t)1 << devlim->log_max_srq;
1079 hdl_info.swi_constructor =
1080 hermon_rsrc_srqhdl_constructor;
1081 hdl_info.swi_destructor = hermon_rsrc_srqhdl_destructor;
1082 hdl_info.swi_flags = HERMON_SWHDL_KMEMCACHE_INIT;
1083 hdl_info.swi_prealloc_sz = sizeof (hermon_srqhdl_t);
1084 ncleanup = HERMON_RSRC_CLEANUP_LEVEL25;
1085 break;
1086
1087 case HERMON_AHHDL:
1088 rsrc_pool->rsrc_quantum =
1089 sizeof (struct hermon_sw_ah_s);
1090 HERMON_RSRC_NAME(rsrc_name, HERMON_AHHDL_CACHE);
1091 hdl_info.swi_num =
1092 (uint64_t)1 << cfgprof->cp_log_num_ah;
1093 hdl_info.swi_max = HERMON_NUM_AH;
1094 hdl_info.swi_constructor =
1095 hermon_rsrc_ahhdl_constructor;
1096 hdl_info.swi_destructor = hermon_rsrc_ahhdl_destructor;
1097 hdl_info.swi_flags = HERMON_SWHDL_KMEMCACHE_INIT;
1098 ncleanup = HERMON_RSRC_CLEANUP_LEVEL26;
1099 break;
1100
1101 case HERMON_QPHDL:
1102 rsrc_pool->rsrc_quantum =
1103 sizeof (struct hermon_sw_qp_s);
1104 HERMON_RSRC_NAME(rsrc_name, HERMON_QPHDL_CACHE);
1105 hdl_info.swi_num =
1106 (uint64_t)1 << cfgprof->cp_log_num_qp;
1107 hdl_info.swi_max = (uint64_t)1 << devlim->log_max_qp;
1108 hdl_info.swi_constructor =
1109 hermon_rsrc_qphdl_constructor;
1110 hdl_info.swi_destructor = hermon_rsrc_qphdl_destructor;
1111 hdl_info.swi_flags = HERMON_SWHDL_KMEMCACHE_INIT;
1112 hdl_info.swi_prealloc_sz = sizeof (hermon_qphdl_t);
1113 ncleanup = HERMON_RSRC_CLEANUP_LEVEL27;
1114 break;
1115
1116 case HERMON_REFCNT:
1117 rsrc_pool->rsrc_quantum = sizeof (hermon_sw_refcnt_t);
1118 HERMON_RSRC_NAME(rsrc_name, HERMON_REFCNT_CACHE);
1119 hdl_info.swi_num =
1120 (uint64_t)1 << cfgprof->cp_log_num_dmpt;
1121 hdl_info.swi_max = (uint64_t)1 << devlim->log_max_dmpt;
1122 hdl_info.swi_constructor =
1123 hermon_rsrc_refcnt_constructor;
1124 hdl_info.swi_destructor = hermon_rsrc_refcnt_destructor;
1125 hdl_info.swi_flags = HERMON_SWHDL_KMEMCACHE_INIT;
1126 ncleanup = HERMON_RSRC_CLEANUP_LEVEL28;
1127 break;
1128
1129 default:
1130 continue;
1131 }
1132
1133 /* Set the common values and call the init routine */
1134 rsrc_pool->rsrc_loc = HERMON_IN_SYSMEM;
1135 rsrc_pool->rsrc_state = state;
1136 hdl_info.swi_rsrcpool = rsrc_pool;
1137 hdl_info.swi_rsrcname = rsrc_name;
1138 status = hermon_rsrc_sw_handles_init(state, &hdl_info);
1139 if (status != DDI_SUCCESS) {
1140 hermon_rsrc_fini(state, cleanup);
1141 status = DDI_FAILURE;
1142 goto rsrcinitp2_fail;
1143 }
1144 cleanup = ncleanup;
1145 }
1146
1147 /*
1148 * Initialize a resource pool for the MCG handles. Notice that for
1149 * these MCG handles, we are allocating a table of structures (used to
1150 * keep track of the MCG entries that are being written to hardware
1151 * and to speed up multicast attach/detach operations).
1152 */
1153 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1154 hdl_info.swi_max = ((uint64_t)1 << devlim->log_max_mcg);
1155 hdl_info.swi_flags = HERMON_SWHDL_TABLE_INIT;
1156 hdl_info.swi_prealloc_sz = sizeof (struct hermon_sw_mcg_list_s);
1157 status = hermon_rsrc_sw_handles_init(state, &hdl_info);
1158 if (status != DDI_SUCCESS) {
1159 hermon_rsrc_fini(state, cleanup);
1160 status = DDI_FAILURE;
1161 goto rsrcinitp2_fail;
1162 }
1163 state->hs_mcghdl = hdl_info.swi_table_ptr;
1164 cleanup = HERMON_RSRC_CLEANUP_LEVEL29;
1165
1166 /*
1167 * Last, initialize the resource pool for the UAR pages, which contain
1168 * the hardware's doorbell registers. Each process supported in User
1169 * Mode is assigned a UAR page. Also coming from this pool are the
1170 * kernel-assigned UAR page, and any hardware-reserved pages. Note
1171 * that the number of UAR pages is configurable, the value must be less
1172 * than the maximum value (obtained from the QUERY_DEV_LIM command) or
1173 * the initialization will fail. Note also that we assign the base
1174 * address of the UAR BAR to the rsrc_start parameter.
1175 */
1176 num = ((uint64_t)1 << cfgprof->cp_log_num_uar);
1177 max = num;
1178 num_prealloc = max(devlim->num_rsvd_uar, 128);
1179 rsrc_pool = &state->hs_rsrc_hdl[HERMON_UARPG];
1180 rsrc_pool->rsrc_loc = HERMON_IN_UAR;
1181 rsrc_pool->rsrc_pool_size = (num << PAGESHIFT);
1182 rsrc_pool->rsrc_shift = PAGESHIFT;
1183 rsrc_pool->rsrc_quantum = (uint_t)PAGESIZE;
1184 rsrc_pool->rsrc_align = PAGESIZE;
1185 rsrc_pool->rsrc_state = state;
1186 rsrc_pool->rsrc_start = (void *)state->hs_reg_uar_baseaddr;
1187 HERMON_RSRC_NAME(rsrc_name, HERMON_UAR_PAGE_VMEM_ATTCH);
1188 entry_info.hwi_num = num;
1189 entry_info.hwi_max = max;
1190 entry_info.hwi_prealloc = num_prealloc;
1191 entry_info.hwi_rsrcpool = rsrc_pool;
1192 entry_info.hwi_rsrcname = rsrc_name;
1193 status = hermon_rsrc_hw_entries_init(state, &entry_info);
1194 if (status != DDI_SUCCESS) {
1195 hermon_rsrc_fini(state, cleanup);
1196 status = DDI_FAILURE;
1197 goto rsrcinitp2_fail;
1198 }
1199
1200 cleanup = HERMON_RSRC_CLEANUP_ALL;
1201
1202 kmem_free(rsrc_name, HERMON_RSRC_NAME_MAXLEN);
1203 return (DDI_SUCCESS);
1204
1205 rsrcinitp2_fail:
1206 kmem_free(rsrc_name, HERMON_RSRC_NAME_MAXLEN);
1207 return (status);
1208 }
1209
1210
1211 /*
1212 * hermon_rsrc_fini()
1213 * Context: Only called from attach() and/or detach() path contexts
1214 */
1215 void
1216 hermon_rsrc_fini(hermon_state_t *state, hermon_rsrc_cleanup_level_t clean)
1217 {
1218 hermon_rsrc_sw_hdl_info_t hdl_info;
1219 hermon_rsrc_hw_entry_info_t entry_info;
1220 hermon_rsrc_mbox_info_t mbox_info;
1221 hermon_cfg_profile_t *cfgprof;
1222
1223 ASSERT(state != NULL);
1224
1225 cfgprof = state->hs_cfg_profile;
1226
1227 /*
1228 * If init code above is shortened up (see comments), then we
1229 * need to establish how to safely and simply clean up from any
1230 * given failure point. Flags, maybe...
1231 */
1232
1233 switch (clean) {
1234 /*
1235 * If we add more resources that need to be cleaned up here, we should
1236 * ensure that HERMON_RSRC_CLEANUP_ALL is still the first entry (i.e.
1237 * corresponds to the last resource allocated).
1238 */
1239
1240 case HERMON_RSRC_CLEANUP_ALL:
1241 case HERMON_RSRC_CLEANUP_LEVEL31:
1242 /* Cleanup the UAR page resource pool, first the dbr pages */
1243 if (state->hs_kern_dbr) {
1244 hermon_dbr_kern_free(state);
1245 state->hs_kern_dbr = NULL;
1246 }
1247
1248 /* NS then, the pool itself */
1249 entry_info.hwi_rsrcpool = &state->hs_rsrc_hdl[HERMON_UARPG];
1250 hermon_rsrc_hw_entries_fini(state, &entry_info);
1251
1252 /* FALLTHROUGH */
1253
1254 case HERMON_RSRC_CLEANUP_LEVEL30:
1255 /* Cleanup the central MCG handle pointers list */
1256 hdl_info.swi_rsrcpool = NULL;
1257 hdl_info.swi_table_ptr = state->hs_mcghdl;
1258 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1259 hdl_info.swi_prealloc_sz = sizeof (struct hermon_sw_mcg_list_s);
1260 hermon_rsrc_sw_handles_fini(state, &hdl_info);
1261 /* FALLTHROUGH */
1262
1263 case HERMON_RSRC_CLEANUP_LEVEL29:
1264 /* Cleanup the reference count resource pool */
1265 hdl_info.swi_rsrcpool = &state->hs_rsrc_hdl[HERMON_REFCNT];
1266 hdl_info.swi_table_ptr = NULL;
1267 hermon_rsrc_sw_handles_fini(state, &hdl_info);
1268 /* FALLTHROUGH */
1269
1270 case HERMON_RSRC_CLEANUP_LEVEL28:
1271 /* Cleanup the QP handle resource pool */
1272 hdl_info.swi_rsrcpool = &state->hs_rsrc_hdl[HERMON_QPHDL];
1273 hdl_info.swi_table_ptr = NULL;
1274 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
1275 hdl_info.swi_prealloc_sz = sizeof (hermon_qphdl_t);
1276 hermon_rsrc_sw_handles_fini(state, &hdl_info);
1277 /* FALLTHROUGH */
1278 case HERMON_RSRC_CLEANUP_LEVEL27:
1279 /* Cleanup the address handle resrouce pool */
1280 hdl_info.swi_rsrcpool = &state->hs_rsrc_hdl[HERMON_AHHDL];
1281 hdl_info.swi_table_ptr = NULL;
1282 hermon_rsrc_sw_handles_fini(state, &hdl_info);
1283 /* FALLTHROUGH */
1284
1285 case HERMON_RSRC_CLEANUP_LEVEL26:
1286 /* Cleanup the SRQ handle resource pool. */
1287 hdl_info.swi_rsrcpool = &state->hs_rsrc_hdl[HERMON_SRQHDL];
1288 hdl_info.swi_table_ptr = NULL;
1289 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_srq);
1290 hdl_info.swi_prealloc_sz = sizeof (hermon_srqhdl_t);
1291 hermon_rsrc_sw_handles_fini(state, &hdl_info);
1292 /* FALLTHROUGH */
1293
1294 case HERMON_RSRC_CLEANUP_LEVEL25:
1295 /* Cleanup the CQ handle resource pool */
1296 hdl_info.swi_rsrcpool = &state->hs_rsrc_hdl[HERMON_CQHDL];
1297 hdl_info.swi_table_ptr = NULL;
1298 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_cq);
1299 hdl_info.swi_prealloc_sz = sizeof (hermon_cqhdl_t);
1300 hermon_rsrc_sw_handles_fini(state, &hdl_info);
1301 /* FALLTHROUGH */
1302
1303 case HERMON_RSRC_CLEANUP_LEVEL24:
1304 /* Cleanup the EQ handle resource pool */
1305 hdl_info.swi_rsrcpool = &state->hs_rsrc_hdl[HERMON_EQHDL];
1306 hdl_info.swi_table_ptr = NULL;
1307 hermon_rsrc_sw_handles_fini(state, &hdl_info);
1308 /* FALLTHROUGH */
1309
1310 case HERMON_RSRC_CLEANUP_LEVEL23:
1311 /* Cleanup the MR handle resource pool */
1312 hdl_info.swi_rsrcpool = &state->hs_rsrc_hdl[HERMON_MRHDL];
1313 hdl_info.swi_table_ptr = NULL;
1314 hermon_rsrc_sw_handles_fini(state, &hdl_info);
1315 /* FALLTHROUGH */
1316
1317 case HERMON_RSRC_CLEANUP_LEVEL22:
1318 /* Cleanup the PD handle resource pool */
1319 hdl_info.swi_rsrcpool = &state->hs_rsrc_hdl[HERMON_PDHDL];
1320 hdl_info.swi_table_ptr = NULL;
1321 hermon_rsrc_pd_handles_fini(state, &hdl_info);
1322 /* FALLTHROUGH */
1323
1324 case HERMON_RSRC_CLEANUP_LEVEL21:
1325 /* Currently unused - FALLTHROUGH */
1326
1327 case HERMON_RSRC_CLEANUP_LEVEL20:
1328 /* Cleanup the outstanding command list */
1329 hermon_outstanding_cmdlist_fini(state);
1330 /* FALLTHROUGH */
1331
1332 case HERMON_RSRC_CLEANUP_LEVEL19:
1333 /* Cleanup the EQC table resource pool */
1334 entry_info.hwi_rsrcpool = &state->hs_rsrc_hdl[HERMON_EQC];
1335 hermon_rsrc_hw_entries_fini(state, &entry_info);
1336 /* FALLTHROUGH */
1337
1338 case HERMON_RSRC_CLEANUP_LEVEL18:
1339 /* Cleanup the MCG table resource pool */
1340 entry_info.hwi_rsrcpool = &state->hs_rsrc_hdl[HERMON_MCG];
1341 hermon_rsrc_hw_entries_fini(state, &entry_info);
1342 /* FALLTHROUGH */
1343
1344 case HERMON_RSRC_CLEANUP_LEVEL17:
1345 /* Currently Unused - fallthrough */
1346 case HERMON_RSRC_CLEANUP_LEVEL16:
1347 /* Cleanup the SRQC table resource pool */
1348 entry_info.hwi_rsrcpool = &state->hs_rsrc_hdl[HERMON_SRQC];
1349 hermon_rsrc_hw_entries_fini(state, &entry_info);
1350 /* FALLTHROUGH */
1351
1352 case HERMON_RSRC_CLEANUP_LEVEL15:
1353 /* Cleanup the AUXC table resource pool */
1354 entry_info.hwi_rsrcpool = &state->hs_rsrc_hdl[HERMON_AUXC];
1355 hermon_rsrc_hw_entries_fini(state, &entry_info);
1356 /* FALLTHROUGH */
1357
1358 case HERMON_RSRC_CLEANUP_LEVEL14:
1359 /* Cleanup the ALTCF table resource pool */
1360 entry_info.hwi_rsrcpool = &state->hs_rsrc_hdl[HERMON_ALTC];
1361 hermon_rsrc_hw_entries_fini(state, &entry_info);
1362 /* FALLTHROUGH */
1363
1364 case HERMON_RSRC_CLEANUP_LEVEL13:
1365 /* Cleanup the CQC table resource pool */
1366 entry_info.hwi_rsrcpool = &state->hs_rsrc_hdl[HERMON_CQC];
1367 hermon_rsrc_hw_entries_fini(state, &entry_info);
1368 /* FALLTHROUGH */
1369
1370 case HERMON_RSRC_CLEANUP_LEVEL12:
1371 /* Cleanup the RDB table resource pool */
1372 entry_info.hwi_rsrcpool = &state->hs_rsrc_hdl[HERMON_RDB];
1373 hermon_rsrc_hw_entries_fini(state, &entry_info);
1374 /* FALLTHROUGH */
1375
1376 case HERMON_RSRC_CLEANUP_LEVEL11:
1377 /* Cleanup the QPC table resource pool */
1378 entry_info.hwi_rsrcpool = &state->hs_rsrc_hdl[HERMON_QPC];
1379 hermon_rsrc_hw_entries_fini(state, &entry_info);
1380 /* FALLTHROUGH */
1381
1382 case HERMON_RSRC_CLEANUP_LEVEL10EQ:
1383 /* Cleanup the cMPTs for the EQs, CQs, SRQs, and QPs */
1384 entry_info.hwi_rsrcpool = &state->hs_rsrc_hdl[HERMON_CMPT_EQC];
1385 hermon_rsrc_hw_entries_fini(state, &entry_info);
1386 /* FALLTHROUGH */
1387
1388 case HERMON_RSRC_CLEANUP_LEVEL10CQ:
1389 /* Cleanup the cMPTs for the EQs, CQs, SRQs, and QPs */
1390 entry_info.hwi_rsrcpool = &state->hs_rsrc_hdl[HERMON_CMPT_CQC];
1391 hermon_rsrc_hw_entries_fini(state, &entry_info);
1392 /* FALLTHROUGH */
1393
1394 case HERMON_RSRC_CLEANUP_LEVEL10SRQ:
1395 /* Cleanup the cMPTs for the EQs, CQs, SRQs, and QPs */
1396 entry_info.hwi_rsrcpool = &state->hs_rsrc_hdl[HERMON_CMPT_SRQC];
1397 hermon_rsrc_hw_entries_fini(state, &entry_info);
1398 /* FALLTHROUGH */
1399
1400 case HERMON_RSRC_CLEANUP_LEVEL10QP:
1401 /* Cleanup the cMPTs for the EQs, CQs, SRQs, and QPs */
1402 entry_info.hwi_rsrcpool = &state->hs_rsrc_hdl[HERMON_CMPT_QPC];
1403 hermon_rsrc_hw_entries_fini(state, &entry_info);
1404 /* FALLTHROUGH */
1405
1406 case HERMON_RSRC_CLEANUP_LEVEL10:
1407 /* Cleanup the dMPT table resource pool */
1408 entry_info.hwi_rsrcpool = &state->hs_rsrc_hdl[HERMON_DMPT];
1409 hermon_rsrc_hw_entries_fini(state, &entry_info);
1410 /* FALLTHROUGH */
1411
1412 case HERMON_RSRC_CLEANUP_LEVEL9:
1413 /* Cleanup the MTT table resource pool */
1414 entry_info.hwi_rsrcpool = &state->hs_rsrc_hdl[HERMON_MTT];
1415 hermon_rsrc_hw_entries_fini(state, &entry_info);
1416 break;
1417
1418 /*
1419 * The cleanup below comes from the "Phase 1" initialization step.
1420 * (see hermon_rsrc_init_phase1() above)
1421 */
1422 case HERMON_RSRC_CLEANUP_PHASE1_COMPLETE:
1423 /* Cleanup the "In" mailbox list */
1424 hermon_intr_inmbox_list_fini(state);
1425 /* FALLTHROUGH */
1426
1427 case HERMON_RSRC_CLEANUP_LEVEL7:
1428 /* Cleanup the interrupt "In" mailbox resource pool */
1429 mbox_info.mbi_rsrcpool =
1430 &state->hs_rsrc_hdl[HERMON_INTR_IN_MBOX];
1431 hermon_rsrc_mbox_fini(state, &mbox_info);
1432 /* FALLTHROUGH */
1433
1434 case HERMON_RSRC_CLEANUP_LEVEL6:
1435 /* Cleanup the "In" mailbox list */
1436 hermon_inmbox_list_fini(state);
1437 /* FALLTHROUGH */
1438
1439 case HERMON_RSRC_CLEANUP_LEVEL5:
1440 /* Cleanup the "In" mailbox resource pool */
1441 mbox_info.mbi_rsrcpool = &state->hs_rsrc_hdl[HERMON_IN_MBOX];
1442 hermon_rsrc_mbox_fini(state, &mbox_info);
1443 /* FALLTHROUGH */
1444
1445 case HERMON_RSRC_CLEANUP_LEVEL4:
1446 /* Cleanup the interrupt "Out" mailbox list */
1447 hermon_intr_outmbox_list_fini(state);
1448 /* FALLTHROUGH */
1449
1450 case HERMON_RSRC_CLEANUP_LEVEL3:
1451 /* Cleanup the "Out" mailbox resource pool */
1452 mbox_info.mbi_rsrcpool =
1453 &state->hs_rsrc_hdl[HERMON_INTR_OUT_MBOX];
1454 hermon_rsrc_mbox_fini(state, &mbox_info);
1455 /* FALLTHROUGH */
1456
1457 case HERMON_RSRC_CLEANUP_LEVEL2:
1458 /* Cleanup the "Out" mailbox list */
1459 hermon_outmbox_list_fini(state);
1460 /* FALLTHROUGH */
1461
1462 case HERMON_RSRC_CLEANUP_LEVEL1:
1463 /* Cleanup the "Out" mailbox resource pool */
1464 mbox_info.mbi_rsrcpool = &state->hs_rsrc_hdl[HERMON_OUT_MBOX];
1465 hermon_rsrc_mbox_fini(state, &mbox_info);
1466 /* FALLTHROUGH */
1467
1468 case HERMON_RSRC_CLEANUP_LEVEL0:
1469 /* Free the array of hermon_rsrc_pool_info_t's */
1470
1471 kmem_free(state->hs_rsrc_hdl, HERMON_NUM_RESOURCES *
1472 sizeof (hermon_rsrc_pool_info_t));
1473
1474 kmem_cache_destroy(state->hs_rsrc_cache);
1475 break;
1476
1477 default:
1478 HERMON_WARNING(state, "unexpected resource cleanup level");
1479 break;
1480 }
1481 }
1482
1483
1484 /*
1485 * hermon_rsrc_mbox_init()
1486 * Context: Only called from attach() path context
1487 */
1488 static int
1489 hermon_rsrc_mbox_init(hermon_state_t *state, hermon_rsrc_mbox_info_t *info)
1490 {
1491 hermon_rsrc_pool_info_t *rsrc_pool;
1492 hermon_rsrc_priv_mbox_t *priv;
1493
1494 ASSERT(state != NULL);
1495 ASSERT(info != NULL);
1496
1497 rsrc_pool = info->mbi_rsrcpool;
1498 ASSERT(rsrc_pool != NULL);
1499
1500 /* Allocate and initialize mailbox private structure */
1501 priv = kmem_zalloc(sizeof (hermon_rsrc_priv_mbox_t), KM_SLEEP);
1502 priv->pmb_dip = state->hs_dip;
1503 priv->pmb_devaccattr = state->hs_reg_accattr;
1504 priv->pmb_xfer_mode = DDI_DMA_CONSISTENT;
1505
1506 /*
1507 * Initialize many of the default DMA attributes. Then set alignment
1508 * and scatter-gather restrictions specific for mailbox memory.
1509 */
1510 hermon_dma_attr_init(state, &priv->pmb_dmaattr);
1511 priv->pmb_dmaattr.dma_attr_align = HERMON_MBOX_ALIGN;
1512 priv->pmb_dmaattr.dma_attr_sgllen = 1;
1513 priv->pmb_dmaattr.dma_attr_flags = 0;
1514 rsrc_pool->rsrc_private = priv;
1515
1516 ASSERT(rsrc_pool->rsrc_loc == HERMON_IN_SYSMEM);
1517
1518 rsrc_pool->rsrc_start = NULL;
1519 rsrc_pool->rsrc_vmp = NULL;
1520
1521 return (DDI_SUCCESS);
1522 }
1523
1524
1525 /*
1526 * hermon_rsrc_mbox_fini()
1527 * Context: Only called from attach() and/or detach() path contexts
1528 */
1529 /* ARGSUSED */
1530 static void
1531 hermon_rsrc_mbox_fini(hermon_state_t *state, hermon_rsrc_mbox_info_t *info)
1532 {
1533 hermon_rsrc_pool_info_t *rsrc_pool;
1534
1535 ASSERT(state != NULL);
1536 ASSERT(info != NULL);
1537
1538 rsrc_pool = info->mbi_rsrcpool;
1539 ASSERT(rsrc_pool != NULL);
1540
1541 /* Free up the private struct */
1542 kmem_free(rsrc_pool->rsrc_private, sizeof (hermon_rsrc_priv_mbox_t));
1543 }
1544
1545
1546 /*
1547 * hermon_rsrc_hw_entries_init()
1548 * Context: Only called from attach() path context
1549 */
1550 int
1551 hermon_rsrc_hw_entries_init(hermon_state_t *state,
1552 hermon_rsrc_hw_entry_info_t *info)
1553 {
1554 hermon_rsrc_pool_info_t *rsrc_pool;
1555 hermon_rsrc_t *rsvd_rsrc = NULL;
1556 vmem_t *vmp;
1557 uint64_t num_hwentry, max_hwentry, num_prealloc;
1558 int status;
1559
1560 ASSERT(state != NULL);
1561 ASSERT(info != NULL);
1562
1563 rsrc_pool = info->hwi_rsrcpool;
1564 ASSERT(rsrc_pool != NULL);
1565 num_hwentry = info->hwi_num;
1566 max_hwentry = info->hwi_max;
1567 num_prealloc = info->hwi_prealloc;
1568
1569 if (hermon_rsrc_verbose) {
1570 IBTF_DPRINTF_L2("hermon", "hermon_rsrc_hw_entries_init: "
1571 "rsrc_type (0x%x) num (%llx) max (0x%llx) prealloc "
1572 "(0x%llx)", rsrc_pool->rsrc_type, (longlong_t)num_hwentry,
1573 (longlong_t)max_hwentry, (longlong_t)num_prealloc);
1574 }
1575
1576 /* Make sure number of HW entries makes sense */
1577 if (num_hwentry > max_hwentry) {
1578 return (DDI_FAILURE);
1579 }
1580
1581 /* Set this pool's rsrc_start from the initial ICM allocation */
1582 if (rsrc_pool->rsrc_start == 0) {
1583
1584 /* use a ROUND value that works on both 32 and 64-bit kernels */
1585 rsrc_pool->rsrc_start = (void *)(uintptr_t)0x10000000;
1586
1587 if (hermon_rsrc_verbose) {
1588 IBTF_DPRINTF_L2("hermon", "hermon_rsrc_hw_entries_init:"
1589 " rsrc_type (0x%x) rsrc_start set (0x%lx)",
1590 rsrc_pool->rsrc_type, rsrc_pool->rsrc_start);
1591 }
1592 }
1593
1594 /*
1595 * Create new vmem arena for the HW entries table if rsrc_quantum
1596 * is non-zero. Otherwise if rsrc_quantum is zero, then these HW
1597 * entries are not going to be dynamically allocatable (i.e. they
1598 * won't be allocated/freed through hermon_rsrc_alloc/free). This
1599 * latter option is used for both ALTC and CMPT resources which
1600 * are managed by hardware.
1601 */
1602 if (rsrc_pool->rsrc_quantum != 0) {
1603 vmp = vmem_create(info->hwi_rsrcname,
1604 (void *)(uintptr_t)rsrc_pool->rsrc_start,
1605 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_quantum,
1606 NULL, NULL, NULL, 0, VM_SLEEP);
1607 if (vmp == NULL) {
1608 /* failed to create vmem arena */
1609 return (DDI_FAILURE);
1610 }
1611 rsrc_pool->rsrc_vmp = vmp;
1612 if (hermon_rsrc_verbose) {
1613 IBTF_DPRINTF_L2("hermon", "hermon_rsrc_hw_entries_init:"
1614 " rsrc_type (0x%x) created vmem arena for rsrc",
1615 rsrc_pool->rsrc_type);
1616 }
1617 } else {
1618 /* we do not require a vmem arena */
1619 rsrc_pool->rsrc_vmp = NULL;
1620 if (hermon_rsrc_verbose) {
1621 IBTF_DPRINTF_L2("hermon", "hermon_rsrc_hw_entries_init:"
1622 " rsrc_type (0x%x) vmem arena not required",
1623 rsrc_pool->rsrc_type);
1624 }
1625 }
1626
1627 /* Allocate hardware reserved resources, if any */
1628 if (num_prealloc != 0) {
1629 status = hermon_rsrc_alloc(state, rsrc_pool->rsrc_type,
1630 num_prealloc, HERMON_SLEEP, &rsvd_rsrc);
1631 if (status != DDI_SUCCESS) {
1632 /* unable to preallocate the reserved entries */
1633 if (rsrc_pool->rsrc_vmp != NULL) {
1634 vmem_destroy(rsrc_pool->rsrc_vmp);
1635 }
1636 return (DDI_FAILURE);
1637 }
1638 }
1639 rsrc_pool->rsrc_private = rsvd_rsrc;
1640
1641 return (DDI_SUCCESS);
1642 }
1643
1644
1645 /*
1646 * hermon_rsrc_hw_entries_fini()
1647 * Context: Only called from attach() and/or detach() path contexts
1648 */
1649 void
1650 hermon_rsrc_hw_entries_fini(hermon_state_t *state,
1651 hermon_rsrc_hw_entry_info_t *info)
1652 {
1653 hermon_rsrc_pool_info_t *rsrc_pool;
1654 hermon_rsrc_t *rsvd_rsrc;
1655
1656 ASSERT(state != NULL);
1657 ASSERT(info != NULL);
1658
1659 rsrc_pool = info->hwi_rsrcpool;
1660 ASSERT(rsrc_pool != NULL);
1661
1662 /* Free up any "reserved" (i.e. preallocated) HW entries */
1663 rsvd_rsrc = (hermon_rsrc_t *)rsrc_pool->rsrc_private;
1664 if (rsvd_rsrc != NULL) {
1665 hermon_rsrc_free(state, &rsvd_rsrc);
1666 }
1667
1668 /*
1669 * If we've actually setup a vmem arena for the HW entries, then
1670 * destroy it now
1671 */
1672 if (rsrc_pool->rsrc_vmp != NULL) {
1673 vmem_destroy(rsrc_pool->rsrc_vmp);
1674 }
1675 }
1676
1677
1678 /*
1679 * hermon_rsrc_sw_handles_init()
1680 * Context: Only called from attach() path context
1681 */
1682 /* ARGSUSED */
1683 static int
1684 hermon_rsrc_sw_handles_init(hermon_state_t *state,
1685 hermon_rsrc_sw_hdl_info_t *info)
1686 {
1687 hermon_rsrc_pool_info_t *rsrc_pool;
1688 uint64_t num_swhdl, max_swhdl, prealloc_sz;
1689
1690 ASSERT(state != NULL);
1691 ASSERT(info != NULL);
1692
1693 rsrc_pool = info->swi_rsrcpool;
1694 ASSERT(rsrc_pool != NULL);
1695 num_swhdl = info->swi_num;
1696 max_swhdl = info->swi_max;
1697 prealloc_sz = info->swi_prealloc_sz;
1698
1699
1700 /* Make sure number of SW handles makes sense */
1701 if (num_swhdl > max_swhdl) {
1702 return (DDI_FAILURE);
1703 }
1704
1705 /*
1706 * Depending on the flags parameter, create a kmem_cache for some
1707 * number of software handle structures. Note: kmem_cache_create()
1708 * will SLEEP until successful.
1709 */
1710 if (info->swi_flags & HERMON_SWHDL_KMEMCACHE_INIT) {
1711 rsrc_pool->rsrc_private = kmem_cache_create(
1712 info->swi_rsrcname, rsrc_pool->rsrc_quantum, 0,
1713 info->swi_constructor, info->swi_destructor, NULL,
1714 rsrc_pool->rsrc_state, NULL, 0);
1715 }
1716
1717
1718 /* Allocate the central list of SW handle pointers */
1719 if (info->swi_flags & HERMON_SWHDL_TABLE_INIT) {
1720 info->swi_table_ptr = kmem_zalloc(num_swhdl * prealloc_sz,
1721 KM_SLEEP);
1722 }
1723
1724 return (DDI_SUCCESS);
1725 }
1726
1727
1728 /*
1729 * hermon_rsrc_sw_handles_fini()
1730 * Context: Only called from attach() and/or detach() path contexts
1731 */
1732 /* ARGSUSED */
1733 static void
1734 hermon_rsrc_sw_handles_fini(hermon_state_t *state,
1735 hermon_rsrc_sw_hdl_info_t *info)
1736 {
1737 hermon_rsrc_pool_info_t *rsrc_pool;
1738 uint64_t num_swhdl, prealloc_sz;
1739
1740 ASSERT(state != NULL);
1741 ASSERT(info != NULL);
1742
1743 rsrc_pool = info->swi_rsrcpool;
1744 num_swhdl = info->swi_num;
1745 prealloc_sz = info->swi_prealloc_sz;
1746
1747 /*
1748 * If a "software handle" kmem_cache exists for this resource, then
1749 * destroy it now
1750 */
1751 if (rsrc_pool != NULL) {
1752 kmem_cache_destroy(rsrc_pool->rsrc_private);
1753 }
1754
1755 /* Free up this central list of SW handle pointers */
1756 if (info->swi_table_ptr != NULL) {
1757 kmem_free(info->swi_table_ptr, num_swhdl * prealloc_sz);
1758 }
1759 }
1760
1761
1762 /*
1763 * hermon_rsrc_pd_handles_init()
1764 * Context: Only called from attach() path context
1765 */
1766 static int
1767 hermon_rsrc_pd_handles_init(hermon_state_t *state,
1768 hermon_rsrc_sw_hdl_info_t *info)
1769 {
1770 hermon_rsrc_pool_info_t *rsrc_pool;
1771 vmem_t *vmp;
1772 char vmem_name[HERMON_RSRC_NAME_MAXLEN];
1773 int status;
1774
1775 ASSERT(state != NULL);
1776 ASSERT(info != NULL);
1777
1778 rsrc_pool = info->swi_rsrcpool;
1779 ASSERT(rsrc_pool != NULL);
1780
1781 /* Initialize the resource pool for software handle table */
1782 status = hermon_rsrc_sw_handles_init(state, info);
1783 if (status != DDI_SUCCESS) {
1784 return (DDI_FAILURE);
1785 }
1786
1787 /* Build vmem arena name from Hermon instance */
1788 HERMON_RSRC_NAME(vmem_name, HERMON_PDHDL_VMEM);
1789
1790 /* Create new vmem arena for PD numbers */
1791 vmp = vmem_create(vmem_name, (caddr_t)1, info->swi_num, 1, NULL,
1792 NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
1793 if (vmp == NULL) {
1794 /* Unable to create vmem arena */
1795 info->swi_table_ptr = NULL;
1796 hermon_rsrc_sw_handles_fini(state, info);
1797 return (DDI_FAILURE);
1798 }
1799 rsrc_pool->rsrc_vmp = vmp;
1800
1801 return (DDI_SUCCESS);
1802 }
1803
1804
1805 /*
1806 * hermon_rsrc_pd_handles_fini()
1807 * Context: Only called from attach() and/or detach() path contexts
1808 */
1809 static void
1810 hermon_rsrc_pd_handles_fini(hermon_state_t *state,
1811 hermon_rsrc_sw_hdl_info_t *info)
1812 {
1813 hermon_rsrc_pool_info_t *rsrc_pool;
1814
1815 ASSERT(state != NULL);
1816 ASSERT(info != NULL);
1817
1818 rsrc_pool = info->swi_rsrcpool;
1819
1820 /* Destroy the specially created UAR scratch table vmem arena */
1821 vmem_destroy(rsrc_pool->rsrc_vmp);
1822
1823 /* Destroy the "hermon_sw_pd_t" kmem_cache */
1824 hermon_rsrc_sw_handles_fini(state, info);
1825 }
1826
1827
1828 /*
1829 * hermon_rsrc_mbox_alloc()
1830 * Context: Only called from attach() path context
1831 */
1832 static int
1833 hermon_rsrc_mbox_alloc(hermon_rsrc_pool_info_t *pool_info, uint_t num,
1834 hermon_rsrc_t *hdl)
1835 {
1836 hermon_rsrc_priv_mbox_t *priv;
1837 caddr_t kaddr;
1838 size_t real_len, temp_len;
1839 int status;
1840
1841 ASSERT(pool_info != NULL);
1842 ASSERT(hdl != NULL);
1843
1844 /* Get the private pointer for the mailboxes */
1845 priv = pool_info->rsrc_private;
1846 ASSERT(priv != NULL);
1847
1848 /* Allocate a DMA handle for the mailbox */
1849 status = ddi_dma_alloc_handle(priv->pmb_dip, &priv->pmb_dmaattr,
1850 DDI_DMA_SLEEP, NULL, &hdl->hr_dmahdl);
1851 if (status != DDI_SUCCESS) {
1852 return (DDI_FAILURE);
1853 }
1854
1855 /* Allocate memory for the mailbox */
1856 temp_len = (num << pool_info->rsrc_shift);
1857 status = ddi_dma_mem_alloc(hdl->hr_dmahdl, temp_len,
1858 &priv->pmb_devaccattr, priv->pmb_xfer_mode, DDI_DMA_SLEEP,
1859 NULL, &kaddr, &real_len, &hdl->hr_acchdl);
1860 if (status != DDI_SUCCESS) {
1861 /* No more memory available for mailbox entries */
1862 ddi_dma_free_handle(&hdl->hr_dmahdl);
1863 return (DDI_FAILURE);
1864 }
1865
1866 hdl->hr_addr = (void *)kaddr;
1867 hdl->hr_len = (uint32_t)real_len;
1868
1869 return (DDI_SUCCESS);
1870 }
1871
1872
1873 /*
1874 * hermon_rsrc_mbox_free()
1875 * Context: Can be called from interrupt or base context.
1876 */
1877 static void
1878 hermon_rsrc_mbox_free(hermon_rsrc_t *hdl)
1879 {
1880 ASSERT(hdl != NULL);
1881
1882 /* Use ddi_dma_mem_free() to free up sys memory for mailbox */
1883 ddi_dma_mem_free(&hdl->hr_acchdl);
1884
1885 /* Free the DMA handle for the mailbox */
1886 ddi_dma_free_handle(&hdl->hr_dmahdl);
1887 }
1888
1889
1890 /*
1891 * hermon_rsrc_hw_entry_alloc()
1892 * Context: Can be called from interrupt or base context.
1893 */
1894 static int
1895 hermon_rsrc_hw_entry_alloc(hermon_rsrc_pool_info_t *pool_info, uint_t num,
1896 uint_t num_align, uint_t sleepflag, hermon_rsrc_t *hdl)
1897 {
1898 void *addr;
1899 uint64_t offset;
1900 uint32_t align;
1901 int status;
1902 int flag;
1903
1904 ASSERT(pool_info != NULL);
1905 ASSERT(hdl != NULL);
1906
1907 /*
1908 * Use vmem_xalloc() to get a properly aligned pointer (based on
1909 * the number requested) to the HW entry(ies). This handles the
1910 * cases (for special QPCs and for RDB entries) where we need more
1911 * than one and need to ensure that they are properly aligned.
1912 */
1913 flag = (sleepflag == HERMON_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
1914 hdl->hr_len = (num << pool_info->rsrc_shift);
1915 align = (num_align << pool_info->rsrc_shift);
1916
1917 addr = vmem_xalloc(pool_info->rsrc_vmp, hdl->hr_len,
1918 align, 0, 0, NULL, NULL, flag | VM_FIRSTFIT);
1919
1920 if (addr == NULL) {
1921 /* No more HW entries available */
1922 return (DDI_FAILURE);
1923 }
1924
1925 hdl->hr_acchdl = NULL; /* only used for mbox resources */
1926
1927 /* Calculate vaddr and HW table index */
1928 offset = (uintptr_t)addr - (uintptr_t)pool_info->rsrc_start;
1929 hdl->hr_addr = addr; /* only used for mbox and uarpg resources */
1930 hdl->hr_indx = offset >> pool_info->rsrc_shift;
1931
1932 if (pool_info->rsrc_loc == HERMON_IN_ICM) {
1933 int num_to_hdl;
1934 hermon_rsrc_type_t rsrc_type = pool_info->rsrc_type;
1935
1936 num_to_hdl = (rsrc_type == HERMON_QPC ||
1937 rsrc_type == HERMON_CQC || rsrc_type == HERMON_SRQC);
1938
1939 /* confirm ICM is mapped, and allocate if necessary */
1940 status = hermon_rsrc_hw_entry_icm_confirm(pool_info, num, hdl,
1941 num_to_hdl);
1942 if (status != DDI_SUCCESS) {
1943 return (DDI_FAILURE);
1944 }
1945 hdl->hr_addr = NULL; /* not used for ICM resources */
1946 }
1947
1948 return (DDI_SUCCESS);
1949 }
1950
1951
1952 /*
1953 * hermon_rsrc_hw_entry_reserve()
1954 * Context: Can be called from interrupt or base context.
1955 */
1956 int
1957 hermon_rsrc_hw_entry_reserve(hermon_rsrc_pool_info_t *pool_info, uint_t num,
1958 uint_t num_align, uint_t sleepflag, hermon_rsrc_t *hdl)
1959 {
1960 void *addr;
1961 uint64_t offset;
1962 uint32_t align;
1963 int flag;
1964
1965 ASSERT(pool_info != NULL);
1966 ASSERT(hdl != NULL);
1967 ASSERT(pool_info->rsrc_loc == HERMON_IN_ICM);
1968
1969 /*
1970 * Use vmem_xalloc() to get a properly aligned pointer (based on
1971 * the number requested) to the HW entry(ies). This handles the
1972 * cases (for special QPCs and for RDB entries) where we need more
1973 * than one and need to ensure that they are properly aligned.
1974 */
1975 flag = (sleepflag == HERMON_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
1976 hdl->hr_len = (num << pool_info->rsrc_shift);
1977 align = (num_align << pool_info->rsrc_shift);
1978
1979 addr = vmem_xalloc(pool_info->rsrc_vmp, hdl->hr_len,
1980 align, 0, 0, NULL, NULL, flag | VM_FIRSTFIT);
1981
1982 if (addr == NULL) {
1983 /* No more HW entries available */
1984 return (DDI_FAILURE);
1985 }
1986
1987 hdl->hr_acchdl = NULL; /* only used for mbox resources */
1988
1989 /* Calculate vaddr and HW table index */
1990 offset = (uintptr_t)addr - (uintptr_t)pool_info->rsrc_start;
1991 hdl->hr_addr = NULL;
1992 hdl->hr_indx = offset >> pool_info->rsrc_shift;
1993
1994 /* ICM will be allocated and mapped if and when it gets used */
1995
1996 return (DDI_SUCCESS);
1997 }
1998
1999
2000 /*
2001 * hermon_rsrc_hw_entry_free()
2002 * Context: Can be called from interrupt or base context.
2003 */
2004 static void
2005 hermon_rsrc_hw_entry_free(hermon_rsrc_pool_info_t *pool_info,
2006 hermon_rsrc_t *hdl)
2007 {
2008 void *addr;
2009 uint64_t offset;
2010 int status;
2011
2012 ASSERT(pool_info != NULL);
2013 ASSERT(hdl != NULL);
2014
2015 /* Calculate the allocated address */
2016 offset = hdl->hr_indx << pool_info->rsrc_shift;
2017 addr = (void *)(uintptr_t)(offset + (uintptr_t)pool_info->rsrc_start);
2018
2019 /* Use vmem_xfree() to free up the HW table entry */
2020 vmem_xfree(pool_info->rsrc_vmp, addr, hdl->hr_len);
2021
2022 if (pool_info->rsrc_loc == HERMON_IN_ICM) {
2023 int num_to_hdl;
2024 hermon_rsrc_type_t rsrc_type = pool_info->rsrc_type;
2025
2026 num_to_hdl = (rsrc_type == HERMON_QPC ||
2027 rsrc_type == HERMON_CQC || rsrc_type == HERMON_SRQC);
2028
2029 /* free ICM references, and free ICM if required */
2030 status = hermon_rsrc_hw_entry_icm_free(pool_info, hdl,
2031 num_to_hdl);
2032 if (status != DDI_SUCCESS)
2033 HERMON_WARNING(pool_info->rsrc_state,
2034 "failure in hw_entry_free");
2035 }
2036 }
2037
2038 /*
2039 * hermon_rsrc_hw_entry_icm_confirm()
2040 * Context: Can be called from interrupt or base context.
2041 */
2042 static int
2043 hermon_rsrc_hw_entry_icm_confirm(hermon_rsrc_pool_info_t *pool_info, uint_t num,
2044 hermon_rsrc_t *hdl, int num_to_hdl)
2045 {
2046 hermon_state_t *state;
2047 hermon_icm_table_t *icm_table;
2048 uint8_t *bitmap;
2049 hermon_dma_info_t *dma_info;
2050 hermon_rsrc_type_t type;
2051 uint32_t rindx, span_offset;
2052 uint32_t span_avail;
2053 int num_backed;
2054 int status;
2055 uint32_t index1, index2;
2056
2057 /*
2058 * Utility routine responsible for ensuring that there is memory
2059 * backing the ICM resources allocated via hermon_rsrc_hw_entry_alloc().
2060 * Confirm existing ICM mapping(s) or allocate ICM memory for the
2061 * given hardware resources being allocated, and increment the
2062 * ICM DMA structure(s) reference count.
2063 *
2064 * We may be allocating more objects than can fit in a single span,
2065 * or more than will fit in the remaining contiguous memory (from
2066 * the offset indicated by hdl->ar_indx) in the span in question.
2067 * In either of these cases, we'll be breaking up our allocation
2068 * into multiple spans.
2069 */
2070 state = pool_info->rsrc_state;
2071 type = pool_info->rsrc_type;
2072 icm_table = &state->hs_icm[type];
2073
2074 rindx = hdl->hr_indx;
2075 hermon_index(index1, index2, rindx, icm_table, span_offset);
2076
2077 if (hermon_rsrc_verbose) {
2078 IBTF_DPRINTF_L2("hermon", "hermon_rsrc_hw_entry_icm_confirm: "
2079 "type (0x%x) num (0x%x) length (0x%x) index (0x%x, 0x%x): ",
2080 type, num, hdl->hr_len, index1, index2);
2081 }
2082
2083 mutex_enter(&icm_table->icm_table_lock);
2084 hermon_bitmap(bitmap, dma_info, icm_table, index1, num_to_hdl);
2085 while (num) {
2086 #ifndef __lock_lint
2087 while (icm_table->icm_busy) {
2088 cv_wait(&icm_table->icm_table_cv,
2089 &icm_table->icm_table_lock);
2090 }
2091 #endif
2092 if (!HERMON_BMAP_BIT_ISSET(bitmap, index2)) {
2093 /* Allocate ICM for this span */
2094 icm_table->icm_busy = 1;
2095 mutex_exit(&icm_table->icm_table_lock);
2096 status = hermon_icm_alloc(state, type, index1, index2);
2097 mutex_enter(&icm_table->icm_table_lock);
2098 icm_table->icm_busy = 0;
2099 cv_broadcast(&icm_table->icm_table_cv);
2100 if (status != DDI_SUCCESS) {
2101 goto fail_alloc;
2102 }
2103 if (hermon_rsrc_verbose) {
2104 IBTF_DPRINTF_L2("hermon", "hermon_rsrc_"
2105 "hw_entry_icm_confirm: ALLOCATED ICM: "
2106 "type (0x%x) index (0x%x, 0x%x)",
2107 type, index1, index2);
2108 }
2109 }
2110
2111 /*
2112 * We need to increment the refcnt of this span by the
2113 * number of objects in this resource allocation that are
2114 * backed by this span. Given that the rsrc allocation is
2115 * contiguous, this value will be the number of objects in
2116 * the span from 'span_offset' onward, either up to a max
2117 * of the total number of objects, or the end of the span.
2118 * So, determine the number of objects that can be backed
2119 * by this span ('span_avail'), then determine the number
2120 * of backed resources.
2121 */
2122 span_avail = icm_table->span - span_offset;
2123 if (num > span_avail) {
2124 num_backed = span_avail;
2125 } else {
2126 num_backed = num;
2127 }
2128
2129 /*
2130 * Now that we know 'num_backed', increment the refcnt,
2131 * decrement the total number, and set 'span_offset' to
2132 * 0 in case we roll over into the next span.
2133 */
2134 dma_info[index2].icm_refcnt += num_backed;
2135 rindx += num_backed;
2136 num -= num_backed;
2137
2138 if (hermon_rsrc_verbose) {
2139 IBTF_DPRINTF_L2("ALLOC", "ICM type (0x%x) index "
2140 "(0x%x, 0x%x) num_backed (0x%x)",
2141 type, index1, index2, num_backed);
2142 IBTF_DPRINTF_L2("ALLOC", "ICM type (0x%x) refcnt now "
2143 "(0x%x) num_remaining (0x%x)", type,
2144 dma_info[index2].icm_refcnt, num);
2145 }
2146 if (num == 0)
2147 break;
2148
2149 hermon_index(index1, index2, rindx, icm_table, span_offset);
2150 hermon_bitmap(bitmap, dma_info, icm_table, index1, num_to_hdl);
2151 }
2152 mutex_exit(&icm_table->icm_table_lock);
2153
2154 return (DDI_SUCCESS);
2155
2156 fail_alloc:
2157 /* JBDB */
2158 if (hermon_rsrc_verbose) {
2159 IBTF_DPRINTF_L2("hermon", "hermon_rsrc_"
2160 "hw_entry_icm_confirm: FAILED ICM ALLOC: "
2161 "type (0x%x) num remaind (0x%x) index (0x%x, 0x%x)"
2162 "refcnt (0x%x)", type, num, index1, index2,
2163 icm_table->icm_dma[index1][index2].icm_refcnt);
2164 }
2165 IBTF_DPRINTF_L2("hermon", "WARNING: "
2166 "unimplemented fail code in hermon_rsrc_hw_entry_icm_alloc\n");
2167
2168 #if needs_work
2169 /* free refcnt's and any spans we've allocated */
2170 while (index-- != start) {
2171 /*
2172 * JBDB - This is a bit tricky. We need to
2173 * free refcnt's on any spans that we've
2174 * incremented them on, and completely free
2175 * spans that we've allocated. How do we do
2176 * this here? Does it need to be as involved
2177 * as the core of icm_free() below, or can
2178 * we leverage breadcrumbs somehow?
2179 */
2180 HERMON_WARNING(state, "unable to allocate ICM memory: "
2181 "UNIMPLEMENTED HANDLING!!");
2182 }
2183 #else
2184 cmn_err(CE_WARN,
2185 "unimplemented fail code in hermon_rsrc_hw_entry_icm_alloc\n");
2186 #endif
2187 mutex_exit(&icm_table->icm_table_lock);
2188
2189 HERMON_WARNING(state, "unable to allocate ICM memory");
2190 return (DDI_FAILURE);
2191 }
2192
2193 /*
2194 * hermon_rsrc_hw_entry_icm_free()
2195 * Context: Can be called from interrupt or base context.
2196 */
2197 static int
2198 hermon_rsrc_hw_entry_icm_free(hermon_rsrc_pool_info_t *pool_info,
2199 hermon_rsrc_t *hdl, int num_to_hdl)
2200 {
2201 hermon_state_t *state;
2202 hermon_icm_table_t *icm_table;
2203 uint8_t *bitmap;
2204 hermon_dma_info_t *dma_info;
2205 hermon_rsrc_type_t type;
2206 uint32_t span_offset;
2207 uint32_t span_remain;
2208 int num_freed;
2209 int num;
2210 uint32_t index1, index2, rindx;
2211
2212 /*
2213 * Utility routine responsible for freeing references to ICM
2214 * DMA spans, and freeing the ICM memory if necessary.
2215 *
2216 * We may have allocated objects in a single contiguous resource
2217 * allocation that reside in a number of spans, at any given
2218 * starting offset within a span. We therefore must determine
2219 * where this allocation starts, and then determine if we need
2220 * to free objects in more than one span.
2221 */
2222 state = pool_info->rsrc_state;
2223 type = pool_info->rsrc_type;
2224 icm_table = &state->hs_icm[type];
2225
2226 rindx = hdl->hr_indx;
2227 hermon_index(index1, index2, rindx, icm_table, span_offset);
2228 hermon_bitmap(bitmap, dma_info, icm_table, index1, num_to_hdl);
2229
2230 /* determine the number of ICM objects in this allocation */
2231 num = hdl->hr_len >> pool_info->rsrc_shift;
2232
2233 if (hermon_rsrc_verbose) {
2234 IBTF_DPRINTF_L2("hermon", "hermon_rsrc_hw_entry_icm_free: "
2235 "type (0x%x) num (0x%x) length (0x%x) index (0x%x, 0x%x)",
2236 type, num, hdl->hr_len, index1, index2);
2237 }
2238 mutex_enter(&icm_table->icm_table_lock);
2239 while (num) {
2240 /*
2241 * As with the ICM confirm code above, we need to
2242 * decrement the ICM span(s) by the number of
2243 * resources being freed. So, determine the number
2244 * of objects that are backed in this span from
2245 * 'span_offset' onward, and set 'num_freed' to
2246 * the smaller of either that number ('span_remain'),
2247 * or the total number of objects being freed.
2248 */
2249 span_remain = icm_table->span - span_offset;
2250 if (num > span_remain) {
2251 num_freed = span_remain;
2252 } else {
2253 num_freed = num;
2254 }
2255
2256 /*
2257 * Now that we know 'num_freed', decrement the refcnt,
2258 * decrement the total number, and set 'span_offset' to
2259 * 0 in case we roll over into the next span.
2260 */
2261 dma_info[index2].icm_refcnt -= num_freed;
2262 num -= num_freed;
2263 rindx += num_freed;
2264
2265 if (hermon_rsrc_verbose) {
2266 IBTF_DPRINTF_L2("FREE", "ICM type (0x%x) index "
2267 "(0x%x, 0x%x) num_freed (0x%x)", type,
2268 index1, index2, num_freed);
2269 IBTF_DPRINTF_L2("FREE", "ICM type (0x%x) refcnt now "
2270 "(0x%x) num remaining (0x%x)", type,
2271 icm_table->icm_dma[index1][index2].icm_refcnt, num);
2272 }
2273
2274 #if HERMON_ICM_FREE_ENABLED
2275 /* If we've freed the last object in this span, free it */
2276 if ((index1 != 0 || index2 != 0) &&
2277 (dma_info[index2].icm_refcnt == 0)) {
2278 if (hermon_rsrc_verbose) {
2279 IBTF_DPRINTF_L2("hermon", "hermon_rsrc_hw_entry"
2280 "_icm_free: freeing ICM type (0x%x) index"
2281 " (0x%x, 0x%x)", type, index1, index2);
2282 }
2283 hermon_icm_free(state, type, index1, index2);
2284 }
2285 #endif
2286 if (num == 0)
2287 break;
2288
2289 hermon_index(index1, index2, rindx, icm_table, span_offset);
2290 hermon_bitmap(bitmap, dma_info, icm_table, index1, num_to_hdl);
2291 }
2292 mutex_exit(&icm_table->icm_table_lock);
2293
2294 return (DDI_SUCCESS);
2295 }
2296
2297
2298
2299 /*
2300 * hermon_rsrc_swhdl_alloc()
2301 * Context: Can be called from interrupt or base context.
2302 */
2303 static int
2304 hermon_rsrc_swhdl_alloc(hermon_rsrc_pool_info_t *pool_info, uint_t sleepflag,
2305 hermon_rsrc_t *hdl)
2306 {
2307 void *addr;
2308 int flag;
2309
2310 ASSERT(pool_info != NULL);
2311 ASSERT(hdl != NULL);
2312
2313 /* Allocate the software handle structure */
2314 flag = (sleepflag == HERMON_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
2315 addr = kmem_cache_alloc(pool_info->rsrc_private, flag);
2316 if (addr == NULL) {
2317 return (DDI_FAILURE);
2318 }
2319 hdl->hr_len = pool_info->rsrc_quantum;
2320 hdl->hr_addr = addr;
2321
2322 return (DDI_SUCCESS);
2323 }
2324
2325
2326 /*
2327 * hermon_rsrc_swhdl_free()
2328 * Context: Can be called from interrupt or base context.
2329 */
2330 static void
2331 hermon_rsrc_swhdl_free(hermon_rsrc_pool_info_t *pool_info, hermon_rsrc_t *hdl)
2332 {
2333 ASSERT(pool_info != NULL);
2334 ASSERT(hdl != NULL);
2335
2336 /* Free the software handle structure */
2337 kmem_cache_free(pool_info->rsrc_private, hdl->hr_addr);
2338 }
2339
2340
2341 /*
2342 * hermon_rsrc_pdhdl_alloc()
2343 * Context: Can be called from interrupt or base context.
2344 */
2345 static int
2346 hermon_rsrc_pdhdl_alloc(hermon_rsrc_pool_info_t *pool_info, uint_t sleepflag,
2347 hermon_rsrc_t *hdl)
2348 {
2349 hermon_pdhdl_t addr;
2350 void *tmpaddr;
2351 int flag, status;
2352
2353 ASSERT(pool_info != NULL);
2354 ASSERT(hdl != NULL);
2355
2356 /* Allocate the software handle */
2357 status = hermon_rsrc_swhdl_alloc(pool_info, sleepflag, hdl);
2358 if (status != DDI_SUCCESS) {
2359 return (DDI_FAILURE);
2360 }
2361 addr = (hermon_pdhdl_t)hdl->hr_addr;
2362 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*addr))
2363
2364 /* Allocate a PD number for the handle */
2365 flag = (sleepflag == HERMON_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
2366 tmpaddr = vmem_alloc(pool_info->rsrc_vmp, 1, flag);
2367 if (tmpaddr == NULL) {
2368 /* No more PD number entries available */
2369 hermon_rsrc_swhdl_free(pool_info, hdl);
2370 return (DDI_FAILURE);
2371 }
2372 addr->pd_pdnum = (uint32_t)(uintptr_t)tmpaddr;
2373 addr->pd_rsrcp = hdl;
2374 hdl->hr_indx = addr->pd_pdnum;
2375
2376 return (DDI_SUCCESS);
2377 }
2378
2379
2380 /*
2381 * hermon_rsrc_pdhdl_free()
2382 * Context: Can be called from interrupt or base context.
2383 */
2384 static void
2385 hermon_rsrc_pdhdl_free(hermon_rsrc_pool_info_t *pool_info, hermon_rsrc_t *hdl)
2386 {
2387 ASSERT(pool_info != NULL);
2388 ASSERT(hdl != NULL);
2389
2390 /* Use vmem_free() to free up the PD number */
2391 vmem_free(pool_info->rsrc_vmp, (void *)(uintptr_t)hdl->hr_indx, 1);
2392
2393 /* Free the software handle structure */
2394 hermon_rsrc_swhdl_free(pool_info, hdl);
2395 }
2396
2397
2398 /*
2399 * hermon_rsrc_pdhdl_constructor()
2400 * Context: Can be called from interrupt or base context.
2401 */
2402 /* ARGSUSED */
2403 static int
2404 hermon_rsrc_pdhdl_constructor(void *pd, void *priv, int flags)
2405 {
2406 hermon_pdhdl_t pdhdl;
2407 hermon_state_t *state;
2408
2409 pdhdl = (hermon_pdhdl_t)pd;
2410 state = (hermon_state_t *)priv;
2411
2412 mutex_init(&pdhdl->pd_lock, NULL, MUTEX_DRIVER,
2413 DDI_INTR_PRI(state->hs_intrmsi_pri));
2414
2415 return (DDI_SUCCESS);
2416 }
2417
2418
2419 /*
2420 * hermon_rsrc_pdhdl_destructor()
2421 * Context: Can be called from interrupt or base context.
2422 */
2423 /* ARGSUSED */
2424 static void
2425 hermon_rsrc_pdhdl_destructor(void *pd, void *priv)
2426 {
2427 hermon_pdhdl_t pdhdl;
2428
2429 pdhdl = (hermon_pdhdl_t)pd;
2430
2431 mutex_destroy(&pdhdl->pd_lock);
2432 }
2433
2434
2435 /*
2436 * hermon_rsrc_cqhdl_constructor()
2437 * Context: Can be called from interrupt or base context.
2438 */
2439 /* ARGSUSED */
2440 static int
2441 hermon_rsrc_cqhdl_constructor(void *cq, void *priv, int flags)
2442 {
2443 hermon_cqhdl_t cqhdl;
2444 hermon_state_t *state;
2445
2446 cqhdl = (hermon_cqhdl_t)cq;
2447 state = (hermon_state_t *)priv;
2448
2449 mutex_init(&cqhdl->cq_lock, NULL, MUTEX_DRIVER,
2450 DDI_INTR_PRI(state->hs_intrmsi_pri));
2451
2452 return (DDI_SUCCESS);
2453 }
2454
2455
2456 /*
2457 * hermon_rsrc_cqhdl_destructor()
2458 * Context: Can be called from interrupt or base context.
2459 */
2460 /* ARGSUSED */
2461 static void
2462 hermon_rsrc_cqhdl_destructor(void *cq, void *priv)
2463 {
2464 hermon_cqhdl_t cqhdl;
2465
2466 cqhdl = (hermon_cqhdl_t)cq;
2467
2468 mutex_destroy(&cqhdl->cq_lock);
2469 }
2470
2471
2472 /*
2473 * hermon_rsrc_qphdl_constructor()
2474 * Context: Can be called from interrupt or base context.
2475 */
2476 /* ARGSUSED */
2477 static int
2478 hermon_rsrc_qphdl_constructor(void *qp, void *priv, int flags)
2479 {
2480 hermon_qphdl_t qphdl;
2481 hermon_state_t *state;
2482
2483 qphdl = (hermon_qphdl_t)qp;
2484 state = (hermon_state_t *)priv;
2485
2486 mutex_init(&qphdl->qp_lock, NULL, MUTEX_DRIVER,
2487 DDI_INTR_PRI(state->hs_intrmsi_pri));
2488
2489 return (DDI_SUCCESS);
2490 }
2491
2492
2493 /*
2494 * hermon_rsrc_qphdl_destructor()
2495 * Context: Can be called from interrupt or base context.
2496 */
2497 /* ARGSUSED */
2498 static void
2499 hermon_rsrc_qphdl_destructor(void *qp, void *priv)
2500 {
2501 hermon_qphdl_t qphdl;
2502
2503 qphdl = (hermon_qphdl_t)qp;
2504
2505 mutex_destroy(&qphdl->qp_lock);
2506 }
2507
2508
2509 /*
2510 * hermon_rsrc_srqhdl_constructor()
2511 * Context: Can be called from interrupt or base context.
2512 */
2513 /* ARGSUSED */
2514 static int
2515 hermon_rsrc_srqhdl_constructor(void *srq, void *priv, int flags)
2516 {
2517 hermon_srqhdl_t srqhdl;
2518 hermon_state_t *state;
2519
2520 srqhdl = (hermon_srqhdl_t)srq;
2521 state = (hermon_state_t *)priv;
2522
2523 mutex_init(&srqhdl->srq_lock, NULL, MUTEX_DRIVER,
2524 DDI_INTR_PRI(state->hs_intrmsi_pri));
2525
2526 return (DDI_SUCCESS);
2527 }
2528
2529
2530 /*
2531 * hermon_rsrc_srqhdl_destructor()
2532 * Context: Can be called from interrupt or base context.
2533 */
2534 /* ARGSUSED */
2535 static void
2536 hermon_rsrc_srqhdl_destructor(void *srq, void *priv)
2537 {
2538 hermon_srqhdl_t srqhdl;
2539
2540 srqhdl = (hermon_srqhdl_t)srq;
2541
2542 mutex_destroy(&srqhdl->srq_lock);
2543 }
2544
2545
2546 /*
2547 * hermon_rsrc_refcnt_constructor()
2548 * Context: Can be called from interrupt or base context.
2549 */
2550 /* ARGSUSED */
2551 static int
2552 hermon_rsrc_refcnt_constructor(void *rc, void *priv, int flags)
2553 {
2554 hermon_sw_refcnt_t *refcnt;
2555 hermon_state_t *state;
2556
2557 refcnt = (hermon_sw_refcnt_t *)rc;
2558 state = (hermon_state_t *)priv;
2559
2560 mutex_init(&refcnt->swrc_lock, NULL, MUTEX_DRIVER,
2561 DDI_INTR_PRI(state->hs_intrmsi_pri));
2562
2563 return (DDI_SUCCESS);
2564 }
2565
2566
2567 /*
2568 * hermon_rsrc_refcnt_destructor()
2569 * Context: Can be called from interrupt or base context.
2570 */
2571 /* ARGSUSED */
2572 static void
2573 hermon_rsrc_refcnt_destructor(void *rc, void *priv)
2574 {
2575 hermon_sw_refcnt_t *refcnt;
2576
2577 refcnt = (hermon_sw_refcnt_t *)rc;
2578
2579 mutex_destroy(&refcnt->swrc_lock);
2580 }
2581
2582
2583 /*
2584 * hermon_rsrc_ahhdl_constructor()
2585 * Context: Can be called from interrupt or base context.
2586 */
2587 /* ARGSUSED */
2588 static int
2589 hermon_rsrc_ahhdl_constructor(void *ah, void *priv, int flags)
2590 {
2591 hermon_ahhdl_t ahhdl;
2592 hermon_state_t *state;
2593
2594 ahhdl = (hermon_ahhdl_t)ah;
2595 state = (hermon_state_t *)priv;
2596
2597 mutex_init(&ahhdl->ah_lock, NULL, MUTEX_DRIVER,
2598 DDI_INTR_PRI(state->hs_intrmsi_pri));
2599 return (DDI_SUCCESS);
2600 }
2601
2602
2603 /*
2604 * hermon_rsrc_ahhdl_destructor()
2605 * Context: Can be called from interrupt or base context.
2606 */
2607 /* ARGSUSED */
2608 static void
2609 hermon_rsrc_ahhdl_destructor(void *ah, void *priv)
2610 {
2611 hermon_ahhdl_t ahhdl;
2612
2613 ahhdl = (hermon_ahhdl_t)ah;
2614
2615 mutex_destroy(&ahhdl->ah_lock);
2616 }
2617
2618
2619 /*
2620 * hermon_rsrc_mrhdl_constructor()
2621 * Context: Can be called from interrupt or base context.
2622 */
2623 /* ARGSUSED */
2624 static int
2625 hermon_rsrc_mrhdl_constructor(void *mr, void *priv, int flags)
2626 {
2627 hermon_mrhdl_t mrhdl;
2628 hermon_state_t *state;
2629
2630 mrhdl = (hermon_mrhdl_t)mr;
2631 state = (hermon_state_t *)priv;
2632
2633 mutex_init(&mrhdl->mr_lock, NULL, MUTEX_DRIVER,
2634 DDI_INTR_PRI(state->hs_intrmsi_pri));
2635
2636 return (DDI_SUCCESS);
2637 }
2638
2639
2640 /*
2641 * hermon_rsrc_mrhdl_destructor()
2642 * Context: Can be called from interrupt or base context.
2643 */
2644 /* ARGSUSED */
2645 static void
2646 hermon_rsrc_mrhdl_destructor(void *mr, void *priv)
2647 {
2648 hermon_mrhdl_t mrhdl;
2649
2650 mrhdl = (hermon_mrhdl_t)mr;
2651
2652 mutex_destroy(&mrhdl->mr_lock);
2653 }
2654
2655
2656 /*
2657 * hermon_rsrc_mcg_entry_get_size()
2658 */
2659 static int
2660 hermon_rsrc_mcg_entry_get_size(hermon_state_t *state, uint_t *mcg_size_shift)
2661 {
2662 uint_t num_qp_per_mcg, max_qp_per_mcg, log2;
2663
2664 /*
2665 * Round the configured number of QP per MCG to next larger
2666 * power-of-2 size and update.
2667 */
2668 num_qp_per_mcg = state->hs_cfg_profile->cp_num_qp_per_mcg + 8;
2669 log2 = highbit(num_qp_per_mcg);
2670 if (ISP2(num_qp_per_mcg)) {
2671 log2 = log2 - 1;
2672 }
2673 state->hs_cfg_profile->cp_num_qp_per_mcg = (1 << log2) - 8;
2674
2675 /* Now make sure number of QP per MCG makes sense */
2676 num_qp_per_mcg = state->hs_cfg_profile->cp_num_qp_per_mcg;
2677 max_qp_per_mcg = (1 << state->hs_devlim.log_max_qp_mcg);
2678 if (num_qp_per_mcg > max_qp_per_mcg) {
2679 return (DDI_FAILURE);
2680 }
2681
2682 /* Return the (shift) size of an individual MCG HW entry */
2683 *mcg_size_shift = log2 + 2;
2684
2685 return (DDI_SUCCESS);
2686 }