1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /*
26 * ibdm.c
27 *
28 * This file contains the InifiniBand Device Manager (IBDM) support functions.
29 * IB nexus driver will only be the client for the IBDM module.
30 *
31 * IBDM registers with IBTF for HCA arrival/removal notification.
32 * IBDM registers with SA access to send DM MADs to discover the IOC's behind
33 * the IOU's.
34 *
35 * IB nexus driver registers with IBDM to find the information about the
36 * HCA's and IOC's (behind the IOU) present on the IB fabric.
37 */
38
39 #include <sys/systm.h>
40 #include <sys/taskq.h>
41 #include <sys/ib/mgt/ibdm/ibdm_impl.h>
42 #include <sys/ib/mgt/ibmf/ibmf_impl.h>
43 #include <sys/ib/ibtl/impl/ibtl_ibnex.h>
44 #include <sys/modctl.h>
45
46 /* Function Prototype declarations */
47 static int ibdm_free_iou_info(ibdm_dp_gidinfo_t *, ibdm_iou_info_t **);
48 static int ibdm_fini(void);
49 static int ibdm_init(void);
50 static int ibdm_get_reachable_ports(ibdm_port_attr_t *,
51 ibdm_hca_list_t *);
52 static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t);
53 static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *);
54 static boolean_t ibdm_is_cisco(ib_guid_t);
55 static boolean_t ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *);
56 static void ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *);
57 static int ibdm_set_classportinfo(ibdm_dp_gidinfo_t *);
58 static int ibdm_send_classportinfo(ibdm_dp_gidinfo_t *);
59 static int ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *);
60 static int ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *);
61 static int ibdm_get_node_port_guids(ibmf_saa_handle_t, ib_lid_t,
62 ib_guid_t *, ib_guid_t *);
63 static int ibdm_retry_command(ibdm_timeout_cb_args_t *);
64 static int ibdm_get_diagcode(ibdm_dp_gidinfo_t *, int);
65 static int ibdm_verify_mad_status(ib_mad_hdr_t *);
66 static int ibdm_handle_redirection(ibmf_msg_t *,
67 ibdm_dp_gidinfo_t *, int *);
68 static void ibdm_wait_probe_completion(void);
69 static void ibdm_sweep_fabric(int);
70 static void ibdm_probe_gid_thread(void *);
71 static void ibdm_wakeup_probe_gid_cv(void);
72 static void ibdm_port_attr_ibmf_init(ibdm_port_attr_t *, ib_pkey_t, int);
73 static int ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *, int);
74 static void ibdm_update_port_attr(ibdm_port_attr_t *);
75 static void ibdm_handle_hca_attach(ib_guid_t);
76 static void ibdm_handle_srventry_mad(ibmf_msg_t *,
77 ibdm_dp_gidinfo_t *, int *);
78 static void ibdm_ibmf_recv_cb(ibmf_handle_t, ibmf_msg_t *, void *);
79 static void ibdm_recv_incoming_mad(void *);
80 static void ibdm_process_incoming_mad(ibmf_handle_t, ibmf_msg_t *, void *);
81 static void ibdm_ibmf_send_cb(ibmf_handle_t, ibmf_msg_t *, void *);
82 static void ibdm_pkt_timeout_hdlr(void *arg);
83 static void ibdm_initialize_port(ibdm_port_attr_t *);
84 static void ibdm_update_port_pkeys(ibdm_port_attr_t *port);
85 static void ibdm_handle_diagcode(ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
86 static void ibdm_probe_gid(ibdm_dp_gidinfo_t *);
87 static void ibdm_alloc_send_buffers(ibmf_msg_t *);
88 static void ibdm_free_send_buffers(ibmf_msg_t *);
89 static void ibdm_handle_hca_detach(ib_guid_t);
90 static void ibdm_handle_port_change_event(ibt_async_event_t *);
91 static int ibdm_fini_port(ibdm_port_attr_t *);
92 static int ibdm_uninit_hca(ibdm_hca_list_t *);
93 static void ibdm_handle_setclassportinfo(ibmf_handle_t, ibmf_msg_t *,
94 ibdm_dp_gidinfo_t *, int *);
95 static void ibdm_handle_iounitinfo(ibmf_handle_t,
96 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
97 static void ibdm_handle_ioc_profile(ibmf_handle_t,
98 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
99 static void ibdm_event_hdlr(void *, ibt_hca_hdl_t,
100 ibt_async_code_t, ibt_async_event_t *);
101 static void ibdm_handle_classportinfo(ibmf_handle_t,
102 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
103 static void ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *,
104 ibdm_dp_gidinfo_t *);
105
106 static ibdm_hca_list_t *ibdm_dup_hca_attr(ibdm_hca_list_t *);
107 static ibdm_ioc_info_t *ibdm_dup_ioc_info(ibdm_ioc_info_t *,
108 ibdm_dp_gidinfo_t *gid_list);
109 static void ibdm_probe_ioc(ib_guid_t, ib_guid_t, int);
110 static ibdm_ioc_info_t *ibdm_is_ioc_present(ib_guid_t,
111 ibdm_dp_gidinfo_t *, int *);
112 static ibdm_port_attr_t *ibdm_get_port_attr(ibt_async_event_t *,
113 ibdm_hca_list_t **);
114 static sa_node_record_t *ibdm_get_node_records(ibmf_saa_handle_t,
115 size_t *, ib_guid_t);
116 static int ibdm_get_node_record_by_port(ibmf_saa_handle_t,
117 ib_guid_t, sa_node_record_t **, size_t *);
118 static sa_portinfo_record_t *ibdm_get_portinfo(ibmf_saa_handle_t, size_t *,
119 ib_lid_t);
120 static ibdm_dp_gidinfo_t *ibdm_create_gid_info(ibdm_port_attr_t *,
121 ib_gid_t, ib_gid_t);
122 static ibdm_dp_gidinfo_t *ibdm_find_gid(ib_guid_t, ib_guid_t);
123 static int ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *, uint8_t);
124 static ibdm_ioc_info_t *ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *, int);
125 static void ibdm_saa_event_cb(ibmf_saa_handle_t, ibmf_saa_subnet_event_t,
126 ibmf_saa_event_details_t *, void *);
127 static void ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *,
128 ibdm_dp_gidinfo_t *);
129 static ibdm_dp_gidinfo_t *ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *);
130 static void ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *,
131 ibdm_dp_gidinfo_t *);
132 static void ibdm_addto_gidlist(ibdm_gid_t **, ibdm_gid_t *);
133 static void ibdm_free_gid_list(ibdm_gid_t *);
134 static void ibdm_rescan_gidlist(ib_guid_t *ioc_guid);
135 static void ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *);
136 static void ibdm_saa_event_taskq(void *);
137 static void ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *);
138 static void ibdm_get_next_port(ibdm_hca_list_t **,
139 ibdm_port_attr_t **, int);
140 static void ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *,
141 ibdm_dp_gidinfo_t *);
142 static void ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *,
143 ibdm_hca_list_t *);
144 static void ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *);
145 static void ibdm_saa_handle_new_gid(void *);
146 static void ibdm_reset_all_dgids(ibmf_saa_handle_t);
147 static void ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *);
148 static void ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *);
149 static void ibdm_fill_srv_attr_mod(ib_mad_hdr_t *, ibdm_timeout_cb_args_t *);
150 static void ibdm_bump_transactionID(ibdm_dp_gidinfo_t *);
151 static ibdm_ioc_info_t *ibdm_handle_prev_iou();
152 static int ibdm_serv_cmp(ibdm_srvents_info_t *, ibdm_srvents_info_t *,
153 int);
154 static ibdm_ioc_info_t *ibdm_get_ioc_info_with_gid(ib_guid_t,
155 ibdm_dp_gidinfo_t **);
156
157 int ibdm_dft_timeout = IBDM_DFT_TIMEOUT;
158 int ibdm_dft_retry_cnt = IBDM_DFT_NRETRIES;
159 #ifdef DEBUG
160 int ibdm_ignore_saa_event = 0;
161 #endif
162 int ibdm_enumerate_iocs = 0;
163
164 /* Modload support */
165 static struct modlmisc ibdm_modlmisc = {
166 &mod_miscops,
167 "InfiniBand Device Manager"
168 };
169
170 struct modlinkage ibdm_modlinkage = {
171 MODREV_1,
172 (void *)&ibdm_modlmisc,
173 NULL
174 };
175
176 static ibt_clnt_modinfo_t ibdm_ibt_modinfo = {
177 IBTI_V_CURR,
178 IBT_DM,
179 ibdm_event_hdlr,
180 NULL,
181 "ibdm"
182 };
183
184 /* Global variables */
185 ibdm_t ibdm;
186 int ibdm_taskq_enable = IBDM_ENABLE_TASKQ_HANDLING;
187 char *ibdm_string = "ibdm";
188
189 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv",
190 ibdm.ibdm_dp_gidlist_head))
191
192 /*
193 * _init
194 * Loadable module init, called before any other module.
195 * Initialize mutex
196 * Register with IBTF
197 */
198 int
199 _init(void)
200 {
201 int err;
202
203 IBTF_DPRINTF_L4("ibdm", "\t_init: addr of ibdm %p", &ibdm);
204
205 if ((err = ibdm_init()) != IBDM_SUCCESS) {
206 IBTF_DPRINTF_L2("ibdm", "_init: ibdm_init failed 0x%x", err);
207 (void) ibdm_fini();
208 return (DDI_FAILURE);
209 }
210
211 if ((err = mod_install(&ibdm_modlinkage)) != 0) {
212 IBTF_DPRINTF_L2("ibdm", "_init: mod_install failed 0x%x", err);
213 (void) ibdm_fini();
214 }
215 return (err);
216 }
217
218
219 int
220 _fini(void)
221 {
222 int err;
223
224 if ((err = ibdm_fini()) != IBDM_SUCCESS) {
225 IBTF_DPRINTF_L2("ibdm", "_fini: ibdm_fini failed 0x%x", err);
226 (void) ibdm_init();
227 return (EBUSY);
228 }
229
230 if ((err = mod_remove(&ibdm_modlinkage)) != 0) {
231 IBTF_DPRINTF_L2("ibdm", "_fini: mod_remove failed 0x%x", err);
232 (void) ibdm_init();
233 }
234 return (err);
235 }
236
237
238 int
239 _info(struct modinfo *modinfop)
240 {
241 return (mod_info(&ibdm_modlinkage, modinfop));
242 }
243
244
245 /*
246 * ibdm_init():
247 * Register with IBTF
248 * Allocate memory for the HCAs
249 * Allocate minor-nodes for the HCAs
250 */
251 static int
252 ibdm_init(void)
253 {
254 int i, hca_count;
255 ib_guid_t *hca_guids;
256 ibt_status_t status;
257
258 IBTF_DPRINTF_L4("ibdm", "\tibdm_init:");
259 if (!(ibdm.ibdm_state & IBDM_LOCKS_ALLOCED)) {
260 mutex_init(&ibdm.ibdm_mutex, NULL, MUTEX_DEFAULT, NULL);
261 mutex_init(&ibdm.ibdm_hl_mutex, NULL, MUTEX_DEFAULT, NULL);
262 mutex_init(&ibdm.ibdm_ibnex_mutex, NULL, MUTEX_DEFAULT, NULL);
263 cv_init(&ibdm.ibdm_port_settle_cv, NULL, CV_DRIVER, NULL);
264 mutex_enter(&ibdm.ibdm_mutex);
265 ibdm.ibdm_state |= IBDM_LOCKS_ALLOCED;
266 }
267
268 if (!(ibdm.ibdm_state & IBDM_IBT_ATTACHED)) {
269 if ((status = ibt_attach(&ibdm_ibt_modinfo, NULL, NULL,
270 (void *)&ibdm.ibdm_ibt_clnt_hdl)) != IBT_SUCCESS) {
271 IBTF_DPRINTF_L2("ibdm", "ibdm_init: ibt_attach "
272 "failed %x", status);
273 mutex_exit(&ibdm.ibdm_mutex);
274 return (IBDM_FAILURE);
275 }
276
277 ibdm.ibdm_state |= IBDM_IBT_ATTACHED;
278 mutex_exit(&ibdm.ibdm_mutex);
279 }
280
281
282 if (!(ibdm.ibdm_state & IBDM_HCA_ATTACHED)) {
283 hca_count = ibt_get_hca_list(&hca_guids);
284 IBTF_DPRINTF_L4("ibdm", "ibdm_init: num_hcas = %d", hca_count);
285 for (i = 0; i < hca_count; i++)
286 (void) ibdm_handle_hca_attach(hca_guids[i]);
287 if (hca_count)
288 ibt_free_hca_list(hca_guids, hca_count);
289
290 mutex_enter(&ibdm.ibdm_mutex);
291 ibdm.ibdm_state |= IBDM_HCA_ATTACHED;
292 mutex_exit(&ibdm.ibdm_mutex);
293 }
294
295 if (!(ibdm.ibdm_state & IBDM_CVS_ALLOCED)) {
296 cv_init(&ibdm.ibdm_probe_cv, NULL, CV_DRIVER, NULL);
297 cv_init(&ibdm.ibdm_busy_cv, NULL, CV_DRIVER, NULL);
298 mutex_enter(&ibdm.ibdm_mutex);
299 ibdm.ibdm_state |= IBDM_CVS_ALLOCED;
300 mutex_exit(&ibdm.ibdm_mutex);
301 }
302 return (IBDM_SUCCESS);
303 }
304
305
306 static int
307 ibdm_free_iou_info(ibdm_dp_gidinfo_t *gid_info, ibdm_iou_info_t **ioup)
308 {
309 int ii, k, niocs;
310 size_t size;
311 ibdm_gid_t *delete, *head;
312 timeout_id_t timeout_id;
313 ibdm_ioc_info_t *ioc;
314 ibdm_iou_info_t *gl_iou = *ioup;
315
316 ASSERT(mutex_owned(&gid_info->gl_mutex));
317 if (gl_iou == NULL) {
318 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: No IOU");
319 return (0);
320 }
321
322 niocs = gl_iou->iou_info.iou_num_ctrl_slots;
323 IBTF_DPRINTF_L4("ibdm", "\tfree_iou_info: gid_info = %p, niocs %d",
324 gid_info, niocs);
325
326 for (ii = 0; ii < niocs; ii++) {
327 ioc = (ibdm_ioc_info_t *)&gl_iou->iou_ioc_info[ii];
328
329 /* handle the case where an ioc_timeout_id is scheduled */
330 if (ioc->ioc_timeout_id) {
331 timeout_id = ioc->ioc_timeout_id;
332 ioc->ioc_timeout_id = 0;
333 mutex_exit(&gid_info->gl_mutex);
334 IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
335 "ioc_timeout_id = 0x%x", timeout_id);
336 if (untimeout(timeout_id) == -1) {
337 IBTF_DPRINTF_L2("ibdm", "free_iou_info: "
338 "untimeout ioc_timeout_id failed");
339 mutex_enter(&gid_info->gl_mutex);
340 return (-1);
341 }
342 mutex_enter(&gid_info->gl_mutex);
343 }
344
345 /* handle the case where an ioc_dc_timeout_id is scheduled */
346 if (ioc->ioc_dc_timeout_id) {
347 timeout_id = ioc->ioc_dc_timeout_id;
348 ioc->ioc_dc_timeout_id = 0;
349 mutex_exit(&gid_info->gl_mutex);
350 IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
351 "ioc_dc_timeout_id = 0x%x", timeout_id);
352 if (untimeout(timeout_id) == -1) {
353 IBTF_DPRINTF_L2("ibdm", "free_iou_info: "
354 "untimeout ioc_dc_timeout_id failed");
355 mutex_enter(&gid_info->gl_mutex);
356 return (-1);
357 }
358 mutex_enter(&gid_info->gl_mutex);
359 }
360
361 /* handle the case where serv[k].se_timeout_id is scheduled */
362 for (k = 0; k < ioc->ioc_profile.ioc_service_entries; k++) {
363 if (ioc->ioc_serv[k].se_timeout_id) {
364 timeout_id = ioc->ioc_serv[k].se_timeout_id;
365 ioc->ioc_serv[k].se_timeout_id = 0;
366 mutex_exit(&gid_info->gl_mutex);
367 IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
368 "ioc->ioc_serv[%d].se_timeout_id = 0x%x",
369 k, timeout_id);
370 if (untimeout(timeout_id) == -1) {
371 IBTF_DPRINTF_L2("ibdm", "free_iou_info:"
372 " untimeout se_timeout_id failed");
373 mutex_enter(&gid_info->gl_mutex);
374 return (-1);
375 }
376 mutex_enter(&gid_info->gl_mutex);
377 }
378 }
379
380 /* delete GID list in IOC */
381 head = ioc->ioc_gid_list;
382 while (head) {
383 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: "
384 "Deleting gid_list struct %p", head);
385 delete = head;
386 head = head->gid_next;
387 kmem_free(delete, sizeof (ibdm_gid_t));
388 }
389 ioc->ioc_gid_list = NULL;
390
391 /* delete ioc_serv */
392 size = ioc->ioc_profile.ioc_service_entries *
393 sizeof (ibdm_srvents_info_t);
394 if (ioc->ioc_serv && size) {
395 kmem_free(ioc->ioc_serv, size);
396 ioc->ioc_serv = NULL;
397 }
398 }
399 /*
400 * Clear the IBDM_CISCO_PROBE_DONE flag to get the IO Unit information
401 * via the switch during the probe process.
402 */
403 gid_info->gl_flag &= ~IBDM_CISCO_PROBE_DONE;
404
405 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: deleting IOU & IOC");
406 size = sizeof (ibdm_iou_info_t) + niocs * sizeof (ibdm_ioc_info_t);
407 kmem_free(gl_iou, size);
408 *ioup = NULL;
409 return (0);
410 }
411
412
413 /*
414 * ibdm_fini():
415 * Un-register with IBTF
416 * De allocate memory for the GID info
417 */
418 static int
419 ibdm_fini()
420 {
421 int ii;
422 ibdm_hca_list_t *hca_list, *temp;
423 ibdm_dp_gidinfo_t *gid_info, *tmp;
424 ibdm_gid_t *head, *delete;
425
426 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini");
427
428 mutex_enter(&ibdm.ibdm_hl_mutex);
429 if (ibdm.ibdm_state & IBDM_IBT_ATTACHED) {
430 if (ibt_detach(ibdm.ibdm_ibt_clnt_hdl) != IBT_SUCCESS) {
431 IBTF_DPRINTF_L2("ibdm", "\t_fini: ibt_detach failed");
432 mutex_exit(&ibdm.ibdm_hl_mutex);
433 return (IBDM_FAILURE);
434 }
435 ibdm.ibdm_state &= ~IBDM_IBT_ATTACHED;
436 ibdm.ibdm_ibt_clnt_hdl = NULL;
437 }
438
439 hca_list = ibdm.ibdm_hca_list_head;
440 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: nhcas %d", ibdm.ibdm_hca_count);
441 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
442 temp = hca_list;
443 hca_list = hca_list->hl_next;
444 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: hca %p", temp);
445 if (ibdm_uninit_hca(temp) != IBDM_SUCCESS) {
446 IBTF_DPRINTF_L2("ibdm", "\tibdm_fini: "
447 "uninit_hca %p failed", temp);
448 mutex_exit(&ibdm.ibdm_hl_mutex);
449 return (IBDM_FAILURE);
450 }
451 }
452 mutex_exit(&ibdm.ibdm_hl_mutex);
453
454 mutex_enter(&ibdm.ibdm_mutex);
455 if (ibdm.ibdm_state & IBDM_HCA_ATTACHED)
456 ibdm.ibdm_state &= ~IBDM_HCA_ATTACHED;
457
458 gid_info = ibdm.ibdm_dp_gidlist_head;
459 while (gid_info) {
460 mutex_enter(&gid_info->gl_mutex);
461 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
462 mutex_exit(&gid_info->gl_mutex);
463 ibdm_delete_glhca_list(gid_info);
464
465 tmp = gid_info;
466 gid_info = gid_info->gl_next;
467 mutex_destroy(&tmp->gl_mutex);
468 head = tmp->gl_gid;
469 while (head) {
470 IBTF_DPRINTF_L4("ibdm",
471 "\tibdm_fini: Deleting gid structs");
472 delete = head;
473 head = head->gid_next;
474 kmem_free(delete, sizeof (ibdm_gid_t));
475 }
476 kmem_free(tmp, sizeof (ibdm_dp_gidinfo_t));
477 }
478 mutex_exit(&ibdm.ibdm_mutex);
479
480 if (ibdm.ibdm_state & IBDM_LOCKS_ALLOCED) {
481 ibdm.ibdm_state &= ~IBDM_LOCKS_ALLOCED;
482 mutex_destroy(&ibdm.ibdm_mutex);
483 mutex_destroy(&ibdm.ibdm_hl_mutex);
484 mutex_destroy(&ibdm.ibdm_ibnex_mutex);
485 cv_destroy(&ibdm.ibdm_port_settle_cv);
486 }
487 if (ibdm.ibdm_state & IBDM_CVS_ALLOCED) {
488 ibdm.ibdm_state &= ~IBDM_CVS_ALLOCED;
489 cv_destroy(&ibdm.ibdm_probe_cv);
490 cv_destroy(&ibdm.ibdm_busy_cv);
491 }
492 return (IBDM_SUCCESS);
493 }
494
495
496 /*
497 * ibdm_event_hdlr()
498 *
499 * IBDM registers this asynchronous event handler at the time of
500 * ibt_attach. IBDM support the following async events. For other
501 * event, simply returns success.
502 * IBT_HCA_ATTACH_EVENT:
503 * Retrieves the information about all the port that are
504 * present on this HCA, allocates the port attributes
505 * structure and calls IB nexus callback routine with
506 * the port attributes structure as an input argument.
507 * IBT_HCA_DETACH_EVENT:
508 * Retrieves the information about all the ports that are
509 * present on this HCA and calls IB nexus callback with
510 * port guid as an argument
511 * IBT_EVENT_PORT_UP:
512 * Register with IBMF and SA access
513 * Setup IBMF receive callback routine
514 * IBT_EVENT_PORT_DOWN:
515 * Un-Register with IBMF and SA access
516 * Teardown IBMF receive callback routine
517 */
518 /*ARGSUSED*/
519 static void
520 ibdm_event_hdlr(void *clnt_hdl,
521 ibt_hca_hdl_t hca_hdl, ibt_async_code_t code, ibt_async_event_t *event)
522 {
523 ibdm_hca_list_t *hca_list;
524 ibdm_port_attr_t *port;
525 ibmf_saa_handle_t port_sa_hdl;
526
527 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: async code 0x%x", code);
528
529 switch (code) {
530 case IBT_HCA_ATTACH_EVENT: /* New HCA registered with IBTF */
531 ibdm_handle_hca_attach(event->ev_hca_guid);
532 break;
533
534 case IBT_HCA_DETACH_EVENT: /* HCA unregistered with IBTF */
535 ibdm_handle_hca_detach(event->ev_hca_guid);
536 mutex_enter(&ibdm.ibdm_ibnex_mutex);
537 if (ibdm.ibdm_ibnex_callback != NULL) {
538 (*ibdm.ibdm_ibnex_callback)((void *)
539 &event->ev_hca_guid, IBDM_EVENT_HCA_REMOVED);
540 }
541 mutex_exit(&ibdm.ibdm_ibnex_mutex);
542 break;
543
544 case IBT_EVENT_PORT_UP:
545 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_UP");
546 mutex_enter(&ibdm.ibdm_hl_mutex);
547 port = ibdm_get_port_attr(event, &hca_list);
548 if (port == NULL) {
549 IBTF_DPRINTF_L2("ibdm",
550 "\tevent_hdlr: HCA not present");
551 mutex_exit(&ibdm.ibdm_hl_mutex);
552 break;
553 }
554 ibdm_initialize_port(port);
555 hca_list->hl_nports_active++;
556 cv_broadcast(&ibdm.ibdm_port_settle_cv);
557 mutex_exit(&ibdm.ibdm_hl_mutex);
558
559 /* Inform IB nexus driver */
560 mutex_enter(&ibdm.ibdm_ibnex_mutex);
561 if (ibdm.ibdm_ibnex_callback != NULL) {
562 (*ibdm.ibdm_ibnex_callback)((void *)
563 &event->ev_hca_guid, IBDM_EVENT_PORT_UP);
564 }
565 mutex_exit(&ibdm.ibdm_ibnex_mutex);
566 break;
567
568 case IBT_ERROR_PORT_DOWN:
569 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_DOWN");
570 mutex_enter(&ibdm.ibdm_hl_mutex);
571 port = ibdm_get_port_attr(event, &hca_list);
572 if (port == NULL) {
573 IBTF_DPRINTF_L2("ibdm",
574 "\tevent_hdlr: HCA not present");
575 mutex_exit(&ibdm.ibdm_hl_mutex);
576 break;
577 }
578 hca_list->hl_nports_active--;
579 port_sa_hdl = port->pa_sa_hdl;
580 (void) ibdm_fini_port(port);
581 port->pa_state = IBT_PORT_DOWN;
582 cv_broadcast(&ibdm.ibdm_port_settle_cv);
583 mutex_exit(&ibdm.ibdm_hl_mutex);
584 ibdm_reset_all_dgids(port_sa_hdl);
585 break;
586
587 case IBT_PORT_CHANGE_EVENT:
588 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_CHANGE");
589 if (event->ev_port_flags & IBT_PORT_CHANGE_PKEY)
590 ibdm_handle_port_change_event(event);
591 break;
592
593 default: /* Ignore all other events/errors */
594 break;
595 }
596 }
597
598 static void
599 ibdm_handle_port_change_event(ibt_async_event_t *event)
600 {
601 ibdm_port_attr_t *port;
602 ibdm_hca_list_t *hca_list;
603
604 IBTF_DPRINTF_L2("ibdm", "\tibdm_handle_port_change_event:"
605 " HCA guid %llx", event->ev_hca_guid);
606 mutex_enter(&ibdm.ibdm_hl_mutex);
607 port = ibdm_get_port_attr(event, &hca_list);
608 if (port == NULL) {
609 IBTF_DPRINTF_L2("ibdm", "\tevent_hdlr: HCA not present");
610 mutex_exit(&ibdm.ibdm_hl_mutex);
611 return;
612 }
613 ibdm_update_port_pkeys(port);
614 cv_broadcast(&ibdm.ibdm_port_settle_cv);
615 mutex_exit(&ibdm.ibdm_hl_mutex);
616
617 /* Inform IB nexus driver */
618 mutex_enter(&ibdm.ibdm_ibnex_mutex);
619 if (ibdm.ibdm_ibnex_callback != NULL) {
620 (*ibdm.ibdm_ibnex_callback)((void *)
621 &event->ev_hca_guid, IBDM_EVENT_PORT_PKEY_CHANGE);
622 }
623 mutex_exit(&ibdm.ibdm_ibnex_mutex);
624 }
625
626 /*
627 * ibdm_update_port_pkeys()
628 * Update the pkey table
629 * Update the port attributes
630 */
631 static void
632 ibdm_update_port_pkeys(ibdm_port_attr_t *port)
633 {
634 uint_t nports, size;
635 uint_t pkey_idx, opkey_idx;
636 uint16_t npkeys;
637 ibt_hca_portinfo_t *pinfop;
638 ib_pkey_t pkey;
639 ibdm_pkey_tbl_t *pkey_tbl;
640 ibdm_port_attr_t newport;
641
642 IBTF_DPRINTF_L4("ibdm", "\tupdate_port_pkeys:");
643 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
644
645 /* Check whether the port is active */
646 if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL,
647 NULL) != IBT_SUCCESS)
648 return;
649
650 if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num,
651 &pinfop, &nports, &size) != IBT_SUCCESS) {
652 /* This should not occur */
653 port->pa_npkeys = 0;
654 port->pa_pkey_tbl = NULL;
655 return;
656 }
657
658 npkeys = pinfop->p_pkey_tbl_sz;
659 pkey_tbl = kmem_zalloc(npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
660 newport.pa_pkey_tbl = pkey_tbl;
661 newport.pa_ibmf_hdl = port->pa_ibmf_hdl;
662
663 for (pkey_idx = 0; pkey_idx < npkeys; pkey_idx++) {
664 pkey = pkey_tbl[pkey_idx].pt_pkey =
665 pinfop->p_pkey_tbl[pkey_idx];
666 /*
667 * Is this pkey present in the current table ?
668 */
669 for (opkey_idx = 0; opkey_idx < port->pa_npkeys; opkey_idx++) {
670 if (pkey == port->pa_pkey_tbl[opkey_idx].pt_pkey) {
671 pkey_tbl[pkey_idx].pt_qp_hdl =
672 port->pa_pkey_tbl[opkey_idx].pt_qp_hdl;
673 port->pa_pkey_tbl[opkey_idx].pt_qp_hdl = NULL;
674 break;
675 }
676 }
677
678 if (opkey_idx == port->pa_npkeys) {
679 pkey = pkey_tbl[pkey_idx].pt_pkey;
680 if (IBDM_INVALID_PKEY(pkey)) {
681 pkey_tbl[pkey_idx].pt_qp_hdl = NULL;
682 continue;
683 }
684 ibdm_port_attr_ibmf_init(&newport, pkey, pkey_idx);
685 }
686 }
687
688 for (opkey_idx = 0; opkey_idx < port->pa_npkeys; opkey_idx++) {
689 if (port->pa_pkey_tbl[opkey_idx].pt_qp_hdl != NULL) {
690 if (ibdm_port_attr_ibmf_fini(port, opkey_idx) !=
691 IBDM_SUCCESS) {
692 IBTF_DPRINTF_L2("ibdm", "\tupdate_port_pkeys: "
693 "ibdm_port_attr_ibmf_fini failed for "
694 "port pkey 0x%x",
695 port->pa_pkey_tbl[opkey_idx].pt_pkey);
696 }
697 }
698 }
699
700 if (port->pa_pkey_tbl != NULL) {
701 kmem_free(port->pa_pkey_tbl,
702 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
703 }
704
705 port->pa_npkeys = npkeys;
706 port->pa_pkey_tbl = pkey_tbl;
707 port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix;
708 port->pa_state = pinfop->p_linkstate;
709 ibt_free_portinfo(pinfop, size);
710 }
711
712 /*
713 * ibdm_initialize_port()
714 * Register with IBMF
715 * Register with SA access
716 * Register a receive callback routine with IBMF. IBMF invokes
717 * this routine whenever a MAD arrives at this port.
718 * Update the port attributes
719 */
720 static void
721 ibdm_initialize_port(ibdm_port_attr_t *port)
722 {
723 int ii;
724 uint_t nports, size;
725 uint_t pkey_idx;
726 ib_pkey_t pkey;
727 ibt_hca_portinfo_t *pinfop;
728 ibmf_register_info_t ibmf_reg;
729 ibmf_saa_subnet_event_args_t event_args;
730
731 IBTF_DPRINTF_L4("ibdm", "\tinitialize_port:");
732 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
733
734 /* Check whether the port is active */
735 if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL,
736 NULL) != IBT_SUCCESS)
737 return;
738
739 if (port->pa_sa_hdl != NULL || port->pa_pkey_tbl != NULL)
740 return;
741
742 if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num,
743 &pinfop, &nports, &size) != IBT_SUCCESS) {
744 /* This should not occur */
745 port->pa_npkeys = 0;
746 port->pa_pkey_tbl = NULL;
747 return;
748 }
749 port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix;
750
751 port->pa_state = pinfop->p_linkstate;
752 port->pa_npkeys = pinfop->p_pkey_tbl_sz;
753 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc(
754 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
755
756 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++)
757 port->pa_pkey_tbl[pkey_idx].pt_pkey =
758 pinfop->p_pkey_tbl[pkey_idx];
759
760 ibt_free_portinfo(pinfop, size);
761
762 if (ibdm_enumerate_iocs) {
763 event_args.is_event_callback = ibdm_saa_event_cb;
764 event_args.is_event_callback_arg = port;
765 if (ibmf_sa_session_open(port->pa_port_guid, 0, &event_args,
766 IBMF_VERSION, 0, &port->pa_sa_hdl) != IBMF_SUCCESS) {
767 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
768 "sa access registration failed");
769 (void) ibdm_fini_port(port);
770 return;
771 }
772
773 ibmf_reg.ir_ci_guid = port->pa_hca_guid;
774 ibmf_reg.ir_port_num = port->pa_port_num;
775 ibmf_reg.ir_client_class = DEV_MGT_MANAGER;
776
777 if (ibmf_register(&ibmf_reg, IBMF_VERSION, 0, NULL, NULL,
778 &port->pa_ibmf_hdl, &port->pa_ibmf_caps) != IBMF_SUCCESS) {
779 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
780 "IBMF registration failed");
781 (void) ibdm_fini_port(port);
782 return;
783 }
784
785 if (ibmf_setup_async_cb(port->pa_ibmf_hdl,
786 IBMF_QP_HANDLE_DEFAULT,
787 ibdm_ibmf_recv_cb, 0, 0) != IBMF_SUCCESS) {
788 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
789 "IBMF setup recv cb failed");
790 (void) ibdm_fini_port(port);
791 return;
792 }
793 } else {
794 port->pa_sa_hdl = NULL;
795 port->pa_ibmf_hdl = NULL;
796 }
797
798 for (ii = 0; ii < port->pa_npkeys; ii++) {
799 pkey = port->pa_pkey_tbl[ii].pt_pkey;
800 if (IBDM_INVALID_PKEY(pkey)) {
801 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
802 continue;
803 }
804 ibdm_port_attr_ibmf_init(port, pkey, ii);
805 }
806 }
807
808
809 /*
810 * ibdm_port_attr_ibmf_init:
811 * With IBMF - Alloc QP Handle and Setup Async callback
812 */
813 static void
814 ibdm_port_attr_ibmf_init(ibdm_port_attr_t *port, ib_pkey_t pkey, int ii)
815 {
816 int ret;
817
818 if (ibdm_enumerate_iocs == 0) {
819 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
820 return;
821 }
822
823 if ((ret = ibmf_alloc_qp(port->pa_ibmf_hdl, pkey, IB_GSI_QKEY,
824 IBMF_ALT_QP_MAD_NO_RMPP, &port->pa_pkey_tbl[ii].pt_qp_hdl)) !=
825 IBMF_SUCCESS) {
826 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: "
827 "IBMF failed to alloc qp %d", ret);
828 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
829 return;
830 }
831
832 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_init: QP handle is %p",
833 port->pa_ibmf_hdl);
834
835 if ((ret = ibmf_setup_async_cb(port->pa_ibmf_hdl,
836 port->pa_pkey_tbl[ii].pt_qp_hdl, ibdm_ibmf_recv_cb, 0, 0)) !=
837 IBMF_SUCCESS) {
838 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: "
839 "IBMF setup recv cb failed %d", ret);
840 (void) ibmf_free_qp(port->pa_ibmf_hdl,
841 &port->pa_pkey_tbl[ii].pt_qp_hdl, 0);
842 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
843 }
844 }
845
846
847 /*
848 * ibdm_get_port_attr()
849 * Get port attributes from HCA guid and port number
850 * Return pointer to ibdm_port_attr_t on Success
851 * and NULL on failure
852 */
853 static ibdm_port_attr_t *
854 ibdm_get_port_attr(ibt_async_event_t *event, ibdm_hca_list_t **retval)
855 {
856 ibdm_hca_list_t *hca_list;
857 ibdm_port_attr_t *port_attr;
858 int ii;
859
860 IBTF_DPRINTF_L4("ibdm", "\tget_port_attr: port# %d", event->ev_port);
861 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
862 hca_list = ibdm.ibdm_hca_list_head;
863 while (hca_list) {
864 if (hca_list->hl_hca_guid == event->ev_hca_guid) {
865 for (ii = 0; ii < hca_list->hl_nports; ii++) {
866 port_attr = &hca_list->hl_port_attr[ii];
867 if (port_attr->pa_port_num == event->ev_port) {
868 *retval = hca_list;
869 return (port_attr);
870 }
871 }
872 }
873 hca_list = hca_list->hl_next;
874 }
875 return (NULL);
876 }
877
878
879 /*
880 * ibdm_update_port_attr()
881 * Update the port attributes
882 */
883 static void
884 ibdm_update_port_attr(ibdm_port_attr_t *port)
885 {
886 uint_t nports, size;
887 uint_t pkey_idx;
888 ibt_hca_portinfo_t *portinfop;
889
890 IBTF_DPRINTF_L4("ibdm", "\tupdate_port_attr: Begin");
891 if (ibt_query_hca_ports(port->pa_hca_hdl,
892 port->pa_port_num, &portinfop, &nports, &size) != IBT_SUCCESS) {
893 /* This should not occur */
894 port->pa_npkeys = 0;
895 port->pa_pkey_tbl = NULL;
896 return;
897 }
898 port->pa_sn_prefix = portinfop->p_sgid_tbl[0].gid_prefix;
899
900 port->pa_state = portinfop->p_linkstate;
901
902 /*
903 * PKey information in portinfo valid only if port is
904 * ACTIVE. Bail out if not.
905 */
906 if (port->pa_state != IBT_PORT_ACTIVE) {
907 port->pa_npkeys = 0;
908 port->pa_pkey_tbl = NULL;
909 ibt_free_portinfo(portinfop, size);
910 return;
911 }
912
913 port->pa_npkeys = portinfop->p_pkey_tbl_sz;
914 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc(
915 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
916
917 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) {
918 port->pa_pkey_tbl[pkey_idx].pt_pkey =
919 portinfop->p_pkey_tbl[pkey_idx];
920 }
921 ibt_free_portinfo(portinfop, size);
922 }
923
924
925 /*
926 * ibdm_handle_hca_attach()
927 */
928 static void
929 ibdm_handle_hca_attach(ib_guid_t hca_guid)
930 {
931 uint_t size;
932 uint_t ii, nports;
933 ibt_status_t status;
934 ibt_hca_hdl_t hca_hdl;
935 ibt_hca_attr_t *hca_attr;
936 ibdm_hca_list_t *hca_list, *temp;
937 ibdm_port_attr_t *port_attr;
938 ibt_hca_portinfo_t *portinfop;
939
940 IBTF_DPRINTF_L4("ibdm",
941 "\thandle_hca_attach: hca_guid = 0x%llX", hca_guid);
942
943 /* open the HCA first */
944 if ((status = ibt_open_hca(ibdm.ibdm_ibt_clnt_hdl, hca_guid,
945 &hca_hdl)) != IBT_SUCCESS) {
946 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
947 "open_hca failed, status 0x%x", status);
948 return;
949 }
950
951 hca_attr = (ibt_hca_attr_t *)
952 kmem_alloc(sizeof (ibt_hca_attr_t), KM_SLEEP);
953 /* ibt_query_hca always returns IBT_SUCCESS */
954 (void) ibt_query_hca(hca_hdl, hca_attr);
955
956 IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x,"
957 " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id,
958 hca_attr->hca_version_id, hca_attr->hca_nports);
959
960 if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports,
961 &size)) != IBT_SUCCESS) {
962 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
963 "ibt_query_hca_ports failed, status 0x%x", status);
964 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
965 (void) ibt_close_hca(hca_hdl);
966 return;
967 }
968 hca_list = (ibdm_hca_list_t *)
969 kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP);
970 hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
971 (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP);
972 hca_list->hl_hca_guid = hca_attr->hca_node_guid;
973 hca_list->hl_nports = hca_attr->hca_nports;
974 hca_list->hl_attach_time = ddi_get_time();
975 hca_list->hl_hca_hdl = hca_hdl;
976
977 /*
978 * Init a dummy port attribute for the HCA node
979 * This is for Per-HCA Node. Initialize port_attr :
980 * hca_guid & port_guid -> hca_guid
981 * npkeys, pkey_tbl is NULL
982 * port_num, sn_prefix is 0
983 * vendorid, product_id, dev_version from HCA
984 * pa_state is IBT_PORT_ACTIVE
985 */
986 hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
987 sizeof (ibdm_port_attr_t), KM_SLEEP);
988 port_attr = hca_list->hl_hca_port_attr;
989 port_attr->pa_vendorid = hca_attr->hca_vendor_id;
990 port_attr->pa_productid = hca_attr->hca_device_id;
991 port_attr->pa_dev_version = hca_attr->hca_version_id;
992 port_attr->pa_hca_guid = hca_attr->hca_node_guid;
993 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl;
994 port_attr->pa_port_guid = hca_attr->hca_node_guid;
995 port_attr->pa_state = IBT_PORT_ACTIVE;
996
997
998 for (ii = 0; ii < nports; ii++) {
999 port_attr = &hca_list->hl_port_attr[ii];
1000 port_attr->pa_vendorid = hca_attr->hca_vendor_id;
1001 port_attr->pa_productid = hca_attr->hca_device_id;
1002 port_attr->pa_dev_version = hca_attr->hca_version_id;
1003 port_attr->pa_hca_guid = hca_attr->hca_node_guid;
1004 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl;
1005 port_attr->pa_port_guid = portinfop[ii].p_sgid_tbl->gid_guid;
1006 port_attr->pa_sn_prefix = portinfop[ii].p_sgid_tbl->gid_prefix;
1007 port_attr->pa_port_num = portinfop[ii].p_port_num;
1008 port_attr->pa_state = portinfop[ii].p_linkstate;
1009
1010 /*
1011 * Register with IBMF, SA access when the port is in
1012 * ACTIVE state. Also register a callback routine
1013 * with IBMF to receive incoming DM MAD's.
1014 * The IBDM event handler takes care of registration of
1015 * port which are not active.
1016 */
1017 IBTF_DPRINTF_L4("ibdm",
1018 "\thandle_hca_attach: port guid %llx Port state 0x%x",
1019 port_attr->pa_port_guid, portinfop[ii].p_linkstate);
1020
1021 if (portinfop[ii].p_linkstate == IBT_PORT_ACTIVE) {
1022 mutex_enter(&ibdm.ibdm_hl_mutex);
1023 hca_list->hl_nports_active++;
1024 ibdm_initialize_port(port_attr);
1025 cv_broadcast(&ibdm.ibdm_port_settle_cv);
1026 mutex_exit(&ibdm.ibdm_hl_mutex);
1027 }
1028 }
1029 mutex_enter(&ibdm.ibdm_hl_mutex);
1030 for (temp = ibdm.ibdm_hca_list_head; temp; temp = temp->hl_next) {
1031 if (temp->hl_hca_guid == hca_guid) {
1032 IBTF_DPRINTF_L2("ibdm", "hca_attach: HCA %llX "
1033 "already seen by IBDM", hca_guid);
1034 mutex_exit(&ibdm.ibdm_hl_mutex);
1035 (void) ibdm_uninit_hca(hca_list);
1036 return;
1037 }
1038 }
1039 ibdm.ibdm_hca_count++;
1040 if (ibdm.ibdm_hca_list_head == NULL) {
1041 ibdm.ibdm_hca_list_head = hca_list;
1042 ibdm.ibdm_hca_list_tail = hca_list;
1043 } else {
1044 ibdm.ibdm_hca_list_tail->hl_next = hca_list;
1045 ibdm.ibdm_hca_list_tail = hca_list;
1046 }
1047 mutex_exit(&ibdm.ibdm_hl_mutex);
1048 mutex_enter(&ibdm.ibdm_ibnex_mutex);
1049 if (ibdm.ibdm_ibnex_callback != NULL) {
1050 (*ibdm.ibdm_ibnex_callback)((void *)
1051 &hca_guid, IBDM_EVENT_HCA_ADDED);
1052 }
1053 mutex_exit(&ibdm.ibdm_ibnex_mutex);
1054
1055 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1056 ibt_free_portinfo(portinfop, size);
1057 }
1058
1059
1060 /*
1061 * ibdm_handle_hca_detach()
1062 */
1063 static void
1064 ibdm_handle_hca_detach(ib_guid_t hca_guid)
1065 {
1066 ibdm_hca_list_t *head, *prev = NULL;
1067 size_t len;
1068 ibdm_dp_gidinfo_t *gidinfo;
1069 ibdm_port_attr_t *port_attr;
1070 int i;
1071
1072 IBTF_DPRINTF_L4("ibdm",
1073 "\thandle_hca_detach: hca_guid = 0x%llx", hca_guid);
1074
1075 /* Make sure no probes are running */
1076 mutex_enter(&ibdm.ibdm_mutex);
1077 while (ibdm.ibdm_busy & IBDM_BUSY)
1078 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
1079 ibdm.ibdm_busy |= IBDM_BUSY;
1080 mutex_exit(&ibdm.ibdm_mutex);
1081
1082 mutex_enter(&ibdm.ibdm_hl_mutex);
1083 head = ibdm.ibdm_hca_list_head;
1084 while (head) {
1085 if (head->hl_hca_guid == hca_guid) {
1086 if (prev == NULL)
1087 ibdm.ibdm_hca_list_head = head->hl_next;
1088 else
1089 prev->hl_next = head->hl_next;
1090 if (ibdm.ibdm_hca_list_tail == head)
1091 ibdm.ibdm_hca_list_tail = prev;
1092 ibdm.ibdm_hca_count--;
1093 break;
1094 }
1095 prev = head;
1096 head = head->hl_next;
1097 }
1098 mutex_exit(&ibdm.ibdm_hl_mutex);
1099 if (ibdm_uninit_hca(head) != IBDM_SUCCESS)
1100 (void) ibdm_handle_hca_attach(hca_guid);
1101
1102 #ifdef DEBUG
1103 if (ibdm_enumerate_iocs == 0) {
1104 ASSERT(ibdm.ibdm_dp_gidlist_head == NULL);
1105 }
1106 #endif
1107
1108 /*
1109 * Now clean up the HCA lists in the gidlist.
1110 */
1111 for (gidinfo = ibdm.ibdm_dp_gidlist_head; gidinfo; gidinfo =
1112 gidinfo->gl_next) {
1113 prev = NULL;
1114 head = gidinfo->gl_hca_list;
1115 while (head) {
1116 if (head->hl_hca_guid == hca_guid) {
1117 if (prev == NULL)
1118 gidinfo->gl_hca_list =
1119 head->hl_next;
1120 else
1121 prev->hl_next = head->hl_next;
1122 for (i = 0; i < head->hl_nports; i++) {
1123 port_attr = &head->hl_port_attr[i];
1124 if (port_attr->pa_pkey_tbl != NULL)
1125 kmem_free(
1126 port_attr->pa_pkey_tbl,
1127 port_attr->pa_npkeys *
1128 sizeof (ibdm_pkey_tbl_t));
1129 }
1130 len = sizeof (ibdm_hca_list_t) +
1131 (head->hl_nports *
1132 sizeof (ibdm_port_attr_t));
1133 kmem_free(head, len);
1134
1135 break;
1136 }
1137 prev = head;
1138 head = head->hl_next;
1139 }
1140 }
1141
1142 mutex_enter(&ibdm.ibdm_mutex);
1143 ibdm.ibdm_busy &= ~IBDM_BUSY;
1144 cv_broadcast(&ibdm.ibdm_busy_cv);
1145 mutex_exit(&ibdm.ibdm_mutex);
1146 }
1147
1148
1149 static int
1150 ibdm_uninit_hca(ibdm_hca_list_t *head)
1151 {
1152 int ii;
1153 ibdm_port_attr_t *port_attr;
1154
1155 for (ii = 0; ii < head->hl_nports; ii++) {
1156 port_attr = &head->hl_port_attr[ii];
1157 if (ibdm_fini_port(port_attr) != IBDM_SUCCESS) {
1158 IBTF_DPRINTF_L2("ibdm", "uninit_hca: HCA %p port 0x%x "
1159 "ibdm_fini_port() failed", head, ii);
1160 return (IBDM_FAILURE);
1161 }
1162 }
1163 if (head->hl_hca_hdl)
1164 if (ibt_close_hca(head->hl_hca_hdl) != IBT_SUCCESS) {
1165 IBTF_DPRINTF_L2("ibdm", "uninit_hca: "
1166 "ibt_close_hca() failed");
1167 return (IBDM_FAILURE);
1168 }
1169 kmem_free(head->hl_port_attr,
1170 head->hl_nports * sizeof (ibdm_port_attr_t));
1171 kmem_free(head->hl_hca_port_attr, sizeof (ibdm_port_attr_t));
1172 kmem_free(head, sizeof (ibdm_hca_list_t));
1173 return (IBDM_SUCCESS);
1174 }
1175
1176
1177 /*
1178 * For each port on the HCA,
1179 * 1) Teardown IBMF receive callback function
1180 * 2) Unregister with IBMF
1181 * 3) Unregister with SA access
1182 */
1183 static int
1184 ibdm_fini_port(ibdm_port_attr_t *port_attr)
1185 {
1186 int ii, ibmf_status;
1187
1188 for (ii = 0; ii < port_attr->pa_npkeys; ii++) {
1189 if (port_attr->pa_pkey_tbl == NULL)
1190 break;
1191 if (!port_attr->pa_pkey_tbl[ii].pt_qp_hdl)
1192 continue;
1193 if (ibdm_port_attr_ibmf_fini(port_attr, ii) != IBDM_SUCCESS) {
1194 IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
1195 "ibdm_port_attr_ibmf_fini failed for "
1196 "port pkey 0x%x", ii);
1197 return (IBDM_FAILURE);
1198 }
1199 }
1200
1201 if (port_attr->pa_ibmf_hdl) {
1202 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl,
1203 IBMF_QP_HANDLE_DEFAULT, 0);
1204 if (ibmf_status != IBMF_SUCCESS) {
1205 IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
1206 "ibmf_tear_down_async_cb failed %d", ibmf_status);
1207 return (IBDM_FAILURE);
1208 }
1209
1210 ibmf_status = ibmf_unregister(&port_attr->pa_ibmf_hdl, 0);
1211 if (ibmf_status != IBMF_SUCCESS) {
1212 IBTF_DPRINTF_L2("ibdm", "\tfini_port: "
1213 "ibmf_unregister failed %d", ibmf_status);
1214 return (IBDM_FAILURE);
1215 }
1216
1217 port_attr->pa_ibmf_hdl = NULL;
1218 }
1219
1220 if (port_attr->pa_sa_hdl) {
1221 ibmf_status = ibmf_sa_session_close(&port_attr->pa_sa_hdl, 0);
1222 if (ibmf_status != IBMF_SUCCESS) {
1223 IBTF_DPRINTF_L2("ibdm", "\tfini_port: "
1224 "ibmf_sa_session_close failed %d", ibmf_status);
1225 return (IBDM_FAILURE);
1226 }
1227 port_attr->pa_sa_hdl = NULL;
1228 }
1229
1230 if (port_attr->pa_pkey_tbl != NULL) {
1231 kmem_free(port_attr->pa_pkey_tbl,
1232 port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
1233 port_attr->pa_pkey_tbl = NULL;
1234 port_attr->pa_npkeys = 0;
1235 }
1236
1237 return (IBDM_SUCCESS);
1238 }
1239
1240
1241 /*
1242 * ibdm_port_attr_ibmf_fini:
1243 * With IBMF - Tear down Async callback and free QP Handle
1244 */
1245 static int
1246 ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *port_attr, int ii)
1247 {
1248 int ibmf_status;
1249
1250 IBTF_DPRINTF_L5("ibdm", "\tport_attr_ibmf_fini:");
1251
1252 if (ibdm_enumerate_iocs == 0) {
1253 ASSERT(port_attr->pa_pkey_tbl[ii].pt_qp_hdl == NULL);
1254 return (IBDM_SUCCESS);
1255 }
1256
1257 if (port_attr->pa_pkey_tbl[ii].pt_qp_hdl) {
1258 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl,
1259 port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0);
1260 if (ibmf_status != IBMF_SUCCESS) {
1261 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: "
1262 "ibmf_tear_down_async_cb failed %d", ibmf_status);
1263 return (IBDM_FAILURE);
1264 }
1265 ibmf_status = ibmf_free_qp(port_attr->pa_ibmf_hdl,
1266 &port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0);
1267 if (ibmf_status != IBMF_SUCCESS) {
1268 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: "
1269 "ibmf_free_qp failed %d", ibmf_status);
1270 return (IBDM_FAILURE);
1271 }
1272 port_attr->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
1273 }
1274 return (IBDM_SUCCESS);
1275 }
1276
1277
1278 /*
1279 * ibdm_gid_decr_pending:
1280 * decrement gl_pending_cmds. If zero wakeup sleeping threads
1281 */
1282 static void
1283 ibdm_gid_decr_pending(ibdm_dp_gidinfo_t *gidinfo)
1284 {
1285 mutex_enter(&ibdm.ibdm_mutex);
1286 mutex_enter(&gidinfo->gl_mutex);
1287 if (--gidinfo->gl_pending_cmds == 0) {
1288 /*
1289 * Handle DGID getting removed.
1290 */
1291 if (gidinfo->gl_disconnected) {
1292 mutex_exit(&gidinfo->gl_mutex);
1293 mutex_exit(&ibdm.ibdm_mutex);
1294
1295 IBTF_DPRINTF_L3(ibdm_string, "\tgid_decr_pending: "
1296 "gidinfo %p hot removal", gidinfo);
1297 ibdm_delete_gidinfo(gidinfo);
1298
1299 mutex_enter(&ibdm.ibdm_mutex);
1300 ibdm.ibdm_ngid_probes_in_progress--;
1301 ibdm_wait_probe_completion();
1302 mutex_exit(&ibdm.ibdm_mutex);
1303 return;
1304 }
1305 mutex_exit(&gidinfo->gl_mutex);
1306 mutex_exit(&ibdm.ibdm_mutex);
1307 ibdm_notify_newgid_iocs(gidinfo);
1308 mutex_enter(&ibdm.ibdm_mutex);
1309 mutex_enter(&gidinfo->gl_mutex);
1310
1311 ibdm.ibdm_ngid_probes_in_progress--;
1312 ibdm_wait_probe_completion();
1313 }
1314 mutex_exit(&gidinfo->gl_mutex);
1315 mutex_exit(&ibdm.ibdm_mutex);
1316 }
1317
1318
1319 /*
1320 * ibdm_wait_probe_completion:
1321 * wait for probing to complete
1322 */
1323 static void
1324 ibdm_wait_probe_completion(void)
1325 {
1326 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1327 if (ibdm.ibdm_ngid_probes_in_progress) {
1328 IBTF_DPRINTF_L4("ibdm", "\twait for probe complete");
1329 ibdm.ibdm_busy |= IBDM_PROBE_IN_PROGRESS;
1330 while (ibdm.ibdm_busy & IBDM_PROBE_IN_PROGRESS)
1331 cv_wait(&ibdm.ibdm_probe_cv, &ibdm.ibdm_mutex);
1332 }
1333 }
1334
1335
1336 /*
1337 * ibdm_wait_cisco_probe_completion:
1338 * wait for the reply from the Cisco FC GW switch after a setclassportinfo
1339 * request is sent. This wait can be achieved on each gid.
1340 */
1341 static void
1342 ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *gidinfo)
1343 {
1344 ASSERT(MUTEX_HELD(&gidinfo->gl_mutex));
1345 IBTF_DPRINTF_L4("ibdm", "\twait for cisco probe complete");
1346 gidinfo->gl_flag |= IBDM_CISCO_PROBE;
1347 while (gidinfo->gl_flag & IBDM_CISCO_PROBE)
1348 cv_wait(&gidinfo->gl_probe_cv, &gidinfo->gl_mutex);
1349 }
1350
1351
1352 /*
1353 * ibdm_wakeup_probe_gid_cv:
1354 * wakeup waiting threads (based on ibdm_ngid_probes_in_progress)
1355 */
1356 static void
1357 ibdm_wakeup_probe_gid_cv(void)
1358 {
1359 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1360 if (!ibdm.ibdm_ngid_probes_in_progress) {
1361 IBTF_DPRINTF_L4("ibdm", "wakeup_probe_gid_thread: Wakeup");
1362 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
1363 cv_broadcast(&ibdm.ibdm_probe_cv);
1364 }
1365
1366 }
1367
1368
1369 /*
1370 * ibdm_sweep_fabric(reprobe_flag)
1371 * Find all possible Managed IOU's and their IOC's that are visible
1372 * to the host. The algorithm used is as follows
1373 *
1374 * Send a "bus walk" request for each port on the host HCA to SA access
1375 * SA returns complete set of GID's that are reachable from
1376 * source port. This is done in parallel.
1377 *
1378 * Initialize GID state to IBDM_GID_PROBE_NOT_DONE
1379 *
1380 * Sort the GID list and eliminate duplicate GID's
1381 * 1) Use DGID for sorting
1382 * 2) use PortGuid for sorting
1383 * Send SA query to retrieve NodeRecord and
1384 * extract PortGuid from that.
1385 *
1386 * Set GID state to IBDM_GID_PROBE_FAILED to all the ports that dont
1387 * support DM MAD's
1388 * Send a "Portinfo" query to get the port capabilities and
1389 * then check for DM MAD's support
1390 *
1391 * Send "ClassPortInfo" request for all the GID's in parallel,
1392 * set the GID state to IBDM_GET_CLASSPORTINFO and wait on the
1393 * cv_signal to complete.
1394 *
1395 * When DM agent on the remote GID sends back the response, IBMF
1396 * invokes DM callback routine.
1397 *
1398 * If the response is proper, send "IOUnitInfo" request and set
1399 * GID state to IBDM_GET_IOUNITINFO.
1400 *
1401 * If the response is proper, send "IocProfileInfo" request to
1402 * all the IOC simultaneously and set GID state to IBDM_GET_IOC_DETAILS.
1403 *
1404 * Send request to get Service entries simultaneously
1405 *
1406 * Signal the waiting thread when received response for all the commands.
1407 *
1408 * Set the GID state to IBDM_GID_PROBE_FAILED when received a error
1409 * response during the probing period.
1410 *
1411 * Note:
1412 * ibdm.ibdm_ngid_probes_in_progress and ibdm_gid_list_t:gl_pending_cmds
1413 * keep track of number commands in progress at any point of time.
1414 * MAD transaction ID is used to identify a particular GID
1415 * TBD: Consider registering the IBMF receive callback on demand
1416 *
1417 * Note: This routine must be called with ibdm.ibdm_mutex held
1418 * TBD: Re probe the failure GID (for certain failures) when requested
1419 * for fabric sweep next time
1420 *
1421 * Parameters : If reprobe_flag is set, All IOCs will be reprobed.
1422 */
1423 static void
1424 ibdm_sweep_fabric(int reprobe_flag)
1425 {
1426 int ii;
1427 int new_paths = 0;
1428 uint8_t niocs;
1429 taskqid_t tid;
1430 ibdm_ioc_info_t *ioc;
1431 ibdm_hca_list_t *hca_list = NULL;
1432 ibdm_port_attr_t *port = NULL;
1433 ibdm_dp_gidinfo_t *gid_info;
1434
1435 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: Enter");
1436 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1437
1438 /*
1439 * Check whether a sweep already in progress. If so, just
1440 * wait for the fabric sweep to complete
1441 */
1442 while (ibdm.ibdm_busy & IBDM_BUSY)
1443 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
1444 ibdm.ibdm_busy |= IBDM_BUSY;
1445 mutex_exit(&ibdm.ibdm_mutex);
1446
1447 ibdm_dump_sweep_fabric_timestamp(0);
1448
1449 /* Rescan the GID list for any removed GIDs for reprobe */
1450 if (reprobe_flag)
1451 ibdm_rescan_gidlist(NULL);
1452
1453 /*
1454 * Get list of all the ports reachable from the local known HCA
1455 * ports which are active
1456 */
1457 mutex_enter(&ibdm.ibdm_hl_mutex);
1458 for (ibdm_get_next_port(&hca_list, &port, 1); port;
1459 ibdm_get_next_port(&hca_list, &port, 1)) {
1460 /*
1461 * Get PATHS to all the reachable ports from
1462 * SGID and update the global ibdm structure.
1463 */
1464 new_paths = ibdm_get_reachable_ports(port, hca_list);
1465 ibdm.ibdm_ngids += new_paths;
1466 }
1467 mutex_exit(&ibdm.ibdm_hl_mutex);
1468
1469 mutex_enter(&ibdm.ibdm_mutex);
1470 ibdm.ibdm_ngid_probes_in_progress += ibdm.ibdm_ngids;
1471 mutex_exit(&ibdm.ibdm_mutex);
1472
1473 /* Send a request to probe GIDs asynchronously. */
1474 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
1475 gid_info = gid_info->gl_next) {
1476 mutex_enter(&gid_info->gl_mutex);
1477 gid_info->gl_reprobe_flag = reprobe_flag;
1478 mutex_exit(&gid_info->gl_mutex);
1479
1480 /* process newly encountered GIDs */
1481 tid = taskq_dispatch(system_taskq, ibdm_probe_gid_thread,
1482 (void *)gid_info, TQ_NOSLEEP);
1483 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: gid_info = %p"
1484 " taskq_id = %x", gid_info, tid);
1485 /* taskq failed to dispatch call it directly */
1486 if (tid == NULL)
1487 ibdm_probe_gid_thread((void *)gid_info);
1488 }
1489
1490 mutex_enter(&ibdm.ibdm_mutex);
1491 ibdm_wait_probe_completion();
1492
1493 /*
1494 * Update the properties, if reprobe_flag is set
1495 * Skip if gl_reprobe_flag is set, this will be
1496 * a re-inserted / new GID, for which notifications
1497 * have already been send.
1498 */
1499 if (reprobe_flag) {
1500 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
1501 gid_info = gid_info->gl_next) {
1502 if (gid_info->gl_iou == NULL)
1503 continue;
1504 if (gid_info->gl_reprobe_flag) {
1505 gid_info->gl_reprobe_flag = 0;
1506 continue;
1507 }
1508
1509 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
1510 for (ii = 0; ii < niocs; ii++) {
1511 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
1512 if (ioc)
1513 ibdm_reprobe_update_port_srv(ioc,
1514 gid_info);
1515 }
1516 }
1517 } else if (ibdm.ibdm_prev_iou) {
1518 ibdm_ioc_info_t *ioc_list;
1519
1520 /*
1521 * Get the list of IOCs which have changed.
1522 * If any IOCs have changed, Notify IBNexus
1523 */
1524 ibdm.ibdm_prev_iou = 0;
1525 ioc_list = ibdm_handle_prev_iou();
1526 if (ioc_list) {
1527 if (ibdm.ibdm_ibnex_callback != NULL) {
1528 (*ibdm.ibdm_ibnex_callback)(
1529 (void *)ioc_list,
1530 IBDM_EVENT_IOC_PROP_UPDATE);
1531 }
1532 }
1533 }
1534
1535 ibdm_dump_sweep_fabric_timestamp(1);
1536
1537 ibdm.ibdm_busy &= ~IBDM_BUSY;
1538 cv_broadcast(&ibdm.ibdm_busy_cv);
1539 IBTF_DPRINTF_L5("ibdm", "\tsweep_fabric: EXIT");
1540 }
1541
1542
1543 /*
1544 * ibdm_is_cisco:
1545 * Check if this is a Cisco device or not.
1546 */
1547 static boolean_t
1548 ibdm_is_cisco(ib_guid_t guid)
1549 {
1550 if ((guid >> IBDM_OUI_GUID_SHIFT) == IBDM_CISCO_COMPANY_ID)
1551 return (B_TRUE);
1552 return (B_FALSE);
1553 }
1554
1555
1556 /*
1557 * ibdm_is_cisco_switch:
1558 * Check if this switch is a CISCO switch or not.
1559 * Note that if this switch is already activated, ibdm_is_cisco_switch()
1560 * returns B_FALSE not to re-activate it again.
1561 */
1562 static boolean_t
1563 ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *gid_info)
1564 {
1565 int company_id, device_id;
1566 ASSERT(gid_info != 0);
1567 ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
1568
1569 /*
1570 * If this switch is already activated, don't re-activate it.
1571 */
1572 if (gid_info->gl_flag & IBDM_CISCO_PROBE_DONE)
1573 return (B_FALSE);
1574
1575 /*
1576 * Check if this switch is a Cisco FC GW or not.
1577 * Use the node guid (the OUI part) instead of the vendor id
1578 * since the vendor id is zero in practice.
1579 */
1580 company_id = gid_info->gl_nodeguid >> IBDM_OUI_GUID_SHIFT;
1581 device_id = gid_info->gl_devid;
1582
1583 if (company_id == IBDM_CISCO_COMPANY_ID &&
1584 device_id == IBDM_CISCO_DEVICE_ID)
1585 return (B_TRUE);
1586 return (B_FALSE);
1587 }
1588
1589
1590 /*
1591 * ibdm_probe_gid_thread:
1592 * thread that does the actual work for sweeping the fabric
1593 * for a given GID
1594 */
1595 static void
1596 ibdm_probe_gid_thread(void *args)
1597 {
1598 int reprobe_flag;
1599 ib_guid_t node_guid;
1600 ib_guid_t port_guid;
1601 ibdm_dp_gidinfo_t *gid_info;
1602
1603 gid_info = (ibdm_dp_gidinfo_t *)args;
1604 reprobe_flag = gid_info->gl_reprobe_flag;
1605 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: gid_info = %p, flag = %d",
1606 gid_info, reprobe_flag);
1607 ASSERT(gid_info != NULL);
1608 ASSERT(gid_info->gl_pending_cmds == 0);
1609
1610 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE &&
1611 reprobe_flag == 0) {
1612 /*
1613 * This GID may have been already probed. Send
1614 * in a CLP to check if IOUnitInfo changed?
1615 * Explicitly set gl_reprobe_flag to 0 so that
1616 * IBnex is not notified on completion
1617 */
1618 if (gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) {
1619 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: "
1620 "get new IOCs information");
1621 mutex_enter(&gid_info->gl_mutex);
1622 gid_info->gl_pending_cmds++;
1623 gid_info->gl_state = IBDM_GET_IOUNITINFO;
1624 gid_info->gl_reprobe_flag = 0;
1625 mutex_exit(&gid_info->gl_mutex);
1626 if (ibdm_send_iounitinfo(gid_info) != IBDM_SUCCESS) {
1627 mutex_enter(&gid_info->gl_mutex);
1628 --gid_info->gl_pending_cmds;
1629 mutex_exit(&gid_info->gl_mutex);
1630 mutex_enter(&ibdm.ibdm_mutex);
1631 --ibdm.ibdm_ngid_probes_in_progress;
1632 ibdm_wakeup_probe_gid_cv();
1633 mutex_exit(&ibdm.ibdm_mutex);
1634 }
1635 } else {
1636 mutex_enter(&ibdm.ibdm_mutex);
1637 --ibdm.ibdm_ngid_probes_in_progress;
1638 ibdm_wakeup_probe_gid_cv();
1639 mutex_exit(&ibdm.ibdm_mutex);
1640 }
1641 return;
1642 } else if (reprobe_flag && gid_info->gl_state ==
1643 IBDM_GID_PROBING_COMPLETE) {
1644 /*
1645 * Reprobe all IOCs for the GID which has completed
1646 * probe. Skip other port GIDs to same IOU.
1647 * Explicitly set gl_reprobe_flag to 0 so that
1648 * IBnex is not notified on completion
1649 */
1650 ibdm_ioc_info_t *ioc_info;
1651 uint8_t niocs, ii;
1652
1653 ASSERT(gid_info->gl_iou);
1654 mutex_enter(&gid_info->gl_mutex);
1655 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
1656 gid_info->gl_state = IBDM_GET_IOC_DETAILS;
1657 gid_info->gl_pending_cmds += niocs;
1658 gid_info->gl_reprobe_flag = 0;
1659 mutex_exit(&gid_info->gl_mutex);
1660 for (ii = 0; ii < niocs; ii++) {
1661 uchar_t slot_info;
1662 ib_dm_io_unitinfo_t *giou_info;
1663
1664 /*
1665 * Check whether IOC is present in the slot
1666 * Series of nibbles (in the field
1667 * iou_ctrl_list) represents a slot in the
1668 * IOU.
1669 * Byte format: 76543210
1670 * Bits 0-3 of first byte represent Slot 2
1671 * bits 4-7 of first byte represent slot 1,
1672 * bits 0-3 of second byte represent slot 4
1673 * and so on
1674 * Each 4-bit nibble has the following meaning
1675 * 0x0 : IOC not installed
1676 * 0x1 : IOC is present
1677 * 0xf : Slot does not exist
1678 * and all other values are reserved.
1679 */
1680 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii);
1681 giou_info = &gid_info->gl_iou->iou_info;
1682 slot_info = giou_info->iou_ctrl_list[(ii/2)];
1683 if ((ii % 2) == 0)
1684 slot_info = (slot_info >> 4);
1685
1686 if ((slot_info & 0xf) != 1) {
1687 ioc_info->ioc_state =
1688 IBDM_IOC_STATE_PROBE_FAILED;
1689 ibdm_gid_decr_pending(gid_info);
1690 continue;
1691 }
1692
1693 if (ibdm_send_ioc_profile(gid_info, ii) !=
1694 IBDM_SUCCESS) {
1695 ibdm_gid_decr_pending(gid_info);
1696 }
1697 }
1698
1699 return;
1700 } else if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) {
1701 mutex_enter(&ibdm.ibdm_mutex);
1702 --ibdm.ibdm_ngid_probes_in_progress;
1703 ibdm_wakeup_probe_gid_cv();
1704 mutex_exit(&ibdm.ibdm_mutex);
1705 return;
1706 }
1707
1708 /*
1709 * Check whether the destination GID supports DM agents. If
1710 * not, stop probing the GID and continue with the next GID
1711 * in the list.
1712 */
1713 if (ibdm_is_dev_mgt_supported(gid_info) != IBDM_SUCCESS) {
1714 mutex_enter(&gid_info->gl_mutex);
1715 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1716 gid_info->gl_is_dm_capable = B_FALSE;
1717 mutex_exit(&gid_info->gl_mutex);
1718 ibdm_delete_glhca_list(gid_info);
1719 mutex_enter(&ibdm.ibdm_mutex);
1720 --ibdm.ibdm_ngid_probes_in_progress;
1721 ibdm_wakeup_probe_gid_cv();
1722 mutex_exit(&ibdm.ibdm_mutex);
1723 return;
1724 }
1725
1726 /*
1727 * This GID is Device management capable
1728 */
1729 mutex_enter(&gid_info->gl_mutex);
1730 gid_info->gl_is_dm_capable = B_TRUE;
1731 mutex_exit(&gid_info->gl_mutex);
1732
1733 /* Get the nodeguid and portguid of the port */
1734 if (ibdm_get_node_port_guids(gid_info->gl_sa_hdl, gid_info->gl_dlid,
1735 &node_guid, &port_guid) != IBDM_SUCCESS) {
1736 mutex_enter(&gid_info->gl_mutex);
1737 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1738 mutex_exit(&gid_info->gl_mutex);
1739 ibdm_delete_glhca_list(gid_info);
1740 mutex_enter(&ibdm.ibdm_mutex);
1741 --ibdm.ibdm_ngid_probes_in_progress;
1742 ibdm_wakeup_probe_gid_cv();
1743 mutex_exit(&ibdm.ibdm_mutex);
1744 return;
1745 }
1746
1747 /*
1748 * Check whether we already knew about this NodeGuid
1749 * If so, do not probe the GID and continue with the
1750 * next GID in the gid list. Set the GID state to
1751 * probing done.
1752 */
1753 mutex_enter(&ibdm.ibdm_mutex);
1754 gid_info->gl_nodeguid = node_guid;
1755 gid_info->gl_portguid = port_guid;
1756 if (ibdm_check_dest_nodeguid(gid_info) != NULL) {
1757 mutex_exit(&ibdm.ibdm_mutex);
1758 mutex_enter(&gid_info->gl_mutex);
1759 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED;
1760 mutex_exit(&gid_info->gl_mutex);
1761 ibdm_delete_glhca_list(gid_info);
1762 mutex_enter(&ibdm.ibdm_mutex);
1763 --ibdm.ibdm_ngid_probes_in_progress;
1764 ibdm_wakeup_probe_gid_cv();
1765 mutex_exit(&ibdm.ibdm_mutex);
1766 return;
1767 }
1768 ibdm_add_to_gl_gid(gid_info, gid_info);
1769 mutex_exit(&ibdm.ibdm_mutex);
1770
1771 /*
1772 * New or reinserted GID : Enable notification to IBnex
1773 */
1774 mutex_enter(&gid_info->gl_mutex);
1775 gid_info->gl_reprobe_flag = 1;
1776
1777 /*
1778 * A Cisco FC GW needs the special handling to get IOUnitInfo.
1779 */
1780 if (ibdm_is_cisco_switch(gid_info)) {
1781 gid_info->gl_pending_cmds++;
1782 gid_info->gl_state = IBDM_SET_CLASSPORTINFO;
1783 mutex_exit(&gid_info->gl_mutex);
1784
1785 if (ibdm_set_classportinfo(gid_info) != IBDM_SUCCESS) {
1786 mutex_enter(&gid_info->gl_mutex);
1787 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1788 --gid_info->gl_pending_cmds;
1789 mutex_exit(&gid_info->gl_mutex);
1790
1791 /* free the hca_list on this gid_info */
1792 ibdm_delete_glhca_list(gid_info);
1793
1794 mutex_enter(&ibdm.ibdm_mutex);
1795 --ibdm.ibdm_ngid_probes_in_progress;
1796 ibdm_wakeup_probe_gid_cv();
1797 mutex_exit(&ibdm.ibdm_mutex);
1798
1799 return;
1800 }
1801
1802 mutex_enter(&gid_info->gl_mutex);
1803 ibdm_wait_cisco_probe_completion(gid_info);
1804
1805 IBTF_DPRINTF_L4("ibdm", "\tibdm_probe_gid_thread: "
1806 "CISCO Wakeup signal received");
1807 }
1808
1809 /* move on to the 'GET_CLASSPORTINFO' stage */
1810 gid_info->gl_pending_cmds++;
1811 gid_info->gl_state = IBDM_GET_CLASSPORTINFO;
1812 mutex_exit(&gid_info->gl_mutex);
1813
1814 IBTF_DPRINTF_L3(ibdm_string, "\tibdm_probe_gid_thread: "
1815 "%d: gid_info %p gl_state %d pending_cmds %d",
1816 __LINE__, gid_info, gid_info->gl_state,
1817 gid_info->gl_pending_cmds);
1818
1819 /*
1820 * Send ClassPortInfo request to the GID asynchronously.
1821 */
1822 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) {
1823
1824 mutex_enter(&gid_info->gl_mutex);
1825 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1826 --gid_info->gl_pending_cmds;
1827 mutex_exit(&gid_info->gl_mutex);
1828
1829 /* free the hca_list on this gid_info */
1830 ibdm_delete_glhca_list(gid_info);
1831
1832 mutex_enter(&ibdm.ibdm_mutex);
1833 --ibdm.ibdm_ngid_probes_in_progress;
1834 ibdm_wakeup_probe_gid_cv();
1835 mutex_exit(&ibdm.ibdm_mutex);
1836
1837 return;
1838 }
1839 }
1840
1841
1842 /*
1843 * ibdm_check_dest_nodeguid
1844 * Searches for the NodeGuid in the GID list
1845 * Returns matching gid_info if found and otherwise NULL
1846 *
1847 * This function is called to handle new GIDs discovered
1848 * during device sweep / probe or for GID_AVAILABLE event.
1849 *
1850 * Parameter :
1851 * gid_info GID to check
1852 */
1853 static ibdm_dp_gidinfo_t *
1854 ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *gid_info)
1855 {
1856 ibdm_dp_gidinfo_t *gid_list;
1857 ibdm_gid_t *tmp;
1858
1859 IBTF_DPRINTF_L4("ibdm", "\tcheck_dest_nodeguid");
1860
1861 gid_list = ibdm.ibdm_dp_gidlist_head;
1862 while (gid_list) {
1863 if ((gid_list != gid_info) &&
1864 (gid_info->gl_nodeguid == gid_list->gl_nodeguid)) {
1865 IBTF_DPRINTF_L4("ibdm",
1866 "\tcheck_dest_nodeguid: NodeGuid is present");
1867
1868 /* Add to gid_list */
1869 tmp = kmem_zalloc(sizeof (ibdm_gid_t),
1870 KM_SLEEP);
1871 tmp->gid_dgid_hi = gid_info->gl_dgid_hi;
1872 tmp->gid_dgid_lo = gid_info->gl_dgid_lo;
1873 tmp->gid_next = gid_list->gl_gid;
1874 gid_list->gl_gid = tmp;
1875 gid_list->gl_ngids++;
1876 return (gid_list);
1877 }
1878
1879 gid_list = gid_list->gl_next;
1880 }
1881
1882 return (NULL);
1883 }
1884
1885
1886 /*
1887 * ibdm_is_dev_mgt_supported
1888 * Get the PortInfo attribute (SA Query)
1889 * Check "CompatabilityMask" field in the Portinfo.
1890 * Return IBDM_SUCCESS if DM MAD's supported (if bit 19 set)
1891 * by the port, otherwise IBDM_FAILURE
1892 */
1893 static int
1894 ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *gid_info)
1895 {
1896 int ret;
1897 size_t length = 0;
1898 sa_portinfo_record_t req, *resp = NULL;
1899 ibmf_saa_access_args_t qargs;
1900
1901 bzero(&req, sizeof (sa_portinfo_record_t));
1902 req.EndportLID = gid_info->gl_dlid;
1903
1904 qargs.sq_attr_id = SA_PORTINFORECORD_ATTRID;
1905 qargs.sq_access_type = IBMF_SAA_RETRIEVE;
1906 qargs.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID;
1907 qargs.sq_template = &req;
1908 qargs.sq_callback = NULL;
1909 qargs.sq_callback_arg = NULL;
1910
1911 ret = ibmf_sa_access(gid_info->gl_sa_hdl,
1912 &qargs, 0, &length, (void **)&resp);
1913
1914 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) {
1915 IBTF_DPRINTF_L2("ibdm", "\tis_dev_mgt_supported:"
1916 "failed to get PORTINFO attribute %d", ret);
1917 return (IBDM_FAILURE);
1918 }
1919
1920 if (resp->PortInfo.CapabilityMask & SM_CAP_MASK_IS_DM_SUPPD) {
1921 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: SUPPD !!");
1922 ret = IBDM_SUCCESS;
1923 } else {
1924 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: "
1925 "Not SUPPD !!, cap 0x%x", resp->PortInfo.CapabilityMask);
1926 ret = IBDM_FAILURE;
1927 }
1928 kmem_free(resp, length);
1929 return (ret);
1930 }
1931
1932
1933 /*
1934 * ibdm_get_node_port_guids()
1935 * Get the NodeInfoRecord of the port
1936 * Save NodeGuid and PortGUID values in the GID list structure.
1937 * Return IBDM_SUCCESS/IBDM_FAILURE
1938 */
1939 static int
1940 ibdm_get_node_port_guids(ibmf_saa_handle_t sa_hdl, ib_lid_t dlid,
1941 ib_guid_t *node_guid, ib_guid_t *port_guid)
1942 {
1943 int ret;
1944 size_t length = 0;
1945 sa_node_record_t req, *resp = NULL;
1946 ibmf_saa_access_args_t qargs;
1947
1948 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids");
1949
1950 bzero(&req, sizeof (sa_node_record_t));
1951 req.LID = dlid;
1952
1953 qargs.sq_attr_id = SA_NODERECORD_ATTRID;
1954 qargs.sq_access_type = IBMF_SAA_RETRIEVE;
1955 qargs.sq_component_mask = SA_NODEINFO_COMPMASK_NODELID;
1956 qargs.sq_template = &req;
1957 qargs.sq_callback = NULL;
1958 qargs.sq_callback_arg = NULL;
1959
1960 ret = ibmf_sa_access(sa_hdl, &qargs, 0, &length, (void **)&resp);
1961 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) {
1962 IBTF_DPRINTF_L2("ibdm", "\tget_node_port_guids:"
1963 " SA Retrieve Failed: %d", ret);
1964 return (IBDM_FAILURE);
1965 }
1966 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids: NodeGuid %llx Port"
1967 "GUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.NodeGUID);
1968
1969 *node_guid = resp->NodeInfo.NodeGUID;
1970 *port_guid = resp->NodeInfo.PortGUID;
1971 kmem_free(resp, length);
1972 return (IBDM_SUCCESS);
1973 }
1974
1975
1976 /*
1977 * ibdm_get_reachable_ports()
1978 * Get list of the destination GID (and its path records) by
1979 * querying the SA access.
1980 *
1981 * Returns Number paths
1982 */
1983 static int
1984 ibdm_get_reachable_ports(ibdm_port_attr_t *portinfo, ibdm_hca_list_t *hca)
1985 {
1986 uint_t ii, jj, nrecs;
1987 uint_t npaths = 0;
1988 size_t length;
1989 ib_gid_t sgid;
1990 ibdm_pkey_tbl_t *pkey_tbl;
1991 sa_path_record_t *result;
1992 sa_path_record_t *precp;
1993 ibdm_dp_gidinfo_t *gid_info;
1994
1995 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
1996 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: portinfo %p", portinfo);
1997
1998 sgid.gid_prefix = portinfo->pa_sn_prefix;
1999 sgid.gid_guid = portinfo->pa_port_guid;
2000
2001 /* get reversible paths */
2002 if (portinfo->pa_sa_hdl && ibmf_saa_paths_from_gid(portinfo->pa_sa_hdl,
2003 sgid, IBMF_SAA_PKEY_WC, B_TRUE, 0, &nrecs, &length, &result)
2004 != IBMF_SUCCESS) {
2005 IBTF_DPRINTF_L2("ibdm",
2006 "\tget_reachable_ports: Getting path records failed");
2007 return (0);
2008 }
2009
2010 for (ii = 0; ii < nrecs; ii++) {
2011 sa_node_record_t *nrec;
2012 size_t length;
2013
2014 precp = &result[ii];
2015 if ((gid_info = ibdm_check_dgid(precp->DGID.gid_guid,
2016 precp->DGID.gid_prefix)) != NULL) {
2017 IBTF_DPRINTF_L5("ibdm", "\tget_reachable_ports: "
2018 "Already exists nrecs %d, ii %d", nrecs, ii);
2019 ibdm_addto_glhcalist(gid_info, hca);
2020 continue;
2021 }
2022 /*
2023 * This is a new GID. Allocate a GID structure and
2024 * initialize the structure
2025 * gl_state is initialized to IBDM_GID_PROBE_NOT_DONE (0)
2026 * by kmem_zalloc call
2027 */
2028 gid_info = kmem_zalloc(sizeof (ibdm_dp_gidinfo_t), KM_SLEEP);
2029 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL);
2030 cv_init(&gid_info->gl_probe_cv, NULL, CV_DRIVER, NULL);
2031 gid_info->gl_dgid_hi = precp->DGID.gid_prefix;
2032 gid_info->gl_dgid_lo = precp->DGID.gid_guid;
2033 gid_info->gl_sgid_hi = precp->SGID.gid_prefix;
2034 gid_info->gl_sgid_lo = precp->SGID.gid_guid;
2035 gid_info->gl_p_key = precp->P_Key;
2036 gid_info->gl_sa_hdl = portinfo->pa_sa_hdl;
2037 gid_info->gl_ibmf_hdl = portinfo->pa_ibmf_hdl;
2038 gid_info->gl_slid = precp->SLID;
2039 gid_info->gl_dlid = precp->DLID;
2040 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID)
2041 << IBDM_GID_TRANSACTIONID_SHIFT;
2042 gid_info->gl_min_transactionID = gid_info->gl_transactionID;
2043 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1)
2044 << IBDM_GID_TRANSACTIONID_SHIFT;
2045 gid_info->gl_SL = precp->SL;
2046
2047 /*
2048 * get the node record with this guid if the destination
2049 * device is a Cisco one.
2050 */
2051 if (ibdm_is_cisco(precp->DGID.gid_guid) &&
2052 (gid_info->gl_nodeguid == 0 || gid_info->gl_devid == 0) &&
2053 ibdm_get_node_record_by_port(portinfo->pa_sa_hdl,
2054 precp->DGID.gid_guid, &nrec, &length) == IBDM_SUCCESS) {
2055 gid_info->gl_nodeguid = nrec->NodeInfo.NodeGUID;
2056 gid_info->gl_devid = nrec->NodeInfo.DeviceID;
2057 kmem_free(nrec, length);
2058 }
2059
2060 ibdm_addto_glhcalist(gid_info, hca);
2061
2062 ibdm_dump_path_info(precp);
2063
2064 gid_info->gl_qp_hdl = NULL;
2065 ASSERT(portinfo->pa_pkey_tbl != NULL &&
2066 portinfo->pa_npkeys != 0);
2067
2068 for (jj = 0; jj < portinfo->pa_npkeys; jj++) {
2069 pkey_tbl = &portinfo->pa_pkey_tbl[jj];
2070 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) &&
2071 (pkey_tbl->pt_qp_hdl != NULL)) {
2072 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
2073 break;
2074 }
2075 }
2076
2077 /*
2078 * QP handle for GID not initialized. No matching Pkey
2079 * was found!! ibdm should *not* hit this case. Flag an
2080 * error and drop the GID if ibdm does encounter this.
2081 */
2082 if (gid_info->gl_qp_hdl == NULL) {
2083 IBTF_DPRINTF_L2(ibdm_string,
2084 "\tget_reachable_ports: No matching Pkey");
2085 ibdm_delete_gidinfo(gid_info);
2086 continue;
2087 }
2088 if (ibdm.ibdm_dp_gidlist_head == NULL) {
2089 ibdm.ibdm_dp_gidlist_head = gid_info;
2090 ibdm.ibdm_dp_gidlist_tail = gid_info;
2091 } else {
2092 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info;
2093 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail;
2094 ibdm.ibdm_dp_gidlist_tail = gid_info;
2095 }
2096 npaths++;
2097 }
2098 kmem_free(result, length);
2099 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: npaths = %d", npaths);
2100 return (npaths);
2101 }
2102
2103
2104 /*
2105 * ibdm_check_dgid()
2106 * Look in the global list to check whether we know this DGID already
2107 * Return IBDM_GID_PRESENT/IBDM_GID_NOT_PRESENT
2108 */
2109 static ibdm_dp_gidinfo_t *
2110 ibdm_check_dgid(ib_guid_t guid, ib_sn_prefix_t prefix)
2111 {
2112 ibdm_dp_gidinfo_t *gid_list;
2113
2114 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
2115 gid_list = gid_list->gl_next) {
2116 if ((guid == gid_list->gl_dgid_lo) &&
2117 (prefix == gid_list->gl_dgid_hi)) {
2118 break;
2119 }
2120 }
2121 return (gid_list);
2122 }
2123
2124
2125 /*
2126 * ibdm_find_gid()
2127 * Look in the global list to find a GID entry with matching
2128 * port & node GUID.
2129 * Return pointer to gidinfo if found, else return NULL
2130 */
2131 static ibdm_dp_gidinfo_t *
2132 ibdm_find_gid(ib_guid_t nodeguid, ib_guid_t portguid)
2133 {
2134 ibdm_dp_gidinfo_t *gid_list;
2135
2136 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid(%llx, %llx)\n",
2137 nodeguid, portguid);
2138
2139 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
2140 gid_list = gid_list->gl_next) {
2141 if ((portguid == gid_list->gl_portguid) &&
2142 (nodeguid == gid_list->gl_nodeguid)) {
2143 break;
2144 }
2145 }
2146
2147 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid : returned %p\n",
2148 gid_list);
2149 return (gid_list);
2150 }
2151
2152
2153 /*
2154 * ibdm_set_classportinfo()
2155 * ibdm_set_classportinfo() is a function to activate a Cisco FC GW
2156 * by sending the setClassPortInfo request with the trapLID, trapGID
2157 * and etc. to the gateway since the gateway doesn't provide the IO
2158 * Unit Information othewise. This behavior is the Cisco specific one,
2159 * and this function is called to a Cisco FC GW only.
2160 * Returns IBDM_SUCCESS/IBDM_FAILURE
2161 */
2162 static int
2163 ibdm_set_classportinfo(ibdm_dp_gidinfo_t *gid_info)
2164 {
2165 ibmf_msg_t *msg;
2166 ib_mad_hdr_t *hdr;
2167 ibdm_timeout_cb_args_t *cb_args;
2168 void *data;
2169 ib_mad_classportinfo_t *cpi;
2170
2171 IBTF_DPRINTF_L4("ibdm",
2172 "\tset_classportinfo: gid info 0x%p", gid_info);
2173
2174 /*
2175 * Send command to set classportinfo attribute. Allocate a IBMF
2176 * packet and initialize the packet.
2177 */
2178 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
2179 &msg) != IBMF_SUCCESS) {
2180 IBTF_DPRINTF_L4("ibdm", "\tset_classportinfo: pkt alloc fail");
2181 return (IBDM_FAILURE);
2182 }
2183
2184 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2185 ibdm_alloc_send_buffers(msg);
2186 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2187
2188 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2189 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2190 msg->im_local_addr.ia_remote_qno = 1;
2191 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2192 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2193 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2194
2195 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
2196 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
2197 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
2198 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
2199 hdr->R_Method = IB_DM_DEVMGT_METHOD_SET;
2200 hdr->Status = 0;
2201 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
2202 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO);
2203 hdr->AttributeModifier = 0;
2204
2205 data = msg->im_msgbufs_send.im_bufs_cl_data;
2206 cpi = (ib_mad_classportinfo_t *)data;
2207
2208 /*
2209 * Set the classportinfo values to activate this Cisco FC GW.
2210 */
2211 cpi->TrapGID_hi = h2b64(gid_info->gl_sgid_hi);
2212 cpi->TrapGID_lo = h2b64(gid_info->gl_sgid_lo);
2213 cpi->TrapLID = h2b16(gid_info->gl_slid);
2214 cpi->TrapSL = gid_info->gl_SL;
2215 cpi->TrapP_Key = h2b16(gid_info->gl_p_key);
2216 cpi->TrapQP = h2b32((((ibmf_alt_qp_t *)gid_info->gl_qp_hdl)->isq_qpn));
2217 cpi->TrapQ_Key = h2b32((((ibmf_alt_qp_t *)
2218 gid_info->gl_qp_hdl)->isq_qkey));
2219
2220 cb_args = &gid_info->gl_cpi_cb_args;
2221 cb_args->cb_gid_info = gid_info;
2222 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
2223 cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO;
2224
2225 mutex_enter(&gid_info->gl_mutex);
2226 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2227 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2228 mutex_exit(&gid_info->gl_mutex);
2229
2230 IBTF_DPRINTF_L5("ibdm", "\tset_classportinfo: "
2231 "timeout id %x", gid_info->gl_timeout_id);
2232
2233 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
2234 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2235 IBTF_DPRINTF_L2("ibdm",
2236 "\tset_classportinfo: ibmf send failed");
2237 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
2238 }
2239
2240 return (IBDM_SUCCESS);
2241 }
2242
2243
2244 /*
2245 * ibdm_send_classportinfo()
2246 * Send classportinfo request. When the request is completed
2247 * IBMF calls ibdm_classportinfo_cb routine to inform about
2248 * the completion.
2249 * Returns IBDM_SUCCESS/IBDM_FAILURE
2250 */
2251 static int
2252 ibdm_send_classportinfo(ibdm_dp_gidinfo_t *gid_info)
2253 {
2254 ibmf_msg_t *msg;
2255 ib_mad_hdr_t *hdr;
2256 ibdm_timeout_cb_args_t *cb_args;
2257
2258 IBTF_DPRINTF_L4("ibdm",
2259 "\tsend_classportinfo: gid info 0x%p", gid_info);
2260
2261 /*
2262 * Send command to get classportinfo attribute. Allocate a IBMF
2263 * packet and initialize the packet.
2264 */
2265 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
2266 &msg) != IBMF_SUCCESS) {
2267 IBTF_DPRINTF_L4("ibdm", "\tsend_classportinfo: pkt alloc fail");
2268 return (IBDM_FAILURE);
2269 }
2270
2271 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2272 ibdm_alloc_send_buffers(msg);
2273 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2274
2275 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2276 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2277 msg->im_local_addr.ia_remote_qno = 1;
2278 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2279 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2280 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2281
2282 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
2283 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
2284 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
2285 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
2286 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
2287 hdr->Status = 0;
2288 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
2289 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO);
2290 hdr->AttributeModifier = 0;
2291
2292 cb_args = &gid_info->gl_cpi_cb_args;
2293 cb_args->cb_gid_info = gid_info;
2294 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
2295 cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO;
2296
2297 mutex_enter(&gid_info->gl_mutex);
2298 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2299 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2300 mutex_exit(&gid_info->gl_mutex);
2301
2302 IBTF_DPRINTF_L5("ibdm", "\tsend_classportinfo: "
2303 "timeout id %x", gid_info->gl_timeout_id);
2304
2305 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
2306 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2307 IBTF_DPRINTF_L2("ibdm",
2308 "\tsend_classportinfo: ibmf send failed");
2309 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
2310 }
2311
2312 return (IBDM_SUCCESS);
2313 }
2314
2315
2316 /*
2317 * ibdm_handle_setclassportinfo()
2318 * Invoked by the IBMF when setClassPortInfo request is completed.
2319 */
2320 static void
2321 ibdm_handle_setclassportinfo(ibmf_handle_t ibmf_hdl,
2322 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2323 {
2324 void *data;
2325 timeout_id_t timeout_id;
2326 ib_mad_classportinfo_t *cpi;
2327
2328 IBTF_DPRINTF_L4("ibdm", "\thandle_setclassportinfo:ibmf hdl "
2329 "%p msg %p gid info %p", ibmf_hdl, msg, gid_info);
2330
2331 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) {
2332 IBTF_DPRINTF_L4("ibdm", "\thandle_setclassportinfo: "
2333 "Not a ClassPortInfo resp");
2334 *flag |= IBDM_IBMF_PKT_UNEXP_RESP;
2335 return;
2336 }
2337
2338 /*
2339 * Verify whether timeout handler is created/active.
2340 * If created/ active, cancel the timeout handler
2341 */
2342 mutex_enter(&gid_info->gl_mutex);
2343 if (gid_info->gl_state != IBDM_SET_CLASSPORTINFO) {
2344 IBTF_DPRINTF_L2("ibdm", "\thandle_setclassportinfo:DUP resp");
2345 *flag |= IBDM_IBMF_PKT_DUP_RESP;
2346 mutex_exit(&gid_info->gl_mutex);
2347 return;
2348 }
2349 ibdm_bump_transactionID(gid_info);
2350
2351 gid_info->gl_iou_cb_args.cb_req_type = 0;
2352 if (gid_info->gl_timeout_id) {
2353 timeout_id = gid_info->gl_timeout_id;
2354 mutex_exit(&gid_info->gl_mutex);
2355 IBTF_DPRINTF_L5("ibdm", "handle_setlassportinfo: "
2356 "gl_timeout_id = 0x%x", timeout_id);
2357 if (untimeout(timeout_id) == -1) {
2358 IBTF_DPRINTF_L2("ibdm", "handle_setclassportinfo: "
2359 "untimeout gl_timeout_id failed");
2360 }
2361 mutex_enter(&gid_info->gl_mutex);
2362 gid_info->gl_timeout_id = 0;
2363 }
2364 mutex_exit(&gid_info->gl_mutex);
2365
2366 data = msg->im_msgbufs_recv.im_bufs_cl_data;
2367 cpi = (ib_mad_classportinfo_t *)data;
2368
2369 ibdm_dump_classportinfo(cpi);
2370 }
2371
2372
2373 /*
2374 * ibdm_handle_classportinfo()
2375 * Invoked by the IBMF when the classportinfo request is completed.
2376 */
2377 static void
2378 ibdm_handle_classportinfo(ibmf_handle_t ibmf_hdl,
2379 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2380 {
2381 void *data;
2382 timeout_id_t timeout_id;
2383 ib_mad_hdr_t *hdr;
2384 ib_mad_classportinfo_t *cpi;
2385
2386 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo:ibmf hdl "
2387 "%p msg %p gid info %p", ibmf_hdl, msg, gid_info);
2388
2389 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) {
2390 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo: "
2391 "Not a ClassPortInfo resp");
2392 *flag |= IBDM_IBMF_PKT_UNEXP_RESP;
2393 return;
2394 }
2395
2396 /*
2397 * Verify whether timeout handler is created/active.
2398 * If created/ active, cancel the timeout handler
2399 */
2400 mutex_enter(&gid_info->gl_mutex);
2401 ibdm_bump_transactionID(gid_info);
2402 if (gid_info->gl_state != IBDM_GET_CLASSPORTINFO) {
2403 IBTF_DPRINTF_L2("ibdm", "\thandle_classportinfo:DUP resp");
2404 *flag |= IBDM_IBMF_PKT_DUP_RESP;
2405 mutex_exit(&gid_info->gl_mutex);
2406 return;
2407 }
2408 gid_info->gl_iou_cb_args.cb_req_type = 0;
2409 if (gid_info->gl_timeout_id) {
2410 timeout_id = gid_info->gl_timeout_id;
2411 mutex_exit(&gid_info->gl_mutex);
2412 IBTF_DPRINTF_L5("ibdm", "handle_ioclassportinfo: "
2413 "gl_timeout_id = 0x%x", timeout_id);
2414 if (untimeout(timeout_id) == -1) {
2415 IBTF_DPRINTF_L2("ibdm", "handle_classportinfo: "
2416 "untimeout gl_timeout_id failed");
2417 }
2418 mutex_enter(&gid_info->gl_mutex);
2419 gid_info->gl_timeout_id = 0;
2420 }
2421 gid_info->gl_state = IBDM_GET_IOUNITINFO;
2422 gid_info->gl_pending_cmds++;
2423 mutex_exit(&gid_info->gl_mutex);
2424
2425 data = msg->im_msgbufs_recv.im_bufs_cl_data;
2426 cpi = (ib_mad_classportinfo_t *)data;
2427
2428 /*
2429 * Cache the "RespTimeValue" and redirection information in the
2430 * global gid list data structure. This cached information will
2431 * be used to send any further requests to the GID.
2432 */
2433 gid_info->gl_resp_timeout =
2434 (b2h32(cpi->RespTimeValue) & 0x1F);
2435
2436 gid_info->gl_redirected = ((IBDM_IN_IBMFMSG_STATUS(msg) &
2437 MAD_STATUS_REDIRECT_REQUIRED) ? B_TRUE: B_FALSE);
2438 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID);
2439 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff);
2440 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key);
2441 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key);
2442 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi);
2443 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo);
2444 gid_info->gl_redirectSL = cpi->RedirectSL;
2445
2446 ibdm_dump_classportinfo(cpi);
2447
2448 /*
2449 * Send IOUnitInfo request
2450 * Reuse previously allocated IBMF packet for sending ClassPortInfo
2451 * Check whether DM agent on the remote node requested redirection
2452 * If so, send the request to the redirect DGID/DLID/PKEY/QP.
2453 */
2454 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2455 ibdm_alloc_send_buffers(msg);
2456 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2457 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2458 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2459
2460 if (gid_info->gl_redirected == B_TRUE) {
2461 if (gid_info->gl_redirect_dlid != 0) {
2462 msg->im_local_addr.ia_remote_lid =
2463 gid_info->gl_redirect_dlid;
2464 }
2465 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
2466 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
2467 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
2468 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
2469 } else {
2470 msg->im_local_addr.ia_remote_qno = 1;
2471 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2472 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2473 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2474 }
2475
2476 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
2477 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
2478 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
2479 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
2480 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
2481 hdr->Status = 0;
2482 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
2483 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO);
2484 hdr->AttributeModifier = 0;
2485
2486 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO;
2487 gid_info->gl_iou_cb_args.cb_gid_info = gid_info;
2488 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt;
2489
2490 mutex_enter(&gid_info->gl_mutex);
2491 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2492 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2493 mutex_exit(&gid_info->gl_mutex);
2494
2495 IBTF_DPRINTF_L5("ibdm", "handle_classportinfo:"
2496 "timeout %x", gid_info->gl_timeout_id);
2497
2498 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, NULL,
2499 ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != IBMF_SUCCESS) {
2500 IBTF_DPRINTF_L2("ibdm",
2501 "\thandle_classportinfo: msg transport failed");
2502 ibdm_ibmf_send_cb(ibmf_hdl, msg, &gid_info->gl_iou_cb_args);
2503 }
2504 (*flag) |= IBDM_IBMF_PKT_REUSED;
2505 }
2506
2507
2508 /*
2509 * ibdm_send_iounitinfo:
2510 * Sends a DM request to get IOU unitinfo.
2511 */
2512 static int
2513 ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *gid_info)
2514 {
2515 ibmf_msg_t *msg;
2516 ib_mad_hdr_t *hdr;
2517
2518 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: gid info 0x%p", gid_info);
2519
2520 /*
2521 * Send command to get iounitinfo attribute. Allocate a IBMF
2522 * packet and initialize the packet.
2523 */
2524 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, &msg) !=
2525 IBMF_SUCCESS) {
2526 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: pkt alloc fail");
2527 return (IBDM_FAILURE);
2528 }
2529
2530 mutex_enter(&gid_info->gl_mutex);
2531 ibdm_bump_transactionID(gid_info);
2532 mutex_exit(&gid_info->gl_mutex);
2533
2534
2535 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2536 ibdm_alloc_send_buffers(msg);
2537 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2538 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2539 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2540 msg->im_local_addr.ia_remote_qno = 1;
2541 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2542 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2543 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2544
2545 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
2546 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
2547 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
2548 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
2549 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
2550 hdr->Status = 0;
2551 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
2552 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO);
2553 hdr->AttributeModifier = 0;
2554
2555 gid_info->gl_iou_cb_args.cb_gid_info = gid_info;
2556 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt;
2557 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO;
2558
2559 mutex_enter(&gid_info->gl_mutex);
2560 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2561 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2562 mutex_exit(&gid_info->gl_mutex);
2563
2564 IBTF_DPRINTF_L5("ibdm", "send_iouunitinfo:"
2565 "timeout %x", gid_info->gl_timeout_id);
2566
2567 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg,
2568 NULL, ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) !=
2569 IBMF_SUCCESS) {
2570 IBTF_DPRINTF_L2("ibdm", "\tsend_iounitinfo: ibmf send failed");
2571 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl,
2572 msg, &gid_info->gl_iou_cb_args);
2573 }
2574 return (IBDM_SUCCESS);
2575 }
2576
2577 /*
2578 * ibdm_handle_iounitinfo()
2579 * Invoked by the IBMF when IO Unitinfo request is completed.
2580 */
2581 static void
2582 ibdm_handle_iounitinfo(ibmf_handle_t ibmf_hdl,
2583 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2584 {
2585 int ii, first = B_TRUE;
2586 int num_iocs;
2587 size_t size;
2588 uchar_t slot_info;
2589 timeout_id_t timeout_id;
2590 ib_mad_hdr_t *hdr;
2591 ibdm_ioc_info_t *ioc_info;
2592 ib_dm_io_unitinfo_t *iou_info;
2593 ib_dm_io_unitinfo_t *giou_info;
2594 ibdm_timeout_cb_args_t *cb_args;
2595
2596 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo:"
2597 " ibmf hdl %p pkt %p gid info %p", ibmf_hdl, msg, gid_info);
2598
2599 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_IO_UNITINFO) {
2600 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: "
2601 "Unexpected response");
2602 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
2603 return;
2604 }
2605
2606 mutex_enter(&gid_info->gl_mutex);
2607 if (gid_info->gl_state != IBDM_GET_IOUNITINFO) {
2608 IBTF_DPRINTF_L4("ibdm",
2609 "\thandle_iounitinfo: DUP resp");
2610 mutex_exit(&gid_info->gl_mutex);
2611 (*flag) = IBDM_IBMF_PKT_DUP_RESP;
2612 return;
2613 }
2614 gid_info->gl_iou_cb_args.cb_req_type = 0;
2615 if (gid_info->gl_timeout_id) {
2616 timeout_id = gid_info->gl_timeout_id;
2617 mutex_exit(&gid_info->gl_mutex);
2618 IBTF_DPRINTF_L5("ibdm", "handle_iounitinfo: "
2619 "gl_timeout_id = 0x%x", timeout_id);
2620 if (untimeout(timeout_id) == -1) {
2621 IBTF_DPRINTF_L2("ibdm", "handle_iounitinfo: "
2622 "untimeout gl_timeout_id failed");
2623 }
2624 mutex_enter(&gid_info->gl_mutex);
2625 gid_info->gl_timeout_id = 0;
2626 }
2627 gid_info->gl_state = IBDM_GET_IOC_DETAILS;
2628
2629 iou_info = IBDM_IN_IBMFMSG2IOU(msg);
2630 ibdm_dump_iounitinfo(iou_info);
2631 num_iocs = iou_info->iou_num_ctrl_slots;
2632 /*
2633 * check if number of IOCs reported is zero? if yes, return.
2634 * when num_iocs are reported zero internal IOC database needs
2635 * to be updated. To ensure that save the number of IOCs in
2636 * the new field "gl_num_iocs". Use a new field instead of
2637 * "giou_info->iou_num_ctrl_slots" as that would prevent
2638 * an unnecessary kmem_alloc/kmem_free when num_iocs is 0.
2639 */
2640 if (num_iocs == 0 && gid_info->gl_num_iocs == 0) {
2641 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: no IOC's");
2642 mutex_exit(&gid_info->gl_mutex);
2643 return;
2644 }
2645 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: num_iocs = %d", num_iocs);
2646
2647 /*
2648 * if there is an existing gl_iou (IOU has been probed before)
2649 * check if the "iou_changeid" is same as saved entry in
2650 * "giou_info->iou_changeid".
2651 * (note: this logic can prevent IOC enumeration if a given
2652 * vendor doesn't support setting iou_changeid field for its IOU)
2653 *
2654 * if there is an existing gl_iou and iou_changeid has changed :
2655 * free up existing gl_iou info and its related structures.
2656 * reallocate gl_iou info all over again.
2657 * if we donot free this up; then this leads to memory leaks
2658 */
2659 if (gid_info->gl_iou) {
2660 giou_info = &gid_info->gl_iou->iou_info;
2661 if (b2h16(iou_info->iou_changeid) ==
2662 giou_info->iou_changeid) {
2663 IBTF_DPRINTF_L3("ibdm",
2664 "\thandle_iounitinfo: no IOCs changed");
2665 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE;
2666 mutex_exit(&gid_info->gl_mutex);
2667 return;
2668 }
2669
2670 /*
2671 * Store the iou info as prev_iou to be used after
2672 * sweep is done.
2673 */
2674 ASSERT(gid_info->gl_prev_iou == NULL);
2675 IBTF_DPRINTF_L4(ibdm_string,
2676 "\thandle_iounitinfo: setting gl_prev_iou %p",
2677 gid_info->gl_prev_iou);
2678 gid_info->gl_prev_iou = gid_info->gl_iou;
2679 ibdm.ibdm_prev_iou = 1;
2680 gid_info->gl_iou = NULL;
2681 }
2682
2683 size = sizeof (ibdm_iou_info_t) + num_iocs * sizeof (ibdm_ioc_info_t);
2684 gid_info->gl_iou = (ibdm_iou_info_t *)kmem_zalloc(size, KM_SLEEP);
2685 giou_info = &gid_info->gl_iou->iou_info;
2686 gid_info->gl_iou->iou_ioc_info = (ibdm_ioc_info_t *)
2687 ((char *)gid_info->gl_iou + sizeof (ibdm_iou_info_t));
2688
2689 giou_info->iou_num_ctrl_slots = gid_info->gl_num_iocs = num_iocs;
2690 giou_info->iou_flag = iou_info->iou_flag;
2691 bcopy(iou_info->iou_ctrl_list, giou_info->iou_ctrl_list, 128);
2692 giou_info->iou_changeid = b2h16(iou_info->iou_changeid);
2693 gid_info->gl_pending_cmds++; /* for diag code */
2694 mutex_exit(&gid_info->gl_mutex);
2695
2696 if (ibdm_get_diagcode(gid_info, 0) != IBDM_SUCCESS) {
2697 mutex_enter(&gid_info->gl_mutex);
2698 gid_info->gl_pending_cmds--;
2699 mutex_exit(&gid_info->gl_mutex);
2700 }
2701 /*
2702 * Parallelize getting IOC controller profiles from here.
2703 * Allocate IBMF packets and send commands to get IOC profile for
2704 * each IOC present on the IOU.
2705 */
2706 for (ii = 0; ii < num_iocs; ii++) {
2707 /*
2708 * Check whether IOC is present in the slot
2709 * Series of nibbles (in the field iou_ctrl_list) represents
2710 * a slot in the IOU.
2711 * Byte format: 76543210
2712 * Bits 0-3 of first byte represent Slot 2
2713 * bits 4-7 of first byte represent slot 1,
2714 * bits 0-3 of second byte represent slot 4 and so on
2715 * Each 4-bit nibble has the following meaning
2716 * 0x0 : IOC not installed
2717 * 0x1 : IOC is present
2718 * 0xf : Slot does not exist
2719 * and all other values are reserved.
2720 */
2721 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii);
2722 slot_info = giou_info->iou_ctrl_list[(ii/2)];
2723 if ((ii % 2) == 0)
2724 slot_info = (slot_info >> 4);
2725
2726 if ((slot_info & 0xf) != 1) {
2727 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: "
2728 "No IOC is present in the slot = %d", ii);
2729 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
2730 continue;
2731 }
2732
2733 mutex_enter(&gid_info->gl_mutex);
2734 ibdm_bump_transactionID(gid_info);
2735 mutex_exit(&gid_info->gl_mutex);
2736
2737 /*
2738 * Re use the already allocated packet (for IOUnitinfo) to
2739 * send the first IOC controller attribute. Allocate new
2740 * IBMF packets for the rest of the IOC's
2741 */
2742 if (first != B_TRUE) {
2743 msg = NULL;
2744 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP,
2745 &msg) != IBMF_SUCCESS) {
2746 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: "
2747 "IBMF packet allocation failed");
2748 continue;
2749 }
2750
2751 }
2752
2753 /* allocate send buffers for all messages */
2754 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2755 ibdm_alloc_send_buffers(msg);
2756 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2757
2758 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2759 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2760 if (gid_info->gl_redirected == B_TRUE) {
2761 if (gid_info->gl_redirect_dlid != 0) {
2762 msg->im_local_addr.ia_remote_lid =
2763 gid_info->gl_redirect_dlid;
2764 }
2765 msg->im_local_addr.ia_remote_qno =
2766 gid_info->gl_redirect_QP;
2767 msg->im_local_addr.ia_p_key =
2768 gid_info->gl_redirect_pkey;
2769 msg->im_local_addr.ia_q_key =
2770 gid_info->gl_redirect_qkey;
2771 msg->im_local_addr.ia_service_level =
2772 gid_info->gl_redirectSL;
2773 } else {
2774 msg->im_local_addr.ia_remote_qno = 1;
2775 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2776 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2777 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2778 }
2779
2780 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
2781 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
2782 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
2783 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
2784 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
2785 hdr->Status = 0;
2786 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
2787 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
2788 hdr->AttributeModifier = h2b32(ii + 1);
2789
2790 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_INVALID;
2791 cb_args = &ioc_info->ioc_cb_args;
2792 cb_args->cb_gid_info = gid_info;
2793 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
2794 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO;
2795 cb_args->cb_ioc_num = ii;
2796
2797 mutex_enter(&gid_info->gl_mutex);
2798 gid_info->gl_pending_cmds++; /* for diag code */
2799
2800 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2801 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2802 mutex_exit(&gid_info->gl_mutex);
2803
2804 IBTF_DPRINTF_L5("ibdm", "\thandle_iounitinfo:"
2805 "timeout 0x%x, ioc_num %d", ioc_info->ioc_timeout_id, ii);
2806
2807 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg,
2808 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2809 IBTF_DPRINTF_L2("ibdm",
2810 "\thandle_iounitinfo: msg transport failed");
2811 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args);
2812 }
2813 (*flag) |= IBDM_IBMF_PKT_REUSED;
2814 first = B_FALSE;
2815 gid_info->gl_iou->iou_niocs_probe_in_progress++;
2816 }
2817 }
2818
2819
2820 /*
2821 * ibdm_handle_ioc_profile()
2822 * Invoked by the IBMF when the IOCControllerProfile request
2823 * gets completed
2824 */
2825 static void
2826 ibdm_handle_ioc_profile(ibmf_handle_t ibmf_hdl,
2827 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2828 {
2829 int first = B_TRUE, reprobe = 0;
2830 uint_t ii, ioc_no, srv_start;
2831 uint_t nserv_entries;
2832 timeout_id_t timeout_id;
2833 ib_mad_hdr_t *hdr;
2834 ibdm_ioc_info_t *ioc_info;
2835 ibdm_timeout_cb_args_t *cb_args;
2836 ib_dm_ioc_ctrl_profile_t *ioc, *gioc;
2837
2838 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:"
2839 " ibmf hdl %p msg %p gid info %p", ibmf_hdl, msg, gid_info);
2840
2841 ioc = IBDM_IN_IBMFMSG2IOC(msg);
2842 /*
2843 * Check whether we know this IOC already
2844 * This will return NULL if reprobe is in progress
2845 * IBDM_IOC_STATE_REPROBE_PROGRESS will be set.
2846 * Do not hold mutexes here.
2847 */
2848 if (ibdm_is_ioc_present(ioc->ioc_guid, gid_info, flag) != NULL) {
2849 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:"
2850 "IOC guid %llx is present", ioc->ioc_guid);
2851 return;
2852 }
2853 ioc_no = IBDM_IN_IBMFMSG_ATTRMOD(msg);
2854 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile: ioc_no = %d", ioc_no-1);
2855
2856 /* Make sure that IOC index is with the valid range */
2857 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) {
2858 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: "
2859 "IOC index Out of range, index %d", ioc);
2860 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
2861 return;
2862 }
2863 ioc_info = &gid_info->gl_iou->iou_ioc_info[ioc_no - 1];
2864 ioc_info->ioc_iou_info = gid_info->gl_iou;
2865
2866 mutex_enter(&gid_info->gl_mutex);
2867 if (ioc_info->ioc_state == IBDM_IOC_STATE_REPROBE_PROGRESS) {
2868 reprobe = 1;
2869 ioc_info->ioc_prev_serv = ioc_info->ioc_serv;
2870 ioc_info->ioc_serv = NULL;
2871 ioc_info->ioc_prev_serv_cnt =
2872 ioc_info->ioc_profile.ioc_service_entries;
2873 } else if (ioc_info->ioc_state != IBDM_IOC_STATE_PROBE_INVALID) {
2874 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: DUP response"
2875 "ioc %d, ioc_state %x", ioc_no - 1, ioc_info->ioc_state);
2876 mutex_exit(&gid_info->gl_mutex);
2877 (*flag) |= IBDM_IBMF_PKT_DUP_RESP;
2878 return;
2879 }
2880 ioc_info->ioc_cb_args.cb_req_type = 0;
2881 if (ioc_info->ioc_timeout_id) {
2882 timeout_id = ioc_info->ioc_timeout_id;
2883 ioc_info->ioc_timeout_id = 0;
2884 mutex_exit(&gid_info->gl_mutex);
2885 IBTF_DPRINTF_L5("ibdm", "handle_ioc_profile: "
2886 "ioc_timeout_id = 0x%x", timeout_id);
2887 if (untimeout(timeout_id) == -1) {
2888 IBTF_DPRINTF_L2("ibdm", "handle_ioc_profile: "
2889 "untimeout ioc_timeout_id failed");
2890 }
2891 mutex_enter(&gid_info->gl_mutex);
2892 }
2893
2894 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_SUCCESS;
2895 if (reprobe == 0) {
2896 ioc_info->ioc_iou_guid = gid_info->gl_nodeguid;
2897 ioc_info->ioc_nodeguid = gid_info->gl_nodeguid;
2898 }
2899
2900 /*
2901 * Save all the IOC information in the global structures.
2902 * Note the wire format is Big Endian and the Sparc process also
2903 * big endian. So, there is no need to convert the data fields
2904 * The conversion routines used below are ineffective on Sparc
2905 * machines where as they will be effective on little endian
2906 * machines such as Intel processors.
2907 */
2908 gioc = (ib_dm_ioc_ctrl_profile_t *)&ioc_info->ioc_profile;
2909
2910 /*
2911 * Restrict updates to onlyport GIDs and service entries during reprobe
2912 */
2913 if (reprobe == 0) {
2914 gioc->ioc_guid = b2h64(ioc->ioc_guid);
2915 gioc->ioc_vendorid =
2916 ((b2h32(ioc->ioc_vendorid) & IB_DM_VENDORID_MASK)
2917 >> IB_DM_VENDORID_SHIFT);
2918 gioc->ioc_deviceid = b2h32(ioc->ioc_deviceid);
2919 gioc->ioc_device_ver = b2h16(ioc->ioc_device_ver);
2920 gioc->ioc_subsys_vendorid =
2921 ((b2h32(ioc->ioc_subsys_vendorid) & IB_DM_VENDORID_MASK)
2922 >> IB_DM_VENDORID_SHIFT);
2923 gioc->ioc_subsys_id = b2h32(ioc->ioc_subsys_id);
2924 gioc->ioc_io_class = b2h16(ioc->ioc_io_class);
2925 gioc->ioc_io_subclass = b2h16(ioc->ioc_io_subclass);
2926 gioc->ioc_protocol = b2h16(ioc->ioc_protocol);
2927 gioc->ioc_protocol_ver = b2h16(ioc->ioc_protocol_ver);
2928 gioc->ioc_send_msg_qdepth =
2929 b2h16(ioc->ioc_send_msg_qdepth);
2930 gioc->ioc_rdma_read_qdepth =
2931 b2h16(ioc->ioc_rdma_read_qdepth);
2932 gioc->ioc_send_msg_sz = b2h32(ioc->ioc_send_msg_sz);
2933 gioc->ioc_rdma_xfer_sz = b2h32(ioc->ioc_rdma_xfer_sz);
2934 gioc->ioc_ctrl_opcap_mask = ioc->ioc_ctrl_opcap_mask;
2935 bcopy(ioc->ioc_id_string, gioc->ioc_id_string,
2936 IB_DM_IOC_ID_STRING_LEN);
2937
2938 ioc_info->ioc_iou_diagcode = gid_info->gl_iou->iou_diagcode;
2939 ioc_info->ioc_iou_dc_valid = gid_info->gl_iou->iou_dc_valid;
2940 ioc_info->ioc_diagdeviceid = (IB_DM_IOU_DEVICEID_MASK &
2941 gid_info->gl_iou->iou_info.iou_flag) ? B_TRUE : B_FALSE;
2942
2943 if (ioc_info->ioc_diagdeviceid == B_TRUE) {
2944 gid_info->gl_pending_cmds++;
2945 IBTF_DPRINTF_L3(ibdm_string,
2946 "\tibdm_handle_ioc_profile: "
2947 "%d: gid_info %p gl_state %d pending_cmds %d",
2948 __LINE__, gid_info, gid_info->gl_state,
2949 gid_info->gl_pending_cmds);
2950 }
2951 }
2952 gioc->ioc_service_entries = ioc->ioc_service_entries;
2953 mutex_exit(&gid_info->gl_mutex);
2954
2955 ibdm_dump_ioc_profile(gioc);
2956
2957 if ((ioc_info->ioc_diagdeviceid == B_TRUE) && (reprobe == 0)) {
2958 if (ibdm_get_diagcode(gid_info, ioc_no) != IBDM_SUCCESS) {
2959 mutex_enter(&gid_info->gl_mutex);
2960 gid_info->gl_pending_cmds--;
2961 mutex_exit(&gid_info->gl_mutex);
2962 }
2963 }
2964 ioc_info->ioc_serv = (ibdm_srvents_info_t *)kmem_zalloc(
2965 (gioc->ioc_service_entries * sizeof (ibdm_srvents_info_t)),
2966 KM_SLEEP);
2967
2968 /*
2969 * In one single request, maximum number of requests that can be
2970 * obtained is 4. If number of service entries are more than four,
2971 * calculate number requests needed and send them parallelly.
2972 */
2973 nserv_entries = ioc->ioc_service_entries;
2974 ii = 0;
2975 while (nserv_entries) {
2976 mutex_enter(&gid_info->gl_mutex);
2977 gid_info->gl_pending_cmds++;
2978 ibdm_bump_transactionID(gid_info);
2979 mutex_exit(&gid_info->gl_mutex);
2980
2981 if (first != B_TRUE) {
2982 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP,
2983 &msg) != IBMF_SUCCESS) {
2984 continue;
2985 }
2986
2987 }
2988 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2989 ibdm_alloc_send_buffers(msg);
2990 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2991 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2992 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2993 if (gid_info->gl_redirected == B_TRUE) {
2994 if (gid_info->gl_redirect_dlid != 0) {
2995 msg->im_local_addr.ia_remote_lid =
2996 gid_info->gl_redirect_dlid;
2997 }
2998 msg->im_local_addr.ia_remote_qno =
2999 gid_info->gl_redirect_QP;
3000 msg->im_local_addr.ia_p_key =
3001 gid_info->gl_redirect_pkey;
3002 msg->im_local_addr.ia_q_key =
3003 gid_info->gl_redirect_qkey;
3004 msg->im_local_addr.ia_service_level =
3005 gid_info->gl_redirectSL;
3006 } else {
3007 msg->im_local_addr.ia_remote_qno = 1;
3008 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
3009 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
3010 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
3011 }
3012
3013 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
3014 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
3015 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
3016 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
3017 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
3018 hdr->Status = 0;
3019 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
3020 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES);
3021
3022 srv_start = ii * 4;
3023 cb_args = &ioc_info->ioc_serv[srv_start].se_cb_args;
3024 cb_args->cb_gid_info = gid_info;
3025 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
3026 cb_args->cb_req_type = IBDM_REQ_TYPE_SRVENTS;
3027 cb_args->cb_srvents_start = srv_start;
3028 cb_args->cb_ioc_num = ioc_no - 1;
3029
3030 if (nserv_entries >= IBDM_MAX_SERV_ENTRIES_PER_REQ) {
3031 nserv_entries -= IBDM_MAX_SERV_ENTRIES_PER_REQ;
3032 cb_args->cb_srvents_end = (cb_args->cb_srvents_start +
3033 IBDM_MAX_SERV_ENTRIES_PER_REQ - 1);
3034 } else {
3035 cb_args->cb_srvents_end =
3036 (cb_args->cb_srvents_start + nserv_entries - 1);
3037 nserv_entries = 0;
3038 }
3039 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr))
3040 ibdm_fill_srv_attr_mod(hdr, cb_args);
3041 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr))
3042
3043 mutex_enter(&gid_info->gl_mutex);
3044 ioc_info->ioc_serv[srv_start].se_timeout_id = timeout(
3045 ibdm_pkt_timeout_hdlr, cb_args,
3046 IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
3047 mutex_exit(&gid_info->gl_mutex);
3048
3049 IBTF_DPRINTF_L5("ibdm", "\thandle_ioc_profile:"
3050 "timeout %x, ioc %d srv %d",
3051 ioc_info->ioc_serv[srv_start].se_timeout_id,
3052 ioc_no - 1, srv_start);
3053
3054 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg,
3055 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
3056 IBTF_DPRINTF_L2("ibdm",
3057 "\thandle_ioc_profile: msg send failed");
3058 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args);
3059 }
3060 (*flag) |= IBDM_IBMF_PKT_REUSED;
3061 first = B_FALSE;
3062 ii++;
3063 }
3064 }
3065
3066
3067 /*
3068 * ibdm_handle_srventry_mad()
3069 */
3070 static void
3071 ibdm_handle_srventry_mad(ibmf_msg_t *msg,
3072 ibdm_dp_gidinfo_t *gid_info, int *flag)
3073 {
3074 uint_t ii, ioc_no, attrmod;
3075 uint_t nentries, start, end;
3076 timeout_id_t timeout_id;
3077 ib_dm_srv_t *srv_ents;
3078 ibdm_ioc_info_t *ioc_info;
3079 ibdm_srvents_info_t *gsrv_ents;
3080
3081 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad:"
3082 " IBMF msg %p gid info %p", msg, gid_info);
3083
3084 srv_ents = IBDM_IN_IBMFMSG2SRVENT(msg);
3085 /*
3086 * Get the start and end index of the service entries
3087 * Upper 16 bits identify the IOC
3088 * Lower 16 bits specify the range of service entries
3089 * LSB specifies (Big endian) end of the range
3090 * MSB specifies (Big endian) start of the range
3091 */
3092 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg);
3093 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK);
3094 end = ((attrmod >> 8) & IBDM_8_BIT_MASK);
3095 start = (attrmod & IBDM_8_BIT_MASK);
3096
3097 /* Make sure that IOC index is with the valid range */
3098 if ((ioc_no < 1) |
3099 (ioc_no > gid_info->gl_iou->iou_info.iou_num_ctrl_slots)) {
3100 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
3101 "IOC index Out of range, index %d", ioc_no);
3102 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3103 return;
3104 }
3105 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1));
3106
3107 /*
3108 * Make sure that the "start" and "end" service indexes are
3109 * with in the valid range
3110 */
3111 nentries = ioc_info->ioc_profile.ioc_service_entries;
3112 if ((start > end) | (start >= nentries) | (end >= nentries)) {
3113 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
3114 "Attr modifier 0x%x, #Serv entries %d", attrmod, nentries);
3115 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3116 return;
3117 }
3118 gsrv_ents = &ioc_info->ioc_serv[start];
3119 mutex_enter(&gid_info->gl_mutex);
3120 if (gsrv_ents->se_state != IBDM_SE_INVALID) {
3121 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
3122 "already known, ioc %d, srv %d, se_state %x",
3123 ioc_no - 1, start, gsrv_ents->se_state);
3124 mutex_exit(&gid_info->gl_mutex);
3125 (*flag) |= IBDM_IBMF_PKT_DUP_RESP;
3126 return;
3127 }
3128 ioc_info->ioc_serv[start].se_cb_args.cb_req_type = 0;
3129 if (ioc_info->ioc_serv[start].se_timeout_id) {
3130 IBTF_DPRINTF_L2("ibdm",
3131 "\thandle_srventry_mad: ioc %d start %d", ioc_no, start);
3132 timeout_id = ioc_info->ioc_serv[start].se_timeout_id;
3133 ioc_info->ioc_serv[start].se_timeout_id = 0;
3134 mutex_exit(&gid_info->gl_mutex);
3135 IBTF_DPRINTF_L5("ibdm", "handle_srverntry_mad: "
3136 "se_timeout_id = 0x%x", timeout_id);
3137 if (untimeout(timeout_id) == -1) {
3138 IBTF_DPRINTF_L2("ibdm", "handle_srventry_mad: "
3139 "untimeout se_timeout_id failed");
3140 }
3141 mutex_enter(&gid_info->gl_mutex);
3142 }
3143
3144 gsrv_ents->se_state = IBDM_SE_VALID;
3145 mutex_exit(&gid_info->gl_mutex);
3146 for (ii = start; ii <= end; ii++, srv_ents++, gsrv_ents++) {
3147 gsrv_ents->se_attr.srv_id = b2h64(srv_ents->srv_id);
3148 bcopy(srv_ents->srv_name,
3149 gsrv_ents->se_attr.srv_name, IB_DM_MAX_SVC_NAME_LEN);
3150 ibdm_dump_service_entries(&gsrv_ents->se_attr);
3151 }
3152 }
3153
3154
3155 /*
3156 * ibdm_get_diagcode:
3157 * Send request to get IOU/IOC diag code
3158 * Returns IBDM_SUCCESS/IBDM_FAILURE
3159 */
3160 static int
3161 ibdm_get_diagcode(ibdm_dp_gidinfo_t *gid_info, int attr)
3162 {
3163 ibmf_msg_t *msg;
3164 ib_mad_hdr_t *hdr;
3165 ibdm_ioc_info_t *ioc;
3166 ibdm_timeout_cb_args_t *cb_args;
3167 timeout_id_t *timeout_id;
3168
3169 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: gid info %p, attr = %d",
3170 gid_info, attr);
3171
3172 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
3173 &msg) != IBMF_SUCCESS) {
3174 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: pkt alloc fail");
3175 return (IBDM_FAILURE);
3176 }
3177
3178 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
3179 ibdm_alloc_send_buffers(msg);
3180 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
3181
3182 mutex_enter(&gid_info->gl_mutex);
3183 ibdm_bump_transactionID(gid_info);
3184 mutex_exit(&gid_info->gl_mutex);
3185
3186 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
3187 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
3188 if (gid_info->gl_redirected == B_TRUE) {
3189 if (gid_info->gl_redirect_dlid != 0) {
3190 msg->im_local_addr.ia_remote_lid =
3191 gid_info->gl_redirect_dlid;
3192 }
3193
3194 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
3195 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
3196 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
3197 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
3198 } else {
3199 msg->im_local_addr.ia_remote_qno = 1;
3200 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
3201 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
3202 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
3203 }
3204
3205 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
3206 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
3207 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
3208 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
3209 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
3210 hdr->Status = 0;
3211 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
3212
3213 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
3214 hdr->AttributeModifier = h2b32(attr);
3215
3216 if (attr == 0) {
3217 cb_args = &gid_info->gl_iou_cb_args;
3218 gid_info->gl_iou->iou_dc_valid = B_FALSE;
3219 cb_args->cb_ioc_num = 0;
3220 cb_args->cb_req_type = IBDM_REQ_TYPE_IOU_DIAGCODE;
3221 timeout_id = &gid_info->gl_timeout_id;
3222 } else {
3223 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attr - 1));
3224 ioc->ioc_dc_valid = B_FALSE;
3225 cb_args = &ioc->ioc_dc_cb_args;
3226 cb_args->cb_ioc_num = attr - 1;
3227 cb_args->cb_req_type = IBDM_REQ_TYPE_IOC_DIAGCODE;
3228 timeout_id = &ioc->ioc_dc_timeout_id;
3229 }
3230 cb_args->cb_gid_info = gid_info;
3231 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
3232 cb_args->cb_srvents_start = 0;
3233
3234 mutex_enter(&gid_info->gl_mutex);
3235 *timeout_id = timeout(ibdm_pkt_timeout_hdlr,
3236 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
3237 mutex_exit(&gid_info->gl_mutex);
3238
3239 IBTF_DPRINTF_L5("ibdm", "\tget_diagcode:"
3240 "timeout %x, ioc %d", *timeout_id, cb_args->cb_ioc_num);
3241
3242 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
3243 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
3244 IBTF_DPRINTF_L2("ibdm", "\tget_diagcode: ibmf send failed");
3245 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
3246 }
3247 return (IBDM_SUCCESS);
3248 }
3249
3250 /*
3251 * ibdm_handle_diagcode:
3252 * Process the DiagCode MAD response and update local DM
3253 * data structure.
3254 */
3255 static void
3256 ibdm_handle_diagcode(ibmf_msg_t *ibmf_msg,
3257 ibdm_dp_gidinfo_t *gid_info, int *flag)
3258 {
3259 uint16_t attrmod, *diagcode;
3260 ibdm_iou_info_t *iou;
3261 ibdm_ioc_info_t *ioc;
3262 timeout_id_t timeout_id;
3263 ibdm_timeout_cb_args_t *cb_args;
3264
3265 diagcode = (uint16_t *)ibmf_msg->im_msgbufs_recv.im_bufs_cl_data;
3266
3267 mutex_enter(&gid_info->gl_mutex);
3268 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(ibmf_msg);
3269 iou = gid_info->gl_iou;
3270 if (attrmod == 0) {
3271 if (iou->iou_dc_valid != B_FALSE) {
3272 (*flag) |= IBDM_IBMF_PKT_DUP_RESP;
3273 IBTF_DPRINTF_L4("ibdm",
3274 "\thandle_diagcode: Duplicate IOU DiagCode");
3275 mutex_exit(&gid_info->gl_mutex);
3276 return;
3277 }
3278 cb_args = &gid_info->gl_iou_cb_args;
3279 cb_args->cb_req_type = 0;
3280 iou->iou_diagcode = b2h16(*diagcode);
3281 iou->iou_dc_valid = B_TRUE;
3282 if (gid_info->gl_timeout_id) {
3283 timeout_id = gid_info->gl_timeout_id;
3284 mutex_exit(&gid_info->gl_mutex);
3285 IBTF_DPRINTF_L5("ibdm", "\thandle_diagcode: "
3286 "gl_timeout_id = 0x%x", timeout_id);
3287 if (untimeout(timeout_id) == -1) {
3288 IBTF_DPRINTF_L2("ibdm", "handle_diagcode: "
3289 "untimeout gl_timeout_id failed");
3290 }
3291 mutex_enter(&gid_info->gl_mutex);
3292 gid_info->gl_timeout_id = 0;
3293 }
3294 } else {
3295 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod - 1));
3296 if (ioc->ioc_dc_valid != B_FALSE) {
3297 (*flag) |= IBDM_IBMF_PKT_DUP_RESP;
3298 IBTF_DPRINTF_L4("ibdm",
3299 "\thandle_diagcode: Duplicate IOC DiagCode");
3300 mutex_exit(&gid_info->gl_mutex);
3301 return;
3302 }
3303 cb_args = &ioc->ioc_dc_cb_args;
3304 cb_args->cb_req_type = 0;
3305 ioc->ioc_diagcode = b2h16(*diagcode);
3306 ioc->ioc_dc_valid = B_TRUE;
3307 timeout_id = iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id;
3308 if (timeout_id) {
3309 iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id = 0;
3310 mutex_exit(&gid_info->gl_mutex);
3311 IBTF_DPRINTF_L5("ibdm", "handle_diagcode: "
3312 "timeout_id = 0x%x", timeout_id);
3313 if (untimeout(timeout_id) == -1) {
3314 IBTF_DPRINTF_L2("ibdm", "\thandle_diagcode: "
3315 "untimeout ioc_dc_timeout_id failed");
3316 }
3317 mutex_enter(&gid_info->gl_mutex);
3318 }
3319 }
3320 mutex_exit(&gid_info->gl_mutex);
3321
3322 IBTF_DPRINTF_L4("ibdm", "\thandle_diagcode: DiagCode : 0x%x"
3323 "attrmod : 0x%x", b2h16(*diagcode), attrmod);
3324 }
3325
3326
3327 /*
3328 * ibdm_is_ioc_present()
3329 * Return ibdm_ioc_info_t if IOC guid is found in the global gid list
3330 */
3331 static ibdm_ioc_info_t *
3332 ibdm_is_ioc_present(ib_guid_t ioc_guid,
3333 ibdm_dp_gidinfo_t *gid_info, int *flag)
3334 {
3335 int ii;
3336 ibdm_ioc_info_t *ioc;
3337 ibdm_dp_gidinfo_t *head;
3338 ib_dm_io_unitinfo_t *iou;
3339
3340 mutex_enter(&ibdm.ibdm_mutex);
3341 head = ibdm.ibdm_dp_gidlist_head;
3342 while (head) {
3343 mutex_enter(&head->gl_mutex);
3344 if (head->gl_iou == NULL) {
3345 mutex_exit(&head->gl_mutex);
3346 head = head->gl_next;
3347 continue;
3348 }
3349 iou = &head->gl_iou->iou_info;
3350 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
3351 ioc = IBDM_GIDINFO2IOCINFO(head, ii);
3352 if ((ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) &&
3353 (ioc->ioc_profile.ioc_guid == ioc_guid)) {
3354 if (gid_info == head) {
3355 *flag |= IBDM_IBMF_PKT_DUP_RESP;
3356 } else if (ibdm_check_dgid(head->gl_dgid_lo,
3357 head->gl_dgid_hi) != NULL) {
3358 IBTF_DPRINTF_L4("ibdm", "\tis_ioc_"
3359 "present: gid not present");
3360 ibdm_add_to_gl_gid(gid_info, head);
3361 }
3362 mutex_exit(&head->gl_mutex);
3363 mutex_exit(&ibdm.ibdm_mutex);
3364 return (ioc);
3365 }
3366 }
3367 mutex_exit(&head->gl_mutex);
3368 head = head->gl_next;
3369 }
3370 mutex_exit(&ibdm.ibdm_mutex);
3371 return (NULL);
3372 }
3373
3374
3375 /*
3376 * ibdm_ibmf_send_cb()
3377 * IBMF invokes this callback routine after posting the DM MAD to
3378 * the HCA.
3379 */
3380 /*ARGSUSED*/
3381 static void
3382 ibdm_ibmf_send_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *ibmf_msg, void *arg)
3383 {
3384 ibdm_dump_ibmf_msg(ibmf_msg, 1);
3385 ibdm_free_send_buffers(ibmf_msg);
3386 if (ibmf_free_msg(ibmf_hdl, &ibmf_msg) != IBMF_SUCCESS) {
3387 IBTF_DPRINTF_L4("ibdm",
3388 "\tibmf_send_cb: IBMF free msg failed");
3389 }
3390 }
3391
3392
3393 /*
3394 * ibdm_ibmf_recv_cb()
3395 * Invoked by the IBMF when a response to the one of the DM requests
3396 * is received.
3397 */
3398 /*ARGSUSED*/
3399 static void
3400 ibdm_ibmf_recv_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg)
3401 {
3402 ibdm_taskq_args_t *taskq_args;
3403
3404 /*
3405 * If the taskq enable is set then dispatch a taskq to process
3406 * the MAD, otherwise just process it on this thread
3407 */
3408 if (ibdm_taskq_enable != IBDM_ENABLE_TASKQ_HANDLING) {
3409 ibdm_process_incoming_mad(ibmf_hdl, msg, arg);
3410 return;
3411 }
3412
3413 /*
3414 * create a taskq and dispatch it to process the incoming MAD
3415 */
3416 taskq_args = kmem_alloc(sizeof (ibdm_taskq_args_t), KM_NOSLEEP);
3417 if (taskq_args == NULL) {
3418 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: kmem_alloc failed for"
3419 "taskq_args");
3420 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3421 IBTF_DPRINTF_L4("ibmf_recv_cb",
3422 "\tibmf_recv_cb: IBMF free msg failed");
3423 }
3424 return;
3425 }
3426 taskq_args->tq_ibmf_handle = ibmf_hdl;
3427 taskq_args->tq_ibmf_msg = msg;
3428 taskq_args->tq_args = arg;
3429
3430 if (taskq_dispatch(system_taskq, ibdm_recv_incoming_mad, taskq_args,
3431 TQ_NOSLEEP) == 0) {
3432 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: taskq_dispatch failed");
3433 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3434 IBTF_DPRINTF_L4("ibmf_recv_cb",
3435 "\tibmf_recv_cb: IBMF free msg failed");
3436 }
3437 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t));
3438 return;
3439 }
3440
3441 /* taskq_args are deleted in ibdm_recv_incoming_mad() */
3442 }
3443
3444
3445 void
3446 ibdm_recv_incoming_mad(void *args)
3447 {
3448 ibdm_taskq_args_t *taskq_args;
3449
3450 taskq_args = (ibdm_taskq_args_t *)args;
3451
3452 IBTF_DPRINTF_L4("ibdm", "\tibdm_recv_incoming_mad: "
3453 "Processing incoming MAD via taskq");
3454
3455 ibdm_process_incoming_mad(taskq_args->tq_ibmf_handle,
3456 taskq_args->tq_ibmf_msg, taskq_args->tq_args);
3457
3458 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t));
3459 }
3460
3461
3462 /*
3463 * Calls ibdm_process_incoming_mad with all function arguments extracted
3464 * from args
3465 */
3466 /*ARGSUSED*/
3467 static void
3468 ibdm_process_incoming_mad(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg)
3469 {
3470 int flag = 0;
3471 int ret;
3472 uint64_t transaction_id;
3473 ib_mad_hdr_t *hdr;
3474 ibdm_dp_gidinfo_t *gid_info = NULL;
3475
3476 IBTF_DPRINTF_L4("ibdm",
3477 "\tprocess_incoming_mad: ibmf hdl %p pkt %p", ibmf_hdl, msg);
3478 ibdm_dump_ibmf_msg(msg, 0);
3479
3480 /*
3481 * IBMF calls this routine for every DM MAD that arrives at this port.
3482 * But we handle only the responses for requests we sent. We drop all
3483 * the DM packets that does not have response bit set in the MAD
3484 * header(this eliminates all the requests sent to this port).
3485 * We handle only DM class version 1 MAD's
3486 */
3487 hdr = IBDM_IN_IBMFMSG_MADHDR(msg);
3488 if (ibdm_verify_mad_status(hdr) != IBDM_SUCCESS) {
3489 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3490 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: "
3491 "IBMF free msg failed DM request drop it");
3492 }
3493 return;
3494 }
3495
3496 transaction_id = b2h64(hdr->TransactionID);
3497
3498 mutex_enter(&ibdm.ibdm_mutex);
3499 gid_info = ibdm.ibdm_dp_gidlist_head;
3500 while (gid_info) {
3501 if ((gid_info->gl_transactionID &
3502 IBDM_GID_TRANSACTIONID_MASK) ==
3503 (transaction_id & IBDM_GID_TRANSACTIONID_MASK))
3504 break;
3505 gid_info = gid_info->gl_next;
3506 }
3507 mutex_exit(&ibdm.ibdm_mutex);
3508
3509 if (gid_info == NULL) {
3510 /* Drop the packet */
3511 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: transaction ID"
3512 " does not match: 0x%llx", transaction_id);
3513 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3514 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3515 "IBMF free msg failed DM request drop it");
3516 }
3517 return;
3518 }
3519
3520 /* Handle redirection for all the MAD's, except ClassPortInfo */
3521 if (((IBDM_IN_IBMFMSG_STATUS(msg) & MAD_STATUS_REDIRECT_REQUIRED)) &&
3522 (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO)) {
3523 ret = ibdm_handle_redirection(msg, gid_info, &flag);
3524 if (ret == IBDM_SUCCESS) {
3525 return;
3526 }
3527 } else {
3528 uint_t gl_state;
3529
3530 mutex_enter(&gid_info->gl_mutex);
3531 gl_state = gid_info->gl_state;
3532 mutex_exit(&gid_info->gl_mutex);
3533
3534 switch (gl_state) {
3535
3536 case IBDM_SET_CLASSPORTINFO:
3537 ibdm_handle_setclassportinfo(
3538 ibmf_hdl, msg, gid_info, &flag);
3539 break;
3540
3541 case IBDM_GET_CLASSPORTINFO:
3542 ibdm_handle_classportinfo(
3543 ibmf_hdl, msg, gid_info, &flag);
3544 break;
3545
3546 case IBDM_GET_IOUNITINFO:
3547 ibdm_handle_iounitinfo(ibmf_hdl, msg, gid_info, &flag);
3548 break;
3549
3550 case IBDM_GET_IOC_DETAILS:
3551 switch (IBDM_IN_IBMFMSG_ATTR(msg)) {
3552
3553 case IB_DM_ATTR_SERVICE_ENTRIES:
3554 ibdm_handle_srventry_mad(msg, gid_info, &flag);
3555 break;
3556
3557 case IB_DM_ATTR_IOC_CTRL_PROFILE:
3558 ibdm_handle_ioc_profile(
3559 ibmf_hdl, msg, gid_info, &flag);
3560 break;
3561
3562 case IB_DM_ATTR_DIAG_CODE:
3563 ibdm_handle_diagcode(msg, gid_info, &flag);
3564 break;
3565
3566 default:
3567 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3568 "Error state, wrong attribute :-(");
3569 (void) ibmf_free_msg(ibmf_hdl, &msg);
3570 return;
3571 }
3572 break;
3573 default:
3574 IBTF_DPRINTF_L2("ibdm",
3575 "process_incoming_mad: Dropping the packet"
3576 " gl_state %x", gl_state);
3577 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3578 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3579 "IBMF free msg failed DM request drop it");
3580 }
3581 return;
3582 }
3583 }
3584
3585 if ((flag & IBDM_IBMF_PKT_DUP_RESP) ||
3586 (flag & IBDM_IBMF_PKT_UNEXP_RESP)) {
3587 IBTF_DPRINTF_L2("ibdm",
3588 "\tprocess_incoming_mad:Dup/unexp resp : 0x%x", flag);
3589 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3590 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3591 "IBMF free msg failed DM request drop it");
3592 }
3593 return;
3594 }
3595
3596 mutex_enter(&gid_info->gl_mutex);
3597 if (gid_info->gl_pending_cmds < 1) {
3598 IBTF_DPRINTF_L2("ibdm",
3599 "\tprocess_incoming_mad: pending commands negative");
3600 }
3601 if (--gid_info->gl_pending_cmds) {
3602 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: "
3603 "gid_info %p pending cmds %d",
3604 gid_info, gid_info->gl_pending_cmds);
3605 mutex_exit(&gid_info->gl_mutex);
3606 } else {
3607 uint_t prev_state;
3608 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: Probing DONE");
3609 prev_state = gid_info->gl_state;
3610 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE;
3611 if (prev_state == IBDM_SET_CLASSPORTINFO) {
3612 IBTF_DPRINTF_L4("ibdm",
3613 "\tprocess_incoming_mad: "
3614 "Setclassportinfo for Cisco FC GW is done.");
3615 gid_info->gl_flag &= ~IBDM_CISCO_PROBE;
3616 gid_info->gl_flag |= IBDM_CISCO_PROBE_DONE;
3617 mutex_exit(&gid_info->gl_mutex);
3618 cv_broadcast(&gid_info->gl_probe_cv);
3619 } else {
3620 mutex_exit(&gid_info->gl_mutex);
3621 ibdm_notify_newgid_iocs(gid_info);
3622 mutex_enter(&ibdm.ibdm_mutex);
3623 if (--ibdm.ibdm_ngid_probes_in_progress == 0) {
3624 IBTF_DPRINTF_L4("ibdm",
3625 "\tprocess_incoming_mad: Wakeup");
3626 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
3627 cv_broadcast(&ibdm.ibdm_probe_cv);
3628 }
3629 mutex_exit(&ibdm.ibdm_mutex);
3630 }
3631 }
3632
3633 /*
3634 * Do not deallocate the IBMF packet if atleast one request
3635 * is posted. IBMF packet is reused.
3636 */
3637 if (!(flag & IBDM_IBMF_PKT_REUSED)) {
3638 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3639 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: "
3640 "IBMF free msg failed DM request drop it");
3641 }
3642 }
3643 }
3644
3645
3646 /*
3647 * ibdm_verify_mad_status()
3648 * Verifies the MAD status
3649 * Returns IBDM_SUCCESS if status is correct
3650 * Returns IBDM_FAILURE for bogus MAD status
3651 */
3652 static int
3653 ibdm_verify_mad_status(ib_mad_hdr_t *hdr)
3654 {
3655 int ret = 0;
3656
3657 if ((hdr->R_Method != IB_DM_DEVMGT_METHOD_GET_RESP) ||
3658 (hdr->ClassVersion != IB_DM_CLASS_VERSION_1)) {
3659 return (IBDM_FAILURE);
3660 }
3661
3662 if (b2h16(hdr->Status) == 0)
3663 ret = IBDM_SUCCESS;
3664 else if ((b2h16(hdr->Status) & 0x1f) == MAD_STATUS_REDIRECT_REQUIRED)
3665 ret = IBDM_SUCCESS;
3666 else {
3667 IBTF_DPRINTF_L2("ibdm",
3668 "\tverify_mad_status: Status : 0x%x", b2h16(hdr->Status));
3669 ret = IBDM_FAILURE;
3670 }
3671 return (ret);
3672 }
3673
3674
3675
3676 /*
3677 * ibdm_handle_redirection()
3678 * Returns IBDM_SUCCESS/IBDM_FAILURE
3679 */
3680 static int
3681 ibdm_handle_redirection(ibmf_msg_t *msg,
3682 ibdm_dp_gidinfo_t *gid_info, int *flag)
3683 {
3684 int attrmod, ioc_no, start;
3685 void *data;
3686 timeout_id_t *timeout_id;
3687 ib_mad_hdr_t *hdr;
3688 ibdm_ioc_info_t *ioc = NULL;
3689 ibdm_timeout_cb_args_t *cb_args;
3690 ib_mad_classportinfo_t *cpi;
3691
3692 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Enter");
3693 mutex_enter(&gid_info->gl_mutex);
3694 switch (gid_info->gl_state) {
3695 case IBDM_GET_IOUNITINFO:
3696 cb_args = &gid_info->gl_iou_cb_args;
3697 timeout_id = &gid_info->gl_timeout_id;
3698 break;
3699
3700 case IBDM_GET_IOC_DETAILS:
3701 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg);
3702 switch (IBDM_IN_IBMFMSG_ATTR(msg)) {
3703
3704 case IB_DM_ATTR_DIAG_CODE:
3705 if (attrmod == 0) {
3706 cb_args = &gid_info->gl_iou_cb_args;
3707 timeout_id = &gid_info->gl_timeout_id;
3708 break;
3709 }
3710 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) {
3711 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3712 "IOC# Out of range %d", attrmod);
3713 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3714 mutex_exit(&gid_info->gl_mutex);
3715 return (IBDM_FAILURE);
3716 }
3717 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1));
3718 cb_args = &ioc->ioc_dc_cb_args;
3719 timeout_id = &ioc->ioc_dc_timeout_id;
3720 break;
3721
3722 case IB_DM_ATTR_IOC_CTRL_PROFILE:
3723 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) {
3724 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3725 "IOC# Out of range %d", attrmod);
3726 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3727 mutex_exit(&gid_info->gl_mutex);
3728 return (IBDM_FAILURE);
3729 }
3730 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1));
3731 cb_args = &ioc->ioc_cb_args;
3732 timeout_id = &ioc->ioc_timeout_id;
3733 break;
3734
3735 case IB_DM_ATTR_SERVICE_ENTRIES:
3736 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK);
3737 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) {
3738 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3739 "IOC# Out of range %d", ioc_no);
3740 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3741 mutex_exit(&gid_info->gl_mutex);
3742 return (IBDM_FAILURE);
3743 }
3744 start = (attrmod & IBDM_8_BIT_MASK);
3745 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1));
3746 if (start > ioc->ioc_profile.ioc_service_entries) {
3747 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3748 " SE index Out of range %d", start);
3749 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3750 mutex_exit(&gid_info->gl_mutex);
3751 return (IBDM_FAILURE);
3752 }
3753 cb_args = &ioc->ioc_serv[start].se_cb_args;
3754 timeout_id = &ioc->ioc_serv[start].se_timeout_id;
3755 break;
3756
3757 default:
3758 /* ERROR State */
3759 IBTF_DPRINTF_L2("ibdm",
3760 "\thandle_redirection: wrong attribute :-(");
3761 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3762 mutex_exit(&gid_info->gl_mutex);
3763 return (IBDM_FAILURE);
3764 }
3765 break;
3766 default:
3767 /* ERROR State */
3768 IBTF_DPRINTF_L2("ibdm",
3769 "\thandle_redirection: Error state :-(");
3770 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3771 mutex_exit(&gid_info->gl_mutex);
3772 return (IBDM_FAILURE);
3773 }
3774 if ((*timeout_id) != 0) {
3775 mutex_exit(&gid_info->gl_mutex);
3776 if (untimeout(*timeout_id) == -1) {
3777 IBTF_DPRINTF_L2("ibdm", "\thandle_redirection: "
3778 "untimeout failed %x", *timeout_id);
3779 } else {
3780 IBTF_DPRINTF_L5("ibdm",
3781 "\thandle_redirection: timeout %x", *timeout_id);
3782 }
3783 mutex_enter(&gid_info->gl_mutex);
3784 *timeout_id = 0;
3785 }
3786
3787 data = msg->im_msgbufs_recv.im_bufs_cl_data;
3788 cpi = (ib_mad_classportinfo_t *)data;
3789
3790 gid_info->gl_resp_timeout =
3791 (b2h32(cpi->RespTimeValue) & 0x1F);
3792
3793 gid_info->gl_redirected = B_TRUE;
3794 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID);
3795 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff);
3796 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key);
3797 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key);
3798 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi);
3799 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo);
3800 gid_info->gl_redirectSL = cpi->RedirectSL;
3801
3802 if (gid_info->gl_redirect_dlid != 0) {
3803 msg->im_local_addr.ia_remote_lid =
3804 gid_info->gl_redirect_dlid;
3805 }
3806 ibdm_bump_transactionID(gid_info);
3807 mutex_exit(&gid_info->gl_mutex);
3808
3809 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg, *hdr))
3810 ibdm_alloc_send_buffers(msg);
3811
3812 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
3813 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
3814 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
3815 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
3816 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
3817 hdr->Status = 0;
3818 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
3819 hdr->AttributeID =
3820 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID;
3821 hdr->AttributeModifier =
3822 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier;
3823 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg, *hdr))
3824
3825 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
3826 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
3827 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
3828 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
3829
3830 mutex_enter(&gid_info->gl_mutex);
3831 *timeout_id = timeout(ibdm_pkt_timeout_hdlr,
3832 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
3833 mutex_exit(&gid_info->gl_mutex);
3834
3835 IBTF_DPRINTF_L5("ibdm", "\thandle_redirect:"
3836 "timeout %x", *timeout_id);
3837
3838 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
3839 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
3840 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection:"
3841 "message transport failed");
3842 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
3843 }
3844 (*flag) |= IBDM_IBMF_PKT_REUSED;
3845 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Exit");
3846 return (IBDM_SUCCESS);
3847 }
3848
3849
3850 /*
3851 * ibdm_pkt_timeout_hdlr
3852 * This timeout handler is registed for every IBMF packet that is
3853 * sent through the IBMF. It gets called when no response is received
3854 * within the specified time for the packet. No retries for the failed
3855 * commands currently. Drops the failed IBMF packet and update the
3856 * pending list commands.
3857 */
3858 static void
3859 ibdm_pkt_timeout_hdlr(void *arg)
3860 {
3861 ibdm_iou_info_t *iou;
3862 ibdm_ioc_info_t *ioc;
3863 ibdm_timeout_cb_args_t *cb_args = arg;
3864 ibdm_dp_gidinfo_t *gid_info;
3865 int srv_ent;
3866 uint_t new_gl_state;
3867
3868 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: gid_info: %p "
3869 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3870 cb_args->cb_req_type, cb_args->cb_ioc_num,
3871 cb_args->cb_srvents_start);
3872
3873 gid_info = cb_args->cb_gid_info;
3874 mutex_enter(&gid_info->gl_mutex);
3875
3876 if ((gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) ||
3877 (cb_args->cb_req_type == 0)) {
3878
3879 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: req completed"
3880 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_req_type,
3881 cb_args->cb_ioc_num, cb_args->cb_srvents_start);
3882
3883 if (gid_info->gl_timeout_id)
3884 gid_info->gl_timeout_id = 0;
3885 mutex_exit(&gid_info->gl_mutex);
3886 return;
3887 }
3888 if (cb_args->cb_retry_count) {
3889 cb_args->cb_retry_count--;
3890 /*
3891 * A new timeout_id is set inside ibdm_retry_command().
3892 * When the function returns an error, the timeout_id
3893 * is reset (to zero) in the switch statement below.
3894 */
3895 if (ibdm_retry_command(cb_args) == IBDM_SUCCESS) {
3896 mutex_exit(&gid_info->gl_mutex);
3897 return;
3898 }
3899 cb_args->cb_retry_count = 0;
3900 }
3901
3902 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: command failed: gid %p"
3903 " rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3904 cb_args->cb_req_type, cb_args->cb_ioc_num,
3905 cb_args->cb_srvents_start);
3906
3907 switch (cb_args->cb_req_type) {
3908
3909 case IBDM_REQ_TYPE_CLASSPORTINFO:
3910 case IBDM_REQ_TYPE_IOUINFO:
3911 new_gl_state = IBDM_GID_PROBING_FAILED;
3912 if (gid_info->gl_timeout_id)
3913 gid_info->gl_timeout_id = 0;
3914 break;
3915
3916 case IBDM_REQ_TYPE_IOCINFO:
3917 new_gl_state = IBDM_GID_PROBING_COMPLETE;
3918 iou = gid_info->gl_iou;
3919 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3920 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
3921 if (ioc->ioc_timeout_id)
3922 ioc->ioc_timeout_id = 0;
3923 break;
3924
3925 case IBDM_REQ_TYPE_SRVENTS:
3926 new_gl_state = IBDM_GID_PROBING_COMPLETE;
3927 iou = gid_info->gl_iou;
3928 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3929 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
3930 srv_ent = cb_args->cb_srvents_start;
3931 if (ioc->ioc_serv[srv_ent].se_timeout_id)
3932 ioc->ioc_serv[srv_ent].se_timeout_id = 0;
3933 break;
3934
3935 case IBDM_REQ_TYPE_IOU_DIAGCODE:
3936 new_gl_state = IBDM_GID_PROBING_COMPLETE;
3937 iou = gid_info->gl_iou;
3938 iou->iou_dc_valid = B_FALSE;
3939 if (gid_info->gl_timeout_id)
3940 gid_info->gl_timeout_id = 0;
3941 break;
3942
3943 case IBDM_REQ_TYPE_IOC_DIAGCODE:
3944 new_gl_state = IBDM_GID_PROBING_COMPLETE;
3945 iou = gid_info->gl_iou;
3946 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3947 ioc->ioc_dc_valid = B_FALSE;
3948 if (ioc->ioc_dc_timeout_id)
3949 ioc->ioc_dc_timeout_id = 0;
3950 break;
3951
3952 default: /* ERROR State */
3953 new_gl_state = IBDM_GID_PROBING_FAILED;
3954 if (gid_info->gl_timeout_id)
3955 gid_info->gl_timeout_id = 0;
3956 IBTF_DPRINTF_L2("ibdm",
3957 "\tpkt_timeout_hdlr: wrong request type.");
3958 break;
3959 }
3960
3961 --gid_info->gl_pending_cmds; /* decrease the counter */
3962
3963 if (gid_info->gl_pending_cmds == 0) {
3964 gid_info->gl_state = new_gl_state;
3965 mutex_exit(&gid_info->gl_mutex);
3966 /*
3967 * Delete this gid_info if the gid probe fails.
3968 */
3969 if (new_gl_state == IBDM_GID_PROBING_FAILED) {
3970 ibdm_delete_glhca_list(gid_info);
3971 }
3972 ibdm_notify_newgid_iocs(gid_info);
3973 mutex_enter(&ibdm.ibdm_mutex);
3974 if (--ibdm.ibdm_ngid_probes_in_progress == 0) {
3975 IBTF_DPRINTF_L4("ibdm", "\tpkt_timeout_hdlr: Wakeup");
3976 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
3977 cv_broadcast(&ibdm.ibdm_probe_cv);
3978 }
3979 mutex_exit(&ibdm.ibdm_mutex);
3980 } else {
3981 /*
3982 * Reset gl_pending_cmd if the extra timeout happens since
3983 * gl_pending_cmd becomes negative as a result.
3984 */
3985 if (gid_info->gl_pending_cmds < 0) {
3986 gid_info->gl_pending_cmds = 0;
3987 IBTF_DPRINTF_L2("ibdm",
3988 "\tpkt_timeout_hdlr: extra timeout request."
3989 " reset gl_pending_cmds");
3990 }
3991 mutex_exit(&gid_info->gl_mutex);
3992 /*
3993 * Delete this gid_info if the gid probe fails.
3994 */
3995 if (new_gl_state == IBDM_GID_PROBING_FAILED) {
3996 ibdm_delete_glhca_list(gid_info);
3997 }
3998 }
3999 }
4000
4001
4002 /*
4003 * ibdm_retry_command()
4004 * Retries the failed command.
4005 * Returns IBDM_FAILURE/IBDM_SUCCESS
4006 */
4007 static int
4008 ibdm_retry_command(ibdm_timeout_cb_args_t *cb_args)
4009 {
4010 int ret;
4011 ibmf_msg_t *msg;
4012 ib_mad_hdr_t *hdr;
4013 ibdm_dp_gidinfo_t *gid_info = cb_args->cb_gid_info;
4014 timeout_id_t *timeout_id;
4015 ibdm_ioc_info_t *ioc;
4016 int ioc_no;
4017 ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
4018
4019 IBTF_DPRINTF_L2("ibdm", "\tretry_command: gid_info: %p "
4020 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
4021 cb_args->cb_req_type, cb_args->cb_ioc_num,
4022 cb_args->cb_srvents_start);
4023
4024 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, &msg);
4025
4026
4027 /*
4028 * Reset the gid if alloc_msg failed with BAD_HANDLE
4029 * ibdm_reset_gidinfo reinits the gid_info
4030 */
4031 if (ret == IBMF_BAD_HANDLE) {
4032 IBTF_DPRINTF_L3(ibdm_string, "\tretry_command: gid %p hdl bad",
4033 gid_info);
4034
4035 mutex_exit(&gid_info->gl_mutex);
4036 ibdm_reset_gidinfo(gid_info);
4037 mutex_enter(&gid_info->gl_mutex);
4038
4039 /* Retry alloc */
4040 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP,
4041 &msg);
4042 }
4043
4044 if (ret != IBDM_SUCCESS) {
4045 IBTF_DPRINTF_L2("ibdm", "\tretry_command: alloc failed: %p "
4046 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
4047 cb_args->cb_req_type, cb_args->cb_ioc_num,
4048 cb_args->cb_srvents_start);
4049 return (IBDM_FAILURE);
4050 }
4051
4052 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
4053 ibdm_alloc_send_buffers(msg);
4054 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
4055
4056 ibdm_bump_transactionID(gid_info);
4057
4058 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
4059 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
4060 if (gid_info->gl_redirected == B_TRUE) {
4061 if (gid_info->gl_redirect_dlid != 0) {
4062 msg->im_local_addr.ia_remote_lid =
4063 gid_info->gl_redirect_dlid;
4064 }
4065 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
4066 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
4067 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
4068 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
4069 } else {
4070 msg->im_local_addr.ia_remote_qno = 1;
4071 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
4072 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
4073 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
4074 }
4075 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
4076 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr))
4077 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
4078 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
4079 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
4080 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
4081 hdr->Status = 0;
4082 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
4083
4084 switch (cb_args->cb_req_type) {
4085 case IBDM_REQ_TYPE_CLASSPORTINFO:
4086 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO);
4087 hdr->AttributeModifier = 0;
4088 timeout_id = &gid_info->gl_timeout_id;
4089 break;
4090 case IBDM_REQ_TYPE_IOUINFO:
4091 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO);
4092 hdr->AttributeModifier = 0;
4093 timeout_id = &gid_info->gl_timeout_id;
4094 break;
4095 case IBDM_REQ_TYPE_IOCINFO:
4096 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
4097 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1);
4098 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num);
4099 timeout_id = &ioc->ioc_timeout_id;
4100 break;
4101 case IBDM_REQ_TYPE_SRVENTS:
4102 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES);
4103 ibdm_fill_srv_attr_mod(hdr, cb_args);
4104 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num);
4105 timeout_id =
4106 &ioc->ioc_serv[cb_args->cb_srvents_start].se_timeout_id;
4107 break;
4108 case IBDM_REQ_TYPE_IOU_DIAGCODE:
4109 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
4110 hdr->AttributeModifier = 0;
4111 timeout_id = &gid_info->gl_timeout_id;
4112 break;
4113 case IBDM_REQ_TYPE_IOC_DIAGCODE:
4114 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
4115 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1);
4116 ioc_no = cb_args->cb_ioc_num;
4117 ioc = &gid_info->gl_iou->iou_ioc_info[ioc_no];
4118 timeout_id = &ioc->ioc_dc_timeout_id;
4119 break;
4120 }
4121 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*hdr))
4122
4123 *timeout_id = timeout(ibdm_pkt_timeout_hdlr,
4124 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
4125
4126 mutex_exit(&gid_info->gl_mutex);
4127
4128 IBTF_DPRINTF_L5("ibdm", "\tretry_command: %p,%x,%d,%d:"
4129 "timeout %x", cb_args->cb_req_type, cb_args->cb_ioc_num,
4130 cb_args->cb_srvents_start, *timeout_id);
4131
4132 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl,
4133 gid_info->gl_qp_hdl, msg, NULL, ibdm_ibmf_send_cb,
4134 cb_args, 0) != IBMF_SUCCESS) {
4135 IBTF_DPRINTF_L2("ibdm", "\tretry_command: send failed: %p "
4136 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
4137 cb_args->cb_req_type, cb_args->cb_ioc_num,
4138 cb_args->cb_srvents_start);
4139 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
4140 }
4141 mutex_enter(&gid_info->gl_mutex);
4142 return (IBDM_SUCCESS);
4143 }
4144
4145
4146 /*
4147 * ibdm_update_ioc_port_gidlist()
4148 */
4149 static void
4150 ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *dest,
4151 ibdm_dp_gidinfo_t *gid_info)
4152 {
4153 int ii, ngid_ents;
4154 ibdm_gid_t *tmp;
4155 ibdm_hca_list_t *gid_hca_head, *temp;
4156 ibdm_hca_list_t *ioc_head = NULL;
4157 ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
4158
4159 IBTF_DPRINTF_L5("ibdm", "\tupdate_ioc_port_gidlist: Enter");
4160
4161 ngid_ents = gid_info->gl_ngids;
4162 dest->ioc_nportgids = ngid_ents;
4163 dest->ioc_gid_list = kmem_zalloc(sizeof (ibdm_gid_t) *
4164 ngid_ents, KM_SLEEP);
4165 tmp = gid_info->gl_gid;
4166 for (ii = 0; (ii < ngid_ents) && (tmp); ii++) {
4167 dest->ioc_gid_list[ii].gid_dgid_hi = tmp->gid_dgid_hi;
4168 dest->ioc_gid_list[ii].gid_dgid_lo = tmp->gid_dgid_lo;
4169 tmp = tmp->gid_next;
4170 }
4171
4172 gid_hca_head = gid_info->gl_hca_list;
4173 while (gid_hca_head) {
4174 temp = ibdm_dup_hca_attr(gid_hca_head);
4175 temp->hl_next = ioc_head;
4176 ioc_head = temp;
4177 gid_hca_head = gid_hca_head->hl_next;
4178 }
4179 dest->ioc_hca_list = ioc_head;
4180 }
4181
4182
4183 /*
4184 * ibdm_alloc_send_buffers()
4185 * Allocates memory for the IBMF send buffer to send and/or receive
4186 * the Device Management MAD packet.
4187 */
4188 static void
4189 ibdm_alloc_send_buffers(ibmf_msg_t *msgp)
4190 {
4191 msgp->im_msgbufs_send.im_bufs_mad_hdr =
4192 kmem_zalloc(IBDM_MAD_SIZE, KM_SLEEP);
4193
4194 msgp->im_msgbufs_send.im_bufs_cl_hdr = (uchar_t *)
4195 msgp->im_msgbufs_send.im_bufs_mad_hdr + sizeof (ib_mad_hdr_t);
4196 msgp->im_msgbufs_send.im_bufs_cl_hdr_len = IBDM_DM_MAD_HDR_SZ;
4197
4198 msgp->im_msgbufs_send.im_bufs_cl_data =
4199 ((char *)msgp->im_msgbufs_send.im_bufs_cl_hdr + IBDM_DM_MAD_HDR_SZ);
4200 msgp->im_msgbufs_send.im_bufs_cl_data_len =
4201 IBDM_MAD_SIZE - sizeof (ib_mad_hdr_t) - IBDM_DM_MAD_HDR_SZ;
4202 }
4203
4204
4205 /*
4206 * ibdm_alloc_send_buffers()
4207 * De-allocates memory for the IBMF send buffer
4208 */
4209 static void
4210 ibdm_free_send_buffers(ibmf_msg_t *msgp)
4211 {
4212 if (msgp->im_msgbufs_send.im_bufs_mad_hdr != NULL)
4213 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, IBDM_MAD_SIZE);
4214 }
4215
4216 /*
4217 * ibdm_probe_ioc()
4218 * 1. Gets the node records for the port GUID. This detects all the port
4219 * to the IOU.
4220 * 2. Selectively probes all the IOC, given it's node GUID
4221 * 3. In case of reprobe, only the IOC to be reprobed is send the IOC
4222 * Controller Profile asynchronously
4223 */
4224 /*ARGSUSED*/
4225 static void
4226 ibdm_probe_ioc(ib_guid_t nodeguid, ib_guid_t ioc_guid, int reprobe_flag)
4227 {
4228 int ii, nrecords;
4229 size_t nr_len = 0, pi_len = 0;
4230 ib_gid_t sgid, dgid;
4231 ibdm_hca_list_t *hca_list = NULL;
4232 sa_node_record_t *nr, *tmp;
4233 ibdm_port_attr_t *port = NULL;
4234 ibdm_dp_gidinfo_t *reprobe_gid, *new_gid, *node_gid;
4235 ibdm_dp_gidinfo_t *temp_gidinfo;
4236 ibdm_gid_t *temp_gid;
4237 sa_portinfo_record_t *pi;
4238
4239 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc(%llx, %llx, %x): Begin",
4240 nodeguid, ioc_guid, reprobe_flag);
4241
4242 /* Rescan the GID list for any removed GIDs for reprobe */
4243 if (reprobe_flag)
4244 ibdm_rescan_gidlist(&ioc_guid);
4245
4246 mutex_enter(&ibdm.ibdm_hl_mutex);
4247 for (ibdm_get_next_port(&hca_list, &port, 1); port;
4248 ibdm_get_next_port(&hca_list, &port, 1)) {
4249 reprobe_gid = new_gid = node_gid = NULL;
4250
4251 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, nodeguid);
4252 if (nr == NULL) {
4253 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc: no records");
4254 continue;
4255 }
4256 nrecords = (nr_len / sizeof (sa_node_record_t));
4257 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) {
4258 if ((pi = ibdm_get_portinfo(
4259 port->pa_sa_hdl, &pi_len, tmp->LID)) == NULL) {
4260 IBTF_DPRINTF_L4("ibdm",
4261 "\tibdm_get_portinfo: no portinfo recs");
4262 continue;
4263 }
4264
4265 /*
4266 * If Device Management is not supported on
4267 * this port, skip the rest.
4268 */
4269 if (!(pi->PortInfo.CapabilityMask &
4270 SM_CAP_MASK_IS_DM_SUPPD)) {
4271 kmem_free(pi, pi_len);
4272 continue;
4273 }
4274
4275 /*
4276 * For reprobes: Check if GID, already in
4277 * the list. If so, set the state to SKIPPED
4278 */
4279 if (((temp_gidinfo = ibdm_find_gid(nodeguid,
4280 tmp->NodeInfo.PortGUID)) != NULL) &&
4281 temp_gidinfo->gl_state ==
4282 IBDM_GID_PROBING_COMPLETE) {
4283 ASSERT(reprobe_gid == NULL);
4284 ibdm_addto_glhcalist(temp_gidinfo,
4285 hca_list);
4286 reprobe_gid = temp_gidinfo;
4287 kmem_free(pi, pi_len);
4288 continue;
4289 } else if (temp_gidinfo != NULL) {
4290 kmem_free(pi, pi_len);
4291 ibdm_addto_glhcalist(temp_gidinfo,
4292 hca_list);
4293 continue;
4294 }
4295
4296 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : "
4297 "create_gid : prefix %llx, guid %llx\n",
4298 pi->PortInfo.GidPrefix,
4299 tmp->NodeInfo.PortGUID);
4300
4301 sgid.gid_prefix = port->pa_sn_prefix;
4302 sgid.gid_guid = port->pa_port_guid;
4303 dgid.gid_prefix = pi->PortInfo.GidPrefix;
4304 dgid.gid_guid = tmp->NodeInfo.PortGUID;
4305 new_gid = ibdm_create_gid_info(port, sgid,
4306 dgid);
4307 if (new_gid == NULL) {
4308 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
4309 "create_gid_info failed\n");
4310 kmem_free(pi, pi_len);
4311 continue;
4312 }
4313 if (node_gid == NULL) {
4314 node_gid = new_gid;
4315 ibdm_add_to_gl_gid(node_gid, node_gid);
4316 } else {
4317 IBTF_DPRINTF_L4("ibdm",
4318 "\tprobe_ioc: new gid");
4319 temp_gid = kmem_zalloc(
4320 sizeof (ibdm_gid_t), KM_SLEEP);
4321 temp_gid->gid_dgid_hi =
4322 new_gid->gl_dgid_hi;
4323 temp_gid->gid_dgid_lo =
4324 new_gid->gl_dgid_lo;
4325 temp_gid->gid_next = node_gid->gl_gid;
4326 node_gid->gl_gid = temp_gid;
4327 node_gid->gl_ngids++;
4328 }
4329 new_gid->gl_is_dm_capable = B_TRUE;
4330 new_gid->gl_nodeguid = nodeguid;
4331 new_gid->gl_portguid = dgid.gid_guid;
4332 ibdm_addto_glhcalist(new_gid, hca_list);
4333
4334 /*
4335 * Set the state to skipped as all these
4336 * gids point to the same node.
4337 * We (re)probe only one GID below and reset
4338 * state appropriately
4339 */
4340 new_gid->gl_state = IBDM_GID_PROBING_SKIPPED;
4341 new_gid->gl_devid = (*tmp).NodeInfo.DeviceID;
4342 kmem_free(pi, pi_len);
4343 }
4344 kmem_free(nr, nr_len);
4345
4346 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : reprobe_flag %d "
4347 "reprobe_gid %p new_gid %p node_gid %p",
4348 reprobe_flag, reprobe_gid, new_gid, node_gid);
4349
4350 if (reprobe_flag != 0 && reprobe_gid != NULL) {
4351 int niocs, jj;
4352 ibdm_ioc_info_t *tmp_ioc;
4353 int ioc_matched = 0;
4354
4355 mutex_exit(&ibdm.ibdm_hl_mutex);
4356 mutex_enter(&reprobe_gid->gl_mutex);
4357 reprobe_gid->gl_state = IBDM_GET_IOC_DETAILS;
4358 niocs =
4359 reprobe_gid->gl_iou->iou_info.iou_num_ctrl_slots;
4360 reprobe_gid->gl_pending_cmds++;
4361 mutex_exit(&reprobe_gid->gl_mutex);
4362
4363 for (jj = 0; jj < niocs; jj++) {
4364 tmp_ioc =
4365 IBDM_GIDINFO2IOCINFO(reprobe_gid, jj);
4366 if (tmp_ioc->ioc_profile.ioc_guid != ioc_guid)
4367 continue;
4368
4369 ioc_matched = 1;
4370
4371 /*
4372 * Explicitly set gl_reprobe_flag to 0 so that
4373 * IBnex is not notified on completion
4374 */
4375 mutex_enter(&reprobe_gid->gl_mutex);
4376 reprobe_gid->gl_reprobe_flag = 0;
4377 mutex_exit(&reprobe_gid->gl_mutex);
4378
4379 mutex_enter(&ibdm.ibdm_mutex);
4380 ibdm.ibdm_ngid_probes_in_progress++;
4381 mutex_exit(&ibdm.ibdm_mutex);
4382 if (ibdm_send_ioc_profile(reprobe_gid, jj) !=
4383 IBDM_SUCCESS) {
4384 IBTF_DPRINTF_L4("ibdm",
4385 "\tprobe_ioc: "
4386 "send_ioc_profile failed "
4387 "for ioc %d", jj);
4388 ibdm_gid_decr_pending(reprobe_gid);
4389 break;
4390 }
4391 mutex_enter(&ibdm.ibdm_mutex);
4392 ibdm_wait_probe_completion();
4393 mutex_exit(&ibdm.ibdm_mutex);
4394 break;
4395 }
4396 if (ioc_matched == 0)
4397 ibdm_gid_decr_pending(reprobe_gid);
4398 else {
4399 mutex_enter(&ibdm.ibdm_hl_mutex);
4400 break;
4401 }
4402 } else if (new_gid != NULL) {
4403 mutex_exit(&ibdm.ibdm_hl_mutex);
4404 node_gid = node_gid ? node_gid : new_gid;
4405
4406 /*
4407 * New or reinserted GID : Enable notification
4408 * to IBnex
4409 */
4410 mutex_enter(&node_gid->gl_mutex);
4411 node_gid->gl_reprobe_flag = 1;
4412 mutex_exit(&node_gid->gl_mutex);
4413
4414 ibdm_probe_gid(node_gid);
4415
4416 mutex_enter(&ibdm.ibdm_hl_mutex);
4417 }
4418 }
4419 mutex_exit(&ibdm.ibdm_hl_mutex);
4420 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : End\n");
4421 }
4422
4423
4424 /*
4425 * ibdm_probe_gid()
4426 * Selectively probes the GID
4427 */
4428 static void
4429 ibdm_probe_gid(ibdm_dp_gidinfo_t *gid_info)
4430 {
4431 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid:");
4432
4433 /*
4434 * A Cisco FC GW needs the special handling to get IOUnitInfo.
4435 */
4436 mutex_enter(&gid_info->gl_mutex);
4437 if (ibdm_is_cisco_switch(gid_info)) {
4438 gid_info->gl_pending_cmds++;
4439 gid_info->gl_state = IBDM_SET_CLASSPORTINFO;
4440 mutex_exit(&gid_info->gl_mutex);
4441
4442 if (ibdm_set_classportinfo(gid_info) != IBDM_SUCCESS) {
4443
4444 mutex_enter(&gid_info->gl_mutex);
4445 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
4446 --gid_info->gl_pending_cmds;
4447 mutex_exit(&gid_info->gl_mutex);
4448
4449 /* free the hca_list on this gid_info */
4450 ibdm_delete_glhca_list(gid_info);
4451 gid_info = gid_info->gl_next;
4452 return;
4453 }
4454
4455 mutex_enter(&gid_info->gl_mutex);
4456 ibdm_wait_cisco_probe_completion(gid_info);
4457
4458 IBTF_DPRINTF_L4("ibdm",
4459 "\tprobe_gid: CISCO Wakeup signal received");
4460 }
4461
4462 /* move on to the 'GET_CLASSPORTINFO' stage */
4463 gid_info->gl_pending_cmds++;
4464 gid_info->gl_state = IBDM_GET_CLASSPORTINFO;
4465 mutex_exit(&gid_info->gl_mutex);
4466
4467 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) {
4468
4469 mutex_enter(&gid_info->gl_mutex);
4470 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
4471 --gid_info->gl_pending_cmds;
4472 mutex_exit(&gid_info->gl_mutex);
4473
4474 /* free the hca_list on this gid_info */
4475 ibdm_delete_glhca_list(gid_info);
4476 gid_info = gid_info->gl_next;
4477 return;
4478 }
4479
4480 mutex_enter(&ibdm.ibdm_mutex);
4481 ibdm.ibdm_ngid_probes_in_progress++;
4482 gid_info = gid_info->gl_next;
4483 ibdm_wait_probe_completion();
4484 mutex_exit(&ibdm.ibdm_mutex);
4485
4486 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid: Wakeup signal received");
4487 }
4488
4489
4490 /*
4491 * ibdm_create_gid_info()
4492 * Allocates a gid_info structure and initializes
4493 * Returns pointer to the structure on success
4494 * and NULL on failure
4495 */
4496 static ibdm_dp_gidinfo_t *
4497 ibdm_create_gid_info(ibdm_port_attr_t *port, ib_gid_t sgid, ib_gid_t dgid)
4498 {
4499 uint8_t ii, npaths;
4500 sa_path_record_t *path;
4501 size_t len;
4502 ibdm_pkey_tbl_t *pkey_tbl;
4503 ibdm_dp_gidinfo_t *gid_info = NULL;
4504 int ret;
4505
4506 IBTF_DPRINTF_L4("ibdm", "\tcreate_gid_info: Begin");
4507 npaths = 1;
4508
4509 /* query for reversible paths */
4510 if (port->pa_sa_hdl)
4511 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl,
4512 sgid, dgid, IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0,
4513 &len, &path);
4514 else
4515 return (NULL);
4516
4517 if (ret == IBMF_SUCCESS && path) {
4518 ibdm_dump_path_info(path);
4519
4520 gid_info = kmem_zalloc(
4521 sizeof (ibdm_dp_gidinfo_t), KM_SLEEP);
4522 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL);
4523 cv_init(&gid_info->gl_probe_cv, NULL, CV_DRIVER, NULL);
4524 gid_info->gl_dgid_hi = path->DGID.gid_prefix;
4525 gid_info->gl_dgid_lo = path->DGID.gid_guid;
4526 gid_info->gl_sgid_hi = path->SGID.gid_prefix;
4527 gid_info->gl_sgid_lo = path->SGID.gid_guid;
4528 gid_info->gl_p_key = path->P_Key;
4529 gid_info->gl_sa_hdl = port->pa_sa_hdl;
4530 gid_info->gl_ibmf_hdl = port->pa_ibmf_hdl;
4531 gid_info->gl_slid = path->SLID;
4532 gid_info->gl_dlid = path->DLID;
4533 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID)
4534 << IBDM_GID_TRANSACTIONID_SHIFT;
4535 gid_info->gl_min_transactionID = gid_info->gl_transactionID;
4536 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1)
4537 << IBDM_GID_TRANSACTIONID_SHIFT;
4538 gid_info->gl_SL = path->SL;
4539
4540 gid_info->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT;
4541 for (ii = 0; ii < port->pa_npkeys; ii++) {
4542 if (port->pa_pkey_tbl == NULL)
4543 break;
4544
4545 pkey_tbl = &port->pa_pkey_tbl[ii];
4546 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) &&
4547 (pkey_tbl->pt_qp_hdl != NULL)) {
4548 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
4549 break;
4550 }
4551 }
4552 kmem_free(path, len);
4553
4554 /*
4555 * QP handle for GID not initialized. No matching Pkey
4556 * was found!! ibdm should *not* hit this case. Flag an
4557 * error and drop the GID if ibdm does encounter this.
4558 */
4559 if (gid_info->gl_qp_hdl == NULL) {
4560 IBTF_DPRINTF_L2(ibdm_string,
4561 "\tcreate_gid_info: No matching Pkey");
4562 ibdm_delete_gidinfo(gid_info);
4563 return (NULL);
4564 }
4565
4566 ibdm.ibdm_ngids++;
4567 if (ibdm.ibdm_dp_gidlist_head == NULL) {
4568 ibdm.ibdm_dp_gidlist_head = gid_info;
4569 ibdm.ibdm_dp_gidlist_tail = gid_info;
4570 } else {
4571 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info;
4572 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail;
4573 ibdm.ibdm_dp_gidlist_tail = gid_info;
4574 }
4575 }
4576
4577 return (gid_info);
4578 }
4579
4580
4581 /*
4582 * ibdm_get_node_records
4583 * Sends a SA query to get the NODE record
4584 * Returns pointer to the sa_node_record_t on success
4585 * and NULL on failure
4586 */
4587 static sa_node_record_t *
4588 ibdm_get_node_records(ibmf_saa_handle_t sa_hdl, size_t *length, ib_guid_t guid)
4589 {
4590 sa_node_record_t req, *resp = NULL;
4591 ibmf_saa_access_args_t args;
4592 int ret;
4593
4594 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: Begin");
4595
4596 bzero(&req, sizeof (sa_node_record_t));
4597 req.NodeInfo.NodeGUID = guid;
4598
4599 args.sq_attr_id = SA_NODERECORD_ATTRID;
4600 args.sq_access_type = IBMF_SAA_RETRIEVE;
4601 args.sq_component_mask = SA_NODEINFO_COMPMASK_NODEGUID;
4602 args.sq_template = &req;
4603 args.sq_callback = NULL;
4604 args.sq_callback_arg = NULL;
4605
4606 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp);
4607 if (ret != IBMF_SUCCESS) {
4608 IBTF_DPRINTF_L2("ibdm", "\tget_node_records:"
4609 " SA Retrieve Failed: %d", ret);
4610 return (NULL);
4611 }
4612 if ((resp == NULL) || (*length == 0)) {
4613 IBTF_DPRINTF_L2("ibdm", "\tget_node_records: No records");
4614 return (NULL);
4615 }
4616
4617 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: NodeGuid %llx "
4618 "PortGUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.PortGUID);
4619
4620 return (resp);
4621 }
4622
4623
4624 /*
4625 * ibdm_get_portinfo()
4626 * Sends a SA query to get the PortInfo record
4627 * Returns pointer to the sa_portinfo_record_t on success
4628 * and NULL on failure
4629 */
4630 static sa_portinfo_record_t *
4631 ibdm_get_portinfo(ibmf_saa_handle_t sa_hdl, size_t *length, ib_lid_t lid)
4632 {
4633 sa_portinfo_record_t req, *resp = NULL;
4634 ibmf_saa_access_args_t args;
4635 int ret;
4636
4637 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: Begin");
4638
4639 bzero(&req, sizeof (sa_portinfo_record_t));
4640 req.EndportLID = lid;
4641
4642 args.sq_attr_id = SA_PORTINFORECORD_ATTRID;
4643 args.sq_access_type = IBMF_SAA_RETRIEVE;
4644 args.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID;
4645 args.sq_template = &req;
4646 args.sq_callback = NULL;
4647 args.sq_callback_arg = NULL;
4648
4649 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp);
4650 if (ret != IBMF_SUCCESS) {
4651 IBTF_DPRINTF_L2("ibdm", "\tget_portinfo:"
4652 " SA Retrieve Failed: 0x%X", ret);
4653 return (NULL);
4654 }
4655 if ((*length == 0) || (resp == NULL))
4656 return (NULL);
4657
4658 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: GidPrefix %llx Cap 0x%x",
4659 resp->PortInfo.GidPrefix, resp->PortInfo.CapabilityMask);
4660 return (resp);
4661 }
4662
4663
4664 /*
4665 * ibdm_ibnex_register_callback
4666 * IB nexus callback routine for HCA attach and detach notification
4667 */
4668 void
4669 ibdm_ibnex_register_callback(ibdm_callback_t ibnex_dm_callback)
4670 {
4671 IBTF_DPRINTF_L4("ibdm", "\tibnex_register_callbacks");
4672 mutex_enter(&ibdm.ibdm_ibnex_mutex);
4673 ibdm.ibdm_ibnex_callback = ibnex_dm_callback;
4674 mutex_exit(&ibdm.ibdm_ibnex_mutex);
4675 }
4676
4677
4678 /*
4679 * ibdm_ibnex_unregister_callbacks
4680 */
4681 void
4682 ibdm_ibnex_unregister_callback()
4683 {
4684 IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks");
4685 mutex_enter(&ibdm.ibdm_ibnex_mutex);
4686 ibdm.ibdm_ibnex_callback = NULL;
4687 mutex_exit(&ibdm.ibdm_ibnex_mutex);
4688 }
4689
4690 /*
4691 * ibdm_get_waittime()
4692 * Calculates the wait time based on the last HCA attach time
4693 */
4694 static time_t
4695 ibdm_get_waittime(ib_guid_t hca_guid, int dft_wait)
4696 {
4697 int ii;
4698 time_t temp, wait_time = 0;
4699 ibdm_hca_list_t *hca;
4700
4701 IBTF_DPRINTF_L4("ibdm", "\tget_waittime hcaguid:%llx"
4702 "\tport settling time %d", hca_guid, dft_wait);
4703
4704 ASSERT(mutex_owned(&ibdm.ibdm_hl_mutex));
4705
4706 hca = ibdm.ibdm_hca_list_head;
4707
4708 if (hca_guid) {
4709 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4710 if ((hca_guid == hca->hl_hca_guid) &&
4711 (hca->hl_nports != hca->hl_nports_active)) {
4712 wait_time =
4713 ddi_get_time() - hca->hl_attach_time;
4714 wait_time = ((wait_time >= dft_wait) ?
4715 0 : (dft_wait - wait_time));
4716 break;
4717 }
4718 hca = hca->hl_next;
4719 }
4720 IBTF_DPRINTF_L2("ibdm", "\tget_waittime: wait_time = %ld secs",
4721 (long)wait_time);
4722 return (wait_time);
4723 }
4724
4725 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4726 if (hca->hl_nports != hca->hl_nports_active) {
4727 temp = ddi_get_time() - hca->hl_attach_time;
4728 temp = ((temp >= dft_wait) ? 0 : (dft_wait - temp));
4729 wait_time = (temp > wait_time) ? temp : wait_time;
4730 }
4731 hca = hca->hl_next;
4732 }
4733 IBTF_DPRINTF_L2("ibdm", "\tget_waittime: wait_time = %ld secs",
4734 (long)wait_time);
4735 return (wait_time);
4736 }
4737
4738 void
4739 ibdm_ibnex_port_settle_wait(ib_guid_t hca_guid, int dft_wait)
4740 {
4741 time_t wait_time;
4742 clock_t delta;
4743
4744 mutex_enter(&ibdm.ibdm_hl_mutex);
4745
4746 while ((wait_time = ibdm_get_waittime(hca_guid, dft_wait)) > 0) {
4747 if (wait_time > dft_wait) {
4748 IBTF_DPRINTF_L1("ibdm",
4749 "\tibnex_port_settle_wait: wait_time = %ld secs; "
4750 "Resetting to %d secs",
4751 (long)wait_time, dft_wait);
4752 wait_time = dft_wait;
4753 }
4754 delta = drv_usectohz(wait_time * 1000000);
4755 (void) cv_reltimedwait(&ibdm.ibdm_port_settle_cv,
4756 &ibdm.ibdm_hl_mutex, delta, TR_CLOCK_TICK);
4757 }
4758
4759 mutex_exit(&ibdm.ibdm_hl_mutex);
4760 }
4761
4762
4763 /*
4764 * ibdm_ibnex_probe_hcaport
4765 * Probes the presence of HCA port (with HCA dip and port number)
4766 * Returns port attributes structure on SUCCESS
4767 */
4768 ibdm_port_attr_t *
4769 ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num)
4770 {
4771 int ii, jj;
4772 ibdm_hca_list_t *hca_list;
4773 ibdm_port_attr_t *port_attr;
4774
4775 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:");
4776
4777 mutex_enter(&ibdm.ibdm_hl_mutex);
4778 hca_list = ibdm.ibdm_hca_list_head;
4779 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4780 if (hca_list->hl_hca_guid == hca_guid) {
4781 for (jj = 0; jj < hca_list->hl_nports; jj++) {
4782 if (hca_list->hl_port_attr[jj].pa_port_num ==
4783 port_num) {
4784 break;
4785 }
4786 }
4787 if (jj != hca_list->hl_nports)
4788 break;
4789 }
4790 hca_list = hca_list->hl_next;
4791 }
4792 if (ii == ibdm.ibdm_hca_count) {
4793 IBTF_DPRINTF_L2("ibdm", "\tibnex_probe_hcaport: not found");
4794 mutex_exit(&ibdm.ibdm_hl_mutex);
4795 return (NULL);
4796 }
4797 port_attr = (ibdm_port_attr_t *)kmem_zalloc(
4798 sizeof (ibdm_port_attr_t), KM_SLEEP);
4799 bcopy((char *)&hca_list->hl_port_attr[jj],
4800 port_attr, sizeof (ibdm_port_attr_t));
4801 ibdm_update_port_attr(port_attr);
4802
4803 mutex_exit(&ibdm.ibdm_hl_mutex);
4804 return (port_attr);
4805 }
4806
4807
4808 /*
4809 * ibdm_ibnex_get_port_attrs
4810 * Scan all HCAs for a matching port_guid.
4811 * Returns "port attributes" structure on success.
4812 */
4813 ibdm_port_attr_t *
4814 ibdm_ibnex_get_port_attrs(ib_guid_t port_guid)
4815 {
4816 int ii, jj;
4817 ibdm_hca_list_t *hca_list;
4818 ibdm_port_attr_t *port_attr;
4819
4820 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_port_attrs:");
4821
4822 mutex_enter(&ibdm.ibdm_hl_mutex);
4823 hca_list = ibdm.ibdm_hca_list_head;
4824
4825 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4826 for (jj = 0; jj < hca_list->hl_nports; jj++) {
4827 if (hca_list->hl_port_attr[jj].pa_port_guid ==
4828 port_guid) {
4829 break;
4830 }
4831 }
4832 if (jj != hca_list->hl_nports)
4833 break;
4834 hca_list = hca_list->hl_next;
4835 }
4836
4837 if (ii == ibdm.ibdm_hca_count) {
4838 IBTF_DPRINTF_L2("ibdm", "\tibnex_get_port_attrs: not found");
4839 mutex_exit(&ibdm.ibdm_hl_mutex);
4840 return (NULL);
4841 }
4842
4843 port_attr = (ibdm_port_attr_t *)kmem_alloc(sizeof (ibdm_port_attr_t),
4844 KM_SLEEP);
4845 bcopy((char *)&hca_list->hl_port_attr[jj], port_attr,
4846 sizeof (ibdm_port_attr_t));
4847 ibdm_update_port_attr(port_attr);
4848
4849 mutex_exit(&ibdm.ibdm_hl_mutex);
4850 return (port_attr);
4851 }
4852
4853
4854 /*
4855 * ibdm_ibnex_free_port_attr()
4856 */
4857 void
4858 ibdm_ibnex_free_port_attr(ibdm_port_attr_t *port_attr)
4859 {
4860 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_port_attr:");
4861 if (port_attr) {
4862 if (port_attr->pa_pkey_tbl != NULL) {
4863 kmem_free(port_attr->pa_pkey_tbl,
4864 (port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t)));
4865 }
4866 kmem_free(port_attr, sizeof (ibdm_port_attr_t));
4867 }
4868 }
4869
4870
4871 /*
4872 * ibdm_ibnex_get_hca_list()
4873 * Returns portinfo for all the port for all the HCA's
4874 */
4875 void
4876 ibdm_ibnex_get_hca_list(ibdm_hca_list_t **hca, int *count)
4877 {
4878 ibdm_hca_list_t *head = NULL, *temp, *temp1;
4879 int ii;
4880
4881 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_list:");
4882
4883 mutex_enter(&ibdm.ibdm_hl_mutex);
4884 temp = ibdm.ibdm_hca_list_head;
4885 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4886 temp1 = ibdm_dup_hca_attr(temp);
4887 temp1->hl_next = head;
4888 head = temp1;
4889 temp = temp->hl_next;
4890 }
4891 *count = ibdm.ibdm_hca_count;
4892 *hca = head;
4893 mutex_exit(&ibdm.ibdm_hl_mutex);
4894 }
4895
4896
4897 /*
4898 * ibdm_ibnex_get_hca_info_by_guid()
4899 */
4900 ibdm_hca_list_t *
4901 ibdm_ibnex_get_hca_info_by_guid(ib_guid_t hca_guid)
4902 {
4903 ibdm_hca_list_t *head = NULL, *hca = NULL;
4904
4905 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip");
4906
4907 mutex_enter(&ibdm.ibdm_hl_mutex);
4908 head = ibdm.ibdm_hca_list_head;
4909 while (head) {
4910 if (head->hl_hca_guid == hca_guid) {
4911 hca = ibdm_dup_hca_attr(head);
4912 hca->hl_next = NULL;
4913 break;
4914 }
4915 head = head->hl_next;
4916 }
4917 mutex_exit(&ibdm.ibdm_hl_mutex);
4918 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip %p", hca);
4919 return (hca);
4920 }
4921
4922
4923 /*
4924 * ibdm_dup_hca_attr()
4925 * Allocate a new HCA attribute strucuture and initialize
4926 * hca attribute structure with the incoming HCA attributes
4927 * returned the allocated hca attributes.
4928 */
4929 static ibdm_hca_list_t *
4930 ibdm_dup_hca_attr(ibdm_hca_list_t *in_hca)
4931 {
4932 int len;
4933 ibdm_hca_list_t *out_hca;
4934
4935 len = sizeof (ibdm_hca_list_t) +
4936 (in_hca->hl_nports * sizeof (ibdm_port_attr_t));
4937 IBTF_DPRINTF_L4("ibdm", "\tdup_hca_attr len %d", len);
4938 out_hca = (ibdm_hca_list_t *)kmem_alloc(len, KM_SLEEP);
4939 bcopy((char *)in_hca,
4940 (char *)out_hca, sizeof (ibdm_hca_list_t));
4941 if (in_hca->hl_nports) {
4942 out_hca->hl_port_attr = (ibdm_port_attr_t *)
4943 ((char *)out_hca + sizeof (ibdm_hca_list_t));
4944 bcopy((char *)in_hca->hl_port_attr,
4945 (char *)out_hca->hl_port_attr,
4946 (in_hca->hl_nports * sizeof (ibdm_port_attr_t)));
4947 for (len = 0; len < out_hca->hl_nports; len++)
4948 ibdm_update_port_attr(&out_hca->hl_port_attr[len]);
4949 }
4950 return (out_hca);
4951 }
4952
4953
4954 /*
4955 * ibdm_ibnex_free_hca_list()
4956 * Free one/more HCA lists
4957 */
4958 void
4959 ibdm_ibnex_free_hca_list(ibdm_hca_list_t *hca_list)
4960 {
4961 int ii;
4962 size_t len;
4963 ibdm_hca_list_t *temp;
4964 ibdm_port_attr_t *port;
4965
4966 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_hca_list:");
4967 ASSERT(hca_list);
4968 while (hca_list) {
4969 temp = hca_list;
4970 hca_list = hca_list->hl_next;
4971 for (ii = 0; ii < temp->hl_nports; ii++) {
4972 port = &temp->hl_port_attr[ii];
4973 len = (port->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
4974 if (len != 0)
4975 kmem_free(port->pa_pkey_tbl, len);
4976 }
4977 len = sizeof (ibdm_hca_list_t) + (temp->hl_nports *
4978 sizeof (ibdm_port_attr_t));
4979 kmem_free(temp, len);
4980 }
4981 }
4982
4983
4984 /*
4985 * ibdm_ibnex_probe_iocguid()
4986 * Probes the IOC on the fabric and returns the IOC information
4987 * if present. Otherwise, NULL is returned
4988 */
4989 /* ARGSUSED */
4990 ibdm_ioc_info_t *
4991 ibdm_ibnex_probe_ioc(ib_guid_t iou, ib_guid_t ioc_guid, int reprobe_flag)
4992 {
4993 int k;
4994 ibdm_ioc_info_t *ioc_info;
4995 ibdm_dp_gidinfo_t *gid_info; /* used as index and arg */
4996 timeout_id_t *timeout_id;
4997
4998 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_ioc: (%llX, %llX, %d) Begin",
4999 iou, ioc_guid, reprobe_flag);
5000
5001 if (ibdm_enumerate_iocs == 0)
5002 return (NULL);
5003
5004 /* Check whether we know this already */
5005 ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info);
5006 if (ioc_info == NULL) {
5007 mutex_enter(&ibdm.ibdm_mutex);
5008 while (ibdm.ibdm_busy & IBDM_BUSY)
5009 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5010 ibdm.ibdm_busy |= IBDM_BUSY;
5011 mutex_exit(&ibdm.ibdm_mutex);
5012 ibdm_probe_ioc(iou, ioc_guid, 0);
5013 mutex_enter(&ibdm.ibdm_mutex);
5014 ibdm.ibdm_busy &= ~IBDM_BUSY;
5015 cv_broadcast(&ibdm.ibdm_busy_cv);
5016 mutex_exit(&ibdm.ibdm_mutex);
5017 ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info);
5018 } else if (reprobe_flag) { /* Handle Reprobe for the IOC */
5019 ASSERT(gid_info != NULL);
5020 /* Free the ioc_list before reprobe; and cancel any timers */
5021 mutex_enter(&ibdm.ibdm_mutex);
5022 mutex_enter(&gid_info->gl_mutex);
5023 if (ioc_info->ioc_timeout_id) {
5024 timeout_id = ioc_info->ioc_timeout_id;
5025 ioc_info->ioc_timeout_id = 0;
5026 mutex_exit(&gid_info->gl_mutex);
5027 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
5028 "ioc_timeout_id = 0x%x", timeout_id);
5029 if (untimeout(timeout_id) == -1) {
5030 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
5031 "untimeout ioc_timeout_id failed");
5032 }
5033 mutex_enter(&gid_info->gl_mutex);
5034 }
5035 if (ioc_info->ioc_dc_timeout_id) {
5036 timeout_id = ioc_info->ioc_dc_timeout_id;
5037 ioc_info->ioc_dc_timeout_id = 0;
5038 mutex_exit(&gid_info->gl_mutex);
5039 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
5040 "ioc_dc_timeout_id = 0x%x", timeout_id);
5041 if (untimeout(timeout_id) == -1) {
5042 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
5043 "untimeout ioc_dc_timeout_id failed");
5044 }
5045 mutex_enter(&gid_info->gl_mutex);
5046 }
5047 for (k = 0; k < ioc_info->ioc_profile.ioc_service_entries; k++)
5048 if (ioc_info->ioc_serv[k].se_timeout_id) {
5049 timeout_id = ioc_info->ioc_serv[k].
5050 se_timeout_id;
5051 ioc_info->ioc_serv[k].se_timeout_id = 0;
5052 mutex_exit(&gid_info->gl_mutex);
5053 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
5054 "ioc_info->ioc_serv[k].se_timeout_id = %x",
5055 k, timeout_id);
5056 if (untimeout(timeout_id) == -1) {
5057 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
5058 "untimeout se_timeout_id %d "
5059 "failed", k);
5060 }
5061 mutex_enter(&gid_info->gl_mutex);
5062 }
5063 mutex_exit(&gid_info->gl_mutex);
5064 mutex_exit(&ibdm.ibdm_mutex);
5065 ibdm_ibnex_free_ioc_list(ioc_info);
5066
5067 mutex_enter(&ibdm.ibdm_mutex);
5068 while (ibdm.ibdm_busy & IBDM_BUSY)
5069 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5070 ibdm.ibdm_busy |= IBDM_BUSY;
5071 mutex_exit(&ibdm.ibdm_mutex);
5072
5073 ibdm_probe_ioc(iou, ioc_guid, 1);
5074
5075 /*
5076 * Skip if gl_reprobe_flag is set, this will be
5077 * a re-inserted / new GID, for which notifications
5078 * have already been send.
5079 */
5080 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
5081 gid_info = gid_info->gl_next) {
5082 uint8_t ii, niocs;
5083 ibdm_ioc_info_t *ioc;
5084
5085 if (gid_info->gl_iou == NULL)
5086 continue;
5087
5088 if (gid_info->gl_reprobe_flag) {
5089 gid_info->gl_reprobe_flag = 0;
5090 continue;
5091 }
5092
5093 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
5094 for (ii = 0; ii < niocs; ii++) {
5095 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
5096 if (ioc->ioc_profile.ioc_guid == ioc_guid) {
5097 mutex_enter(&ibdm.ibdm_mutex);
5098 ibdm_reprobe_update_port_srv(ioc,
5099 gid_info);
5100 mutex_exit(&ibdm.ibdm_mutex);
5101 }
5102 }
5103 }
5104 mutex_enter(&ibdm.ibdm_mutex);
5105 ibdm.ibdm_busy &= ~IBDM_BUSY;
5106 cv_broadcast(&ibdm.ibdm_busy_cv);
5107 mutex_exit(&ibdm.ibdm_mutex);
5108
5109 ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info);
5110 }
5111 return (ioc_info);
5112 }
5113
5114
5115 /*
5116 * ibdm_get_ioc_info_with_gid()
5117 * Returns pointer to ibdm_ioc_info_t if it finds
5118 * matching record for the ioc_guid. Otherwise NULL is returned.
5119 * The pointer to gid_info is set to the second argument in case that
5120 * the non-NULL value returns (and the second argument is not NULL).
5121 *
5122 * Note. use the same strings as "ibnex_get_ioc_info" in
5123 * IBTF_DPRINTF() to keep compatibility.
5124 */
5125 static ibdm_ioc_info_t *
5126 ibdm_get_ioc_info_with_gid(ib_guid_t ioc_guid,
5127 ibdm_dp_gidinfo_t **gid_info)
5128 {
5129 int ii;
5130 ibdm_ioc_info_t *ioc = NULL, *tmp = NULL;
5131 ibdm_dp_gidinfo_t *gid_list;
5132 ib_dm_io_unitinfo_t *iou;
5133
5134 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_ioc_info: GUID %llx", ioc_guid);
5135
5136 mutex_enter(&ibdm.ibdm_mutex);
5137 while (ibdm.ibdm_busy & IBDM_BUSY)
5138 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5139 ibdm.ibdm_busy |= IBDM_BUSY;
5140
5141 if (gid_info)
5142 *gid_info = NULL; /* clear the value of gid_info */
5143
5144 gid_list = ibdm.ibdm_dp_gidlist_head;
5145 while (gid_list) {
5146 mutex_enter(&gid_list->gl_mutex);
5147 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) {
5148 mutex_exit(&gid_list->gl_mutex);
5149 gid_list = gid_list->gl_next;
5150 continue;
5151 }
5152 if (gid_list->gl_iou == NULL) {
5153 IBTF_DPRINTF_L2("ibdm",
5154 "\tget_ioc_info: No IOU info");
5155 mutex_exit(&gid_list->gl_mutex);
5156 gid_list = gid_list->gl_next;
5157 continue;
5158 }
5159 iou = &gid_list->gl_iou->iou_info;
5160 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
5161 tmp = IBDM_GIDINFO2IOCINFO(gid_list, ii);
5162 if ((tmp->ioc_profile.ioc_guid == ioc_guid) &&
5163 (tmp->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)) {
5164 ioc = ibdm_dup_ioc_info(tmp, gid_list);
5165 if (gid_info)
5166 *gid_info = gid_list; /* set this ptr */
5167 mutex_exit(&gid_list->gl_mutex);
5168 ibdm.ibdm_busy &= ~IBDM_BUSY;
5169 cv_broadcast(&ibdm.ibdm_busy_cv);
5170 mutex_exit(&ibdm.ibdm_mutex);
5171 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: End");
5172 return (ioc);
5173 }
5174 }
5175 if (ii == iou->iou_num_ctrl_slots)
5176 ioc = NULL;
5177
5178 mutex_exit(&gid_list->gl_mutex);
5179 gid_list = gid_list->gl_next;
5180 }
5181
5182 ibdm.ibdm_busy &= ~IBDM_BUSY;
5183 cv_broadcast(&ibdm.ibdm_busy_cv);
5184 mutex_exit(&ibdm.ibdm_mutex);
5185 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: failure End");
5186 return (ioc);
5187 }
5188
5189 /*
5190 * ibdm_ibnex_get_ioc_info()
5191 * Returns pointer to ibdm_ioc_info_t if it finds
5192 * matching record for the ioc_guid, otherwise NULL
5193 * is returned
5194 *
5195 * Note. this is a wrapper function to ibdm_get_ioc_info_with_gid() now.
5196 */
5197 ibdm_ioc_info_t *
5198 ibdm_ibnex_get_ioc_info(ib_guid_t ioc_guid)
5199 {
5200 if (ibdm_enumerate_iocs == 0)
5201 return (NULL);
5202
5203 /* will not use the gid_info pointer, so the second arg is NULL */
5204 return (ibdm_get_ioc_info_with_gid(ioc_guid, NULL));
5205 }
5206
5207 /*
5208 * ibdm_ibnex_get_ioc_count()
5209 * Returns number of ibdm_ioc_info_t it finds
5210 */
5211 int
5212 ibdm_ibnex_get_ioc_count(void)
5213 {
5214 int count = 0, k;
5215 ibdm_ioc_info_t *ioc;
5216 ibdm_dp_gidinfo_t *gid_list;
5217
5218 if (ibdm_enumerate_iocs == 0)
5219 return (0);
5220
5221 mutex_enter(&ibdm.ibdm_mutex);
5222 ibdm_sweep_fabric(0);
5223
5224 while (ibdm.ibdm_busy & IBDM_BUSY)
5225 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5226 ibdm.ibdm_busy |= IBDM_BUSY;
5227
5228 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
5229 gid_list = gid_list->gl_next) {
5230 mutex_enter(&gid_list->gl_mutex);
5231 if ((gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) ||
5232 (gid_list->gl_iou == NULL)) {
5233 mutex_exit(&gid_list->gl_mutex);
5234 continue;
5235 }
5236 for (k = 0; k < gid_list->gl_iou->iou_info.iou_num_ctrl_slots;
5237 k++) {
5238 ioc = IBDM_GIDINFO2IOCINFO(gid_list, k);
5239 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)
5240 ++count;
5241 }
5242 mutex_exit(&gid_list->gl_mutex);
5243 }
5244 ibdm.ibdm_busy &= ~IBDM_BUSY;
5245 cv_broadcast(&ibdm.ibdm_busy_cv);
5246 mutex_exit(&ibdm.ibdm_mutex);
5247
5248 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_count: count = %d", count);
5249 return (count);
5250 }
5251
5252
5253 /*
5254 * ibdm_ibnex_get_ioc_list()
5255 * Returns information about all the IOCs present on the fabric.
5256 * Reprobes the IOCs and the GID list if list_flag is set to REPROBE_ALL.
5257 * Does not sweep fabric if DONOT_PROBE is set
5258 */
5259 ibdm_ioc_info_t *
5260 ibdm_ibnex_get_ioc_list(ibdm_ibnex_get_ioclist_mtd_t list_flag)
5261 {
5262 int ii;
5263 ibdm_ioc_info_t *ioc_list = NULL, *tmp, *ioc;
5264 ibdm_dp_gidinfo_t *gid_list;
5265 ib_dm_io_unitinfo_t *iou;
5266
5267 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: Enter");
5268
5269 if (ibdm_enumerate_iocs == 0)
5270 return (NULL);
5271
5272 mutex_enter(&ibdm.ibdm_mutex);
5273 if (list_flag != IBDM_IBNEX_DONOT_PROBE)
5274 ibdm_sweep_fabric(list_flag == IBDM_IBNEX_REPROBE_ALL);
5275
5276 while (ibdm.ibdm_busy & IBDM_BUSY)
5277 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5278 ibdm.ibdm_busy |= IBDM_BUSY;
5279
5280 gid_list = ibdm.ibdm_dp_gidlist_head;
5281 while (gid_list) {
5282 mutex_enter(&gid_list->gl_mutex);
5283 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) {
5284 mutex_exit(&gid_list->gl_mutex);
5285 gid_list = gid_list->gl_next;
5286 continue;
5287 }
5288 if (gid_list->gl_iou == NULL) {
5289 IBTF_DPRINTF_L2("ibdm",
5290 "\tget_ioc_list: No IOU info");
5291 mutex_exit(&gid_list->gl_mutex);
5292 gid_list = gid_list->gl_next;
5293 continue;
5294 }
5295 iou = &gid_list->gl_iou->iou_info;
5296 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
5297 ioc = IBDM_GIDINFO2IOCINFO(gid_list, ii);
5298 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) {
5299 tmp = ibdm_dup_ioc_info(ioc, gid_list);
5300 tmp->ioc_next = ioc_list;
5301 ioc_list = tmp;
5302 }
5303 }
5304 mutex_exit(&gid_list->gl_mutex);
5305 gid_list = gid_list->gl_next;
5306 }
5307 ibdm.ibdm_busy &= ~IBDM_BUSY;
5308 cv_broadcast(&ibdm.ibdm_busy_cv);
5309 mutex_exit(&ibdm.ibdm_mutex);
5310
5311 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: End");
5312 return (ioc_list);
5313 }
5314
5315 /*
5316 * ibdm_dup_ioc_info()
5317 * Duplicate the IOC information and return the IOC
5318 * information.
5319 */
5320 static ibdm_ioc_info_t *
5321 ibdm_dup_ioc_info(ibdm_ioc_info_t *in_ioc, ibdm_dp_gidinfo_t *gid_list)
5322 {
5323 ibdm_ioc_info_t *out_ioc;
5324 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*out_ioc));
5325 ASSERT(MUTEX_HELD(&gid_list->gl_mutex));
5326
5327 out_ioc = kmem_alloc(sizeof (ibdm_ioc_info_t), KM_SLEEP);
5328 bcopy(in_ioc, out_ioc, sizeof (ibdm_ioc_info_t));
5329 ibdm_update_ioc_port_gidlist(out_ioc, gid_list);
5330 out_ioc->ioc_iou_dc_valid = gid_list->gl_iou->iou_dc_valid;
5331 out_ioc->ioc_iou_diagcode = gid_list->gl_iou->iou_diagcode;
5332
5333 return (out_ioc);
5334 }
5335
5336
5337 /*
5338 * ibdm_free_ioc_list()
5339 * Deallocate memory for IOC list structure
5340 */
5341 void
5342 ibdm_ibnex_free_ioc_list(ibdm_ioc_info_t *ioc)
5343 {
5344 ibdm_ioc_info_t *temp;
5345
5346 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_ioc_list:");
5347 while (ioc) {
5348 temp = ioc;
5349 ioc = ioc->ioc_next;
5350 kmem_free(temp->ioc_gid_list,
5351 (sizeof (ibdm_gid_t) * temp->ioc_nportgids));
5352 if (temp->ioc_hca_list)
5353 ibdm_ibnex_free_hca_list(temp->ioc_hca_list);
5354 kmem_free(temp, sizeof (ibdm_ioc_info_t));
5355 }
5356 }
5357
5358
5359 /*
5360 * ibdm_ibnex_update_pkey_tbls
5361 * Updates the DM P_Key database.
5362 * NOTE: Two cases are handled here: P_Key being added or removed.
5363 *
5364 * Arguments : NONE
5365 * Return Values : NONE
5366 */
5367 void
5368 ibdm_ibnex_update_pkey_tbls(void)
5369 {
5370 int h, pp, pidx;
5371 uint_t nports;
5372 uint_t size;
5373 ib_pkey_t new_pkey;
5374 ib_pkey_t *orig_pkey;
5375 ibdm_hca_list_t *hca_list;
5376 ibdm_port_attr_t *port;
5377 ibt_hca_portinfo_t *pinfop;
5378
5379 IBTF_DPRINTF_L4("ibdm", "\tibnex_update_pkey_tbls:");
5380
5381 mutex_enter(&ibdm.ibdm_hl_mutex);
5382 hca_list = ibdm.ibdm_hca_list_head;
5383
5384 for (h = 0; h < ibdm.ibdm_hca_count; h++) {
5385
5386 /* This updates P_Key Tables for all ports of this HCA */
5387 (void) ibt_query_hca_ports(hca_list->hl_hca_hdl, 0, &pinfop,
5388 &nports, &size);
5389
5390 /* number of ports shouldn't have changed */
5391 ASSERT(nports == hca_list->hl_nports);
5392
5393 for (pp = 0; pp < hca_list->hl_nports; pp++) {
5394 port = &hca_list->hl_port_attr[pp];
5395
5396 /*
5397 * First figure out the P_Keys from IBTL.
5398 * Three things could have happened:
5399 * New P_Keys added
5400 * Existing P_Keys removed
5401 * Both of the above two
5402 *
5403 * Loop through the P_Key Indices and check if a
5404 * give P_Key_Ix matches that of the one seen by
5405 * IBDM. If they match no action is needed.
5406 *
5407 * If they don't match:
5408 * 1. if orig_pkey is invalid and new_pkey is valid
5409 * ---> add new_pkey to DM database
5410 * 2. if orig_pkey is valid and new_pkey is invalid
5411 * ---> remove orig_pkey from DM database
5412 * 3. if orig_pkey and new_pkey are both valid:
5413 * ---> remov orig_pkey from DM database
5414 * ---> add new_pkey to DM database
5415 * 4. if orig_pkey and new_pkey are both invalid:
5416 * ---> do nothing. Updated DM database.
5417 */
5418
5419 for (pidx = 0; pidx < port->pa_npkeys; pidx++) {
5420 new_pkey = pinfop[pp].p_pkey_tbl[pidx];
5421 orig_pkey = &port->pa_pkey_tbl[pidx].pt_pkey;
5422
5423 /* keys match - do nothing */
5424 if (*orig_pkey == new_pkey)
5425 continue;
5426
5427 if (IBDM_INVALID_PKEY(*orig_pkey) &&
5428 !IBDM_INVALID_PKEY(new_pkey)) {
5429 /* P_Key was added */
5430 IBTF_DPRINTF_L5("ibdm",
5431 "\tibnex_update_pkey_tbls: new "
5432 "P_Key added = 0x%x", new_pkey);
5433 *orig_pkey = new_pkey;
5434 ibdm_port_attr_ibmf_init(port,
5435 new_pkey, pp);
5436 } else if (!IBDM_INVALID_PKEY(*orig_pkey) &&
5437 IBDM_INVALID_PKEY(new_pkey)) {
5438 /* P_Key was removed */
5439 IBTF_DPRINTF_L5("ibdm",
5440 "\tibnex_update_pkey_tbls: P_Key "
5441 "removed = 0x%x", *orig_pkey);
5442 *orig_pkey = new_pkey;
5443 (void) ibdm_port_attr_ibmf_fini(port,
5444 pidx);
5445 } else if (!IBDM_INVALID_PKEY(*orig_pkey) &&
5446 !IBDM_INVALID_PKEY(new_pkey)) {
5447 /* P_Key were replaced */
5448 IBTF_DPRINTF_L5("ibdm",
5449 "\tibnex_update_pkey_tbls: P_Key "
5450 "replaced 0x%x with 0x%x",
5451 *orig_pkey, new_pkey);
5452 (void) ibdm_port_attr_ibmf_fini(port,
5453 pidx);
5454 *orig_pkey = new_pkey;
5455 ibdm_port_attr_ibmf_init(port,
5456 new_pkey, pp);
5457 } else {
5458 /*
5459 * P_Keys are invalid
5460 * set anyway to reflect if
5461 * INVALID_FULL was changed to
5462 * INVALID_LIMITED or vice-versa.
5463 */
5464 *orig_pkey = new_pkey;
5465 } /* end of else */
5466
5467 } /* loop of p_key index */
5468
5469 } /* loop of #ports of HCA */
5470
5471 ibt_free_portinfo(pinfop, size);
5472 hca_list = hca_list->hl_next;
5473
5474 } /* loop for all HCAs in the system */
5475
5476 mutex_exit(&ibdm.ibdm_hl_mutex);
5477 }
5478
5479
5480 /*
5481 * ibdm_send_ioc_profile()
5482 * Send IOC Controller Profile request. When the request is completed
5483 * IBMF calls ibdm_process_incoming_mad routine to inform about
5484 * the completion.
5485 */
5486 static int
5487 ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *gid_info, uint8_t ioc_no)
5488 {
5489 ibmf_msg_t *msg;
5490 ib_mad_hdr_t *hdr;
5491 ibdm_ioc_info_t *ioc_info = &(gid_info->gl_iou->iou_ioc_info[ioc_no]);
5492 ibdm_timeout_cb_args_t *cb_args;
5493
5494 IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: "
5495 "gid info 0x%p, ioc_no = %d", gid_info, ioc_no);
5496
5497 /*
5498 * Send command to get IOC profile.
5499 * Allocate a IBMF packet and initialize the packet.
5500 */
5501 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
5502 &msg) != IBMF_SUCCESS) {
5503 IBTF_DPRINTF_L2("ibdm", "\tsend_ioc_profile: pkt alloc fail");
5504 return (IBDM_FAILURE);
5505 }
5506
5507 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
5508 ibdm_alloc_send_buffers(msg);
5509 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
5510
5511 mutex_enter(&gid_info->gl_mutex);
5512 ibdm_bump_transactionID(gid_info);
5513 mutex_exit(&gid_info->gl_mutex);
5514
5515 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
5516 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
5517 if (gid_info->gl_redirected == B_TRUE) {
5518 if (gid_info->gl_redirect_dlid != 0) {
5519 msg->im_local_addr.ia_remote_lid =
5520 gid_info->gl_redirect_dlid;
5521 }
5522 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
5523 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
5524 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
5525 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
5526 } else {
5527 msg->im_local_addr.ia_remote_qno = 1;
5528 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
5529 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
5530 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
5531 }
5532
5533 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
5534 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
5535 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
5536 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
5537 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
5538 hdr->Status = 0;
5539 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
5540 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
5541 hdr->AttributeModifier = h2b32(ioc_no + 1);
5542
5543 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS;
5544 cb_args = &ioc_info->ioc_cb_args;
5545 cb_args->cb_gid_info = gid_info;
5546 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
5547 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO;
5548 cb_args->cb_ioc_num = ioc_no;
5549
5550 mutex_enter(&gid_info->gl_mutex);
5551 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
5552 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
5553 mutex_exit(&gid_info->gl_mutex);
5554
5555 IBTF_DPRINTF_L5("ibdm", "\tsend_ioc_profile:"
5556 "timeout %x", ioc_info->ioc_timeout_id);
5557
5558 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg,
5559 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
5560 IBTF_DPRINTF_L2("ibdm",
5561 "\tsend_ioc_profile: msg transport failed");
5562 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
5563 }
5564 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS;
5565 return (IBDM_SUCCESS);
5566 }
5567
5568
5569 /*
5570 * ibdm_port_reachable
5571 * Returns B_TRUE if the port GID is reachable by sending
5572 * a SA query to get the NODE record for this port GUID.
5573 */
5574 static boolean_t
5575 ibdm_port_reachable(ibmf_saa_handle_t sa_hdl, ib_guid_t guid)
5576 {
5577 sa_node_record_t *resp;
5578 size_t length;
5579
5580 /*
5581 * Verify if it's reachable by getting the node record.
5582 */
5583 if (ibdm_get_node_record_by_port(sa_hdl, guid, &resp, &length) ==
5584 IBDM_SUCCESS) {
5585 kmem_free(resp, length);
5586 return (B_TRUE);
5587 }
5588 return (B_FALSE);
5589 }
5590
5591 /*
5592 * ibdm_get_node_record_by_port
5593 * Sends a SA query to get the NODE record for port GUID
5594 * Returns IBDM_SUCCESS if the port GID is reachable.
5595 *
5596 * Note: the caller must be responsible for freeing the resource
5597 * by calling kmem_free(resp, length) later.
5598 */
5599 static int
5600 ibdm_get_node_record_by_port(ibmf_saa_handle_t sa_hdl, ib_guid_t guid,
5601 sa_node_record_t **resp, size_t *length)
5602 {
5603 sa_node_record_t req;
5604 ibmf_saa_access_args_t args;
5605 int ret;
5606 ASSERT(resp != NULL && length != NULL);
5607
5608 IBTF_DPRINTF_L4("ibdm", "\tport_reachable: port_guid %llx",
5609 guid);
5610
5611 bzero(&req, sizeof (sa_node_record_t));
5612 req.NodeInfo.PortGUID = guid;
5613
5614 args.sq_attr_id = SA_NODERECORD_ATTRID;
5615 args.sq_access_type = IBMF_SAA_RETRIEVE;
5616 args.sq_component_mask = SA_NODEINFO_COMPMASK_PORTGUID;
5617 args.sq_template = &req;
5618 args.sq_callback = NULL;
5619 args.sq_callback_arg = NULL;
5620
5621 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) resp);
5622 if (ret != IBMF_SUCCESS) {
5623 IBTF_DPRINTF_L2("ibdm", "\tport_reachable:"
5624 " SA Retrieve Failed: %d", ret);
5625 return (IBDM_FAILURE);
5626 }
5627 if (*resp == NULL || *length == 0) {
5628 IBTF_DPRINTF_L2("ibdm", "\tport_reachable: No records");
5629 return (IBDM_FAILURE);
5630 }
5631 /*
5632 * There is one NodeRecord on each endport on a subnet.
5633 */
5634 ASSERT(*length == sizeof (sa_node_record_t));
5635
5636 return (IBDM_SUCCESS);
5637 }
5638
5639
5640 /*
5641 * Update the gidlist for all affected IOCs when GID becomes
5642 * available/unavailable.
5643 *
5644 * Parameters :
5645 * gidinfo - Incoming / Outgoing GID.
5646 * add_flag - 1 for GID added, 0 for GID removed.
5647 * - (-1) : IOC gid list updated, ioc_list required.
5648 *
5649 * This function gets the GID for the node GUID corresponding to the
5650 * port GID. Gets the IOU info
5651 */
5652 static ibdm_ioc_info_t *
5653 ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *gid_info, int avail_flag)
5654 {
5655 ibdm_dp_gidinfo_t *node_gid = NULL;
5656 uint8_t niocs, ii;
5657 ibdm_ioc_info_t *ioc, *ioc_list = NULL, *tmp;
5658
5659 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist");
5660
5661 switch (avail_flag) {
5662 case 1 :
5663 node_gid = ibdm_check_dest_nodeguid(gid_info);
5664 break;
5665 case 0 :
5666 node_gid = ibdm_handle_gid_rm(gid_info);
5667 break;
5668 case -1 :
5669 node_gid = gid_info;
5670 break;
5671 default :
5672 break;
5673 }
5674
5675 if (node_gid == NULL) {
5676 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist: "
5677 "No node GID found, port gid 0x%p, avail_flag %d",
5678 gid_info, avail_flag);
5679 return (NULL);
5680 }
5681
5682 mutex_enter(&node_gid->gl_mutex);
5683 if ((node_gid->gl_state != IBDM_GID_PROBING_COMPLETE &&
5684 node_gid->gl_state != IBDM_GID_PROBING_SKIPPED) ||
5685 node_gid->gl_iou == NULL) {
5686 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist "
5687 "gl_state %x, gl_iou %p", node_gid->gl_state,
5688 node_gid->gl_iou);
5689 mutex_exit(&node_gid->gl_mutex);
5690 return (NULL);
5691 }
5692
5693 niocs = node_gid->gl_iou->iou_info.iou_num_ctrl_slots;
5694 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : niocs %x",
5695 niocs);
5696 for (ii = 0; ii < niocs; ii++) {
5697 ioc = IBDM_GIDINFO2IOCINFO(node_gid, ii);
5698 /*
5699 * Skip IOCs for which probe is not complete or
5700 * reprobe is progress
5701 */
5702 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) {
5703 tmp = ibdm_dup_ioc_info(ioc, node_gid);
5704 tmp->ioc_info_updated.ib_gid_prop_updated = 1;
5705 tmp->ioc_next = ioc_list;
5706 ioc_list = tmp;
5707 }
5708 }
5709 mutex_exit(&node_gid->gl_mutex);
5710
5711 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : return %p",
5712 ioc_list);
5713 return (ioc_list);
5714 }
5715
5716 /*
5717 * ibdm_saa_event_cb :
5718 * Event handling which does *not* require ibdm_hl_mutex to be
5719 * held are executed in the same thread. This is to prevent
5720 * deadlocks with HCA port down notifications which hold the
5721 * ibdm_hl_mutex.
5722 *
5723 * GID_AVAILABLE event is handled here. A taskq is spawned to
5724 * handle GID_UNAVAILABLE.
5725 *
5726 * A new mutex ibdm_ibnex_mutex has been introduced to protect
5727 * ibnex_callback. This has been done to prevent any possible
5728 * deadlock (described above) while handling GID_AVAILABLE.
5729 *
5730 * IBMF calls the event callback for a HCA port. The SA handle
5731 * for this port would be valid, till the callback returns.
5732 * IBDM calling IBDM using the above SA handle should be valid.
5733 *
5734 * IBDM will additionally check (SA handle != NULL), before
5735 * calling IBMF.
5736 */
5737 /*ARGSUSED*/
5738 static void
5739 ibdm_saa_event_cb(ibmf_saa_handle_t ibmf_saa_handle,
5740 ibmf_saa_subnet_event_t ibmf_saa_event,
5741 ibmf_saa_event_details_t *event_details, void *callback_arg)
5742 {
5743 ibdm_saa_event_arg_t *event_arg;
5744 ib_gid_t sgid, dgid;
5745 ibdm_port_attr_t *hca_port;
5746 ibdm_dp_gidinfo_t *gid_info, *node_gid_info = NULL;
5747 sa_node_record_t *nrec;
5748 size_t length;
5749
5750 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg));
5751
5752 hca_port = (ibdm_port_attr_t *)callback_arg;
5753
5754 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_cb(%x, %x, %x, %x)\n",
5755 ibmf_saa_handle, ibmf_saa_event, event_details,
5756 callback_arg);
5757
5758 #ifdef DEBUG
5759 if (ibdm_ignore_saa_event)
5760 return;
5761 #endif
5762
5763 if (ibmf_saa_event == IBMF_SAA_EVENT_GID_AVAILABLE) {
5764 /*
5765 * Ensure no other probe / sweep fabric is in
5766 * progress.
5767 */
5768 mutex_enter(&ibdm.ibdm_mutex);
5769 while (ibdm.ibdm_busy & IBDM_BUSY)
5770 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5771 ibdm.ibdm_busy |= IBDM_BUSY;
5772 mutex_exit(&ibdm.ibdm_mutex);
5773
5774 /*
5775 * If we already know about this GID, return.
5776 * GID_AVAILABLE may be reported for multiple HCA
5777 * ports.
5778 */
5779 if ((ibdm_check_dgid(event_details->ie_gid.gid_guid,
5780 event_details->ie_gid.gid_prefix)) != NULL) {
5781 mutex_enter(&ibdm.ibdm_mutex);
5782 ibdm.ibdm_busy &= ~IBDM_BUSY;
5783 cv_broadcast(&ibdm.ibdm_busy_cv);
5784 mutex_exit(&ibdm.ibdm_mutex);
5785 return;
5786 }
5787
5788 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) "
5789 "Insertion notified",
5790 event_details->ie_gid.gid_prefix,
5791 event_details->ie_gid.gid_guid);
5792
5793 /* This is a new gid, insert it to GID list */
5794 sgid.gid_prefix = hca_port->pa_sn_prefix;
5795 sgid.gid_guid = hca_port->pa_port_guid;
5796 dgid.gid_prefix = event_details->ie_gid.gid_prefix;
5797 dgid.gid_guid = event_details->ie_gid.gid_guid;
5798 gid_info = ibdm_create_gid_info(hca_port, sgid, dgid);
5799 if (gid_info == NULL) {
5800 IBTF_DPRINTF_L4("ibdm", "\tGID_AVAILABLE: "
5801 "create_gid_info returned NULL");
5802 mutex_enter(&ibdm.ibdm_mutex);
5803 ibdm.ibdm_busy &= ~IBDM_BUSY;
5804 cv_broadcast(&ibdm.ibdm_busy_cv);
5805 mutex_exit(&ibdm.ibdm_mutex);
5806 return;
5807 }
5808 mutex_enter(&gid_info->gl_mutex);
5809 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED;
5810 mutex_exit(&gid_info->gl_mutex);
5811
5812 /* Get the node GUID */
5813 if (ibdm_get_node_record_by_port(ibmf_saa_handle, dgid.gid_guid,
5814 &nrec, &length) != IBDM_SUCCESS) {
5815 /*
5816 * Set the state to PROBE_NOT_DONE for the
5817 * next sweep to probe it
5818 */
5819 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_taskq: "
5820 "Skipping GID : port GUID not found");
5821 mutex_enter(&gid_info->gl_mutex);
5822 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE;
5823 mutex_exit(&gid_info->gl_mutex);
5824 mutex_enter(&ibdm.ibdm_mutex);
5825 ibdm.ibdm_busy &= ~IBDM_BUSY;
5826 cv_broadcast(&ibdm.ibdm_busy_cv);
5827 mutex_exit(&ibdm.ibdm_mutex);
5828 return;
5829 }
5830 gid_info->gl_nodeguid = nrec->NodeInfo.NodeGUID;
5831 gid_info->gl_devid = nrec->NodeInfo.DeviceID;
5832 kmem_free(nrec, length);
5833 gid_info->gl_portguid = dgid.gid_guid;
5834
5835 /*
5836 * Get the gid info with the same node GUID.
5837 */
5838 mutex_enter(&ibdm.ibdm_mutex);
5839 node_gid_info = ibdm.ibdm_dp_gidlist_head;
5840 while (node_gid_info) {
5841 if (node_gid_info->gl_nodeguid ==
5842 gid_info->gl_nodeguid &&
5843 node_gid_info->gl_iou != NULL) {
5844 break;
5845 }
5846 node_gid_info = node_gid_info->gl_next;
5847 }
5848 mutex_exit(&ibdm.ibdm_mutex);
5849
5850 /*
5851 * Handling a new GID requires filling of gl_hca_list.
5852 * This require ibdm hca_list to be parsed and hence
5853 * holding the ibdm_hl_mutex. Spawning a new thread to
5854 * handle this.
5855 */
5856 if (node_gid_info == NULL) {
5857 if (taskq_dispatch(system_taskq,
5858 ibdm_saa_handle_new_gid, (void *)gid_info,
5859 TQ_NOSLEEP) == NULL) {
5860 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
5861 "new_gid taskq_dispatch failed");
5862 return;
5863 }
5864 }
5865
5866 mutex_enter(&ibdm.ibdm_mutex);
5867 ibdm.ibdm_busy &= ~IBDM_BUSY;
5868 cv_broadcast(&ibdm.ibdm_busy_cv);
5869 mutex_exit(&ibdm.ibdm_mutex);
5870 return;
5871 }
5872
5873 if (ibmf_saa_event != IBMF_SAA_EVENT_GID_UNAVAILABLE)
5874 return;
5875
5876 /*
5877 * GID UNAVAIL EVENT: Try to locate the GID in the GID list.
5878 * If we don't find it we just return.
5879 */
5880 mutex_enter(&ibdm.ibdm_mutex);
5881 gid_info = ibdm.ibdm_dp_gidlist_head;
5882 while (gid_info) {
5883 if (gid_info->gl_portguid ==
5884 event_details->ie_gid.gid_guid) {
5885 break;
5886 }
5887 gid_info = gid_info->gl_next;
5888 }
5889 mutex_exit(&ibdm.ibdm_mutex);
5890 if (gid_info == NULL) {
5891 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
5892 "GID for GUID %llX not found during GID UNAVAIL event",
5893 event_details->ie_gid.gid_guid);
5894 return;
5895 }
5896
5897 /*
5898 * If this GID is DM capable, we'll have to check whether this DGID
5899 * is reachable via another port.
5900 */
5901 if (gid_info->gl_is_dm_capable == B_TRUE) {
5902 event_arg = (ibdm_saa_event_arg_t *)kmem_alloc(
5903 sizeof (ibdm_saa_event_arg_t), KM_SLEEP);
5904 event_arg->ibmf_saa_handle = ibmf_saa_handle;
5905 event_arg->ibmf_saa_event = ibmf_saa_event;
5906 bcopy(event_details, &event_arg->event_details,
5907 sizeof (ibmf_saa_event_details_t));
5908 event_arg->callback_arg = callback_arg;
5909
5910 if (taskq_dispatch(system_taskq, ibdm_saa_event_taskq,
5911 (void *)event_arg, TQ_NOSLEEP) == NULL) {
5912 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
5913 "taskq_dispatch failed");
5914 ibdm_free_saa_event_arg(event_arg);
5915 return;
5916 }
5917 }
5918 }
5919
5920 /*
5921 * Handle a new GID discovered by GID_AVAILABLE saa event.
5922 */
5923 void
5924 ibdm_saa_handle_new_gid(void *arg)
5925 {
5926 ibdm_dp_gidinfo_t *gid_info;
5927 ibdm_hca_list_t *hca_list = NULL;
5928 ibdm_port_attr_t *port = NULL;
5929 ibdm_ioc_info_t *ioc_list = NULL;
5930
5931 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid(%p)", arg);
5932
5933 gid_info = (ibdm_dp_gidinfo_t *)arg;
5934
5935 /*
5936 * Ensure that no other sweep / probe has completed
5937 * probing this gid.
5938 */
5939 mutex_enter(&gid_info->gl_mutex);
5940 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) {
5941 mutex_exit(&gid_info->gl_mutex);
5942 return;
5943 }
5944 mutex_exit(&gid_info->gl_mutex);
5945
5946 /*
5947 * Parse HCAs to fill gl_hca_list
5948 */
5949 mutex_enter(&ibdm.ibdm_hl_mutex);
5950 for (ibdm_get_next_port(&hca_list, &port, 1); port;
5951 ibdm_get_next_port(&hca_list, &port, 1)) {
5952 if (ibdm_port_reachable(port->pa_sa_hdl,
5953 gid_info->gl_portguid) == B_TRUE) {
5954 ibdm_addto_glhcalist(gid_info, hca_list);
5955 }
5956 }
5957 mutex_exit(&ibdm.ibdm_hl_mutex);
5958
5959 /*
5960 * Ensure no other probe / sweep fabric is in
5961 * progress.
5962 */
5963 mutex_enter(&ibdm.ibdm_mutex);
5964 while (ibdm.ibdm_busy & IBDM_BUSY)
5965 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5966 ibdm.ibdm_busy |= IBDM_BUSY;
5967 mutex_exit(&ibdm.ibdm_mutex);
5968
5969 /*
5970 * New IOU probe it, to check if new IOCs
5971 */
5972 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid: "
5973 "new GID : probing");
5974 mutex_enter(&ibdm.ibdm_mutex);
5975 ibdm.ibdm_ngid_probes_in_progress++;
5976 mutex_exit(&ibdm.ibdm_mutex);
5977 mutex_enter(&gid_info->gl_mutex);
5978 gid_info->gl_reprobe_flag = 0;
5979 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE;
5980 mutex_exit(&gid_info->gl_mutex);
5981 ibdm_probe_gid_thread((void *)gid_info);
5982
5983 mutex_enter(&ibdm.ibdm_mutex);
5984 ibdm_wait_probe_completion();
5985 mutex_exit(&ibdm.ibdm_mutex);
5986
5987 if (gid_info->gl_iou == NULL) {
5988 mutex_enter(&ibdm.ibdm_mutex);
5989 ibdm.ibdm_busy &= ~IBDM_BUSY;
5990 cv_broadcast(&ibdm.ibdm_busy_cv);
5991 mutex_exit(&ibdm.ibdm_mutex);
5992 return;
5993 }
5994
5995 /*
5996 * Update GID list in all IOCs affected by this
5997 */
5998 ioc_list = ibdm_update_ioc_gidlist(gid_info, 1);
5999
6000 /*
6001 * Pass on the IOCs with updated GIDs to IBnexus
6002 */
6003 if (ioc_list) {
6004 mutex_enter(&ibdm.ibdm_ibnex_mutex);
6005 if (ibdm.ibdm_ibnex_callback != NULL) {
6006 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list,
6007 IBDM_EVENT_IOC_PROP_UPDATE);
6008 }
6009 mutex_exit(&ibdm.ibdm_ibnex_mutex);
6010 }
6011
6012 mutex_enter(&ibdm.ibdm_mutex);
6013 ibdm.ibdm_busy &= ~IBDM_BUSY;
6014 cv_broadcast(&ibdm.ibdm_busy_cv);
6015 mutex_exit(&ibdm.ibdm_mutex);
6016 }
6017
6018 /*
6019 * ibdm_saa_event_taskq :
6020 * GID_UNAVAILABLE Event handling requires ibdm_hl_mutex to be
6021 * held. The GID_UNAVAILABLE handling is done in a taskq to
6022 * prevent deadlocks with HCA port down notifications which hold
6023 * ibdm_hl_mutex.
6024 */
6025 void
6026 ibdm_saa_event_taskq(void *arg)
6027 {
6028 ibdm_saa_event_arg_t *event_arg;
6029 ibmf_saa_handle_t ibmf_saa_handle;
6030 ibmf_saa_subnet_event_t ibmf_saa_event;
6031 ibmf_saa_event_details_t *event_details;
6032 void *callback_arg;
6033
6034 ibdm_dp_gidinfo_t *gid_info;
6035 ibdm_port_attr_t *hca_port, *port = NULL;
6036 ibdm_hca_list_t *hca_list = NULL;
6037 int sa_handle_valid = 0;
6038 ibdm_ioc_info_t *ioc_list = NULL;
6039
6040 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg));
6041
6042 event_arg = (ibdm_saa_event_arg_t *)arg;
6043 ibmf_saa_handle = event_arg->ibmf_saa_handle;
6044 ibmf_saa_event = event_arg->ibmf_saa_event;
6045 event_details = &event_arg->event_details;
6046 callback_arg = event_arg->callback_arg;
6047
6048 ASSERT(callback_arg != NULL);
6049 ASSERT(ibmf_saa_event == IBMF_SAA_EVENT_GID_UNAVAILABLE);
6050 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_taskq(%x, %x, %x, %x)",
6051 ibmf_saa_handle, ibmf_saa_event, event_details,
6052 callback_arg);
6053
6054 hca_port = (ibdm_port_attr_t *)callback_arg;
6055
6056 /* Check if the port_attr is still valid */
6057 mutex_enter(&ibdm.ibdm_hl_mutex);
6058 for (ibdm_get_next_port(&hca_list, &port, 0); port;
6059 ibdm_get_next_port(&hca_list, &port, 0)) {
6060 if (port == hca_port && port->pa_port_guid ==
6061 hca_port->pa_port_guid) {
6062 if (ibmf_saa_handle == hca_port->pa_sa_hdl)
6063 sa_handle_valid = 1;
6064 break;
6065 }
6066 }
6067 mutex_exit(&ibdm.ibdm_hl_mutex);
6068 if (sa_handle_valid == 0) {
6069 ibdm_free_saa_event_arg(event_arg);
6070 return;
6071 }
6072
6073 if (hca_port && (hca_port->pa_sa_hdl == NULL ||
6074 ibmf_saa_handle != hca_port->pa_sa_hdl)) {
6075 ibdm_free_saa_event_arg(event_arg);
6076 return;
6077 }
6078 hca_list = NULL;
6079 port = NULL;
6080
6081 /*
6082 * Check if the GID is visible to other HCA ports.
6083 * Return if so.
6084 */
6085 mutex_enter(&ibdm.ibdm_hl_mutex);
6086 for (ibdm_get_next_port(&hca_list, &port, 1); port;
6087 ibdm_get_next_port(&hca_list, &port, 1)) {
6088 if (ibdm_port_reachable(port->pa_sa_hdl,
6089 event_details->ie_gid.gid_guid) == B_TRUE) {
6090 mutex_exit(&ibdm.ibdm_hl_mutex);
6091 ibdm_free_saa_event_arg(event_arg);
6092 return;
6093 }
6094 }
6095 mutex_exit(&ibdm.ibdm_hl_mutex);
6096
6097 /*
6098 * Ensure no other probe / sweep fabric is in
6099 * progress.
6100 */
6101 mutex_enter(&ibdm.ibdm_mutex);
6102 while (ibdm.ibdm_busy & IBDM_BUSY)
6103 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
6104 ibdm.ibdm_busy |= IBDM_BUSY;
6105 mutex_exit(&ibdm.ibdm_mutex);
6106
6107 /*
6108 * If this GID is no longer in GID list, return
6109 * GID_UNAVAILABLE may be reported for multiple HCA
6110 * ports.
6111 */
6112 mutex_enter(&ibdm.ibdm_mutex);
6113 gid_info = ibdm.ibdm_dp_gidlist_head;
6114 while (gid_info) {
6115 if (gid_info->gl_portguid ==
6116 event_details->ie_gid.gid_guid) {
6117 break;
6118 }
6119 gid_info = gid_info->gl_next;
6120 }
6121 mutex_exit(&ibdm.ibdm_mutex);
6122 if (gid_info == NULL) {
6123 mutex_enter(&ibdm.ibdm_mutex);
6124 ibdm.ibdm_busy &= ~IBDM_BUSY;
6125 cv_broadcast(&ibdm.ibdm_busy_cv);
6126 mutex_exit(&ibdm.ibdm_mutex);
6127 ibdm_free_saa_event_arg(event_arg);
6128 return;
6129 }
6130
6131 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) "
6132 "Unavailable notification",
6133 event_details->ie_gid.gid_prefix,
6134 event_details->ie_gid.gid_guid);
6135
6136 /*
6137 * Update GID list in all IOCs affected by this
6138 */
6139 if (gid_info->gl_state == IBDM_GID_PROBING_SKIPPED ||
6140 gid_info->gl_state == IBDM_GID_PROBING_COMPLETE)
6141 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0);
6142
6143 /*
6144 * Remove GID from the global GID list
6145 * Handle the case where all port GIDs for an
6146 * IOU have been hot-removed. Check both gid_info
6147 * & ioc_info for checking ngids.
6148 */
6149 mutex_enter(&ibdm.ibdm_mutex);
6150 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) {
6151 mutex_enter(&gid_info->gl_mutex);
6152 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
6153 mutex_exit(&gid_info->gl_mutex);
6154 }
6155 if (gid_info->gl_prev != NULL)
6156 gid_info->gl_prev->gl_next = gid_info->gl_next;
6157 if (gid_info->gl_next != NULL)
6158 gid_info->gl_next->gl_prev = gid_info->gl_prev;
6159
6160 if (gid_info == ibdm.ibdm_dp_gidlist_head)
6161 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next;
6162 if (gid_info == ibdm.ibdm_dp_gidlist_tail)
6163 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev;
6164 ibdm.ibdm_ngids--;
6165
6166 ibdm.ibdm_busy &= ~IBDM_BUSY;
6167 cv_broadcast(&ibdm.ibdm_busy_cv);
6168 mutex_exit(&ibdm.ibdm_mutex);
6169
6170 /* free the hca_list on this gid_info */
6171 ibdm_delete_glhca_list(gid_info);
6172
6173 mutex_destroy(&gid_info->gl_mutex);
6174 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t));
6175
6176 /*
6177 * Pass on the IOCs with updated GIDs to IBnexus
6178 */
6179 if (ioc_list) {
6180 IBTF_DPRINTF_L4("ibdm", "\tGID_UNAVAILABLE "
6181 "IOC_PROP_UPDATE for %p\n", ioc_list);
6182 mutex_enter(&ibdm.ibdm_ibnex_mutex);
6183 if (ibdm.ibdm_ibnex_callback != NULL) {
6184 (*ibdm.ibdm_ibnex_callback)((void *)
6185 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
6186 }
6187 mutex_exit(&ibdm.ibdm_ibnex_mutex);
6188 }
6189
6190 ibdm_free_saa_event_arg(event_arg);
6191 }
6192
6193
6194 static int
6195 ibdm_cmp_gid_list(ibdm_gid_t *new, ibdm_gid_t *prev)
6196 {
6197 ibdm_gid_t *scan_new, *scan_prev;
6198 int cmp_failed = 0;
6199
6200 ASSERT(new != NULL);
6201 ASSERT(prev != NULL);
6202
6203 /*
6204 * Search for each new gid anywhere in the prev GID list.
6205 * Note that the gid list could have been re-ordered.
6206 */
6207 for (scan_new = new; scan_new; scan_new = scan_new->gid_next) {
6208 for (scan_prev = prev, cmp_failed = 1; scan_prev;
6209 scan_prev = scan_prev->gid_next) {
6210 if (scan_prev->gid_dgid_hi == scan_new->gid_dgid_hi &&
6211 scan_prev->gid_dgid_lo == scan_new->gid_dgid_lo) {
6212 cmp_failed = 0;
6213 break;
6214 }
6215 }
6216
6217 if (cmp_failed)
6218 return (1);
6219 }
6220 return (0);
6221 }
6222
6223 /*
6224 * This is always called in a single thread
6225 * This function updates the gid_list and serv_list of IOC
6226 * The current gid_list is in ioc_info_t(contains only port
6227 * guids for which probe is done) & gidinfo_t(other port gids)
6228 * The gids in both locations are used for comparision.
6229 */
6230 static void
6231 ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *ioc, ibdm_dp_gidinfo_t *gidinfo)
6232 {
6233 ibdm_gid_t *cur_gid_list;
6234 uint_t cur_nportgids;
6235
6236 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
6237
6238 ioc->ioc_info_updated.ib_prop_updated = 0;
6239
6240
6241 /* Current GID list in gid_info only */
6242 cur_gid_list = gidinfo->gl_gid;
6243 cur_nportgids = gidinfo->gl_ngids;
6244
6245 if (ioc->ioc_prev_serv_cnt !=
6246 ioc->ioc_profile.ioc_service_entries ||
6247 ibdm_serv_cmp(&ioc->ioc_serv[0], &ioc->ioc_prev_serv[0],
6248 ioc->ioc_prev_serv_cnt))
6249 ioc->ioc_info_updated.ib_srv_prop_updated = 1;
6250
6251 if (ioc->ioc_prev_nportgids != cur_nportgids ||
6252 ioc->ioc_prev_gid_list == NULL || cur_gid_list == NULL) {
6253 ioc->ioc_info_updated.ib_gid_prop_updated = 1;
6254 } else if (ibdm_cmp_gid_list(ioc->ioc_prev_gid_list, cur_gid_list)) {
6255 ioc->ioc_info_updated.ib_gid_prop_updated = 1;
6256 }
6257
6258 /* Zero out previous entries */
6259 ibdm_free_gid_list(ioc->ioc_prev_gid_list);
6260 if (ioc->ioc_prev_serv)
6261 kmem_free(ioc->ioc_prev_serv, ioc->ioc_prev_serv_cnt *
6262 sizeof (ibdm_srvents_info_t));
6263 ioc->ioc_prev_serv_cnt = 0;
6264 ioc->ioc_prev_nportgids = 0;
6265 ioc->ioc_prev_serv = NULL;
6266 ioc->ioc_prev_gid_list = NULL;
6267 }
6268
6269 /*
6270 * Handle GID removal. This returns gid_info of an GID for the same
6271 * node GUID, if found. For an GID with IOU information, the same
6272 * gid_info is returned if no gid_info with same node_guid is found.
6273 */
6274 static ibdm_dp_gidinfo_t *
6275 ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *rm_gid)
6276 {
6277 ibdm_dp_gidinfo_t *gid_list;
6278
6279 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm(0x%p)", rm_gid);
6280
6281 if (rm_gid->gl_iou == NULL) {
6282 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm NO iou");
6283 /*
6284 * Search for a GID with same node_guid and
6285 * gl_iou != NULL
6286 */
6287 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
6288 gid_list = gid_list->gl_next) {
6289 if (gid_list->gl_iou != NULL && (gid_list->gl_nodeguid
6290 == rm_gid->gl_nodeguid))
6291 break;
6292 }
6293
6294 if (gid_list)
6295 ibdm_rmfrom_glgid_list(gid_list, rm_gid);
6296
6297 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list);
6298 return (gid_list);
6299 } else {
6300 /*
6301 * Search for a GID with same node_guid and
6302 * gl_iou == NULL
6303 */
6304 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm with iou");
6305 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
6306 gid_list = gid_list->gl_next) {
6307 if (gid_list->gl_iou == NULL && (gid_list->gl_nodeguid
6308 == rm_gid->gl_nodeguid))
6309 break;
6310 }
6311
6312 if (gid_list) {
6313 /*
6314 * Copy the following fields from rm_gid :
6315 * 1. gl_state
6316 * 2. gl_iou
6317 * 3. gl_gid & gl_ngids
6318 *
6319 * Note : Function is synchronized by
6320 * ibdm_busy flag.
6321 *
6322 * Note : Redirect info is initialized if
6323 * any MADs for the GID fail
6324 */
6325 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm "
6326 "copying info to GID with gl_iou != NULl");
6327 gid_list->gl_state = rm_gid->gl_state;
6328 gid_list->gl_iou = rm_gid->gl_iou;
6329 gid_list->gl_gid = rm_gid->gl_gid;
6330 gid_list->gl_ngids = rm_gid->gl_ngids;
6331
6332 /* Remove the GID from gl_gid list */
6333 ibdm_rmfrom_glgid_list(gid_list, rm_gid);
6334 } else {
6335 /*
6336 * Handle a case where all GIDs to the IOU have
6337 * been removed.
6338 */
6339 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm 0 GID "
6340 "to IOU");
6341
6342 ibdm_rmfrom_glgid_list(rm_gid, rm_gid);
6343 return (rm_gid);
6344 }
6345 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list);
6346 return (gid_list);
6347 }
6348 }
6349
6350 static void
6351 ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *gid_info,
6352 ibdm_dp_gidinfo_t *rm_gid)
6353 {
6354 ibdm_gid_t *tmp, *prev;
6355
6356 IBTF_DPRINTF_L4("ibdm", "\trmfrom_glgid (%p, %p)",
6357 gid_info, rm_gid);
6358
6359 for (tmp = gid_info->gl_gid, prev = NULL; tmp; ) {
6360 if (tmp->gid_dgid_hi == rm_gid->gl_dgid_hi &&
6361 tmp->gid_dgid_lo == rm_gid->gl_dgid_lo) {
6362 if (prev == NULL)
6363 gid_info->gl_gid = tmp->gid_next;
6364 else
6365 prev->gid_next = tmp->gid_next;
6366
6367 kmem_free(tmp, sizeof (ibdm_gid_t));
6368 gid_info->gl_ngids--;
6369 break;
6370 } else {
6371 prev = tmp;
6372 tmp = tmp->gid_next;
6373 }
6374 }
6375 }
6376
6377 static void
6378 ibdm_addto_gidlist(ibdm_gid_t **src_ptr, ibdm_gid_t *dest)
6379 {
6380 ibdm_gid_t *head = NULL, *new, *tail;
6381
6382 /* First copy the destination */
6383 for (; dest; dest = dest->gid_next) {
6384 new = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP);
6385 new->gid_dgid_hi = dest->gid_dgid_hi;
6386 new->gid_dgid_lo = dest->gid_dgid_lo;
6387 new->gid_next = head;
6388 head = new;
6389 }
6390
6391 /* Insert this to the source */
6392 if (*src_ptr == NULL)
6393 *src_ptr = head;
6394 else {
6395 for (tail = *src_ptr; tail->gid_next != NULL;
6396 tail = tail->gid_next)
6397 ;
6398
6399 tail->gid_next = head;
6400 }
6401 }
6402
6403 static void
6404 ibdm_free_gid_list(ibdm_gid_t *head)
6405 {
6406 ibdm_gid_t *delete;
6407
6408 for (delete = head; delete; ) {
6409 head = delete->gid_next;
6410 kmem_free(delete, sizeof (ibdm_gid_t));
6411 delete = head;
6412 }
6413 }
6414
6415 /*
6416 * This function rescans the DM capable GIDs (gl_state is
6417 * GID_PROBE_COMPLETE or IBDM_GID_PROBING_SKIPPED.This
6418 * basically checks if the DM capable GID is reachable. If
6419 * not this is handled the same way as GID_UNAVAILABLE,
6420 * except that notifications are not send to IBnexus.
6421 *
6422 * This function also initializes the ioc_prev_list for
6423 * a particular IOC (when called from probe_ioc, with
6424 * ioc_guidp != NULL) or all IOCs for the gid (called from
6425 * sweep_fabric, ioc_guidp == NULL).
6426 */
6427 static void
6428 ibdm_rescan_gidlist(ib_guid_t *ioc_guidp)
6429 {
6430 ibdm_dp_gidinfo_t *gid_info, *tmp;
6431 int ii, niocs, found;
6432 ibdm_hca_list_t *hca_list = NULL;
6433 ibdm_port_attr_t *port = NULL;
6434 ibdm_ioc_info_t *ioc_list;
6435
6436 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) {
6437 found = 0;
6438 if (gid_info->gl_state != IBDM_GID_PROBING_SKIPPED &&
6439 gid_info->gl_state != IBDM_GID_PROBING_COMPLETE) {
6440 gid_info = gid_info->gl_next;
6441 continue;
6442 }
6443
6444 /*
6445 * Check if the GID is visible to any HCA ports.
6446 * Return if so.
6447 */
6448 mutex_enter(&ibdm.ibdm_hl_mutex);
6449 for (ibdm_get_next_port(&hca_list, &port, 1); port;
6450 ibdm_get_next_port(&hca_list, &port, 1)) {
6451 if (ibdm_port_reachable(port->pa_sa_hdl,
6452 gid_info->gl_dgid_lo) == B_TRUE) {
6453 found = 1;
6454 break;
6455 }
6456 }
6457 mutex_exit(&ibdm.ibdm_hl_mutex);
6458
6459 if (found) {
6460 if (gid_info->gl_iou == NULL) {
6461 gid_info = gid_info->gl_next;
6462 continue;
6463 }
6464
6465 /* Intialize the ioc_prev_gid_list */
6466 niocs =
6467 gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
6468 for (ii = 0; ii < niocs; ii++) {
6469 ioc_list = IBDM_GIDINFO2IOCINFO(gid_info, ii);
6470
6471 if (ioc_guidp == NULL || (*ioc_guidp ==
6472 ioc_list->ioc_profile.ioc_guid)) {
6473 /* Add info of GIDs in gid_info also */
6474 ibdm_addto_gidlist(
6475 &ioc_list->ioc_prev_gid_list,
6476 gid_info->gl_gid);
6477 ioc_list->ioc_prev_nportgids =
6478 gid_info->gl_ngids;
6479 }
6480 }
6481 gid_info = gid_info->gl_next;
6482 continue;
6483 }
6484
6485 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist "
6486 "deleted port GUID %llx",
6487 gid_info->gl_dgid_lo);
6488
6489 /*
6490 * Update GID list in all IOCs affected by this
6491 */
6492 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0);
6493
6494 /*
6495 * Remove GID from the global GID list
6496 * Handle the case where all port GIDs for an
6497 * IOU have been hot-removed.
6498 */
6499 mutex_enter(&ibdm.ibdm_mutex);
6500 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) {
6501 mutex_enter(&gid_info->gl_mutex);
6502 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
6503 mutex_exit(&gid_info->gl_mutex);
6504 }
6505
6506 tmp = gid_info->gl_next;
6507 if (gid_info->gl_prev != NULL)
6508 gid_info->gl_prev->gl_next = gid_info->gl_next;
6509 if (gid_info->gl_next != NULL)
6510 gid_info->gl_next->gl_prev = gid_info->gl_prev;
6511
6512 if (gid_info == ibdm.ibdm_dp_gidlist_head)
6513 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next;
6514 if (gid_info == ibdm.ibdm_dp_gidlist_tail)
6515 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev;
6516 ibdm.ibdm_ngids--;
6517 mutex_exit(&ibdm.ibdm_mutex);
6518
6519 /* free the hca_list on this gid_info */
6520 ibdm_delete_glhca_list(gid_info);
6521
6522 mutex_destroy(&gid_info->gl_mutex);
6523 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t));
6524
6525 gid_info = tmp;
6526
6527 /*
6528 * Pass on the IOCs with updated GIDs to IBnexus
6529 */
6530 if (ioc_list) {
6531 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist "
6532 "IOC_PROP_UPDATE for %p\n", ioc_list);
6533 mutex_enter(&ibdm.ibdm_ibnex_mutex);
6534 if (ibdm.ibdm_ibnex_callback != NULL) {
6535 (*ibdm.ibdm_ibnex_callback)((void *)
6536 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
6537 }
6538 mutex_exit(&ibdm.ibdm_ibnex_mutex);
6539 }
6540 }
6541 }
6542
6543 /*
6544 * This function notifies IBnex of IOCs on this GID.
6545 * Notification is for GIDs with gl_reprobe_flag set.
6546 * The flag is set when IOC probe / fabric sweep
6547 * probes a GID starting from CLASS port info.
6548 *
6549 * IBnexus will have information of a reconnected IOC
6550 * if it had probed it before. If this is a new IOC,
6551 * IBnexus ignores the notification.
6552 *
6553 * This function should be called with no locks held.
6554 */
6555 static void
6556 ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *gid_info)
6557 {
6558 ibdm_ioc_info_t *ioc_list;
6559
6560 if (gid_info->gl_reprobe_flag == 0 ||
6561 gid_info->gl_iou == NULL)
6562 return;
6563
6564 ioc_list = ibdm_update_ioc_gidlist(gid_info, -1);
6565
6566 /*
6567 * Pass on the IOCs with updated GIDs to IBnexus
6568 */
6569 if (ioc_list) {
6570 mutex_enter(&ibdm.ibdm_ibnex_mutex);
6571 if (ibdm.ibdm_ibnex_callback != NULL) {
6572 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list,
6573 IBDM_EVENT_IOC_PROP_UPDATE);
6574 }
6575 mutex_exit(&ibdm.ibdm_ibnex_mutex);
6576 }
6577 }
6578
6579
6580 static void
6581 ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *arg)
6582 {
6583 if (arg != NULL)
6584 kmem_free(arg, sizeof (ibdm_saa_event_arg_t));
6585 }
6586
6587 /*
6588 * This function parses the list of HCAs and HCA ports
6589 * to return the port_attr of the next HCA port. A port
6590 * connected to IB fabric (port_state active) is returned,
6591 * if connected_flag is set.
6592 */
6593 static void
6594 ibdm_get_next_port(ibdm_hca_list_t **inp_hcap,
6595 ibdm_port_attr_t **inp_portp, int connect_flag)
6596 {
6597 int ii;
6598 ibdm_port_attr_t *port, *next_port = NULL;
6599 ibdm_port_attr_t *inp_port;
6600 ibdm_hca_list_t *hca_list;
6601 int found = 0;
6602
6603 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
6604 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port(%p, %p, %x)",
6605 inp_hcap, inp_portp, connect_flag);
6606
6607 hca_list = *inp_hcap;
6608 inp_port = *inp_portp;
6609
6610 if (hca_list == NULL)
6611 hca_list = ibdm.ibdm_hca_list_head;
6612
6613 for (; hca_list; hca_list = hca_list->hl_next) {
6614 for (ii = 0; ii < hca_list->hl_nports; ii++) {
6615 port = &hca_list->hl_port_attr[ii];
6616
6617 /*
6618 * inp_port != NULL;
6619 * Skip till we find the matching port
6620 */
6621 if (inp_port && !found) {
6622 if (inp_port == port)
6623 found = 1;
6624 continue;
6625 }
6626
6627 if (!connect_flag) {
6628 next_port = port;
6629 break;
6630 }
6631
6632 if (port->pa_sa_hdl == NULL)
6633 ibdm_initialize_port(port);
6634 if (port->pa_sa_hdl == NULL)
6635 (void) ibdm_fini_port(port);
6636 else if (next_port == NULL &&
6637 port->pa_sa_hdl != NULL &&
6638 port->pa_state == IBT_PORT_ACTIVE) {
6639 next_port = port;
6640 break;
6641 }
6642 }
6643
6644 if (next_port)
6645 break;
6646 }
6647
6648 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port : "
6649 "returns hca_list %p port %p", hca_list, next_port);
6650 *inp_hcap = hca_list;
6651 *inp_portp = next_port;
6652 }
6653
6654 static void
6655 ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *nodegid, ibdm_dp_gidinfo_t *addgid)
6656 {
6657 ibdm_gid_t *tmp;
6658
6659 tmp = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP);
6660 tmp->gid_dgid_hi = addgid->gl_dgid_hi;
6661 tmp->gid_dgid_lo = addgid->gl_dgid_lo;
6662
6663 mutex_enter(&nodegid->gl_mutex);
6664 tmp->gid_next = nodegid->gl_gid;
6665 nodegid->gl_gid = tmp;
6666 nodegid->gl_ngids++;
6667 mutex_exit(&nodegid->gl_mutex);
6668 }
6669
6670 static void
6671 ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *gid_info,
6672 ibdm_hca_list_t *hca)
6673 {
6674 ibdm_hca_list_t *head, *prev = NULL, *temp;
6675
6676 IBTF_DPRINTF_L4(ibdm_string, "\taddto_glhcalist(%p, %p) "
6677 ": gl_hca_list %p", gid_info, hca, gid_info->gl_hca_list);
6678 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex));
6679
6680 mutex_enter(&gid_info->gl_mutex);
6681 head = gid_info->gl_hca_list;
6682 if (head == NULL) {
6683 head = ibdm_dup_hca_attr(hca);
6684 head->hl_next = NULL;
6685 gid_info->gl_hca_list = head;
6686 mutex_exit(&gid_info->gl_mutex);
6687 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: "
6688 "gid %p, gl_hca_list %p", gid_info,
6689 gid_info->gl_hca_list);
6690 return;
6691 }
6692
6693 /* Check if already in the list */
6694 while (head) {
6695 if (head->hl_hca_guid == hca->hl_hca_guid) {
6696 mutex_exit(&gid_info->gl_mutex);
6697 IBTF_DPRINTF_L4(ibdm_string,
6698 "\taddto_glhcalist : gid %p hca %p dup",
6699 gid_info, hca);
6700 return;
6701 }
6702 prev = head;
6703 head = head->hl_next;
6704 }
6705
6706 /* Add this HCA to gl_hca_list */
6707 temp = ibdm_dup_hca_attr(hca);
6708 temp->hl_next = NULL;
6709 prev->hl_next = temp;
6710 mutex_exit(&gid_info->gl_mutex);
6711
6712 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: "
6713 "gid %p, gl_hca_list %p", gid_info, gid_info->gl_hca_list);
6714 }
6715
6716 static void
6717 ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *gid_info)
6718 {
6719 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex));
6720 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex));
6721
6722 mutex_enter(&gid_info->gl_mutex);
6723 if (gid_info->gl_hca_list)
6724 ibdm_ibnex_free_hca_list(gid_info->gl_hca_list);
6725 gid_info->gl_hca_list = NULL;
6726 mutex_exit(&gid_info->gl_mutex);
6727 }
6728
6729
6730 static void
6731 ibdm_reset_all_dgids(ibmf_saa_handle_t port_sa_hdl)
6732 {
6733 IBTF_DPRINTF_L4(ibdm_string, "\treset_all_dgids(%X)",
6734 port_sa_hdl);
6735
6736 if (ibdm_enumerate_iocs == 0)
6737 return;
6738
6739 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex));
6740 ASSERT(!MUTEX_HELD(&ibdm.ibdm_hl_mutex));
6741
6742 /* Check : Not busy in another probe / sweep */
6743 mutex_enter(&ibdm.ibdm_mutex);
6744 if ((ibdm.ibdm_busy & IBDM_BUSY) == 0) {
6745 ibdm_dp_gidinfo_t *gid_info;
6746
6747 ibdm.ibdm_busy |= IBDM_BUSY;
6748 mutex_exit(&ibdm.ibdm_mutex);
6749
6750 /*
6751 * Check if any GID is using the SA & IBMF handle
6752 * of HCA port going down. Reset ibdm_dp_gidinfo_t
6753 * using another HCA port which can reach the GID.
6754 * This is for DM capable GIDs only, no need to do
6755 * this for others
6756 *
6757 * Delete the GID if no alternate HCA port to reach
6758 * it is found.
6759 */
6760 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) {
6761 ibdm_dp_gidinfo_t *tmp;
6762
6763 IBTF_DPRINTF_L4(ibdm_string, "\tevent_hdlr "
6764 "checking gidinfo %p", gid_info);
6765
6766 if (gid_info->gl_sa_hdl == port_sa_hdl) {
6767 IBTF_DPRINTF_L3(ibdm_string,
6768 "\tevent_hdlr: down HCA port hdl "
6769 "matches gid %p", gid_info);
6770
6771 /*
6772 * The non-DM GIDs can come back
6773 * with a new subnet prefix, when
6774 * the HCA port commes up again. To
6775 * avoid issues, delete non-DM
6776 * capable GIDs, if the gid was
6777 * discovered using the HCA port
6778 * going down. This is ensured by
6779 * setting gl_disconnected to 1.
6780 */
6781 if (gid_info->gl_is_dm_capable == B_FALSE)
6782 gid_info->gl_disconnected = 1;
6783 else
6784 ibdm_reset_gidinfo(gid_info);
6785
6786 if (gid_info->gl_disconnected) {
6787 IBTF_DPRINTF_L3(ibdm_string,
6788 "\tevent_hdlr: deleting"
6789 " gid %p", gid_info);
6790 tmp = gid_info;
6791 gid_info = gid_info->gl_next;
6792 ibdm_delete_gidinfo(tmp);
6793 } else
6794 gid_info = gid_info->gl_next;
6795 } else
6796 gid_info = gid_info->gl_next;
6797 }
6798
6799 mutex_enter(&ibdm.ibdm_mutex);
6800 ibdm.ibdm_busy &= ~IBDM_BUSY;
6801 cv_signal(&ibdm.ibdm_busy_cv);
6802 }
6803 mutex_exit(&ibdm.ibdm_mutex);
6804 }
6805
6806 static void
6807 ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *gidinfo)
6808 {
6809 ibdm_hca_list_t *hca_list = NULL;
6810 ibdm_port_attr_t *port = NULL;
6811 int gid_reinited = 0;
6812 sa_node_record_t *nr, *tmp;
6813 sa_portinfo_record_t *pi;
6814 size_t nr_len = 0, pi_len = 0;
6815 size_t path_len;
6816 ib_gid_t sgid, dgid;
6817 int ret, ii, nrecords;
6818 sa_path_record_t *path;
6819 uint8_t npaths = 1;
6820 ibdm_pkey_tbl_t *pkey_tbl;
6821
6822 IBTF_DPRINTF_L4(ibdm_string, "\treset_gidinfo(%p)", gidinfo);
6823
6824 /*
6825 * Get list of all the ports reachable from the local known HCA
6826 * ports which are active
6827 */
6828 mutex_enter(&ibdm.ibdm_hl_mutex);
6829 for (ibdm_get_next_port(&hca_list, &port, 1); port;
6830 ibdm_get_next_port(&hca_list, &port, 1)) {
6831
6832
6833 /*
6834 * Get the path and re-populate the gidinfo.
6835 * Getting the path is the same probe_ioc
6836 * Init the gid info as in ibdm_create_gidinfo()
6837 */
6838 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len,
6839 gidinfo->gl_nodeguid);
6840 if (nr == NULL) {
6841 IBTF_DPRINTF_L4(ibdm_string,
6842 "\treset_gidinfo : no records");
6843 continue;
6844 }
6845
6846 nrecords = (nr_len / sizeof (sa_node_record_t));
6847 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) {
6848 if (tmp->NodeInfo.PortGUID == gidinfo->gl_portguid)
6849 break;
6850 }
6851
6852 if (ii == nrecords) {
6853 IBTF_DPRINTF_L4(ibdm_string,
6854 "\treset_gidinfo : no record for portguid");
6855 kmem_free(nr, nr_len);
6856 continue;
6857 }
6858
6859 pi = ibdm_get_portinfo(port->pa_sa_hdl, &pi_len, tmp->LID);
6860 if (pi == NULL) {
6861 IBTF_DPRINTF_L4(ibdm_string,
6862 "\treset_gidinfo : no portinfo");
6863 kmem_free(nr, nr_len);
6864 continue;
6865 }
6866
6867 sgid.gid_prefix = port->pa_sn_prefix;
6868 sgid.gid_guid = port->pa_port_guid;
6869 dgid.gid_prefix = pi->PortInfo.GidPrefix;
6870 dgid.gid_guid = tmp->NodeInfo.PortGUID;
6871
6872 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, sgid, dgid,
6873 IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, &path_len, &path);
6874
6875 if ((ret != IBMF_SUCCESS) || path == NULL) {
6876 IBTF_DPRINTF_L4(ibdm_string,
6877 "\treset_gidinfo : no paths");
6878 kmem_free(pi, pi_len);
6879 kmem_free(nr, nr_len);
6880 continue;
6881 }
6882
6883 gidinfo->gl_dgid_hi = path->DGID.gid_prefix;
6884 gidinfo->gl_dgid_lo = path->DGID.gid_guid;
6885 gidinfo->gl_sgid_hi = path->SGID.gid_prefix;
6886 gidinfo->gl_sgid_lo = path->SGID.gid_guid;
6887 gidinfo->gl_p_key = path->P_Key;
6888 gidinfo->gl_sa_hdl = port->pa_sa_hdl;
6889 gidinfo->gl_ibmf_hdl = port->pa_ibmf_hdl;
6890 gidinfo->gl_slid = path->SLID;
6891 gidinfo->gl_dlid = path->DLID;
6892 /* Reset redirect info, next MAD will set if redirected */
6893 gidinfo->gl_redirected = 0;
6894 gidinfo->gl_devid = (*tmp).NodeInfo.DeviceID;
6895 gidinfo->gl_SL = path->SL;
6896
6897 gidinfo->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT;
6898 for (ii = 0; ii < port->pa_npkeys; ii++) {
6899 if (port->pa_pkey_tbl == NULL)
6900 break;
6901
6902 pkey_tbl = &port->pa_pkey_tbl[ii];
6903 if ((gidinfo->gl_p_key == pkey_tbl->pt_pkey) &&
6904 (pkey_tbl->pt_qp_hdl != NULL)) {
6905 gidinfo->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
6906 break;
6907 }
6908 }
6909
6910 if (gidinfo->gl_qp_hdl == NULL)
6911 IBTF_DPRINTF_L2(ibdm_string,
6912 "\treset_gid_info: No matching Pkey");
6913 else
6914 gid_reinited = 1;
6915
6916 kmem_free(path, path_len);
6917 kmem_free(pi, pi_len);
6918 kmem_free(nr, nr_len);
6919 break;
6920 }
6921 mutex_exit(&ibdm.ibdm_hl_mutex);
6922
6923 if (!gid_reinited)
6924 gidinfo->gl_disconnected = 1;
6925 }
6926
6927 static void
6928 ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *gidinfo)
6929 {
6930 ibdm_ioc_info_t *ioc_list;
6931 int in_gidlist = 0;
6932
6933 /*
6934 * Check if gidinfo has been inserted into the
6935 * ibdm_dp_gidlist_head list. gl_next or gl_prev
6936 * != NULL, if gidinfo is the list.
6937 */
6938 if (gidinfo->gl_prev != NULL ||
6939 gidinfo->gl_next != NULL ||
6940 ibdm.ibdm_dp_gidlist_head == gidinfo)
6941 in_gidlist = 1;
6942
6943 ioc_list = ibdm_update_ioc_gidlist(gidinfo, 0);
6944
6945 /*
6946 * Remove GID from the global GID list
6947 * Handle the case where all port GIDs for an
6948 * IOU have been hot-removed.
6949 */
6950 mutex_enter(&ibdm.ibdm_mutex);
6951 if (gidinfo->gl_iou != NULL && gidinfo->gl_ngids == 0) {
6952 mutex_enter(&gidinfo->gl_mutex);
6953 (void) ibdm_free_iou_info(gidinfo, &gidinfo->gl_iou);
6954 mutex_exit(&gidinfo->gl_mutex);
6955 }
6956
6957 /* Delete gl_hca_list */
6958 mutex_exit(&ibdm.ibdm_mutex);
6959 ibdm_delete_glhca_list(gidinfo);
6960 mutex_enter(&ibdm.ibdm_mutex);
6961
6962 if (in_gidlist) {
6963 if (gidinfo->gl_prev != NULL)
6964 gidinfo->gl_prev->gl_next = gidinfo->gl_next;
6965 if (gidinfo->gl_next != NULL)
6966 gidinfo->gl_next->gl_prev = gidinfo->gl_prev;
6967
6968 if (gidinfo == ibdm.ibdm_dp_gidlist_head)
6969 ibdm.ibdm_dp_gidlist_head = gidinfo->gl_next;
6970 if (gidinfo == ibdm.ibdm_dp_gidlist_tail)
6971 ibdm.ibdm_dp_gidlist_tail = gidinfo->gl_prev;
6972 ibdm.ibdm_ngids--;
6973 }
6974 mutex_exit(&ibdm.ibdm_mutex);
6975
6976 mutex_destroy(&gidinfo->gl_mutex);
6977 cv_destroy(&gidinfo->gl_probe_cv);
6978 kmem_free(gidinfo, sizeof (ibdm_dp_gidinfo_t));
6979
6980 /*
6981 * Pass on the IOCs with updated GIDs to IBnexus
6982 */
6983 if (ioc_list) {
6984 IBTF_DPRINTF_L4("ibdm", "\tdelete_gidinfo "
6985 "IOC_PROP_UPDATE for %p\n", ioc_list);
6986 mutex_enter(&ibdm.ibdm_ibnex_mutex);
6987 if (ibdm.ibdm_ibnex_callback != NULL) {
6988 (*ibdm.ibdm_ibnex_callback)((void *)
6989 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
6990 }
6991 mutex_exit(&ibdm.ibdm_ibnex_mutex);
6992 }
6993 }
6994
6995
6996 static void
6997 ibdm_fill_srv_attr_mod(ib_mad_hdr_t *hdr, ibdm_timeout_cb_args_t *cb_args)
6998 {
6999 uint32_t attr_mod;
7000
7001 attr_mod = (cb_args->cb_ioc_num + 1) << 16;
7002 attr_mod |= cb_args->cb_srvents_start;
7003 attr_mod |= (cb_args->cb_srvents_end) << 8;
7004 hdr->AttributeModifier = h2b32(attr_mod);
7005 }
7006
7007 static void
7008 ibdm_bump_transactionID(ibdm_dp_gidinfo_t *gid_info)
7009 {
7010 ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
7011 gid_info->gl_transactionID++;
7012 if (gid_info->gl_transactionID == gid_info->gl_max_transactionID) {
7013 IBTF_DPRINTF_L4(ibdm_string,
7014 "\tbump_transactionID(%p), wrapup", gid_info);
7015 gid_info->gl_transactionID = gid_info->gl_min_transactionID;
7016 }
7017 }
7018
7019 /*
7020 * gl_prev_iou is set for *non-reprobe* sweeep requests, which
7021 * detected that ChangeID in IOU info has changed. The service
7022 * entry also may have changed. Check if service entry in IOC
7023 * has changed wrt the prev iou, if so notify to IB Nexus.
7024 */
7025 static ibdm_ioc_info_t *
7026 ibdm_handle_prev_iou()
7027 {
7028 ibdm_dp_gidinfo_t *gid_info;
7029 ibdm_ioc_info_t *ioc_list_head = NULL, *ioc_list;
7030 ibdm_ioc_info_t *prev_ioc, *ioc;
7031 int ii, jj, niocs, prev_niocs;
7032
7033 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
7034
7035 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou enter");
7036 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
7037 gid_info = gid_info->gl_next) {
7038 if (gid_info->gl_prev_iou == NULL)
7039 continue;
7040
7041 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou gid %p",
7042 gid_info);
7043 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
7044 prev_niocs =
7045 gid_info->gl_prev_iou->iou_info.iou_num_ctrl_slots;
7046 for (ii = 0; ii < niocs; ii++) {
7047 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
7048
7049 /* Find matching IOC */
7050 for (jj = 0; jj < prev_niocs; jj++) {
7051 prev_ioc = (ibdm_ioc_info_t *)
7052 &gid_info->gl_prev_iou->iou_ioc_info[jj];
7053 if (prev_ioc->ioc_profile.ioc_guid ==
7054 ioc->ioc_profile.ioc_guid)
7055 break;
7056 }
7057 if (jj == prev_niocs)
7058 prev_ioc = NULL;
7059 if (ioc == NULL || prev_ioc == NULL)
7060 continue;
7061 if ((ioc->ioc_profile.ioc_service_entries !=
7062 prev_ioc->ioc_profile.ioc_service_entries) ||
7063 ibdm_serv_cmp(&ioc->ioc_serv[0],
7064 &prev_ioc->ioc_serv[0],
7065 ioc->ioc_profile.ioc_service_entries) != 0) {
7066 IBTF_DPRINTF_L4(ibdm_string,
7067 "/thandle_prev_iou modified IOC: "
7068 "current ioc %p, old ioc %p",
7069 ioc, prev_ioc);
7070 mutex_enter(&gid_info->gl_mutex);
7071 ioc_list = ibdm_dup_ioc_info(ioc, gid_info);
7072 mutex_exit(&gid_info->gl_mutex);
7073 ioc_list->ioc_info_updated.ib_prop_updated
7074 = 0;
7075 ioc_list->ioc_info_updated.ib_srv_prop_updated
7076 = 1;
7077
7078 if (ioc_list_head == NULL)
7079 ioc_list_head = ioc_list;
7080 else {
7081 ioc_list_head->ioc_next = ioc_list;
7082 ioc_list_head = ioc_list;
7083 }
7084 }
7085 }
7086
7087 mutex_enter(&gid_info->gl_mutex);
7088 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_prev_iou);
7089 mutex_exit(&gid_info->gl_mutex);
7090 }
7091 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iouret %p",
7092 ioc_list_head);
7093 return (ioc_list_head);
7094 }
7095
7096 /*
7097 * Compares two service entries lists, returns 0 if same, returns 1
7098 * if no match.
7099 */
7100 static int
7101 ibdm_serv_cmp(ibdm_srvents_info_t *serv1, ibdm_srvents_info_t *serv2,
7102 int nserv)
7103 {
7104 int ii;
7105
7106 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: enter");
7107 for (ii = 0; ii < nserv; ii++, serv1++, serv2++) {
7108 if (serv1->se_attr.srv_id != serv2->se_attr.srv_id ||
7109 bcmp(serv1->se_attr.srv_name,
7110 serv2->se_attr.srv_name,
7111 IB_DM_MAX_SVC_NAME_LEN) != 0) {
7112 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 1");
7113 return (1);
7114 }
7115 }
7116 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 0");
7117 return (0);
7118 }
7119
7120 /* For debugging purpose only */
7121 #ifdef DEBUG
7122 void
7123 ibdm_dump_mad_hdr(ib_mad_hdr_t *mad_hdr)
7124 {
7125 IBTF_DPRINTF_L4("ibdm", "\t\t MAD Header info");
7126 IBTF_DPRINTF_L4("ibdm", "\t\t ---------------");
7127
7128 IBTF_DPRINTF_L4("ibdm", "\tBase version : 0x%x"
7129 "\tMgmt Class : 0x%x", mad_hdr->BaseVersion, mad_hdr->MgmtClass);
7130 IBTF_DPRINTF_L4("ibdm", "\tClass version : 0x%x"
7131 "\tR Method : 0x%x",
7132 mad_hdr->ClassVersion, mad_hdr->R_Method);
7133 IBTF_DPRINTF_L4("ibdm", "\tMAD Status : 0x%x"
7134 "\tTransaction ID : 0x%llx",
7135 b2h16(mad_hdr->Status), b2h64(mad_hdr->TransactionID));
7136 IBTF_DPRINTF_L4("ibdm", "\t Attribute ID : 0x%x"
7137 "\tAttribute Modified : 0x%lx",
7138 b2h16(mad_hdr->AttributeID), b2h32(mad_hdr->AttributeModifier));
7139 }
7140
7141
7142 void
7143 ibdm_dump_ibmf_msg(ibmf_msg_t *ibmf_msg, int flag)
7144 {
7145 ib_mad_hdr_t *mad_hdr;
7146
7147 IBTF_DPRINTF_L4("ibdm", "\t\t(IBMF_PKT): Local address info");
7148 IBTF_DPRINTF_L4("ibdm", "\t\t ------------------");
7149
7150 IBTF_DPRINTF_L4("ibdm", "\tLocal Lid : 0x%x\tRemote Lid : 0x%x"
7151 " Remote Qp : 0x%x", ibmf_msg->im_local_addr.ia_local_lid,
7152 ibmf_msg->im_local_addr.ia_remote_lid,
7153 ibmf_msg->im_local_addr.ia_remote_qno);
7154 IBTF_DPRINTF_L4("ibdm", "\tP_key : 0x%x\tQ_key : 0x%x"
7155 " SL : 0x%x", ibmf_msg->im_local_addr.ia_p_key,
7156 ibmf_msg->im_local_addr.ia_q_key,
7157 ibmf_msg->im_local_addr.ia_service_level);
7158
7159 if (flag)
7160 mad_hdr = (ib_mad_hdr_t *)IBDM_OUT_IBMFMSG_MADHDR(ibmf_msg);
7161 else
7162 mad_hdr = IBDM_IN_IBMFMSG_MADHDR(ibmf_msg);
7163
7164 ibdm_dump_mad_hdr(mad_hdr);
7165 }
7166
7167
7168 void
7169 ibdm_dump_path_info(sa_path_record_t *path)
7170 {
7171 IBTF_DPRINTF_L4("ibdm", "\t\t Path information");
7172 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------");
7173
7174 IBTF_DPRINTF_L4("ibdm", "\t DGID hi : %llx\tDGID lo : %llx",
7175 path->DGID.gid_prefix, path->DGID.gid_guid);
7176 IBTF_DPRINTF_L4("ibdm", "\t SGID hi : %llx\tSGID lo : %llx",
7177 path->SGID.gid_prefix, path->SGID.gid_guid);
7178 IBTF_DPRINTF_L4("ibdm", "\t SLID : %x\t\tDlID : %x",
7179 path->SLID, path->DLID);
7180 IBTF_DPRINTF_L4("ibdm", "\t P Key : %x\t\tSL : %x",
7181 path->P_Key, path->SL);
7182 }
7183
7184
7185 void
7186 ibdm_dump_classportinfo(ib_mad_classportinfo_t *classportinfo)
7187 {
7188 IBTF_DPRINTF_L4("ibdm", "\t\t CLASSPORT INFO");
7189 IBTF_DPRINTF_L4("ibdm", "\t\t --------------");
7190
7191 IBTF_DPRINTF_L4("ibdm", "\t Response Time Value : 0x%x",
7192 ((b2h32(classportinfo->RespTimeValue)) & 0x1F));
7193
7194 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID hi : 0x%llx",
7195 b2h64(classportinfo->RedirectGID_hi));
7196 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID lo : 0x%llx",
7197 b2h64(classportinfo->RedirectGID_lo));
7198 IBTF_DPRINTF_L4("ibdm", "\t Redirected TC : 0x%x",
7199 classportinfo->RedirectTC);
7200 IBTF_DPRINTF_L4("ibdm", "\t Redirected SL : 0x%x",
7201 classportinfo->RedirectSL);
7202 IBTF_DPRINTF_L4("ibdm", "\t Redirected FL : 0x%x",
7203 classportinfo->RedirectFL);
7204 IBTF_DPRINTF_L4("ibdm", "\t Redirected LID : 0x%x",
7205 b2h16(classportinfo->RedirectLID));
7206 IBTF_DPRINTF_L4("ibdm", "\t Redirected P KEY : 0x%x",
7207 b2h16(classportinfo->RedirectP_Key));
7208 IBTF_DPRINTF_L4("ibdm", "\t Redirected QP : 0x%x",
7209 classportinfo->RedirectQP);
7210 IBTF_DPRINTF_L4("ibdm", "\t Redirected Q KEY : 0x%x",
7211 b2h32(classportinfo->RedirectQ_Key));
7212 IBTF_DPRINTF_L4("ibdm", "\t Trap GID hi : 0x%llx",
7213 b2h64(classportinfo->TrapGID_hi));
7214 IBTF_DPRINTF_L4("ibdm", "\t Trap GID lo : 0x%llx",
7215 b2h64(classportinfo->TrapGID_lo));
7216 IBTF_DPRINTF_L4("ibdm", "\t Trap TC : 0x%x",
7217 classportinfo->TrapTC);
7218 IBTF_DPRINTF_L4("ibdm", "\t Trap SL : 0x%x",
7219 classportinfo->TrapSL);
7220 IBTF_DPRINTF_L4("ibdm", "\t Trap FL : 0x%x",
7221 classportinfo->TrapFL);
7222 IBTF_DPRINTF_L4("ibdm", "\t Trap LID : 0x%x",
7223 b2h16(classportinfo->TrapLID));
7224 IBTF_DPRINTF_L4("ibdm", "\t Trap P_Key : 0x%x",
7225 b2h16(classportinfo->TrapP_Key));
7226 IBTF_DPRINTF_L4("ibdm", "\t Trap HL : 0x%x",
7227 classportinfo->TrapHL);
7228 IBTF_DPRINTF_L4("ibdm", "\t Trap QP : 0x%x",
7229 classportinfo->TrapQP);
7230 IBTF_DPRINTF_L4("ibdm", "\t Trap Q_Key : 0x%x",
7231 b2h32(classportinfo->TrapQ_Key));
7232 }
7233
7234
7235 void
7236 ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *iou_info)
7237 {
7238 IBTF_DPRINTF_L4("ibdm", "\t\t I/O UnitInfo");
7239 IBTF_DPRINTF_L4("ibdm", "\t\t ------------");
7240
7241 IBTF_DPRINTF_L4("ibdm", "\tChange ID : 0x%x",
7242 b2h16(iou_info->iou_changeid));
7243 IBTF_DPRINTF_L4("ibdm", "\t#of ctrl slots : %d",
7244 iou_info->iou_num_ctrl_slots);
7245 IBTF_DPRINTF_L4("ibdm", "\tIOU flag : 0x%x",
7246 iou_info->iou_flag);
7247 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 0 : 0x%x",
7248 iou_info->iou_ctrl_list[0]);
7249 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 1 : 0x%x",
7250 iou_info->iou_ctrl_list[1]);
7251 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 2 : 0x%x",
7252 iou_info->iou_ctrl_list[2]);
7253 }
7254
7255
7256 void
7257 ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *ioc)
7258 {
7259 IBTF_DPRINTF_L4("ibdm", "\t\t IOC Controller Profile");
7260 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------------");
7261
7262 IBTF_DPRINTF_L4("ibdm", "\tIOC Guid : %llx", ioc->ioc_guid);
7263 IBTF_DPRINTF_L4("ibdm", "\tVendorID : 0x%x", ioc->ioc_vendorid);
7264 IBTF_DPRINTF_L4("ibdm", "\tDevice Id : 0x%x", ioc->ioc_deviceid);
7265 IBTF_DPRINTF_L4("ibdm", "\tDevice Ver : 0x%x", ioc->ioc_device_ver);
7266 IBTF_DPRINTF_L4("ibdm", "\tSubsys ID : 0x%x", ioc->ioc_subsys_id);
7267 IBTF_DPRINTF_L4("ibdm", "\tIO class : 0x%x", ioc->ioc_io_class);
7268 IBTF_DPRINTF_L4("ibdm", "\tIO subclass : 0x%x", ioc->ioc_io_subclass);
7269 IBTF_DPRINTF_L4("ibdm", "\tProtocol : 0x%x", ioc->ioc_protocol);
7270 IBTF_DPRINTF_L4("ibdm", "\tProtocolV : 0x%x", ioc->ioc_protocol_ver);
7271 IBTF_DPRINTF_L4("ibdm", "\tmsg qdepth : %d", ioc->ioc_send_msg_qdepth);
7272 IBTF_DPRINTF_L4("ibdm", "\trdma qdepth : %d",
7273 ioc->ioc_rdma_read_qdepth);
7274 IBTF_DPRINTF_L4("ibdm", "\tsndmsg sz : %d", ioc->ioc_send_msg_sz);
7275 IBTF_DPRINTF_L4("ibdm", "\trdma xfersz : %d", ioc->ioc_rdma_xfer_sz);
7276 IBTF_DPRINTF_L4("ibdm", "\topcal mask : 0x%x",
7277 ioc->ioc_ctrl_opcap_mask);
7278 IBTF_DPRINTF_L4("ibdm", "\tsrventries : %x", ioc->ioc_service_entries);
7279 }
7280
7281
7282 void
7283 ibdm_dump_service_entries(ib_dm_srv_t *srv_ents)
7284 {
7285 IBTF_DPRINTF_L4("ibdm",
7286 "\thandle_srventry_mad: service id : %llx", srv_ents->srv_id);
7287
7288 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad: "
7289 "Service Name : %s", srv_ents->srv_name);
7290 }
7291
7292 int ibdm_allow_sweep_fabric_timestamp = 1;
7293
7294 void
7295 ibdm_dump_sweep_fabric_timestamp(int flag)
7296 {
7297 static hrtime_t x;
7298 if (flag) {
7299 if (ibdm_allow_sweep_fabric_timestamp) {
7300 IBTF_DPRINTF_L4("ibdm", "\tTime taken to complete "
7301 "sweep %lld ms", ((gethrtime() - x)/ 1000000));
7302 }
7303 x = 0;
7304 } else
7305 x = gethrtime();
7306 }
7307 #endif