Print this page
XXXX introduce drv_sectohz
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_saa_impl.c
+++ new/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_saa_impl.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26 #include <sys/ib/mgt/ibmf/ibmf_saa_impl.h>
27 27 #include <sys/ib/mgt/ibmf/ibmf_saa_utils.h>
28 28
29 29 /* Global sa_access State Pointer */
30 30 saa_state_t *saa_statep;
31 31 _NOTE(READ_ONLY_DATA(saa_statep))
32 32
33 33 extern int ibmf_trace_level;
34 34
35 35 extern int ibmf_taskq_max_tasks;
36 36
37 37 static int
38 38 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
39 39 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags);
40 40 static int
41 41 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp,
42 42 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags);
43 43 static int
44 44 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
45 45 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg,
46 46 hrtime_t trans_send_time, int transport_flags);
47 47 static int ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info,
48 48 boolean_t sleep_flag, ibmf_msg_t **msgp, uint32_t *transport_flagsp,
49 49 ibmf_retrans_t *ibmf_retransp);
50 50 static int ibmf_saa_must_purge(saa_port_t *saa_portp);
51 51 static void ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp);
52 52 static void ibmf_saa_impl_destroy_port(saa_port_t *saa_portp);
53 53 static void ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp);
54 54 static void ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer,
55 55 int status);
56 56 static void ibmf_saa_impl_async_event_cb(ibmf_handle_t ibmf_handle,
57 57 void *clnt_private, ibmf_async_event_t event_type);
58 58 static void ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num);
59 59 static void ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num);
60 60 static void ibmf_saa_impl_port_chg(ibt_async_event_t *event);
61 61 static void ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num);
62 62 static void ibmf_saa_impl_hca_detach(saa_port_t *saa_removed);
63 63 static void ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle,
64 64 ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result,
65 65 size_t *length, boolean_t sleep_flag);
66 66 static int ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id);
67 67 static uint_t ibmf_saa_impl_get_attr_id_length(uint16_t attr_id);
68 68 static void ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp);
69 69 static int ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop,
70 70 ib_guid_t *guid_ret);
71 71 static void ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp,
72 72 ibt_hca_portinfo_t *portinfop);
73 73 static void ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp,
74 74 ibmf_msg_t *msgp);
75 75 static int ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp);
76 76
77 77 int ibmf_saa_max_wait_time = IBMF_SAA_MAX_WAIT_TIME_IN_SECS;
78 78 int ibmf_saa_trans_wait_time = IBMF_SAA_TRANS_WAIT_TIME_IN_SECS;
79 79 int ibmf_saa_max_resp_time = IBMF_SAA_MAX_RESP_TIME;
80 80 int ibmf_saa_max_subnet_timeout = IBMF_SAA_MAX_SUBNET_TIMEOUT;
81 81 int ibmf_saa_retrans_retries = IBMF_SAA_RETRANS_RETRIES;
82 82
83 83 /*
84 84 * ibmf_saa_impl_init:
85 85 * Allocates memory for the ibmf_saa state structure and initializes the taskq.
86 86 * Called from the modules init() routine.
87 87 *
88 88 * Input Arguments
89 89 * none
90 90 *
91 91 * Output Arguments
92 92 * none
93 93 *
94 94 * Returns
95 95 * IBMF_NO_RESOURCES if taskq could not be created.
96 96 * IBMF_SUCCESS on success
97 97 *
98 98 */
99 99 int
100 100 ibmf_saa_impl_init()
101 101 {
102 102 int res;
103 103
104 104 /* CONSTCOND */
105 105 ASSERT(NO_COMPETING_THREADS);
106 106
107 107 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_init_start,
108 108 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() enter\n");
109 109
110 110 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_statep))
111 111
112 112 saa_statep = kmem_zalloc(sizeof (saa_state_t), KM_SLEEP);
113 113
114 114 /* create taskq for notifying event subscribers */
115 115 saa_statep->saa_event_taskq = taskq_create(
116 116 "ibmf_saa_event_taskq", IBMF_TASKQ_NTHREADS,
117 117 MINCLSYSPRI, 1, ibmf_taskq_max_tasks, TASKQ_DYNAMIC |
118 118 TASKQ_PREPOPULATE);
119 119 if (saa_statep->saa_event_taskq == NULL) {
120 120
121 121 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L4,
122 122 ibmf_saa_impl_init_end_err,
123 123 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init(): %s\n",
124 124 tnf_string, msg, "event taskq create failed");
125 125
126 126 kmem_free(saa_statep, sizeof (saa_state_t));
127 127
128 128 res = IBMF_NO_RESOURCES;
129 129
130 130 goto bail;
131 131 }
132 132
133 133 mutex_init(&saa_statep->saa_port_list_mutex, NULL, MUTEX_DRIVER,
134 134 NULL);
135 135
136 136 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_statep))
137 137
138 138 res = IBMF_SUCCESS;
139 139 bail:
140 140
141 141 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_init_end,
142 142 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() exit: status = %d\n",
143 143 tnf_int, res, res);
144 144
145 145 return (res);
146 146 }
147 147
148 148 /*
149 149 * ibmf_saa_impl_fini:
150 150 * If there are no registered clients, cleans up all memory associated with the
151 151 * state, including each of the port list entries.
152 152 * Called from the modules fini() routine.
153 153 *
154 154 * Input Arguments
155 155 * none
156 156 *
157 157 * Output Arguments
158 158 * none
159 159 *
160 160 * Returns
161 161 * EBUSY if there are outstanding transactions or registered clients
162 162 * 0 if cleanup was sucessfull
163 163 *
164 164 */
165 165 int
166 166 ibmf_saa_impl_fini()
167 167 {
168 168 int ret = 0;
169 169 saa_port_t *saa_portp;
170 170 saa_port_t *next;
171 171
172 172 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_start,
173 173 IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() enter\n");
174 174
175 175 /* make sure there are no registered clients */
176 176 mutex_enter(&saa_statep->saa_port_list_mutex);
177 177
178 178 saa_portp = saa_statep->saa_port_list;
179 179 while (saa_portp != NULL) {
180 180
181 181 mutex_enter(&saa_portp->saa_pt_mutex);
182 182
183 183 if (saa_portp->saa_pt_reference_count > 0) {
184 184
185 185 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
186 186 ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "",
187 187 "ibmf_saa_impl_fini: %s, port %016" PRIx64 "\n",
188 188 tnf_string, msg,
189 189 "cannot unload ibmf_saa. Client on port still"
190 190 " registered", tnf_opaque, port,
191 191 saa_portp->saa_pt_port_guid);
192 192
193 193 mutex_exit(&saa_portp->saa_pt_mutex);
194 194
195 195 mutex_exit(&saa_statep->saa_port_list_mutex);
196 196
197 197 ret = EBUSY;
198 198 goto bail;
199 199 }
200 200
201 201 /* make sure there are no outstanding transactions */
202 202
203 203 if (saa_portp->saa_pt_num_outstanding_trans > 0) {
204 204
205 205 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1,
206 206 ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "",
207 207 "ibmf_saa_impl_fini: %s, port = %016" PRIx64
208 208 ", num transactions = %d\n",
209 209 tnf_string, msg, "Cannot unload ibmf_saa."
210 210 " Outstanding transactions on port.",
211 211 tnf_opaque, port,
212 212 saa_portp->saa_pt_port_guid,
213 213 tnf_uint, outstanding_transactions,
214 214 saa_portp->saa_pt_num_outstanding_trans);
215 215
216 216 mutex_exit(&saa_portp->saa_pt_mutex);
217 217
218 218 mutex_exit(&saa_statep->saa_port_list_mutex);
219 219
220 220 ret = EBUSY;
221 221 goto bail;
222 222 }
223 223
224 224 mutex_exit(&saa_portp->saa_pt_mutex);
225 225
226 226 saa_portp = saa_portp->next;
227 227 }
228 228
229 229 mutex_exit(&saa_statep->saa_port_list_mutex);
230 230
231 231 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(saa_statep->saa_port_list,
232 232 *saa_portp))
233 233
234 234 /*
235 235 * no more clients nor pending transaction:
236 236 * unregister ibmf and destroy port entries
237 237 */
238 238 while (saa_statep->saa_port_list != NULL) {
239 239
240 240 saa_portp = saa_statep->saa_port_list;
241 241 next = saa_portp->next;
242 242
243 243 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
244 244 ibmf_saa_impl_fini, IBMF_TNF_TRACE, "",
245 245 "ibmf_saa_impl_fini: %s, prefix = %016" PRIx64 "\n",
246 246 tnf_string, msg, "deinitializing port",
247 247 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid);
248 248
249 249 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp))
250 250
251 251 mutex_enter(&saa_portp->saa_pt_mutex);
252 252
253 253 /* unregister from ibmf */
254 254 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) {
255 255
256 256 mutex_exit(&saa_portp->saa_pt_mutex);
257 257
258 258 if (ibmf_saa_impl_ibmf_unreg(saa_portp)
259 259 != IBMF_SUCCESS) {
260 260 ret = EBUSY;
261 261 goto bail;
262 262 }
263 263 } else
264 264 mutex_exit(&saa_portp->saa_pt_mutex);
265 265
266 266 ibmf_saa_impl_destroy_port(saa_portp);
267 267
268 268 saa_statep->saa_port_list = next;
269 269 }
270 270
271 271 taskq_destroy(saa_statep->saa_event_taskq);
272 272
273 273 mutex_destroy(&saa_statep->saa_port_list_mutex);
274 274
275 275 kmem_free(saa_statep, sizeof (saa_state_t));
276 276
277 277 bail:
278 278 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_end,
279 279 IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() exit\n");
280 280
281 281 return (ret);
282 282 }
283 283
284 284 /*
285 285 * ibmf_saa_is_valid
286 286 * Returns true the entry is valid.
287 287 *
288 288 * Input Arguments
289 289 * saa_portp pointer to state structure
290 290 * add_ref if B_TRUE ref count is incremented on a valid portp
291 291 *
292 292 * Output Arguments
293 293 * none
294 294 *
295 295 * Returns
296 296 * B_TRUE if entry was in a valid state, B_FALSE otherwise
297 297 */
298 298 boolean_t
299 299 ibmf_saa_is_valid(saa_port_t *saa_portp, int add_ref)
300 300 {
301 301 boolean_t is_valid = B_TRUE;
302 302
303 303 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_start,
304 304 IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() enter\n");
305 305
306 306 mutex_enter(&saa_portp->saa_pt_mutex);
307 307
308 308 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID ||
309 309 saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_PURGING) {
310 310
311 311 is_valid = B_FALSE;
312 312
313 313 } else if (add_ref == B_TRUE) {
314 314 /*
315 315 * increment reference count here to ensure that
316 316 * entry does not get purged behind our backs
317 317 */
318 318 saa_portp->saa_pt_reference_count++;
319 319 }
320 320 mutex_exit(&saa_portp->saa_pt_mutex);
321 321
322 322 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_end,
323 323 IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() exit\n");
324 324
325 325 return (is_valid);
326 326 }
327 327
328 328 /*
329 329 * ibmf_saa_must_purge
330 330 * Determines if we can purge a portp (remove it from the list) based on the
331 331 * state and number of clients
332 332 *
333 333 * Input Arguments
334 334 * saa_portp pointer to state structure
335 335 *
336 336 * Output Arguments
337 337 * none
338 338 *
339 339 * Returns
340 340 * B_TRUE if the entry can be removed, B_FALSE otherwise
341 341 */
342 342 static int
343 343 ibmf_saa_must_purge(saa_port_t *saa_portp)
344 344 {
345 345 int must_purge = B_FALSE;
346 346
347 347 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_start,
348 348 IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() enter\n");
349 349
350 350 mutex_enter(&saa_portp->saa_pt_mutex);
351 351
352 352 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID &&
353 353 saa_portp->saa_pt_reference_count == 0) {
354 354
355 355 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_PURGING;
356 356 must_purge = B_TRUE;
357 357 }
358 358
359 359 mutex_exit(&saa_portp->saa_pt_mutex);
360 360
361 361 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_end,
362 362 IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() exit\n");
363 363
364 364 return (must_purge);
365 365 }
366 366
367 367
368 368 /*
369 369 * ibmf_saa_impl_purge:
370 370 * Removes invalid port state entries from the list
371 371 *
372 372 * Input Arguments
373 373 * none
374 374 *
375 375 * Output Arguments
376 376 * none
377 377 *
378 378 * Returns
379 379 * void
380 380 */
381 381 void
382 382 ibmf_saa_impl_purge()
383 383 {
384 384 saa_port_t *cur_portp = NULL;
385 385 saa_port_t *prev_portp = NULL;
386 386 saa_port_t *rem_portp = NULL;
387 387
388 388 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_start,
389 389 IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() enter\n");
390 390
391 391 mutex_enter(&saa_statep->saa_port_list_mutex);
392 392
393 393 cur_portp = saa_statep->saa_port_list;
394 394 prev_portp = cur_portp;
395 395
396 396 while (cur_portp != NULL) {
397 397
398 398 if (ibmf_saa_must_purge(cur_portp) == B_TRUE) {
399 399
400 400 rem_portp = cur_portp;
401 401
402 402 /* unlink entry */
403 403 if (cur_portp == saa_statep->saa_port_list) {
404 404
405 405 saa_statep->saa_port_list = cur_portp->next;
406 406 cur_portp = saa_statep->saa_port_list;
407 407 prev_portp = cur_portp;
408 408
409 409 } else {
410 410
411 411 prev_portp->next = cur_portp->next;
412 412 cur_portp = cur_portp->next;
413 413 }
414 414
415 415 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*rem_portp))
416 416
417 417 /* destroy entry */
418 418 ASSERT(rem_portp != NULL);
419 419 ibmf_saa_impl_destroy_port(rem_portp);
420 420
421 421 } else {
422 422
423 423 prev_portp = cur_portp;
424 424 cur_portp = cur_portp->next;
425 425 }
426 426 }
427 427
428 428 mutex_exit(&saa_statep->saa_port_list_mutex);
429 429
430 430 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_end,
431 431 IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() exit\n");
432 432 }
433 433
434 434 /*
435 435 * saa_impl_add_client:
436 436 * Adds a client for a particular portp. Reference count has been incremented
437 437 * before this call. It is decremented by saa_impl_add_client() if the call
438 438 * fails.
439 439 *
440 440 * Input Arguments
441 441 * none
442 442 *
443 443 * Output Arguments
444 444 * none
445 445 *
446 446 * Returns
447 447 * IBMF_BUSY if there are already too many clients registered,
448 448 * IBMF_BAD_PORT_STATE if the port is invalid (generally because a previous
449 449 * client failed during registration for this port)
450 450 * IBMF_SUCCESS otherwise
451 451 */
452 452 int
453 453 ibmf_saa_impl_add_client(saa_port_t *saa_portp)
454 454 {
455 455 int status = IBMF_SUCCESS;
456 456
457 457 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_add_client_start,
458 458 IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client() enter\n");
459 459
460 460 mutex_enter(&saa_portp->saa_pt_mutex);
461 461
462 462 /*
463 463 * check that we don't exceed max clients
464 464 */
465 465 if (saa_portp->saa_pt_reference_count >
466 466 SAA_MAX_CLIENTS_PER_PORT) {
467 467
468 468 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
469 469 ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, "",
470 470 "ibmf_saa_impl_add_client: %s, num_reg_clients %d\n",
471 471 tnf_string, msg, "too many clients registered for"
472 472 " port", tnf_uint, num_reg_clients,
473 473 saa_portp->saa_pt_reference_count);
474 474
475 475 status = IBMF_BUSY;
476 476 goto bail;
477 477 }
478 478
479 479 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
480 480 ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "",
481 481 "ibmf_saa_impl_add_client: num_registered_clients %d\n",
482 482 tnf_uint, num_registered_clients,
483 483 saa_portp->saa_pt_reference_count);
484 484
485 485 /*
486 486 * wait until anyone who is currently registering
487 487 * this port with ibmf is done
488 488 */
489 489 while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) {
490 490
491 491 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
492 492 ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "",
493 493 "ibmf_saa_impl_add_client: %s\n",
494 494 tnf_string, msg, "someone is registering. waiting"
495 495 " for them to finish");
496 496
497 497 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv,
498 498 &saa_portp->saa_pt_mutex);
499 499
500 500 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
501 501 ibmf_saa_impl_add_client,
502 502 IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client: %s\n",
503 503 tnf_string, msg, "done waiting");
504 504 }
505 505
506 506 /*
507 507 * if port isn't ready here, fail.
508 508 */
509 509 if (saa_portp->saa_pt_state != IBMF_SAA_PORT_STATE_READY) {
510 510
511 511 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
512 512 ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR,
513 513 "", "ibmf_saa_impl_add_client: %s\n",
514 514 tnf_string, msg, "port state not ready,"
515 515 " removing client.");
516 516
517 517 status = IBMF_BAD_PORT_STATE;
518 518 goto bail;
519 519 }
520 520
521 521 bail:
522 522 mutex_exit(&saa_portp->saa_pt_mutex);
523 523
524 524 if (status != IBMF_SUCCESS) {
525 525
526 526 mutex_enter(&saa_portp->saa_pt_kstat_mutex);
527 527
528 528 IBMF_SAA_ADD32_KSTATS(saa_portp,
529 529 clients_reg_failed, 1);
530 530
531 531 mutex_exit(&saa_portp->saa_pt_kstat_mutex);
532 532
533 533 /* decrementing refcount is last thing we do on entry */
534 534
535 535 mutex_enter(&saa_portp->saa_pt_mutex);
536 536
537 537 ASSERT(saa_portp->saa_pt_reference_count > 0);
538 538 saa_portp->saa_pt_reference_count--;
539 539
540 540 mutex_exit(&saa_portp->saa_pt_mutex);
541 541 }
542 542
543 543 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
544 544 ibmf_saa_impl_add_client_end, IBMF_TNF_TRACE, "",
545 545 "ibmf_saa_impl_add_client() exit\n");
546 546
547 547 return (status);
548 548 }
549 549
550 550 /*
551 551 * ibmf_saa_impl_create_port()
552 552 * Create port entry with mimimal inits because
553 553 * we're holding the list mutex: NO BLOCKING CALLS HERE, please.
554 554 *
555 555 * Initialize port state to "registering", so that clients accessing
556 556 * same port concurrently will wait for the end of the ibmf registration.
557 557 * Note: this thread will access port members without locking mutex.
558 558 *
559 559 * Input Arguments
560 560 * pt_guid guid of port
561 561 *
562 562 * Output Arguments
563 563 * saa_portpp pointer to new saa_portp structure
564 564 *
565 565 * Returns
566 566 * IBMF_NO_MEMORY if memory could not be allocated
567 567 * IBMF_SUCCESS otherwise
568 568 */
569 569 int
570 570 ibmf_saa_impl_create_port(ib_guid_t pt_guid, saa_port_t **saa_portpp)
571 571 {
572 572 int status = IBMF_SUCCESS;
573 573 saa_port_t *saa_portp = NULL;
574 574
575 575 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_create_port_start,
576 576 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port:"
577 577 " guid %016" PRIx64 "\n",
578 578 tnf_opaque, port_guid, pt_guid);
579 579
580 580 ASSERT(MUTEX_HELD(&saa_statep->saa_port_list_mutex));
581 581
582 582 /* create & initialize new port */
583 583 saa_portp = kmem_zalloc(sizeof (saa_port_t), KM_NOSLEEP);
584 584
585 585 if (saa_portp == NULL) {
586 586
587 587 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
588 588 ibmf_sa_session_open_err, IBMF_TNF_ERROR, "",
589 589 "ibmf_saa_impl_create_port: %s\n",
590 590 tnf_string, msg, "could not allocate memory for "
591 591 "new port");
592 592
593 593 status = IBMF_NO_MEMORY;
594 594 goto bail;
595 595 }
596 596
597 597 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_sa_session_open,
598 598 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port: %s\n",
599 599 tnf_string, msg, "first client registering, initializing");
600 600
601 601 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp))
602 602
603 603 /* tell everyone that kstats are not initialized */
604 604 saa_portp->saa_pt_kstatp = NULL;
605 605
606 606 /*
607 607 * set up mutexe and state variable to indicate to
608 608 * other clients that were currently in the process of
609 609 * setting up the port data. This will prevent a subsequent
610 610 * client from trying to to register with ibmf before the
611 611 * port data has been initialized.
612 612 */
613 613 mutex_init(&saa_portp->saa_pt_mutex, NULL, MUTEX_DRIVER, NULL);
614 614 cv_init(&saa_portp->saa_pt_ibmf_reg_cv, NULL, CV_DRIVER, NULL);
615 615
616 616 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_REGISTERING;
617 617
618 618 /* create other mutexes */
619 619 mutex_init(&saa_portp->saa_pt_kstat_mutex, NULL, MUTEX_DRIVER, NULL);
620 620
621 621 mutex_init(&saa_portp->saa_pt_event_sub_mutex, NULL, MUTEX_DRIVER,
622 622 NULL);
623 623
624 624 /*
625 625 * clients assume all arrive; set mask to this so we only notify
626 626 * if something failed
627 627 */
628 628 saa_portp->saa_pt_event_sub_last_success_mask =
629 629 IBMF_SAA_PORT_EVENT_SUB_ALL_ARRIVE;
630 630
631 631 /*
632 632 * set port_guid now so any immediately subsequent clients
633 633 * registering on this port, guid will know we're already here
634 634 */
635 635 saa_portp->saa_pt_port_guid = pt_guid;
636 636 saa_portp->saa_pt_reference_count = 1;
637 637 saa_portp->saa_pt_current_tid = pt_guid << 32;
638 638
639 639 saa_portp->saa_pt_redirect_active = B_FALSE;
640 640
641 641 /* set sa_uptime now in case we never receive anything from SA */
642 642 saa_portp->saa_pt_sa_uptime = gethrtime();
643 643
644 644 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_portp))
645 645
646 646 /* Set new pointer in caller's */
647 647 *saa_portpp = saa_portp;
648 648
649 649 bail:
650 650 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_create_port_end,
651 651 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port() exit\n");
652 652
653 653 return (status);
654 654 }
655 655
656 656 /*
657 657 * ibmf_saa_impl_invalidate_port:
658 658 * invalidates port entry (assumes exist) and deletes kstat object
659 659 * kstat object is destroyed in order to allow creating port entry
660 660 * even if this entry is not purged
661 661 */
662 662 static void
663 663 ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp)
664 664 {
665 665 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
666 666 ibmf_saa_impl_invalidate_port_start,
667 667 IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() enter\n");
668 668
669 669 ASSERT(saa_portp != NULL);
670 670 ASSERT(MUTEX_HELD(&saa_portp->saa_pt_mutex));
671 671
672 672 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_INVALID;
673 673 ibmf_saa_impl_uninit_kstats(saa_portp);
674 674
675 675 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
676 676 ibmf_saa_impl_invalidate_port_end,
677 677 IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() exit\n");
678 678 }
679 679
680 680 /*
681 681 * ibmf_saa_impl_destroy_port:
682 682 * Frees the resources associated with an saa_portp structure. Assumes the
683 683 * saa_portp exists
684 684 *
685 685 * Input Arguments
686 686 * saa_portp pointer to saa_portp structure
687 687 *
688 688 * Output Arguments
689 689 * none
690 690 *
691 691 * Returns
692 692 * void
693 693 */
694 694 static void
695 695 ibmf_saa_impl_destroy_port(saa_port_t *saa_portp)
696 696 {
697 697 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_start,
698 698 IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() enter\n");
699 699
700 700 ASSERT(saa_portp != NULL);
701 701
702 702 _NOTE(ASSUMING_PROTECTED(*saa_portp))
703 703
704 704 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
705 705 ibmf_saa_impl_destroy, IBMF_TNF_TRACE, "",
706 706 "ibmf_saa_impl_destroy(): destroying port_guid %016" PRIx64 "\n",
707 707 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid);
708 708
709 709 ibmf_saa_impl_uninit_kstats(saa_portp);
710 710
711 711 /* uninit synchronization variables used for registration */
712 712 mutex_destroy(&saa_portp->saa_pt_mutex);
713 713 cv_destroy(&saa_portp->saa_pt_ibmf_reg_cv);
714 714
715 715 mutex_destroy(&saa_portp->saa_pt_event_sub_mutex);
716 716 mutex_destroy(&saa_portp->saa_pt_kstat_mutex);
717 717
718 718 kmem_free(saa_portp, sizeof (saa_port_t));
719 719
720 720 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_end,
721 721 IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() exit\n");
722 722 }
723 723
724 724 /*
725 725 * ibmf_saa_impl_init_kstats:
726 726 * Create kstats structure. Should be called when memory is alloced for a new
727 727 * port entry.
728 728 */
729 729 int
730 730 ibmf_saa_impl_init_kstats(saa_port_t *saa_portp)
731 731 {
732 732 char buf[128];
733 733 ibmf_saa_kstat_t *ksp;
734 734
735 735 _NOTE(ASSUMING_PROTECTED(saa_portp->saa_pt_kstatp))
736 736
737 737 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
738 738 ibmf_saa_impl_init_kstats_start,
739 739 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() enter\n");
740 740
741 741 /* set up kstats structure */
742 742 (void) sprintf(buf, "ibmf_saa_%016" PRIx64 "_stat",
743 743 saa_portp->saa_pt_port_guid);
744 744
745 745 saa_portp->saa_pt_kstatp = kstat_create("ibmf_saa",
746 746 0, buf, "misc", KSTAT_TYPE_NAMED,
747 747 sizeof (ibmf_saa_kstat_t) / sizeof (kstat_named_t),
748 748 KSTAT_FLAG_WRITABLE);
749 749
750 750 if (saa_portp->saa_pt_kstatp == NULL)
751 751 return (IBMF_NO_RESOURCES);
752 752
753 753 ksp = (ibmf_saa_kstat_t *)saa_portp->saa_pt_kstatp->ks_data;
754 754
755 755 kstat_named_init(&ksp->clients_registered,
756 756 "clients_registered", KSTAT_DATA_UINT32);
757 757
758 758 kstat_named_init(&ksp->clients_reg_failed,
759 759 "clients_reg_failed", KSTAT_DATA_UINT32);
760 760
761 761 kstat_named_init(&ksp->outstanding_requests,
762 762 "outstanding_requests", KSTAT_DATA_UINT32);
763 763
764 764 kstat_named_init(&ksp->total_requests,
765 765 "total_requests", KSTAT_DATA_UINT32);
766 766
767 767 kstat_named_init(&ksp->failed_requests,
768 768 "failed_requests", KSTAT_DATA_UINT32);
769 769
770 770 kstat_named_init(&ksp->requests_timedout,
771 771 "requests_timedout", KSTAT_DATA_UINT32);
772 772
773 773 kstat_install(saa_portp->saa_pt_kstatp);
774 774
775 775 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
776 776 ibmf_saa_impl_init_kstats_end,
777 777 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() exit\n");
778 778
779 779 return (IBMF_SUCCESS);
780 780 }
781 781
782 782 /*
783 783 * ibmf_saa_impl_uninit_kstats:
784 784 * Free kstats context. Should be called when port is either destroyed
785 785 * or invalidated.
786 786 */
787 787 static void
788 788 ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp)
789 789 {
790 790 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
791 791 ibmf_saa_impl_uninit_kstats_start,
792 792 IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() enter\n");
793 793
794 794 mutex_enter(&saa_portp->saa_pt_kstat_mutex);
795 795
796 796 if (saa_portp->saa_pt_kstatp != NULL) {
797 797 kstat_delete(saa_portp->saa_pt_kstatp);
798 798 }
799 799 saa_portp->saa_pt_kstatp = NULL;
800 800
801 801 mutex_exit(&saa_portp->saa_pt_kstat_mutex);
802 802
803 803 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
804 804 ibmf_saa_impl_uninit_kstats_end,
805 805 IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() exit\n");
806 806 }
807 807
808 808 /*
809 809 * ibmf_saa_impl_register_failed:
810 810 * invalidate entry and kick waiters
811 811 */
812 812 void
813 813 ibmf_saa_impl_register_failed(saa_port_t *saa_portp)
814 814 {
815 815 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
816 816 ibmf_saa_impl_register_failed_start,
817 817 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() enter\n");
818 818
819 819 mutex_enter(&saa_portp->saa_pt_mutex);
820 820
821 821 ibmf_saa_impl_invalidate_port(saa_portp);
822 822
823 823 cv_broadcast(&saa_portp->saa_pt_ibmf_reg_cv);
824 824
825 825 /* decrementing refcount is last thing we do on entry */
826 826
827 827 ASSERT(saa_portp->saa_pt_reference_count > 0);
828 828 saa_portp->saa_pt_reference_count--;
829 829
830 830 mutex_exit(&saa_portp->saa_pt_mutex);
831 831
832 832 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
833 833 ibmf_saa_impl_register_failed_end,
834 834 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() exit\n");
835 835 }
836 836
837 837 static int
838 838 ibmf_saa_impl_setup_qp_async_cb(saa_port_t *saa_portp, int setup_async_cb_only)
839 839 {
840 840 int status;
841 841 int unreg_status;
842 842 ib_pkey_t p_key;
843 843 ib_qkey_t q_key;
844 844 uint8_t portnum;
845 845 boolean_t qp_alloced = B_FALSE;
846 846
847 847 if (setup_async_cb_only == 0) {
848 848
849 849 /* allocate a qp through ibmf */
850 850 status = ibmf_alloc_qp(saa_portp->saa_pt_ibmf_handle,
851 851 IB_PKEY_DEFAULT_LIMITED, IB_GSI_QKEY,
852 852 IBMF_ALT_QP_MAD_RMPP, &saa_portp->saa_pt_qp_handle);
853 853
854 854 if (status != IBMF_SUCCESS) {
855 855
856 856 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
857 857 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
858 858 "ibmf_saa_impl_setup_qp_async_cb: %s, "
859 859 "ibmf_status = %d\n",
860 860 tnf_string, msg, "Cannot alloc qp with ibmf",
861 861 tnf_int, status, status);
862 862
863 863 return (status);
864 864 }
865 865
866 866 qp_alloced = B_TRUE;
867 867
868 868 /*
869 869 * query the queue pair number; we will need it to unsubscribe
870 870 * from notice reports
871 871 */
872 872 status = ibmf_query_qp(saa_portp->saa_pt_ibmf_handle,
873 873 saa_portp->saa_pt_qp_handle, &saa_portp->saa_pt_qpn,
874 874 &p_key, &q_key, &portnum, 0);
875 875
876 876 if (status != IBMF_SUCCESS) {
877 877
878 878 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
879 879 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
880 880 "ibmf_saa_impl_setup_qp_async_cb: %s, "
881 881 "ibmf_status = %d\n",
882 882 tnf_string, msg,
883 883 "Cannot query alt qp to get qp num",
884 884 tnf_int, status, status);
885 885
886 886 goto bail;
887 887 }
888 888 }
889 889
890 890 /*
891 891 * core ibmf is taking advantage of the fact that saa_portp is our
892 892 * callback arg. If this changes, the code in ibmf_recv would need to
893 893 * change as well
894 894 */
895 895 status = ibmf_setup_async_cb(saa_portp->saa_pt_ibmf_handle,
896 896 saa_portp->saa_pt_qp_handle, ibmf_saa_report_cb, saa_portp, 0);
897 897 if (status != IBMF_SUCCESS) {
898 898
899 899 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
900 900 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
901 901 "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status = %d\n",
902 902 tnf_string, msg, "Cannot register async cb with ibmf",
903 903 tnf_int, status, status);
904 904
905 905 goto bail;
906 906 }
907 907
908 908 return (IBMF_SUCCESS);
909 909
910 910 bail:
911 911 if (qp_alloced == B_TRUE) {
912 912 /* free alternate qp */
913 913 unreg_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle,
914 914 &saa_portp->saa_pt_qp_handle, 0);
915 915 if (unreg_status != IBMF_SUCCESS) {
916 916
917 917 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
918 918 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
919 919 "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status ="
920 920 " %d\n", tnf_string, msg,
921 921 "Cannot free alternate queue pair with ibmf",
922 922 tnf_int, unreg_status, unreg_status);
923 923 }
924 924 }
925 925
926 926 return (status);
927 927 }
928 928
929 929 /*
930 930 * ibmf_saa_impl_register_port:
931 931 */
932 932 int
933 933 ibmf_saa_impl_register_port(
934 934 saa_port_t *saa_portp)
935 935 {
936 936 uint_t hca_count = 0;
937 937 ib_guid_t *hca_list = NULL;
938 938 int status = IBMF_SUCCESS;
939 939 int unreg_status = IBMF_SUCCESS;
940 940 int ibt_status = IBT_SUCCESS;
941 941 ibt_hca_portinfo_t *port_info_list = NULL;
942 942 uint_t port_count = 0;
943 943 uint_t port_size = 0;
944 944 int ihca, iport;
945 945 ib_guid_t port_guid;
946 946 boolean_t ibmf_reg = B_FALSE;
947 947
948 948 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
949 949 ibmf_saa_impl_register_port_start, IBMF_TNF_TRACE, "",
950 950 "ibmf_saa_impl_register_port() enter\n");
951 951
952 952 ASSERT(saa_portp != NULL);
953 953
954 954 _NOTE(ASSUMING_PROTECTED(*saa_portp))
955 955
956 956 /* get the HCA list */
957 957
958 958 hca_count = ibt_get_hca_list(&hca_list);
959 959
960 960 if (hca_count == 0) {
961 961
962 962 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
963 963 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
964 964 "ibmf_saa_impl_register_port: %s\n",
965 965 tnf_string, msg, "cannot register port (no HCAs).\n");
966 966
967 967 status = IBMF_BAD_PORT;
968 968 goto bail;
969 969 }
970 970
971 971 /* lookup requested port guid in hca list */
972 972 for (ihca = 0; ihca != hca_count; ihca++) {
973 973
974 974 ibt_status = ibt_query_hca_ports_byguid(hca_list[ihca],
975 975 0 /* all ports */, &port_info_list,
976 976 &port_count, &port_size);
977 977
978 978 if (ibt_status != IBT_SUCCESS) {
979 979
980 980 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
981 981 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
982 982 "ibmf_saa_impl_register_port: %s, %016" PRIx64 "\n",
983 983 tnf_string, msg, "Could not query hca. Exiting.",
984 984 tnf_opaque, guid, hca_list[ihca]);
985 985
986 986 status = IBMF_TRANSPORT_FAILURE;
987 987 break;
988 988 }
989 989
990 990 for (iport = 0; iport < port_count; iport++) {
991 991
992 992 /* get port guid associated with hca guid, port num */
993 993 if (ibmf_saa_impl_get_port_guid(
994 994 port_info_list + iport, &port_guid) != IBMF_SUCCESS)
995 995 continue;
996 996
997 997 if (saa_portp->saa_pt_port_guid != port_guid)
998 998 continue;
999 999
1000 1000 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
1001 1001 ibmf_saa_impl_register_port,
1002 1002 IBMF_TNF_TRACE, "",
1003 1003 "ibmf_saa_impl_register_port: %s, hca_guid = %016"
1004 1004 PRIx64 ", port_guid = %016" PRIx64
1005 1005 ", number = %d\n",
1006 1006 tnf_string, msg, "found port",
1007 1007 tnf_opaque, hca_guid, hca_list[ihca],
1008 1008 tnf_opaque, port_guid, port_guid,
1009 1009 tnf_uint, port, iport + 1);
1010 1010
1011 1011 /*
1012 1012 * we're here? then we found our port:
1013 1013 * fill in ibmf registration info
1014 1014 * and address parameters from the portinfo
1015 1015 */
1016 1016
1017 1017 saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid
1018 1018 = hca_list[ihca];
1019 1019 saa_portp->saa_pt_ibmf_reginfo.ir_port_num = iport+1;
1020 1020 saa_portp->saa_pt_ibmf_reginfo.ir_client_class
1021 1021 = SUBN_ADM_MANAGER;
1022 1022
1023 1023 saa_portp->saa_pt_node_guid = hca_list[ihca];
1024 1024 saa_portp->saa_pt_port_num = iport + 1;
1025 1025
1026 1026 ibmf_saa_impl_set_transaction_params(
1027 1027 saa_portp, port_info_list + iport);
1028 1028 break;
1029 1029 }
1030 1030
1031 1031 ibt_free_portinfo(port_info_list, port_size);
1032 1032
1033 1033 if (iport != port_count)
1034 1034 break; /* found our port */
1035 1035 }
1036 1036
1037 1037 ibt_free_hca_list(hca_list, hca_count);
1038 1038
1039 1039 if (ihca == hca_count) {
1040 1040
1041 1041 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1042 1042 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
1043 1043 "ibmf_saa_impl_register_port: %s, port_guid %016"
1044 1044 PRIx64 "\n",
1045 1045 tnf_string, msg, "Could not find port, exiting",
1046 1046 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid);
1047 1047
1048 1048 status = IBMF_BAD_PORT;
1049 1049 }
1050 1050
1051 1051 if (status != IBMF_SUCCESS) {
1052 1052
1053 1053 goto bail;
1054 1054 }
1055 1055
1056 1056 /*
1057 1057 * Now we found the port we searched for,
1058 1058 * and open an ibmf session on that port.
1059 1059 */
1060 1060
1061 1061 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1062 1062 ibmf_saa_impl_register_port, IBMF_TNF_TRACE, "",
1063 1063 "ibmf_saa_impl_register_port: %s, port_guid = %016" PRIx64
1064 1064 ", port = %d\n", tnf_string, msg, "Registering with ibmf",
1065 1065 tnf_opaque, port_guid, saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid,
1066 1066 tnf_uint, port, saa_portp->saa_pt_ibmf_reginfo.ir_port_num);
1067 1067
1068 1068 status = ibmf_register(&saa_portp->saa_pt_ibmf_reginfo,
1069 1069 IBMF_VERSION, IBMF_REG_FLAG_RMPP,
1070 1070 ibmf_saa_impl_async_event_cb, saa_portp,
1071 1071 &saa_portp->saa_pt_ibmf_handle,
1072 1072 &saa_portp->saa_pt_ibmf_impl_features);
1073 1073
1074 1074 if (status != IBMF_SUCCESS) {
1075 1075
1076 1076 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1077 1077 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
1078 1078 "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n",
1079 1079 tnf_string, msg, "Could not register with ibmf",
1080 1080 tnf_int, status, status);
1081 1081
1082 1082 goto bail;
1083 1083 }
1084 1084
1085 1085 ibmf_reg = B_TRUE;
1086 1086
1087 1087 if (ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0) == IBMF_SUCCESS)
1088 1088 return (IBMF_SUCCESS);
1089 1089
1090 1090 bail:
1091 1091 if (ibmf_reg == B_TRUE) {
1092 1092 /* unregister from ibmf */
1093 1093 unreg_status = ibmf_unregister(
1094 1094 &saa_portp->saa_pt_ibmf_handle, 0);
1095 1095
1096 1096 if (unreg_status != IBMF_SUCCESS) {
1097 1097
1098 1098 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1099 1099 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
1100 1100 "ibmf_saa_impl_register_port: %s, ibmf_status ="
1101 1101 " %d\n", tnf_string, msg,
1102 1102 "Cannot unregister from ibmf",
1103 1103 tnf_int, unreg_status, unreg_status);
1104 1104 }
1105 1105 }
1106 1106
1107 1107 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_register_port_end,
1108 1108 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_port() exit\n");
1109 1109
1110 1110 return (status);
1111 1111 }
1112 1112
1113 1113 /*
1114 1114 * ibmf_saa_impl_getclassportinfo:
1115 1115 */
1116 1116 void
1117 1117 ibmf_saa_impl_get_classportinfo(saa_port_t *saa_portp)
1118 1118 {
1119 1119 int res;
1120 1120 saa_impl_trans_info_t *trans_info;
1121 1121
1122 1122 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1123 1123 ibmf_saa_impl_get_classportinfo_start,
1124 1124 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() enter\n");
1125 1125
1126 1126 /*
1127 1127 * allocate memory for trans_info; send_request's callback will free up
1128 1128 * memory since request is asynchronous
1129 1129 */
1130 1130 trans_info = kmem_zalloc(sizeof (saa_impl_trans_info_t), KM_NOSLEEP);
1131 1131 if (trans_info == NULL) {
1132 1132
1133 1133 mutex_enter(&saa_portp->saa_pt_mutex);
1134 1134
1135 1135 /* cpi transaction is handled as a client, decrement refcount */
1136 1136 ASSERT(saa_portp->saa_pt_reference_count > 0);
1137 1137 saa_portp->saa_pt_reference_count--;
1138 1138
1139 1139 mutex_exit(&saa_portp->saa_pt_mutex);
1140 1140
1141 1141 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
1142 1142 ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_ERROR, "",
1143 1143 "ibmf_saa_impl_get_classportinfo: %s\n", tnf_string, msg,
1144 1144 "Could not allocate memory for classportinfo trans_info");
1145 1145
1146 1146 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1147 1147 ibmf_saa_impl_get_classportinfo_end, IBMF_TNF_TRACE, "",
1148 1148 "ibmf_saa_impl_get_classportinfo() exiting\n");
1149 1149
1150 1150 return;
1151 1151 }
1152 1152
1153 1153 /* no specific client associated with this transaction */
1154 1154 trans_info->si_trans_client_data = NULL;
1155 1155 trans_info->si_trans_port = saa_portp;
1156 1156 trans_info->si_trans_method = SA_SUBN_ADM_GET;
1157 1157 trans_info->si_trans_attr_id = MAD_ATTR_ID_CLASSPORTINFO;
1158 1158
1159 1159 trans_info->si_trans_callback = ibmf_saa_impl_get_cpi_cb;
1160 1160 trans_info->si_trans_callback_arg = saa_portp;
1161 1161
1162 1162 mutex_enter(&saa_portp->saa_pt_kstat_mutex);
1163 1163
1164 1164 IBMF_SAA_ADD32_KSTATS(saa_portp, outstanding_requests, 1);
1165 1165 IBMF_SAA_ADD32_KSTATS(saa_portp, total_requests, 1);
1166 1166
1167 1167 mutex_exit(&saa_portp->saa_pt_kstat_mutex);
1168 1168
1169 1169 res = ibmf_saa_impl_send_request(trans_info);
1170 1170
1171 1171 if (res != IBMF_SUCCESS) {
1172 1172
1173 1173 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1174 1174 ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_TRACE, "",
1175 1175 "ibmf_saa_impl_get_classportinfo: %s, res = 0x%x\n",
1176 1176 tnf_string, msg, "ibmf_saa_impl_send_request failed",
1177 1177 tnf_opaque, res, res);
1178 1178
1179 1179 mutex_enter(&saa_portp->saa_pt_kstat_mutex);
1180 1180
1181 1181 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1);
1182 1182 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1);
1183 1183
1184 1184 mutex_exit(&saa_portp->saa_pt_kstat_mutex);
1185 1185
1186 1186 mutex_enter(&saa_portp->saa_pt_mutex);
1187 1187
1188 1188 /* cpi transaction is handled as a client, decrement refcount */
1189 1189 ASSERT(saa_portp->saa_pt_reference_count > 0);
1190 1190 saa_portp->saa_pt_reference_count--;
1191 1191
1192 1192 mutex_exit(&saa_portp->saa_pt_mutex);
1193 1193
1194 1194 }
1195 1195
1196 1196 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1197 1197 ibmf_saa_impl_get_classportinfo_end,
1198 1198 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() exit\n");
1199 1199 }
1200 1200
1201 1201 /*
1202 1202 * ibmf_saa_impl_get_cpi_cb:
1203 1203 *
1204 1204 * Called when the asynchronous getportinfo request receives its response.
1205 1205 * Checks the status. If success, updates the times in the port's
1206 1206 * ibmf_retrans structure that is used in ibmf_msg_transport calls. If failure,
1207 1207 * just use default values.
1208 1208 *
1209 1209 * Input Arguments
1210 1210 * arg user-specified pointer (points to the current port data)
1211 1211 * length length of payload returned (should be size of classportinfo_t)
1212 1212 * buffer pointer to classportinfo returned (should not be null)
1213 1213 * status status of sa access request
1214 1214 *
1215 1215 * Output Arguments
1216 1216 * none
1217 1217 *
1218 1218 * Returns void
1219 1219 */
1220 1220 static void
1221 1221 ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, int status)
1222 1222 {
1223 1223 saa_port_t *saa_portp;
1224 1224 uint64_t base_time, resp_timeout, rttv_timeout;
1225 1225 ib_mad_classportinfo_t *classportinfo;
1226 1226 int resp_time_value;
1227 1227 uint16_t sa_cap_mask;
1228 1228
1229 1229 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_start,
1230 1230 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() enter\n");
1231 1231
1232 1232 /*
1233 1233 * access port entry: note that it may have become invalid
1234 1234 * but we hold a ref count for cpi and the interactions on
1235 1235 * the entry are harmless
1236 1236 */
1237 1237 saa_portp = (saa_port_t *)arg;
1238 1238
1239 1239 /* process response */
1240 1240
1241 1241 if ((status != IBMF_SUCCESS) || (buffer == NULL)) {
1242 1242
1243 1243 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1,
1244 1244 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_ERROR, "",
1245 1245 "ibmf_saa_impl_get_cpi_cb: %s, status = %d, buffer = "
1246 1246 " 0x%p, length = %d\n", tnf_string, msg,
1247 1247 "could not get classportinfo. Check node and path to sm"
1248 1248 " lid", tnf_int, status, status,
1249 1249 tnf_opaque, buffer, buffer, tnf_uint, length, length);
1250 1250
1251 1251 /*
1252 1252 * IB spec (C13-13) indicates 20 can be used as default or
1253 1253 * intial value for classportinfo->resptimeout value
1254 1254 */
1255 1255 resp_time_value = 20;
1256 1256
1257 1257 sa_cap_mask = 0xFFFF;
1258 1258
1259 1259 } else if (buffer != NULL) {
1260 1260
1261 1261 classportinfo = (ib_mad_classportinfo_t *)buffer;
1262 1262
1263 1263 resp_time_value = classportinfo->RespTimeValue & 0x1f;
1264 1264
1265 1265 /*
1266 1266 * Because some subnet managers might not provide sane
1267 1267 * value for "resp_time_value", we limit it here. In
1268 1268 * case this limit is too restrictive (very large fabric),
1269 1269 * we allow the limit to be raised (/etc/system).
1270 1270 */
1271 1271 if (resp_time_value > ibmf_saa_max_resp_time) {
1272 1272 cmn_err(CE_CONT, "!ibmf_saa_max_resp_time (%d) "
1273 1273 "exceeded.", ibmf_saa_max_resp_time);
1274 1274 cmn_err(CE_CONT, "!Reducing subnet administrator "
1275 1275 "resp_time value from %d to %d.",
1276 1276 resp_time_value, ibmf_saa_max_resp_time);
1277 1277 resp_time_value = ibmf_saa_max_resp_time;
1278 1278 }
1279 1279
1280 1280 sa_cap_mask = classportinfo->CapabilityMask;
1281 1281
1282 1282 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1283 1283 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "",
1284 1284 "ibmf_saa_impl_get_cpi_cb: %s, timeout = 0x%x,"
1285 1285 " cap_mask = 0x%x\n",
1286 1286 tnf_string, msg, "got classportinfo",
1287 1287 tnf_opaque, timeout, resp_time_value,
1288 1288 tnf_opaque, cap_mask, sa_cap_mask);
1289 1289
1290 1290 kmem_free(buffer, length);
1291 1291 }
1292 1292
1293 1293 /*
1294 1294 * using IB spec calculation from 13.4.6.2
1295 1295 * use bit shifting for 2^x.
1296 1296 */
1297 1297 base_time = (1 << resp_time_value);
1298 1298
1299 1299 resp_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000;
1300 1300
1301 1301 mutex_enter(&saa_portp->saa_pt_mutex);
1302 1302
1303 1303 base_time = 2 * (1 << saa_portp->saa_pt_timeout);
1304 1304
1305 1305 rttv_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000;
1306 1306
1307 1307 saa_portp->saa_pt_ibmf_retrans.retrans_rtv = resp_timeout;
1308 1308 saa_portp->saa_pt_ibmf_retrans.retrans_rttv = rttv_timeout;
1309 1309 saa_portp->saa_pt_sa_cap_mask = sa_cap_mask;
1310 1310
1311 1311 /*
1312 1312 * cpi transaction is handled as a client,
1313 1313 * decrement refcount; make sure it's the last
1314 1314 * thing we do on this entry
1315 1315 */
1316 1316 ASSERT(saa_portp->saa_pt_reference_count > 0);
1317 1317 saa_portp->saa_pt_reference_count--;
1318 1318
1319 1319 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1320 1320 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "",
1321 1321 "ibmf_saa_impl_get_cpi_cb: %s, subnet_timeout = 0x%x, "
1322 1322 "resp_time_value = 0x%x\n",
1323 1323 tnf_string, msg, "updated resp timeout",
1324 1324 tnf_opaque, subnet_timeout, saa_portp->saa_pt_timeout,
1325 1325 tnf_opaque, resp_time_value, resp_time_value);
1326 1326
1327 1327 mutex_exit(&saa_portp->saa_pt_mutex);
1328 1328
1329 1329 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_end,
1330 1330 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() exit\n");
1331 1331 }
1332 1332
1333 1333 /*
1334 1334 * ibmf_saa_impl_send_request:
1335 1335 * Sends a request to the sa. Can be used for both classportinfo and record
1336 1336 * requests. Will set up all data structures for using the multi-packet
1337 1337 * protocol, create the mad, and send it. Returns SA_SUCCESS if msg transport
1338 1338 * worked, meaning succesful send for the async case and a succesful send and
1339 1339 * recv for the sync case.
1340 1340 */
1341 1341 int
1342 1342 ibmf_saa_impl_send_request(saa_impl_trans_info_t *trans_info)
1343 1343 {
1344 1344 uint16_t attr_id;
1345 1345 saa_client_data_t *client_data;
1346 1346 saa_port_t *saa_portp;
1347 1347 uint32_t transport_flags;
1348 1348 ibmf_msg_cb_t ibmf_callback;
1349 1349 void *ibmf_callback_arg;
1350 1350 ibmf_msg_t *msgp;
1351 1351 ibmf_retrans_t ibmf_retrans;
1352 1352 uint16_t sa_cap_mask;
1353 1353 boolean_t sleep_flag;
1354 1354 int ibmf_status = IBMF_SUCCESS;
1355 1355 int retry_count;
1356 1356 uint16_t mad_status;
1357 1357 boolean_t sa_is_redirected = B_FALSE;
1358 1358
1359 1359 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1360 1360 ibmf_saa_impl_send_request_start,
1361 1361 IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() enter\n");
1362 1362
1363 1363 attr_id = trans_info->si_trans_attr_id;
1364 1364 client_data = trans_info->si_trans_client_data;
1365 1365 saa_portp = trans_info->si_trans_port;
1366 1366
1367 1367 /*
1368 1368 * don't send on invalid entry
1369 1369 * Note that there is a window where it could become
1370 1370 * invalid after this test is done, but we'd rely on ibmf errors...
1371 1371 */
1372 1372 if (ibmf_saa_is_valid(saa_portp, B_FALSE) == B_FALSE) {
1373 1373
1374 1374 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1,
1375 1375 ibmf_saa_impl_send_request,
1376 1376 IBMF_TNF_ERROR, "",
1377 1377 "ibmf_saa_impl_send_request: %s, hca_guid = %016"
1378 1378 PRIx64 ", port_guid = %016" PRIx64
1379 1379 ", number = %d\n",
1380 1380 tnf_string, msg, "sending on invalid port",
1381 1381 tnf_opaque, hca_guid,
1382 1382 saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid,
1383 1383 tnf_opaque, port_guid,
1384 1384 saa_portp->saa_pt_port_guid,
1385 1385 tnf_uint, port,
1386 1386 saa_portp->saa_pt_ibmf_reginfo.ir_port_num);
1387 1387
1388 1388 ibmf_status = IBMF_REQ_INVALID;
1389 1389 goto bail;
1390 1390 }
1391 1391
1392 1392 /* check whether SA supports this attribute */
1393 1393 mutex_enter(&saa_portp->saa_pt_mutex);
1394 1394
1395 1395 sa_cap_mask = saa_portp->saa_pt_sa_cap_mask;
1396 1396 sa_is_redirected = saa_portp->saa_pt_redirect_active;
1397 1397
1398 1398 mutex_exit(&saa_portp->saa_pt_mutex);
1399 1399
1400 1400 ibmf_status = ibmf_saa_impl_check_sa_support(sa_cap_mask, attr_id);
1401 1401
1402 1402 if (ibmf_status != IBMF_SUCCESS) {
1403 1403
1404 1404 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1405 1405 ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "",
1406 1406 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n",
1407 1407 tnf_string, msg, "SA does not support attribute",
1408 1408 tnf_int, ibmf_status, ibmf_status);
1409 1409
1410 1410 goto bail;
1411 1411 }
1412 1412
1413 1413 /* make only non-blocking calls if this is an async request */
1414 1414 if ((trans_info->si_trans_callback == NULL) &&
1415 1415 (trans_info->si_trans_sub_callback == NULL)) {
1416 1416 ibmf_callback = NULL;
1417 1417 ibmf_callback_arg = NULL;
1418 1418 sleep_flag = B_TRUE;
1419 1419 } else {
1420 1420 ibmf_callback = ibmf_saa_async_cb;
1421 1421 ibmf_callback_arg = (void *)trans_info;
1422 1422 sleep_flag = B_FALSE;
1423 1423 }
1424 1424
1425 1425 ibmf_status = ibmf_saa_impl_init_msg(trans_info, sleep_flag, &msgp,
1426 1426 &transport_flags, &ibmf_retrans);
1427 1427 if (ibmf_status != IBMF_SUCCESS) {
1428 1428
1429 1429 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1430 1430 ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "",
1431 1431 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n",
1432 1432 tnf_string, msg, "init_msg() failed",
1433 1433 tnf_int, ibmf_status, ibmf_status);
1434 1434
1435 1435 goto bail;
1436 1436 }
1437 1437
1438 1438 mutex_enter(&saa_portp->saa_pt_mutex);
1439 1439
1440 1440 saa_portp->saa_pt_num_outstanding_trans++;
1441 1441
1442 1442 mutex_exit(&saa_portp->saa_pt_mutex);
1443 1443
1444 1444 /*
1445 1445 * increment the number of outstanding transaction so
1446 1446 * ibmf_close_sa_session() will wait. classportinfo requests
1447 1447 * don't have associated clients so check for valid clientp
1448 1448 */
1449 1449 if (client_data != NULL) {
1450 1450
1451 1451 mutex_enter(&client_data->saa_client_mutex);
1452 1452
1453 1453 client_data->saa_client_num_pending_trans++;
1454 1454
1455 1455 mutex_exit(&client_data->saa_client_mutex);
1456 1456 }
1457 1457
1458 1458 /*
1459 1459 * make the call to msg_transport. If synchronous and success,
1460 1460 * check that the response mad isn't status busy. If so, repeat the
1461 1461 * call
1462 1462 */
1463 1463 retry_count = 0;
1464 1464
1465 1465 /*
1466 1466 * set the send time here. We only set this once at the beginning of
1467 1467 * the transaction. Retrying because of busys or mastersmlid changes
1468 1468 * does not change the original send time. It is meant to be an
1469 1469 * absolute time out value and will only be used if there are other
1470 1470 * problems (i.e. a buggy SA)
1471 1471 */
1472 1472 trans_info->si_trans_send_time = gethrtime();
1473 1473
1474 1474 for (;;) {
1475 1475
1476 1476 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle,
1477 1477 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
1478 1478 ibmf_callback, ibmf_callback_arg, transport_flags);
1479 1479
1480 1480 if (ibmf_callback != NULL)
1481 1481 break;
1482 1482
1483 1483 /*
1484 1484 * stop here for non-sequenced transactions since they wouldn't
1485 1485 * receive a timeout or busy response
1486 1486 */
1487 1487 if (!(transport_flags & IBMF_MSG_TRANS_FLAG_SEQ))
1488 1488 break;
1489 1489
1490 1490 /*
1491 1491 * if the transaction timed out and this was a synchronous
1492 1492 * request there's a possiblity we were talking to the wrong
1493 1493 * master smlid or that the SA has stopped responding on the
1494 1494 * redirected desination (if redirect is active).
1495 1495 * Check this and retry if necessary.
1496 1496 */
1497 1497 if ((ibmf_status == IBMF_TRANS_TIMEOUT) &&
1498 1498 (sleep_flag == B_TRUE)) {
1499 1499 if (sa_is_redirected == B_TRUE) {
1500 1500 ibmf_status = ibmf_saa_impl_revert_to_qp1(
1501 1501 saa_portp, msgp, ibmf_callback,
1502 1502 ibmf_callback_arg, transport_flags);
1503 1503 } else {
1504 1504 ibmf_status = ibmf_saa_impl_new_smlid_retry(
1505 1505 saa_portp, msgp, ibmf_callback,
1506 1506 ibmf_callback_arg, transport_flags);
1507 1507 }
1508 1508 }
1509 1509
1510 1510 /*
1511 1511 * if the transaction timed out (and retrying with a new SM LID
1512 1512 * didn't help) check how long it's been since we received an SA
1513 1513 * packet. If it hasn't been max_wait_time then retry the
1514 1514 * request.
1515 1515 */
1516 1516 if ((ibmf_status == IBMF_TRANS_TIMEOUT) &&
1517 1517 (sleep_flag == B_TRUE)) {
1518 1518
1519 1519 ibmf_status = ibmf_saa_check_sa_and_retry(
1520 1520 saa_portp, msgp, ibmf_callback, ibmf_callback_arg,
1521 1521 trans_info->si_trans_send_time, transport_flags);
1522 1522 }
1523 1523
1524 1524 if (ibmf_status != IBMF_SUCCESS)
1525 1525 break;
1526 1526
1527 1527 if (retry_count >= IBMF_SAA_MAX_BUSY_RETRY_COUNT)
1528 1528 break;
1529 1529
1530 1530 /* sync transaction with status SUCCESS should have response */
1531 1531 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL);
1532 1532
1533 1533 mad_status = b2h16(msgp->im_msgbufs_recv.
1534 1534 im_bufs_mad_hdr->Status);
1535 1535
1536 1536 if ((mad_status != MAD_STATUS_BUSY) &&
1537 1537 (mad_status != MAD_STATUS_REDIRECT_REQUIRED))
1538 1538 break;
1539 1539
1540 1540 if (mad_status == MAD_STATUS_REDIRECT_REQUIRED) {
1541 1541
1542 1542 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1543 1543 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1544 1544 "ibmf_saa_impl_send_request: %s, retry_count %d\n",
1545 1545 tnf_string, msg,
1546 1546 "response returned redirect status",
1547 1547 tnf_int, retry_count, retry_count);
1548 1548
1549 1549 /* update address info and copy it into msgp */
1550 1550 ibmf_saa_impl_update_sa_address_info(saa_portp, msgp);
1551 1551 } else {
1552 1552 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1553 1553 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1554 1554 "ibmf_saa_impl_send_request: %s, retry_count %d\n",
1555 1555 tnf_string, msg, "response returned busy status",
|
↓ open down ↓ |
1555 lines elided |
↑ open up ↑ |
1556 1556 tnf_int, retry_count, retry_count);
1557 1557 }
1558 1558
1559 1559 retry_count++;
1560 1560
1561 1561 /*
1562 1562 * since this is a blocking call, sleep for some time
1563 1563 * to allow SA to transition from busy state (if busy)
1564 1564 */
1565 1565 if (mad_status == MAD_STATUS_BUSY)
1566 - delay(drv_usectohz(
1567 - IBMF_SAA_BUSY_RETRY_SLEEP_SECS * 1000000));
1566 + delay(drv_sectohz(IBMF_SAA_BUSY_RETRY_SLEEP_SECS));
1568 1567 }
1569 1568
1570 1569 if (ibmf_status != IBMF_SUCCESS) {
1571 1570
1572 1571 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1573 1572 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1574 1573 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n",
1575 1574 tnf_string, msg, "ibmf_msg_transport() failed",
1576 1575 tnf_int, ibmf_status, ibmf_status);
1577 1576
1578 1577 ibmf_saa_impl_free_msg(saa_portp->saa_pt_ibmf_handle, msgp);
1579 1578
1580 1579 mutex_enter(&saa_portp->saa_pt_mutex);
1581 1580
1582 1581 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0);
1583 1582 saa_portp->saa_pt_num_outstanding_trans--;
1584 1583
1585 1584 mutex_exit(&saa_portp->saa_pt_mutex);
1586 1585
1587 1586 if (client_data != NULL) {
1588 1587
1589 1588 mutex_enter(&client_data->saa_client_mutex);
1590 1589
1591 1590 ASSERT(client_data->saa_client_num_pending_trans > 0);
1592 1591 client_data->saa_client_num_pending_trans--;
1593 1592
1594 1593 if ((client_data->saa_client_num_pending_trans == 0) &&
1595 1594 (client_data->saa_client_state ==
1596 1595 SAA_CLIENT_STATE_WAITING))
1597 1596 cv_signal(&client_data->saa_client_state_cv);
1598 1597
1599 1598 mutex_exit(&client_data->saa_client_mutex);
1600 1599 }
1601 1600
1602 1601 } else if (sleep_flag == B_TRUE) {
1603 1602
1604 1603 mutex_enter(&saa_portp->saa_pt_mutex);
1605 1604
1606 1605 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0);
1607 1606 saa_portp->saa_pt_num_outstanding_trans--;
1608 1607
1609 1608 mutex_exit(&saa_portp->saa_pt_mutex);
1610 1609
1611 1610 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1612 1611 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1613 1612 "ibmf_saa_impl_send_request: %s\n",
1614 1613 tnf_string, msg, "Message sent and received successfully");
1615 1614
1616 1615 /* fill in response values and free the message */
1617 1616 ibmf_saa_impl_prepare_response(saa_portp->saa_pt_ibmf_handle,
1618 1617 msgp, B_FALSE, &trans_info->si_trans_status,
1619 1618 &trans_info->si_trans_result,
1620 1619 &trans_info->si_trans_length, sleep_flag);
1621 1620
1622 1621 if (client_data != NULL) {
1623 1622 mutex_enter(&client_data->saa_client_mutex);
1624 1623
1625 1624 ASSERT(client_data->saa_client_num_pending_trans > 0);
1626 1625 client_data->saa_client_num_pending_trans--;
1627 1626
1628 1627 if ((client_data->saa_client_num_pending_trans == 0) &&
1629 1628 (client_data->saa_client_state ==
1630 1629 SAA_CLIENT_STATE_WAITING))
1631 1630 cv_signal(&client_data->saa_client_state_cv);
1632 1631
1633 1632 mutex_exit(&client_data->saa_client_mutex);
1634 1633 }
1635 1634 } else {
1636 1635
1637 1636 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1638 1637 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1639 1638 "ibmf_saa_impl_send_request: %s\n",
1640 1639 tnf_string, msg, "Message sent successfully");
1641 1640 }
1642 1641
1643 1642 bail:
1644 1643 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1645 1644 ibmf_saa_impl_send_request_end,
1646 1645 IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() exiting"
1647 1646 " ibmf_status = %d\n", tnf_int, result, ibmf_status);
1648 1647
1649 1648 return (ibmf_status);
1650 1649 }
1651 1650
1652 1651 /*
1653 1652 * ibmf_saa_impl_init_msg:
1654 1653 * Allocates an ibmf message and fills out the header fields and formatted data
1655 1654 * fields. Also sets up the correct transport_flags and retrans argument for
1656 1655 * the message transport call based on the request information.
1657 1656 *
1658 1657 * Input Arguments
1659 1658 * trans_info saa_trans_info structure passed to send_request
1660 1659 * sleep_flag B_TRUE if init_msg can sleep in function calls
1661 1660 *
1662 1661 * Output Arguments
1663 1662 * msgp ibmf message that should be given to msg_transport
1664 1663 * transport_flagsp transport flags that should be given to msg_transport
1665 1664 * ibmf_retrans_t retrans parameter that should be given to msg_transport
1666 1665 *
1667 1666 * Returns
1668 1667 * ibmf_status
1669 1668 */
1670 1669 static int
1671 1670 ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, boolean_t sleep_flag,
1672 1671 ibmf_msg_t **msgp, uint32_t *transport_flagsp,
1673 1672 ibmf_retrans_t *ibmf_retransp)
1674 1673 {
1675 1674 int ibmf_status;
1676 1675 ibmf_msg_bufs_t *req_mad;
1677 1676 ib_mad_hdr_t *mad_hdr;
1678 1677 int ibmf_sleep_flag, km_sleep_flag;
1679 1678 int free_res;
1680 1679 ib_sa_hdr_t sa_hdr;
1681 1680 ibmf_msg_t *ibmf_msg;
1682 1681 uint16_t attr_id, pack_attr_id;
1683 1682 uint8_t method;
1684 1683 saa_client_data_t *client_data;
1685 1684 saa_port_t *saa_portp;
1686 1685 sa_multipath_record_t *multipath_template;
1687 1686 size_t payload_length;
1688 1687 uint32_t transport_flags;
1689 1688
1690 1689 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1691 1690 ibmf_saa_impl_init_msg_start,
1692 1691 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() entering\n");
1693 1692
1694 1693 attr_id = trans_info->si_trans_attr_id;
1695 1694 method = trans_info->si_trans_method;
1696 1695 client_data = trans_info->si_trans_client_data;
1697 1696 saa_portp = trans_info->si_trans_port;
1698 1697
1699 1698 if (sleep_flag == B_TRUE) {
1700 1699 ibmf_sleep_flag = IBMF_ALLOC_SLEEP;
1701 1700 km_sleep_flag = KM_SLEEP;
1702 1701 } else {
1703 1702 ibmf_sleep_flag = IBMF_ALLOC_NOSLEEP;
1704 1703 km_sleep_flag = KM_NOSLEEP;
1705 1704 }
1706 1705
1707 1706 ibmf_status = ibmf_alloc_msg(saa_portp->saa_pt_ibmf_handle,
1708 1707 ibmf_sleep_flag, &ibmf_msg);
1709 1708 if (ibmf_status != IBMF_SUCCESS) {
1710 1709
1711 1710 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1712 1711 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1713 1712 "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n",
1714 1713 tnf_string, msg, "Cannot allocate msg_buf.",
1715 1714 tnf_int, ibmf_status, ibmf_status);
1716 1715
1717 1716 goto bail;
1718 1717 }
1719 1718
1720 1719 req_mad = &ibmf_msg->im_msgbufs_send;
1721 1720
1722 1721 /* create a template (SA MAD) */
1723 1722 mad_hdr = kmem_zalloc(sizeof (ib_mad_hdr_t), km_sleep_flag);
1724 1723
1725 1724 if (mad_hdr == NULL) {
1726 1725
1727 1726 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
1728 1727 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1729 1728 "ibmf_saa_impl_init_msg: %s\n",
1730 1729 tnf_string, msg, "Cannot allocate mad header.");
1731 1730
1732 1731 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1733 1732 &ibmf_msg);
1734 1733 ASSERT(free_res == IBMF_SUCCESS);
1735 1734
1736 1735 ibmf_status = IBMF_NO_MEMORY;
1737 1736 goto bail;
1738 1737 }
1739 1738
1740 1739 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mad_hdr,
1741 1740 *req_mad))
1742 1741
1743 1742 bzero(mad_hdr, sizeof (ib_mad_hdr_t));
1744 1743 mad_hdr->BaseVersion = SAA_MAD_BASE_VERSION;
1745 1744 mad_hdr->MgmtClass = MAD_MGMT_CLASS_SUBN_ADM;
1746 1745 mad_hdr->ClassVersion = SAA_MAD_CLASS_VERSION;
1747 1746 mad_hdr->R_Method = method;
1748 1747 mad_hdr->AttributeID = h2b16(attr_id);
1749 1748
1750 1749 /* attribute modifier is all Fs since RIDs are no longer used */
1751 1750 mad_hdr->AttributeModifier = h2b32(0xffffffff);
1752 1751
1753 1752 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
1754 1753 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "",
1755 1754 "ibmf_saa_impl_init_msg: %s, class = 0x%x, method = 0x%x,"
1756 1755 " attr_id = 0x%x\n", tnf_string, msg, "Sending MAD",
1757 1756 tnf_opaque, class, mad_hdr->MgmtClass,
1758 1757 tnf_opaque, method, mad_hdr->R_Method,
1759 1758 tnf_opaque, attr_id, attr_id);
1760 1759
1761 1760 bzero(&sa_hdr, sizeof (ib_sa_hdr_t));
1762 1761 sa_hdr.ComponentMask = trans_info->si_trans_component_mask;
1763 1762
1764 1763 if (client_data != NULL)
1765 1764 sa_hdr.SM_KEY = client_data->saa_client_sm_key;
1766 1765
1767 1766 /*
1768 1767 * pack data for IB wire format; req_mad will have different pointers to
1769 1768 * sa header and payload, mad_hdr will be the same
1770 1769 */
1771 1770 req_mad->im_bufs_mad_hdr = mad_hdr;
1772 1771
1773 1772 ibmf_status = ibmf_saa_utils_pack_sa_hdr(&sa_hdr,
1774 1773 &req_mad->im_bufs_cl_hdr, &req_mad->im_bufs_cl_hdr_len,
1775 1774 km_sleep_flag);
1776 1775
1777 1776 if (ibmf_status != IBMF_SUCCESS) {
1778 1777
1779 1778 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1780 1779 ibmf_saa_impl_init_msg, IBMF_TNF_ERROR, "",
1781 1780 "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n",
1782 1781 tnf_string, msg, "ibmf_saa_utils_pack_sa_hdr() failed",
1783 1782 tnf_int, ibmf_status, ibmf_status);
1784 1783
1785 1784 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t));
1786 1785
1787 1786 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1788 1787 &ibmf_msg);
1789 1788 ASSERT(free_res == IBMF_SUCCESS);
1790 1789
1791 1790 goto bail;
1792 1791 }
1793 1792
1794 1793 if (attr_id == SA_MULTIPATHRECORD_ATTRID) {
1795 1794
1796 1795 multipath_template =
1797 1796 (sa_multipath_record_t *)trans_info->si_trans_template;
1798 1797
1799 1798 payload_length = sizeof (sa_multipath_record_t) +
1800 1799 ((multipath_template->SGIDCount +
1801 1800 multipath_template->DGIDCount) * sizeof (ib_gid_t));
1802 1801
1803 1802 pack_attr_id = attr_id;
1804 1803 } else {
1805 1804
1806 1805 /* trace record template is a path record */
1807 1806 pack_attr_id = (attr_id == SA_TRACERECORD_ATTRID) ?
1808 1807 SA_PATHRECORD_ATTRID : attr_id;
1809 1808
1810 1809 payload_length = ibmf_saa_impl_get_attr_id_length(pack_attr_id);
1811 1810
1812 1811 if (payload_length == 0) {
1813 1812 payload_length = trans_info->si_trans_template_length;
1814 1813
1815 1814 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
1816 1815 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "",
1817 1816 "ibmf_saa_impl_init_msg: %s, length = %d\n",
1818 1817 tnf_string, msg,
1819 1818 "Unknown attribute. Using user-defined length.",
1820 1819 tnf_uint, length, payload_length)
1821 1820 }
1822 1821 }
1823 1822
1824 1823 /* transport type depends on method */
1825 1824 switch (method) {
1826 1825
1827 1826 case SA_SUBN_ADM_GET:
1828 1827 case SA_SUBN_ADM_DELETE:
1829 1828 case SA_SUBN_ADM_GET_TABLE:
1830 1829 case SA_SUBN_ADM_GET_TRACE_TABLE:
1831 1830 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ;
1832 1831 break;
1833 1832 case SA_SUBN_ADM_SET:
1834 1833 /* unsubscribes can be sequenced or unsequenced */
1835 1834 if (trans_info->si_trans_unseq_unsubscribe == B_TRUE) {
1836 1835 transport_flags = 0;
1837 1836 } else {
1838 1837 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ;
1839 1838 }
1840 1839 break;
1841 1840 case SA_SUBN_ADM_GET_MULTI:
1842 1841 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ |
1843 1842 IBMF_MSG_TRANS_FLAG_RMPP;
1844 1843 break;
1845 1844 default :
1846 1845 ibmf_status = IBMF_UNSUPP_METHOD;
1847 1846 goto bail;
1848 1847 }
1849 1848
1850 1849 trans_info->si_trans_transport_flags = transport_flags;
1851 1850
1852 1851 if (trans_info->si_trans_template != NULL) {
1853 1852
1854 1853 ibmf_status = ibmf_saa_utils_pack_payload(
1855 1854 trans_info->si_trans_template, payload_length, pack_attr_id,
1856 1855 &req_mad->im_bufs_cl_data, &req_mad->im_bufs_cl_data_len,
1857 1856 km_sleep_flag);
1858 1857 if (ibmf_status != IBMF_SUCCESS) {
1859 1858
1860 1859 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1861 1860 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1862 1861 "ibmf_saa_impl_init_msg: %s, ibmf_status ="
1863 1862 " %d\n", tnf_string, msg,
1864 1863 "ibmf_saa_utils_pack_payload() failed",
1865 1864 tnf_int, ibmf_status, ibmf_status);
1866 1865
1867 1866 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t));
1868 1867
1869 1868 kmem_free(req_mad->im_bufs_cl_hdr,
1870 1869 req_mad->im_bufs_cl_hdr_len);
1871 1870
1872 1871 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1873 1872 &ibmf_msg);
1874 1873 ASSERT(free_res == IBMF_SUCCESS);
1875 1874
1876 1875 goto bail;
1877 1876 }
1878 1877
1879 1878 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1880 1879 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "",
1881 1880 "ibmf_saa_impl_init_msg: %s, attr_id = 0x%x, length ="
1882 1881 " %d\n", tnf_string, msg, "Packed payload successfully",
1883 1882 tnf_opaque, attr_id, attr_id,
1884 1883 tnf_uint, length, req_mad->im_bufs_cl_data_len);
1885 1884
1886 1885 /* non-RMPP transactions have template size limit */
1887 1886 if (((transport_flags & IBMF_MSG_TRANS_FLAG_RMPP) == 0) &&
1888 1887 ((req_mad->im_bufs_cl_data_len + req_mad->im_bufs_cl_hdr_len
1889 1888 + sizeof (ib_mad_hdr_t)) > IBMF_MAD_SIZE)) {
1890 1889
1891 1890 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
1892 1891 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1893 1892 "ibmf_saa_impl_init_msg: %s\n", tnf_string, msg,
1894 1893 "Template too large to fit in single packet");
1895 1894
1896 1895 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t));
1897 1896
1898 1897 kmem_free(req_mad->im_bufs_cl_hdr,
1899 1898 req_mad->im_bufs_cl_hdr_len);
1900 1899
1901 1900 kmem_free(req_mad->im_bufs_cl_data,
1902 1901 req_mad->im_bufs_cl_data_len);
1903 1902
1904 1903 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1905 1904 &ibmf_msg);
1906 1905 ASSERT(free_res == IBMF_SUCCESS);
1907 1906
1908 1907 ibmf_status = IBMF_REQ_INVALID;
1909 1908 goto bail;
1910 1909 }
1911 1910 }
1912 1911
1913 1912 mutex_enter(&saa_portp->saa_pt_mutex);
1914 1913
1915 1914 mad_hdr->TransactionID = h2b64(saa_portp->saa_pt_current_tid++);
1916 1915
1917 1916 bcopy(&saa_portp->saa_pt_ibmf_retrans, ibmf_retransp,
1918 1917 sizeof (ibmf_retrans_t));
1919 1918
1920 1919 /* copy local addressing information to message */
1921 1920 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &ibmf_msg->im_local_addr,
1922 1921 sizeof (ibmf_addr_info_t));
1923 1922
1924 1923 /* copy global addressing information to message if in use */
1925 1924 if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) {
1926 1925
1927 1926 ibmf_msg->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS;
1928 1927
1929 1928 bcopy(&saa_portp->saa_pt_ibmf_global_addr,
1930 1929 &ibmf_msg->im_global_addr,
1931 1930 sizeof (ibmf_global_addr_info_t));
1932 1931 } else {
1933 1932 ibmf_msg->im_msg_flags = 0;
1934 1933 }
1935 1934
1936 1935 mutex_exit(&saa_portp->saa_pt_mutex);
1937 1936
1938 1937 *msgp = ibmf_msg;
1939 1938 *transport_flagsp = transport_flags;
1940 1939 bail:
1941 1940 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1942 1941 ibmf_saa_impl_init_msg_end,
1943 1942 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() exiting"
1944 1943 " ibmf_status = %d\n", tnf_int, result, ibmf_status);
1945 1944
1946 1945 return (ibmf_status);
1947 1946
1948 1947 }
1949 1948
1950 1949 /*
1951 1950 * ibmf_saa_impl_new_smlid_retry:
1952 1951 *
1953 1952 * It's possible for the MasterSMLID to change while ibmf_saa is running. The
1954 1953 * MasterSMLID is set when we first register with ibmf_saa. If a request
1955 1954 * timesout, this function should be called to check whether the SM LID changed.
1956 1955 * If so, it will call msg_transport again with the request.
1957 1956 *
1958 1957 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the
1959 1958 * same values passed to the original ibmf_msg_transport that timedout. The
1960 1959 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure.
1961 1960 *
1962 1961 * If the lid did not change then this function returns IBMF_TRANS_TIMEOUT.
1963 1962 * That way, callers can simply return the result of this function.
1964 1963 *
1965 1964 * Input Arguments
1966 1965 * saa_portp pointer to saa_port structure
1967 1966 * msgp ibmf message that timedout
1968 1967 * ibmf_callback callback that should be called by msg_transport
1969 1968 * ibmf_callback_arg args for ibmf_callback
1970 1969 * transport_flags flags for ibmf_msg_transport
1971 1970 *
1972 1971 * Output Arguments
1973 1972 * none
1974 1973 *
1975 1974 * Returns
1976 1975 * IBMF_SUCCESS if lid changed and request was resent successfully,
1977 1976 * IBMF_TRANS_TIMEOUT if lid did not change,
1978 1977 * same values as ibmf_msg_transport() if lid changed but request could not be
1979 1978 * resent.
1980 1979 */
1981 1980 static int
1982 1981 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
1983 1982 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags)
1984 1983 {
1985 1984 ibt_hca_portinfo_t *ibt_portinfop;
1986 1985 ib_lid_t master_sm_lid;
1987 1986 int subnet_timeout;
1988 1987 uint_t nports, size;
1989 1988 ibmf_retrans_t ibmf_retrans;
1990 1989 int ibmf_status;
1991 1990 ibt_status_t ibt_status;
1992 1991
1993 1992 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1994 1993 ibmf_saa_impl_new_smlid_retry_start,
1995 1994 IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() enter\n");
1996 1995
1997 1996 _NOTE(ASSUMING_PROTECTED(*msgp))
1998 1997 _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr))
1999 1998
2000 1999 /* first query the portinfo to see if the lid changed */
2001 2000 ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid,
2002 2001 saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size);
2003 2002
2004 2003 if (ibt_status != IBT_SUCCESS) {
2005 2004
2006 2005 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2007 2006 ibmf_saa_impl_new_smlid_retry_err, IBMF_TNF_ERROR, "",
2008 2007 "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status ="
2009 2008 " %d\n", tnf_string, msg,
2010 2009 "ibt_query_hca_ports_byguid() failed",
2011 2010 tnf_int, ibt_status, ibt_status);
2012 2011
2013 2012 ibmf_status = IBMF_TRANSPORT_FAILURE;
2014 2013
2015 2014 goto bail;
2016 2015 }
2017 2016
2018 2017 master_sm_lid = ibt_portinfop->p_sm_lid;
2019 2018 subnet_timeout = ibt_portinfop->p_subnet_timeout;
2020 2019
2021 2020 ibt_free_portinfo(ibt_portinfop, size);
2022 2021
2023 2022 /* if master smlid is different than the remote lid we sent to */
2024 2023 if (master_sm_lid != msgp->im_local_addr.ia_remote_lid) {
2025 2024
2026 2025 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L2,
2027 2026 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "",
2028 2027 "ibmf_saa_impl_new_smlid_retry: %s, new_lid 0x%x,"
2029 2028 " old_lid 0x%x\n", tnf_string, msg,
2030 2029 "master smlid has changed. retrying msg_transport",
2031 2030 tnf_opaque, new_lid, master_sm_lid,
2032 2031 tnf_opaque, old_lid, msgp->im_local_addr.ia_remote_lid);
2033 2032
2034 2033 mutex_enter(&saa_portp->saa_pt_mutex);
2035 2034
2036 2035 /* update the master sm lid value in ibmf_saa */
2037 2036 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2038 2037 master_sm_lid;
2039 2038
2040 2039 /* new tid needed */
2041 2040 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID =
2042 2041 h2b64(saa_portp->saa_pt_current_tid++);
2043 2042
2044 2043 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2045 2044 sizeof (ibmf_retrans_t));
2046 2045
2047 2046 /* update the subnet timeout since this may be a new sm/sa */
2048 2047 saa_portp->saa_pt_timeout = subnet_timeout;
2049 2048
2050 2049 /* place upper bound on subnet timeout in case of faulty SM */
2051 2050 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout)
2052 2051 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout;
2053 2052
2054 2053 /* increment the reference count to account for the cpi call */
2055 2054 saa_portp->saa_pt_reference_count++;
2056 2055
2057 2056 mutex_exit(&saa_portp->saa_pt_mutex);
2058 2057
2059 2058 /* update the remote lid for this particular message */
2060 2059 msgp->im_local_addr.ia_remote_lid = master_sm_lid;
2061 2060
2062 2061 /* get the classportinfo again since this may be a new sm/sa */
2063 2062 ibmf_saa_impl_get_classportinfo(saa_portp);
2064 2063
2065 2064 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle,
2066 2065 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2067 2066 ibmf_callback, ibmf_callback_arg, transport_flags);
2068 2067
2069 2068 if (ibmf_status != IBMF_SUCCESS) {
2070 2069
2071 2070 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
2072 2071 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "",
2073 2072 "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status = "
2074 2073 "%d\n", tnf_string, msg,
2075 2074 "ibmf_msg_transport() failed",
2076 2075 tnf_int, ibmf_status, ibmf_status);
2077 2076 }
2078 2077
2079 2078 goto bail;
2080 2079 }
2081 2080
2082 2081 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
2083 2082 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "",
2084 2083 "ibmf_saa_impl_new_smlid_retry: %s, master_smlid = 0x%x\n",
2085 2084 tnf_string, msg,
2086 2085 "master smlid did not change. returning failure",
2087 2086 tnf_opaque, master_smlid, master_sm_lid);
2088 2087
2089 2088 /* mark status as timeout since that was original failure */
2090 2089 ibmf_status = IBMF_TRANS_TIMEOUT;
2091 2090
2092 2091 bail:
2093 2092 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2094 2093 ibmf_saa_impl_new_smlid_retry_end,
2095 2094 IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() exiting"
2096 2095 " ibmf_status = %d\n", tnf_int, result, ibmf_status);
2097 2096
2098 2097 return (ibmf_status);
2099 2098 }
2100 2099
2101 2100 /*
2102 2101 * ibmf_saa_impl_revert_to_qp1()
2103 2102 *
2104 2103 * The SA that we had contact with via redirect may fail to respond. If this
2105 2104 * occurs SA should revert back to qp1 and the SMLID set in the port.
2106 2105 * msg_transport for the message that timed out will be retried with
2107 2106 * these new parameters.
2108 2107 *
2109 2108 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the
2110 2109 * same values passed to the original ibmf_msg_transport that timedout. The
2111 2110 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure.
2112 2111 *
2113 2112 * Input Arguments
2114 2113 * saa_portp pointer to saa_port structure
2115 2114 * msgp ibmf message that timedout
2116 2115 * ibmf_callback callback that should be called by msg_transport
2117 2116 * ibmf_callback_arg args for ibmf_callback
2118 2117 * transport_flags flags for ibmf_msg_transport
2119 2118 *
2120 2119 * Output Arguments
2121 2120 * none
2122 2121 *
2123 2122 * Returns
2124 2123 * none
2125 2124 */
2126 2125 static int
2127 2126 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp,
2128 2127 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags)
2129 2128 {
2130 2129 ibt_hca_portinfo_t *ibt_portinfop;
2131 2130 ib_lid_t master_sm_lid, base_lid;
2132 2131 uint8_t sm_sl;
2133 2132 int subnet_timeout;
2134 2133 uint_t nports, size;
2135 2134 ibmf_retrans_t ibmf_retrans;
2136 2135 int ibmf_status;
2137 2136 ibt_status_t ibt_status;
2138 2137
2139 2138 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
2140 2139 ibmf_saa_impl_revert_to_qp1_start,
2141 2140 IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() enter\n");
2142 2141
2143 2142 _NOTE(ASSUMING_PROTECTED(*msgp))
2144 2143 _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr))
2145 2144
2146 2145 /* first query the portinfo to see if the lid changed */
2147 2146 ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid,
2148 2147 saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size);
2149 2148
2150 2149 if (ibt_status != IBT_SUCCESS) {
2151 2150
2152 2151 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2153 2152 ibmf_saa_impl_revert_to_qp1_err, IBMF_TNF_ERROR, "",
2154 2153 "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status ="
2155 2154 " %d\n", tnf_string, msg,
2156 2155 "ibt_query_hca_ports_byguid() failed",
2157 2156 tnf_int, ibt_status, ibt_status);
2158 2157
2159 2158 ibmf_status = IBMF_TRANSPORT_FAILURE;
2160 2159
2161 2160 goto bail;
2162 2161 }
2163 2162
2164 2163 master_sm_lid = ibt_portinfop->p_sm_lid;
2165 2164 base_lid = ibt_portinfop->p_base_lid;
2166 2165 sm_sl = ibt_portinfop->p_sm_sl;
2167 2166 subnet_timeout = ibt_portinfop->p_subnet_timeout;
2168 2167
2169 2168 ibt_free_portinfo(ibt_portinfop, size);
2170 2169
2171 2170
2172 2171 mutex_enter(&saa_portp->saa_pt_mutex);
2173 2172
2174 2173 saa_portp->saa_pt_redirect_active = B_FALSE;
2175 2174
2176 2175 /* update the address info in ibmf_saa */
2177 2176 saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = base_lid;
2178 2177 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = master_sm_lid;
2179 2178 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = sm_sl;
2180 2179 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1;
2181 2180 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = IB_PKEY_DEFAULT_LIMITED;
2182 2181 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY;
2183 2182 saa_portp->saa_pt_ibmf_msg_flags = 0;
2184 2183
2185 2184 /* new tid needed */
2186 2185 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID =
2187 2186 h2b64(saa_portp->saa_pt_current_tid++);
2188 2187
2189 2188 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2190 2189 sizeof (ibmf_retrans_t));
2191 2190
2192 2191 /* update the subnet timeout since this may be a new sm/sa */
2193 2192 saa_portp->saa_pt_timeout = subnet_timeout;
2194 2193
2195 2194 /* place upper bound on subnet timeout in case of faulty SM */
2196 2195 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout)
2197 2196 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout;
2198 2197
2199 2198 /* increment the reference count to account for the cpi call */
2200 2199 saa_portp->saa_pt_reference_count++;
2201 2200
2202 2201 mutex_exit(&saa_portp->saa_pt_mutex);
2203 2202
2204 2203 /* update the address info for this particular message */
2205 2204 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr,
2206 2205 sizeof (ibmf_addr_info_t));
2207 2206 msgp->im_msg_flags = 0; /* No GRH */
2208 2207
2209 2208 /* get the classportinfo again since this may be a new sm/sa */
2210 2209 ibmf_saa_impl_get_classportinfo(saa_portp);
2211 2210
2212 2211 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle,
2213 2212 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2214 2213 ibmf_callback, ibmf_callback_args, transport_flags);
2215 2214
2216 2215 if (ibmf_status != IBMF_SUCCESS) {
2217 2216
2218 2217 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
2219 2218 ibmf_saa_impl_revert_to_qp1, IBMF_TNF_TRACE, "",
2220 2219 "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status = "
2221 2220 "%d\n", tnf_string, msg,
2222 2221 "ibmf_msg_transport() failed",
2223 2222 tnf_int, ibmf_status, ibmf_status);
2224 2223 }
2225 2224
2226 2225 bail:
2227 2226
2228 2227 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2229 2228 ibmf_saa_impl_revert_to_qp1_end,
2230 2229 IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() exiting"
2231 2230 " ibmf_status = %d\n", tnf_int, result, ibmf_status);
2232 2231
2233 2232 return (ibmf_status);
2234 2233 }
2235 2234
2236 2235 /*
2237 2236 * ibmf_saa_impl_async_event_cb:
2238 2237 * ibmf event callback, argument to ibmf_register
2239 2238 * ibmf_handle is unused
2240 2239 */
2241 2240 /* ARGSUSED */
2242 2241 static void
2243 2242 ibmf_saa_impl_async_event_cb(
2244 2243 ibmf_handle_t ibmf_handle,
2245 2244 void *clnt_private,
2246 2245 ibmf_async_event_t event_type)
2247 2246 {
2248 2247 saa_port_t *saa_portp;
2249 2248
2250 2249 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2251 2250 ibmf_saa_impl_async_event_cb_start, IBMF_TNF_TRACE, "",
2252 2251 "ibmf_saa_impl_async_event_cb: Handling event type 0x%x\n",
2253 2252 tnf_opaque, event_type, event_type);
2254 2253
2255 2254 saa_portp = (saa_port_t *)clnt_private;
2256 2255 ASSERT(saa_portp != NULL);
2257 2256
2258 2257 switch (event_type) {
2259 2258
2260 2259 case IBMF_CI_OFFLINE:
2261 2260 ibmf_saa_impl_hca_detach(saa_portp);
2262 2261 break;
2263 2262 default:
2264 2263 break;
2265 2264 }
2266 2265
2267 2266 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3,
2268 2267 ibmf_saa_impl_async_event_cb_end,
2269 2268 IBMF_TNF_TRACE, "", "ibmf_saa_impl_async_event_cb() exit\n");
2270 2269 }
2271 2270
2272 2271
2273 2272 /*
2274 2273 * ibmf_saa_impl_ibt_async_handler:
2275 2274 * MUST NOT BE STATIC (referred from within IBMF)
2276 2275 */
2277 2276 void
2278 2277 ibmf_saa_impl_ibt_async_handler(ibt_async_code_t code, ibt_async_event_t *event)
2279 2278 {
2280 2279 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2281 2280 ibmf_saa_impl_ibt_async_handler_start, IBMF_TNF_TRACE, "",
2282 2281 "ibmf_saa_impl_ibt_async_handler: Handling event code 0x%x\n",
2283 2282 tnf_opaque, code, code);
2284 2283
2285 2284 switch (code) {
2286 2285
2287 2286 case IBT_EVENT_PORT_UP:
2288 2287 ibmf_saa_impl_port_up(event->ev_hca_guid, event->ev_port);
2289 2288 break;
2290 2289 case IBT_ERROR_PORT_DOWN:
2291 2290 ibmf_saa_impl_port_down(event->ev_hca_guid, event->ev_port);
2292 2291 break;
2293 2292 case IBT_PORT_CHANGE_EVENT:
2294 2293 ibmf_saa_impl_port_chg(event);
2295 2294 break;
2296 2295 case IBT_CLNT_REREG_EVENT:
2297 2296 ibmf_saa_impl_client_rereg(event->ev_hca_guid, event->ev_port);
2298 2297 break;
2299 2298 default:
2300 2299 break;
2301 2300 }
2302 2301
2303 2302 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_async_handler_end,
2304 2303 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibt_async_handler() exit\n");
2305 2304 }
2306 2305
2307 2306 /*
2308 2307 * ibmf_saa_impl_port_chg:
2309 2308 */
2310 2309 static void
2311 2310 ibmf_saa_impl_port_chg(ibt_async_event_t *event)
2312 2311 {
2313 2312 saa_port_t *saa_portp = NULL;
2314 2313 boolean_t is_ready = B_FALSE;
2315 2314 ibt_hca_portinfo_t *ibt_portinfop;
2316 2315 uint_t nports, size;
2317 2316 ibt_status_t ibt_status;
2318 2317 ib_guid_t ci_guid;
2319 2318 int port_num;
2320 2319
2321 2320 ci_guid = event->ev_hca_guid;
2322 2321 port_num = event->ev_port;
2323 2322
2324 2323 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_chg_start,
2325 2324 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg: Handling port chg"
2326 2325 " guid %016" PRIx64 " port %d\n",
2327 2326 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2328 2327
2329 2328 /* Get classportinfo of corresponding entry */
2330 2329 mutex_enter(&saa_statep->saa_port_list_mutex);
2331 2330
2332 2331 saa_portp = saa_statep->saa_port_list;
2333 2332 while (saa_portp != NULL) {
2334 2333 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
2335 2334 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
2336 2335 mutex_enter(&saa_portp->saa_pt_mutex);
2337 2336
2338 2337 is_ready = (saa_portp->saa_pt_state
2339 2338 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
2340 2339
2341 2340 /*
2342 2341 * increment reference count to account for cpi and
2343 2342 * informinfos. All 4 informinfo's sent are treated as
2344 2343 * one port client reference
2345 2344 */
2346 2345 if (is_ready)
2347 2346 saa_portp->saa_pt_reference_count ++;
2348 2347
2349 2348 mutex_exit(&saa_portp->saa_pt_mutex);
2350 2349
2351 2350 if (is_ready)
2352 2351 break; /* normally, only 1 port entry */
2353 2352 }
2354 2353 saa_portp = saa_portp->next;
2355 2354 }
2356 2355
2357 2356 mutex_exit(&saa_statep->saa_port_list_mutex);
2358 2357
2359 2358 if (saa_portp != NULL) {
2360 2359 /* first query the portinfo to see if the lid changed */
2361 2360 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
2362 2361 &ibt_portinfop, &nports, &size);
2363 2362
2364 2363 if (ibt_status != IBT_SUCCESS) {
2365 2364
2366 2365 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2367 2366 ibmf_saa_impl_port_chg_err, IBMF_TNF_ERROR, "",
2368 2367 "ibmf_saa_impl_port_chg: %s, ibmf_status ="
2369 2368 " %d\n", tnf_string, msg,
2370 2369 "ibt_query_hca_ports_byguid() failed",
2371 2370 tnf_int, ibt_status, ibt_status);
2372 2371
2373 2372 goto bail;
2374 2373 }
2375 2374
2376 2375 mutex_enter(&saa_portp->saa_pt_mutex);
2377 2376 if (event->ev_port_flags & IBT_PORT_CHANGE_SM_LID) {
2378 2377 /* update the Master SM Lid value in ibmf_saa */
2379 2378 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2380 2379 ibt_portinfop->p_sm_lid;
2381 2380 }
2382 2381 if (event->ev_port_flags & IBT_PORT_CHANGE_SM_SL) {
2383 2382 /* update the Master SM SL value in ibmf_saa */
2384 2383 saa_portp->saa_pt_ibmf_addr_info.ia_service_level =
2385 2384 ibt_portinfop->p_sm_sl;
2386 2385 }
2387 2386 if (event->ev_port_flags & IBT_PORT_CHANGE_SUB_TIMEOUT) {
2388 2387 /* update the Subnet timeout value in ibmf_saa */
2389 2388 saa_portp->saa_pt_timeout =
2390 2389 ibt_portinfop->p_subnet_timeout;
2391 2390 }
2392 2391 mutex_exit(&saa_portp->saa_pt_mutex);
2393 2392
2394 2393 ibt_free_portinfo(ibt_portinfop, size);
2395 2394
2396 2395 /* get the classportinfo again */
2397 2396 ibmf_saa_impl_get_classportinfo(saa_portp);
2398 2397 }
2399 2398 bail:
2400 2399
2401 2400 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_chg_end,
2402 2401 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg() exit\n");
2403 2402 }
2404 2403 /*
2405 2404 * ibmf_saa_impl_client_rereg:
2406 2405 */
2407 2406 static void
2408 2407 ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num)
2409 2408 {
2410 2409 saa_port_t *saa_portp = NULL;
2411 2410 boolean_t is_ready = B_FALSE;
2412 2411 ibt_hca_portinfo_t *ibt_portinfop;
2413 2412 ib_lid_t master_sm_lid;
2414 2413 uint_t nports, size;
2415 2414 ibt_status_t ibt_status;
2416 2415 boolean_t event_subs = B_FALSE;
2417 2416
2418 2417 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_rereg_start,
2419 2418 IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg: Handling clnt "
2420 2419 "rereg guid %016" PRIx64 " port %d\n",
2421 2420 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2422 2421
2423 2422 /* Get classportinfo of corresponding entry */
2424 2423 mutex_enter(&saa_statep->saa_port_list_mutex);
2425 2424
2426 2425 saa_portp = saa_statep->saa_port_list;
2427 2426 while (saa_portp != NULL) {
2428 2427
2429 2428 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
2430 2429 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
2431 2430
2432 2431 mutex_enter(&saa_portp->saa_pt_mutex);
2433 2432
2434 2433 is_ready = (saa_portp->saa_pt_state
2435 2434 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
2436 2435
2437 2436 /*
2438 2437 * increment reference count to account for cpi and
2439 2438 * informinfos. All 4 informinfo's sent are treated as
2440 2439 * one port client reference
2441 2440 */
2442 2441 if (is_ready)
2443 2442 saa_portp->saa_pt_reference_count += 2;
2444 2443
2445 2444 mutex_exit(&saa_portp->saa_pt_mutex);
2446 2445
2447 2446 if (is_ready)
2448 2447 break; /* normally, only 1 port entry */
2449 2448 }
2450 2449 saa_portp = saa_portp->next;
2451 2450 }
2452 2451
2453 2452 mutex_exit(&saa_statep->saa_port_list_mutex);
2454 2453
2455 2454 if (saa_portp != NULL && is_ready == B_TRUE) {
2456 2455
2457 2456 /* verify whether master sm lid changed */
2458 2457
2459 2458 /* first query the portinfo to see if the lid changed */
2460 2459 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
2461 2460 &ibt_portinfop, &nports, &size);
2462 2461
2463 2462 if (ibt_status != IBT_SUCCESS) {
2464 2463
2465 2464 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2466 2465 ibmf_saa_impl_port_rereg_err, IBMF_TNF_ERROR, "",
2467 2466 "ibmf_saa_impl_client_rereg: %s, ibmf_status ="
2468 2467 " %d\n", tnf_string, msg,
2469 2468 "ibt_query_hca_ports_byguid() failed",
2470 2469 tnf_int, ibt_status, ibt_status);
2471 2470
2472 2471 goto bail;
2473 2472 }
2474 2473
2475 2474 master_sm_lid = ibt_portinfop->p_sm_lid;
2476 2475
2477 2476 ibt_free_portinfo(ibt_portinfop, size);
2478 2477
2479 2478 /* check whether we need to subscribe for events */
2480 2479 mutex_enter(&saa_portp->saa_pt_event_sub_mutex);
2481 2480
2482 2481 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ?
2483 2482 B_TRUE : B_FALSE;
2484 2483
2485 2484 mutex_exit(&saa_portp->saa_pt_event_sub_mutex);
2486 2485
2487 2486 /* update the master smlid */
2488 2487 mutex_enter(&saa_portp->saa_pt_mutex);
2489 2488
2490 2489 /* update the master sm lid value in ibmf_saa */
2491 2490 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2492 2491 master_sm_lid;
2493 2492
2494 2493 /* if we're not subscribed for events, dec reference count */
2495 2494 if (event_subs == B_FALSE)
2496 2495 saa_portp->saa_pt_reference_count--;
2497 2496
2498 2497 mutex_exit(&saa_portp->saa_pt_mutex);
2499 2498
2500 2499 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
2501 2500 ibmf_saa_impl_port_rereg, IBMF_TNF_TRACE, "",
2502 2501 "ibmf_saa_impl_client_rereg: %s, master_sm_lid = 0x%x\n",
2503 2502 tnf_string, msg,
2504 2503 "port is up. Sending classportinfo request",
2505 2504 tnf_opaque, master_sm_lid, master_sm_lid);
2506 2505
2507 2506 /* get the classportinfo again */
2508 2507 ibmf_saa_impl_get_classportinfo(saa_portp);
2509 2508
2510 2509 /*
2511 2510 * resubscribe to events if there are subscribers since SA may
2512 2511 * have removed our subscription records when the port went down
2513 2512 */
2514 2513 if (event_subs == B_TRUE)
2515 2514 ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE);
2516 2515 }
2517 2516
2518 2517 bail:
2519 2518
2520 2519 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_rereg_end,
2521 2520 IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg() exit\n");
2522 2521 }
2523 2522 /*
2524 2523 * ibmf_saa_impl_port_up:
2525 2524 */
2526 2525 static void
2527 2526 ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num)
2528 2527 {
2529 2528 saa_port_t *saa_portp = NULL;
2530 2529 int is_ready;
2531 2530 ibt_hca_portinfo_t *ibt_portinfop;
2532 2531 ib_lid_t master_sm_lid;
2533 2532 uint_t nports, size;
2534 2533 ibt_status_t ibt_status;
2535 2534 boolean_t event_subs = B_FALSE;
2536 2535
2537 2536 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_up_start,
2538 2537 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up: Handling port up"
2539 2538 " guid %016" PRIx64 " port %d\n",
2540 2539 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2541 2540
2542 2541 /* Get classportinfo of corresponding entry */
2543 2542 mutex_enter(&saa_statep->saa_port_list_mutex);
2544 2543
2545 2544 saa_portp = saa_statep->saa_port_list;
2546 2545 while (saa_portp != NULL) {
2547 2546
2548 2547 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
2549 2548 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
2550 2549
2551 2550 mutex_enter(&saa_portp->saa_pt_mutex);
2552 2551
2553 2552 is_ready = (saa_portp->saa_pt_state
2554 2553 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
2555 2554
2556 2555 /*
2557 2556 * increment reference count to account for cpi and
2558 2557 * informinfos. All 4 informinfo's sent are treated as
2559 2558 * one port client reference
2560 2559 */
2561 2560 if (is_ready == B_TRUE)
2562 2561 saa_portp->saa_pt_reference_count += 2;
2563 2562
2564 2563 mutex_exit(&saa_portp->saa_pt_mutex);
2565 2564
2566 2565 if (is_ready == B_TRUE)
2567 2566 break; /* normally, only 1 port entry */
2568 2567 }
2569 2568 saa_portp = saa_portp->next;
2570 2569 }
2571 2570
2572 2571 mutex_exit(&saa_statep->saa_port_list_mutex);
2573 2572
2574 2573 if (saa_portp != NULL && is_ready == B_TRUE) {
2575 2574
2576 2575 /* verify whether master sm lid changed */
2577 2576
2578 2577 /* first query the portinfo to see if the lid changed */
2579 2578 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
2580 2579 &ibt_portinfop, &nports, &size);
2581 2580
2582 2581 if (ibt_status != IBT_SUCCESS) {
2583 2582
2584 2583 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2585 2584 ibmf_saa_impl_port_up_err, IBMF_TNF_ERROR, "",
2586 2585 "ibmf_saa_impl_port_up: %s, ibmf_status ="
2587 2586 " %d\n", tnf_string, msg,
2588 2587 "ibt_query_hca_ports_byguid() failed",
2589 2588 tnf_int, ibt_status, ibt_status);
2590 2589
2591 2590 goto bail;
2592 2591 }
2593 2592
2594 2593 master_sm_lid = ibt_portinfop->p_sm_lid;
2595 2594
2596 2595 ibt_free_portinfo(ibt_portinfop, size);
2597 2596
2598 2597 /* check whether we need to subscribe for events */
2599 2598 mutex_enter(&saa_portp->saa_pt_event_sub_mutex);
2600 2599
2601 2600 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ?
2602 2601 B_TRUE : B_FALSE;
2603 2602
2604 2603 mutex_exit(&saa_portp->saa_pt_event_sub_mutex);
2605 2604
2606 2605 /* update the master smlid */
2607 2606 mutex_enter(&saa_portp->saa_pt_mutex);
2608 2607
2609 2608 /* update the master sm lid value in ibmf_saa */
2610 2609 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2611 2610 master_sm_lid;
2612 2611
2613 2612 /* if we're not subscribed for events, dec reference count */
2614 2613 if (event_subs == B_FALSE)
2615 2614 saa_portp->saa_pt_reference_count--;
2616 2615
2617 2616 mutex_exit(&saa_portp->saa_pt_mutex);
2618 2617
2619 2618 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
2620 2619 ibmf_saa_impl_port_up, IBMF_TNF_TRACE, "",
2621 2620 "ibmf_saa_impl_port_up: %s, master_sm_lid = 0x%x\n",
2622 2621 tnf_string, msg,
2623 2622 "port is up. Sending classportinfo request",
2624 2623 tnf_opaque, master_sm_lid, master_sm_lid);
2625 2624
2626 2625 /* get the classportinfo again */
2627 2626 ibmf_saa_impl_get_classportinfo(saa_portp);
2628 2627
2629 2628 /*
2630 2629 * resubscribe to events if there are subscribers since SA may
2631 2630 * have removed our subscription records when the port went down
2632 2631 */
2633 2632 if (event_subs == B_TRUE)
2634 2633 ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE);
2635 2634 }
2636 2635
2637 2636 bail:
2638 2637
2639 2638 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_up_end,
2640 2639 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up() exit\n");
2641 2640 }
2642 2641
2643 2642 /*
2644 2643 * ibmf_saa_impl_port_down:
2645 2644 */
2646 2645 static void
2647 2646 ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num)
2648 2647 {
2649 2648
2650 2649 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_down_start,
2651 2650 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down: Handling port down"
2652 2651 " guid %016" PRIx64 " port %d\n",
2653 2652 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2654 2653
2655 2654
2656 2655 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_down_end,
2657 2656 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down() exit\n");
2658 2657 }
2659 2658
2660 2659 /*
2661 2660 * ibmf_saa_impl_hca_detach:
2662 2661 * find entry, unregister if there are no clients
2663 2662 * have to unregister since ibmf needs to close the hca and will only do this if
2664 2663 * no clients are registered
2665 2664 */
2666 2665 static void
2667 2666 ibmf_saa_impl_hca_detach(saa_port_t *saa_removed)
2668 2667 {
2669 2668 saa_port_t *saa_portp;
2670 2669 boolean_t must_unreg, must_unsub;
2671 2670
2672 2671 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_start,
2673 2672 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: Detaching"
2674 2673 " entry %016" PRIx64 "\n", tnf_opaque, entry, saa_removed);
2675 2674
2676 2675 /* find this entry */
2677 2676 mutex_enter(&saa_statep->saa_port_list_mutex);
2678 2677
2679 2678 saa_portp = saa_statep->saa_port_list;
2680 2679 while (saa_portp != NULL) {
2681 2680
2682 2681 if (saa_portp == saa_removed)
2683 2682 break;
2684 2683
2685 2684 saa_portp = saa_portp->next;
2686 2685 }
2687 2686 mutex_exit(&saa_statep->saa_port_list_mutex);
2688 2687
2689 2688 ASSERT(saa_portp != NULL);
2690 2689
2691 2690 if (saa_portp == NULL) {
2692 2691
2693 2692 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
2694 2693 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "",
2695 2694 "ibmf_saa_impl_hca_detach: %s, entry %016"
2696 2695 PRIx64 "\n",
2697 2696 tnf_string, msg,
2698 2697 "Port entry NOT found",
2699 2698 tnf_opaque, entryp, saa_removed);
2700 2699
2701 2700 goto bail;
2702 2701 }
2703 2702
2704 2703 /* if there are clients expecting Reports(), unsusbscribe */
2705 2704 mutex_enter(&saa_portp->saa_pt_event_sub_mutex);
2706 2705
2707 2706 must_unsub = (saa_portp->saa_pt_event_sub_client_list != NULL) ?
2708 2707 B_TRUE : B_FALSE;
2709 2708
2710 2709 mutex_exit(&saa_portp->saa_pt_event_sub_mutex);
2711 2710
2712 2711 /* fail if outstanding transactions */
2713 2712 mutex_enter(&saa_portp->saa_pt_mutex);
2714 2713
2715 2714 if (saa_portp->saa_pt_num_outstanding_trans > 0) {
2716 2715
2717 2716 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L1,
2718 2717 ibmf_saa_impl_fini_err, IBMF_TNF_TRACE, "",
2719 2718 "ibmf_saa_impl_fini: %s, port = %016" PRIx64
2720 2719 ", num transactions = %d\n",
2721 2720 tnf_string, msg, "Detaching HCA."
2722 2721 " Outstanding transactions on port.",
2723 2722 tnf_opaque, port,
2724 2723 saa_portp->saa_pt_port_guid,
2725 2724 tnf_uint, outstanding_transactions,
2726 2725 saa_portp->saa_pt_num_outstanding_trans);
2727 2726
2728 2727 mutex_exit(&saa_portp->saa_pt_mutex);
2729 2728
2730 2729 goto bail;
2731 2730 }
2732 2731
2733 2732
2734 2733 /*
2735 2734 * increment reference count by one to account for unsubscribe requests
2736 2735 * that are about to be sent. All four informinfo's are treated as one
2737 2736 * port client reference. The count will be decremented by
2738 2737 * subscribe_events() before the call returns.
2739 2738 */
2740 2739 if (must_unsub == B_TRUE)
2741 2740 saa_portp->saa_pt_reference_count++;
2742 2741
2743 2742 mutex_exit(&saa_portp->saa_pt_mutex);
2744 2743
2745 2744 /*
2746 2745 * try and unsubscribe from SA. Generate synchronous, unsequenced
2747 2746 * unsubscribe requests.
2748 2747 */
2749 2748 if (must_unsub == B_TRUE)
2750 2749 ibmf_saa_subscribe_events(saa_portp, B_FALSE, B_TRUE);
2751 2750
2752 2751 /* warning if registered clients */
2753 2752 mutex_enter(&saa_portp->saa_pt_mutex);
2754 2753
2755 2754 if (saa_portp->saa_pt_reference_count > 0) {
2756 2755
2757 2756 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
2758 2757 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "",
2759 2758 "ibmf_saa_impl_hca_detach: %s, port %016"
2760 2759 PRIx64 "\n",
2761 2760 tnf_string, msg,
2762 2761 "Detaching HCA for port with clients still"
2763 2762 " registered", tnf_opaque, port,
2764 2763 saa_portp->saa_pt_port_guid);
2765 2764 }
2766 2765
2767 2766 /* synchronize on end of registration */
2768 2767 while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) {
2769 2768
2770 2769 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1,
2771 2770 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "",
2772 2771 "ibmf_saa_impl_hca_detach: %s\n",
2773 2772 tnf_string, msg, "someone is registering. waiting"
2774 2773 " for them to finish");
2775 2774
2776 2775 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv,
2777 2776 &saa_portp->saa_pt_mutex);
2778 2777
2779 2778 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1,
2780 2779 ibmf_saa_impl_hca_detach,
2781 2780 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: %s\n",
2782 2781 tnf_string, msg, "done waiting");
2783 2782 }
2784 2783
2785 2784 /* unregister from ibmf */
2786 2785 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) {
2787 2786 must_unreg = B_TRUE;
2788 2787 } else
2789 2788 must_unreg = B_FALSE;
2790 2789
2791 2790 ibmf_saa_impl_invalidate_port(saa_portp);
2792 2791
2793 2792 mutex_exit(&saa_portp->saa_pt_mutex);
2794 2793
2795 2794 if (must_unreg == B_TRUE) {
2796 2795 if (ibmf_saa_impl_ibmf_unreg(saa_portp) != IBMF_SUCCESS) {
2797 2796 mutex_enter(&saa_portp->saa_pt_mutex);
2798 2797 mutex_enter(&saa_portp->saa_pt_kstat_mutex);
2799 2798 (void) ibmf_saa_impl_init_kstats(saa_portp);
2800 2799 mutex_exit(&saa_portp->saa_pt_kstat_mutex);
2801 2800 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_READY;
2802 2801 if (must_unsub == B_TRUE)
2803 2802 saa_portp->saa_pt_reference_count++;
2804 2803 mutex_exit(&saa_portp->saa_pt_mutex);
2805 2804
2806 2805 if (must_unsub == B_TRUE) {
2807 2806 ibmf_saa_subscribe_events(saa_portp, B_TRUE,
2808 2807 B_FALSE);
2809 2808 }
2810 2809 }
2811 2810 }
2812 2811 bail:
2813 2812 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_end,
2814 2813 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach() exit\n");
2815 2814 }
2816 2815
2817 2816 /* ARGSUSED */
2818 2817 void
2819 2818 ibmf_saa_async_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, void *args)
2820 2819 {
2821 2820 saa_impl_trans_info_t *trans_info;
2822 2821 int status;
2823 2822 size_t length;
2824 2823 void *result;
2825 2824 saa_port_t *saa_portp;
2826 2825 saa_client_data_t *client_data;
2827 2826 int ibmf_status;
2828 2827 boolean_t ignore_data;
2829 2828 ibmf_retrans_t ibmf_retrans;
2830 2829 boolean_t sa_is_redirected = B_FALSE;
2831 2830
2832 2831 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_start,
2833 2832 IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() enter\n");
2834 2833
2835 2834 trans_info = (saa_impl_trans_info_t *)args;
2836 2835
2837 2836 client_data = trans_info->si_trans_client_data;
2838 2837 saa_portp = trans_info->si_trans_port;
2839 2838
2840 2839 mutex_enter(&saa_portp->saa_pt_mutex);
2841 2840 sa_is_redirected = saa_portp->saa_pt_redirect_active;
2842 2841 mutex_exit(&saa_portp->saa_pt_mutex);
2843 2842
2844 2843 if ((msgp->im_msg_status == IBMF_TRANS_TIMEOUT) &&
2845 2844 (sa_is_redirected == B_TRUE)) {
2846 2845
2847 2846 /*
2848 2847 * We should retry the request using SM_LID and QP1 if we
2849 2848 * have been using redirect up until now
2850 2849 */
2851 2850 ibmf_status = ibmf_saa_impl_revert_to_qp1(
2852 2851 saa_portp, msgp, ibmf_saa_async_cb, args,
2853 2852 trans_info->si_trans_transport_flags);
2854 2853
2855 2854 /*
2856 2855 * If revert_to_qp1 returns success msg was resent.
2857 2856 * Otherwise msg could not be resent. Continue normally
2858 2857 */
2859 2858 if (ibmf_status == IBMF_SUCCESS)
2860 2859 goto bail;
2861 2860
2862 2861 } else if (msgp->im_msg_status == IBMF_TRANS_TIMEOUT) {
2863 2862
2864 2863
2865 2864 ibmf_status = ibmf_saa_impl_new_smlid_retry(saa_portp, msgp,
2866 2865 ibmf_saa_async_cb, args,
2867 2866 trans_info->si_trans_transport_flags);
2868 2867
2869 2868 /*
2870 2869 * if smlid_retry() returns success sm lid changed and msg
2871 2870 * was resent. Otherwise, lid did not change or msg could not
2872 2871 * be resent. Continue normally.
2873 2872 */
2874 2873 if (ibmf_status == IBMF_SUCCESS)
2875 2874 goto bail;
2876 2875
2877 2876 /*
2878 2877 * check whether we've received anything from the SA in a while.
2879 2878 * If we have, this function will retry and return success. If
2880 2879 * we haven't continue normally so that we return a timeout to
2881 2880 * the client
2882 2881 */
2883 2882 ibmf_status = ibmf_saa_check_sa_and_retry(
2884 2883 saa_portp, msgp, ibmf_saa_async_cb, args,
2885 2884 trans_info->si_trans_send_time,
2886 2885 trans_info->si_trans_transport_flags);
2887 2886
2888 2887 if (ibmf_status == IBMF_SUCCESS)
2889 2888 goto bail;
2890 2889 }
2891 2890
2892 2891 /*
2893 2892 * If SA returned success but mad status is busy, retry a few times.
2894 2893 * If SA returned success but mad status says redirect is required,
2895 2894 * update the address info and retry the request to the new SA address
2896 2895 */
2897 2896 if (msgp->im_msg_status == IBMF_SUCCESS) {
2898 2897
2899 2898 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL);
2900 2899
2901 2900 if ((b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status) ==
2902 2901 MAD_STATUS_BUSY) &&
2903 2902 (trans_info->si_trans_retry_busy_count <
2904 2903 IBMF_SAA_MAX_BUSY_RETRY_COUNT)) {
2905 2904
2906 2905 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
2907 2906 ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
2908 2907 "ibmf_saa_async_cb: %s, retry_count = %d\n",
2909 2908 tnf_string, msg,
2910 2909 "async response returned busy status",
2911 2910 tnf_int, retry_count,
2912 2911 trans_info->si_trans_retry_busy_count);
2913 2912
2914 2913 trans_info->si_trans_retry_busy_count++;
2915 2914
2916 2915 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2917 2916 sizeof (ibmf_retrans_t));
2918 2917
2919 2918 ibmf_status = ibmf_msg_transport(
2920 2919 saa_portp->saa_pt_ibmf_handle,
2921 2920 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2922 2921 ibmf_saa_async_cb, args,
2923 2922 trans_info->si_trans_transport_flags);
2924 2923
2925 2924 /*
2926 2925 * if retry is successful, quit here since async_cb will
2927 2926 * get called again; otherwise, let this function call
2928 2927 * handle the cleanup
2929 2928 */
2930 2929 if (ibmf_status == IBMF_SUCCESS)
2931 2930 goto bail;
2932 2931 } else if (b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status)
2933 2932 == MAD_STATUS_REDIRECT_REQUIRED) {
2934 2933
2935 2934 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L2,
2936 2935 ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
2937 2936 "ibmf_saa_async_cb: "
2938 2937 "async response returned redirect status\n");
2939 2938
2940 2939 /* update address info and copy it into msgp */
2941 2940 ibmf_saa_impl_update_sa_address_info(saa_portp, msgp);
2942 2941
2943 2942 /* retry with new address info */
2944 2943 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2945 2944 sizeof (ibmf_retrans_t));
2946 2945
2947 2946 ibmf_status = ibmf_msg_transport(
2948 2947 saa_portp->saa_pt_ibmf_handle,
2949 2948 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2950 2949 ibmf_saa_async_cb, args,
2951 2950 trans_info->si_trans_transport_flags);
2952 2951
2953 2952 /*
2954 2953 * if retry is successful, quit here since async_cb will
2955 2954 * get called again; otherwise, let this function call
2956 2955 * handle the cleanup
2957 2956 */
2958 2957 if (ibmf_status == IBMF_SUCCESS)
2959 2958 goto bail;
2960 2959 }
2961 2960 }
2962 2961
2963 2962 mutex_enter(&saa_portp->saa_pt_mutex);
2964 2963
2965 2964 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0);
2966 2965 saa_portp->saa_pt_num_outstanding_trans--;
2967 2966
2968 2967 mutex_exit(&saa_portp->saa_pt_mutex);
2969 2968
2970 2969 if ((trans_info->si_trans_callback == NULL) &&
2971 2970 (trans_info->si_trans_sub_callback == NULL))
2972 2971 ignore_data = B_TRUE;
2973 2972 else
2974 2973 ignore_data = B_FALSE;
2975 2974
2976 2975 ibmf_saa_impl_prepare_response(ibmf_handle, msgp, ignore_data, &status,
2977 2976 &result, &length, B_FALSE);
2978 2977
2979 2978 mutex_enter(&saa_portp->saa_pt_kstat_mutex);
2980 2979
2981 2980 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1);
2982 2981
2983 2982 if (status != IBMF_SUCCESS)
2984 2983 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1);
2985 2984
2986 2985 if (status == IBMF_TRANS_TIMEOUT)
2987 2986 IBMF_SAA_ADD32_KSTATS(saa_portp, requests_timedout, 1);
2988 2987
2989 2988 mutex_exit(&saa_portp->saa_pt_kstat_mutex);
2990 2989
2991 2990 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2992 2991 ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
2993 2992 "ibmf_saa_async_cb: %s\n", tnf_string, msg,
2994 2993 "Calling ibmf_saa client's callback");
2995 2994
2996 2995 /*
2997 2996 * there are three classes or trans_info users: ibmf_saa clients and
2998 2997 * classportinfo requests; informinfo subscribe requests, and report
2999 2998 * responses. For the first two, call the correct callback. For report
3000 2999 * responses there's no need to notify anyone.
3001 3000 */
3002 3001 if (trans_info->si_trans_callback != NULL) {
3003 3002 /* ibmf_saa client or classportinfo request */
3004 3003 trans_info->si_trans_callback(trans_info->si_trans_callback_arg,
3005 3004 length, result, status);
3006 3005 } else if (trans_info->si_trans_sub_callback != NULL) {
3007 3006 /* informinfo subscribe request */
3008 3007 trans_info->si_trans_sub_callback(
3009 3008 trans_info->si_trans_callback_arg, length, result, status,
3010 3009 trans_info->si_trans_sub_producer_type);
3011 3010 }
3012 3011
3013 3012 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3014 3013 ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
3015 3014 "ibmf_saa_async_cb: %s\n", tnf_string, msg,
3016 3015 "Returned from callback");
3017 3016
3018 3017 if (client_data != NULL) {
3019 3018 mutex_enter(&client_data->saa_client_mutex);
3020 3019
3021 3020 ASSERT(client_data->saa_client_num_pending_trans > 0);
3022 3021 client_data->saa_client_num_pending_trans--;
3023 3022
3024 3023 if ((client_data->saa_client_num_pending_trans == 0) &&
3025 3024 (client_data->saa_client_state == SAA_CLIENT_STATE_WAITING))
3026 3025 cv_signal(&client_data->saa_client_state_cv);
3027 3026
3028 3027 mutex_exit(&client_data->saa_client_mutex);
3029 3028 }
3030 3029
3031 3030 kmem_free(trans_info, sizeof (saa_impl_trans_info_t));
3032 3031
3033 3032 bail:
3034 3033
3035 3034 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_end,
3036 3035 IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() exit\n");
3037 3036 }
3038 3037
3039 3038 /*
3040 3039 * ibmf_saa_check_sa_and_retry:
3041 3040 *
3042 3041 * If a particular transaction times out, we don't want to give up if we know
3043 3042 * the SA is responding. Check the time since we last received a response. If
3044 3043 * it's less than ibmf_saa_max_wait_time retry the request.
3045 3044 *
3046 3045 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the
3047 3046 * same values passed to the original ibmf_msg_transport that timed out. The
3048 3047 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure.
3049 3048 *
3050 3049 * If max_wait_time seconds have passed, this function returns IBMF_TIMEOUT.
3051 3050 * That way, callers can simply return the result of this function.
3052 3051 *
3053 3052 * Input Arguments
3054 3053 * saa_portp pointer to saa_port structure
3055 3054 * msgp ibmf message that timedout
3056 3055 * ibmf_callback callback that should be called by msg_transport
3057 3056 * ibmf_callback_arg args for ibmf_callback
3058 3057 * transport_flags flags for ibmf_msg_transport
3059 3058 *
3060 3059 * Output Arguments
3061 3060 * none
3062 3061 *
3063 3062 * Returns
3064 3063 * IBMF_SUCCESS if we've recently received data from the SA and request was
3065 3064 * resent.
3066 3065 * IBMF_TRANS_TIMEOUT if no data has been received from the SA in max_wait_time
3067 3066 * same values as ibmf_msg_transport() if data has been received but request
3068 3067 * could not be resent.
3069 3068 */
3070 3069 static int
3071 3070 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
3072 3071 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg,
3073 3072 hrtime_t trans_send_time, int transport_flags)
3074 3073 {
3075 3074 hrtime_t curr_time, sa_uptime;
3076 3075 ibmf_retrans_t ibmf_retrans;
3077 3076 int ibmf_status;
3078 3077
3079 3078 do {
3080 3079
3081 3080 mutex_enter(&saa_portp->saa_pt_mutex);
3082 3081
3083 3082 sa_uptime = saa_portp->saa_pt_sa_uptime;
3084 3083
3085 3084 /* if nothing received from SA since we sent */
3086 3085 curr_time = gethrtime();
3087 3086
3088 3087 /*
3089 3088 * check if it's been a very long time since this
3090 3089 * particular transaction was sent
3091 3090 */
3092 3091 if (((curr_time - trans_send_time) / 1000000000) >
3093 3092 ibmf_saa_trans_wait_time) {
3094 3093
3095 3094 mutex_exit(&saa_portp->saa_pt_mutex);
3096 3095
3097 3096 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1,
3098 3097 ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "",
3099 3098 "ibmf_saa_check_sa_and_retry: %s, msgp = "
3100 3099 "%p sa_uptime = %" PRIu64 ", trans send time = %"
3101 3100 PRIu64 ", curr_time = %" PRIu64 "\n",
3102 3101 tnf_string, msg,
3103 3102 "Nothing received for this transaction",
3104 3103 tnf_opaque, msgp, msgp,
3105 3104 tnf_long, sa_uptime, sa_uptime,
3106 3105 tnf_long, trans_send_time, trans_send_time,
3107 3106 tnf_long, curr_time, curr_time);
3108 3107
3109 3108 ibmf_status = IBMF_TRANS_TIMEOUT;
3110 3109
3111 3110 break;
3112 3111 }
3113 3112
3114 3113 /*
3115 3114 * check time since we received something,
3116 3115 * and make sure that it hasn't been an extra long
3117 3116 * time for this particular transaction
3118 3117 */
3119 3118 if (((curr_time - sa_uptime) / 1000000000) <
3120 3119 ibmf_saa_max_wait_time) {
3121 3120
3122 3121 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L2,
3123 3122 ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "",
3124 3123 "ibmf_saa_check_sa_and_retry: %s, msgp = "
3125 3124 "%p sa_uptime = %" PRIu64 " trans_send_time = %"
3126 3125 PRIu64 " curr_time = %" PRIu64 "\n",
3127 3126 tnf_string, msg, "Something received. Retrying",
3128 3127 tnf_opaque, msgp, msgp,
3129 3128 tnf_long, sa_uptime, sa_uptime,
3130 3129 tnf_long, trans_send_time, trans_send_time,
3131 3130 tnf_long, curr_time, curr_time);
3132 3131
3133 3132 /*
3134 3133 * something received in WAIT_TIME_IN_SECS;
3135 3134 * resend request
3136 3135 */
3137 3136
3138 3137 /* new tid needed */
3139 3138 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID =
3140 3139 h2b64(saa_portp->saa_pt_current_tid++);
3141 3140
3142 3141 /*
3143 3142 * We are going to retry the access to the SM but
3144 3143 * Master SMLID could have changed due to a port change
3145 3144 * event. So update the remote_lid of the message with
3146 3145 * the SMLID from saa_portp for this port before the
3147 3146 * retry.
3148 3147 */
3149 3148 msgp->im_local_addr.ia_remote_lid =
3150 3149 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid;
3151 3150
3152 3151 bcopy(&saa_portp->saa_pt_ibmf_retrans,
3153 3152 &ibmf_retrans, sizeof (ibmf_retrans_t));
3154 3153
3155 3154 mutex_exit(&saa_portp->saa_pt_mutex);
3156 3155
3157 3156 ibmf_status = ibmf_msg_transport(
3158 3157 saa_portp->saa_pt_ibmf_handle,
3159 3158 saa_portp->saa_pt_qp_handle, msgp,
3160 3159 &ibmf_retrans, ibmf_callback, ibmf_callback_arg,
3161 3160 transport_flags);
3162 3161
3163 3162 if (ibmf_status == IBMF_SUCCESS)
3164 3163 goto bail;
3165 3164
3166 3165 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
3167 3166 ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "",
3168 3167 "ibmf_saa_check_sa_and_retry: %s, ibmf_status = "
3169 3168 "%d\n", tnf_string, msg,
3170 3169 "ibmf_msg_transport() failed",
3171 3170 tnf_int, ibmf_status, ibmf_status);
3172 3171 } else {
3173 3172
3174 3173 mutex_exit(&saa_portp->saa_pt_mutex);
3175 3174
3176 3175 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L1,
3177 3176 ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "",
3178 3177 "ibmf_saa_check_sa_and_retry: %s, msgp = "
3179 3178 "%p sa_uptime = %" PRIu64 " curr_time = %"
3180 3179 PRIu64 "\n", tnf_string, msg,
3181 3180 "Nothing received. Timing out",
3182 3181 tnf_opaque, msgp, msgp,
3183 3182 tnf_long, sa_uptime, sa_uptime,
3184 3183 tnf_long, curr_time, curr_time);
3185 3184
3186 3185 ibmf_status = IBMF_TRANS_TIMEOUT;
3187 3186
3188 3187 break;
3189 3188 }
3190 3189 } while (ibmf_status == IBMF_TRANS_TIMEOUT);
3191 3190
3192 3191 bail:
3193 3192 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3194 3193 ibmf_saa_check_sa_and_retry_end,
3195 3194 IBMF_TNF_TRACE, "", "ibmf_saa_check_sa_and_retry() exiting"
3196 3195 " ibmf_status = %d\n", tnf_int, result, ibmf_status);
3197 3196
3198 3197 return (ibmf_status);
3199 3198 }
3200 3199
3201 3200
3202 3201 /*
3203 3202 * ibmf_saa_impl_prepare_response:
3204 3203 */
3205 3204 static void
3206 3205 ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle,
3207 3206 ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result,
3208 3207 size_t *length, boolean_t sleep_flag)
3209 3208 {
3210 3209 ibmf_msg_bufs_t *resp_buf;
3211 3210 uint16_t attr_id;
3212 3211 uint8_t method;
3213 3212 boolean_t is_get_resp;
3214 3213 uint16_t mad_status;
3215 3214 uint16_t attr_offset;
3216 3215 ib_sa_hdr_t *sa_hdr;
3217 3216
3218 3217 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3219 3218 ibmf_saa_impl_prepare_response_start,
3220 3219 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() enter\n");
3221 3220
3222 3221 _NOTE(ASSUMING_PROTECTED(*msgp))
3223 3222
3224 3223 *result = NULL;
3225 3224 *length = 0;
3226 3225 sa_hdr = NULL;
3227 3226
3228 3227 resp_buf = &msgp->im_msgbufs_recv;
3229 3228
3230 3229 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*resp_buf))
3231 3230
3232 3231 if (msgp->im_msg_status != IBMF_SUCCESS) {
3233 3232
3234 3233 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
3235 3234 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3236 3235 "ibmf_saa_impl_prepare_response: %s, msg_status = %d\n",
3237 3236 tnf_string, msg, "Bad ibmf status",
3238 3237 tnf_int, msg_status, msgp->im_msg_status);
3239 3238
3240 3239 *status = msgp->im_msg_status;
3241 3240
3242 3241 goto exit;
3243 3242 }
3244 3243
3245 3244 if (resp_buf->im_bufs_mad_hdr == NULL) {
3246 3245
3247 3246 /*
3248 3247 * this was an unsequenced transaction (from an unsubscribe for
3249 3248 * following a CI_OFFLINE event)
3250 3249 */
3251 3250 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3252 3251 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3253 3252 "ibmf_saa_impl_prepare_response: %s\n",
3254 3253 tnf_string, msg, "Unsequenced transaction callback");
3255 3254
3256 3255 goto exit;
3257 3256 }
3258 3257
3259 3258 if ((mad_status = b2h16(resp_buf->im_bufs_mad_hdr->Status)) !=
3260 3259 MAD_STATUS_NO_INVALID_FIELDS) {
3261 3260
3262 3261 /* convert mad packet status to IBMF status */
3263 3262 switch (mad_status) {
3264 3263
3265 3264 case SA_STATUS_ERR_NO_RESOURCES:
3266 3265 *status = IBMF_NO_RESOURCES;
3267 3266 break;
3268 3267 case SA_STATUS_ERR_REQ_INVALID:
3269 3268 *status = IBMF_REQ_INVALID;
3270 3269 break;
3271 3270 case SA_STATUS_ERR_NO_RECORDS:
3272 3271 *status = IBMF_NO_RECORDS;
3273 3272 break;
3274 3273 case SA_STATUS_ERR_TOO_MANY_RECORDS:
3275 3274 *status = IBMF_TOO_MANY_RECORDS;
3276 3275 break;
3277 3276 case SA_STATUS_ERR_REQ_INVALID_GID:
3278 3277 *status = IBMF_INVALID_GID;
3279 3278 break;
3280 3279 case SA_STATUS_ERR_REQ_INSUFFICIENT_COMPONENTS:
3281 3280 *status = IBMF_INSUFF_COMPS;
3282 3281 break;
3283 3282 case MAD_STATUS_UNSUPP_METHOD:
3284 3283 *status = IBMF_UNSUPP_METHOD;
3285 3284 break;
3286 3285 case MAD_STATUS_UNSUPP_METHOD_ATTR:
3287 3286 *status = IBMF_UNSUPP_METHOD_ATTR;
3288 3287 break;
3289 3288 case MAD_STATUS_INVALID_FIELD:
3290 3289 *status = IBMF_INVALID_FIELD;
3291 3290 break;
3292 3291 default:
3293 3292 *status = IBMF_REQ_INVALID;
3294 3293 break;
3295 3294 }
3296 3295
3297 3296 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
3298 3297 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3299 3298 "ibmf_saa_impl_prepare_response: %s, mad_status = %x\n",
3300 3299 tnf_string, msg, "Bad MAD status",
3301 3300 tnf_int, mad_status, mad_status);
3302 3301
3303 3302 goto exit;
3304 3303 }
3305 3304
3306 3305 attr_id = b2h16(resp_buf->im_bufs_mad_hdr->AttributeID);
3307 3306 method = resp_buf->im_bufs_mad_hdr->R_Method;
3308 3307
3309 3308 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
3310 3309 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3311 3310 "ibmf_saa_impl_prepare_response: attr_id = 0x%x, method = "
3312 3311 "0x%x\n",
3313 3312 tnf_opaque, attr_id, attr_id,
3314 3313 tnf_opaque, method, method);
3315 3314
3316 3315 /*
3317 3316 * ignore any data from deleteresp since there's no way to know whether
3318 3317 * real data was returned; also ignore data if this was a Report
3319 3318 * response
3320 3319 */
3321 3320 if (method == SA_SUBN_ADM_DELETE_RESP) {
3322 3321
3323 3322 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3324 3323 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3325 3324 "impf_saa_impl_prepare_response: %s\n",
3326 3325 tnf_string, msg,
3327 3326 "DeleteResp or NoticeResp returned. "
3328 3327 "Ignoring response data");
3329 3328
3330 3329 *status = IBMF_SUCCESS;
3331 3330
3332 3331 *length = 0;
3333 3332 *result = NULL;
3334 3333
3335 3334 goto exit;
3336 3335 }
3337 3336
3338 3337 if (attr_id == SA_MULTIPATHRECORD_ATTRID) {
3339 3338
3340 3339 /*
3341 3340 * getmulti is only for requests; attribute should not
3342 3341 * be returned from SA
3343 3342 */
3344 3343 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
3345 3344 ibmf_saa_impl_prepare_response_err, IBMF_TNF_ERROR,
3346 3345 "", "ibmf_saa_impl_prepare_response: %s\n",
3347 3346 tnf_string, msg, "SA returned getmulti record");
3348 3347
3349 3348 *status = IBMF_REQ_INVALID;
3350 3349
3351 3350 goto exit;
3352 3351 }
3353 3352
3354 3353 /* if we are supposed to ignore data, stop here */
3355 3354 if (ignore_data == B_TRUE) {
3356 3355
3357 3356 *status = IBMF_SUCCESS;
3358 3357
3359 3358 goto exit;
3360 3359 }
3361 3360
3362 3361 is_get_resp = resp_buf->im_bufs_mad_hdr->R_Method ==
3363 3362 SA_SUBN_ADM_GET_RESP ? B_TRUE: B_FALSE;
3364 3363
3365 3364 /* unpack the sa header to get the attribute offset */
3366 3365 *status = ibmf_saa_utils_unpack_sa_hdr(resp_buf->im_bufs_cl_hdr,
3367 3366 resp_buf->im_bufs_cl_hdr_len, &sa_hdr, sleep_flag);
3368 3367 if (*status != IBMF_SUCCESS) {
3369 3368
3370 3369 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3371 3370 ibmf_saa_impl_prepare_response_err,
3372 3371 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s,"
3373 3372 " ibmf_status = %d\n", tnf_string, msg,
3374 3373 "Could not unpack sa hdr", tnf_int, ibmf_status, *status);
3375 3374
3376 3375 goto exit;
3377 3376 }
3378 3377
3379 3378 attr_offset = sa_hdr->AttributeOffset;
3380 3379
3381 3380 /*
3382 3381 * unpack data payload; if unpack function doesn't return success
3383 3382 * (because it could not allocate memory) forward this status to waiting
3384 3383 * client
3385 3384 */
3386 3385 *status = ibmf_saa_utils_unpack_payload(resp_buf->im_bufs_cl_data,
3387 3386 resp_buf->im_bufs_cl_data_len, attr_id, result, length,
3388 3387 attr_offset, is_get_resp, sleep_flag);
3389 3388 if (*status == IBMF_SUCCESS) {
3390 3389
3391 3390 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
3392 3391 ibmf_saa_impl_prepare_response,
3393 3392 IBMF_TNF_TRACE, "",
3394 3393 "ibmf_saa_impl_prepare_response: attr_id = "
3395 3394 "0x%x, attr_offset = %d, packed_payload_len = %d, "
3396 3395 "unpacked_payload_len = %d\n",
3397 3396 tnf_opaque, attr_id, attr_id,
3398 3397 tnf_opaque, attr_offset, attr_offset,
3399 3398 tnf_opaque, packed_payload_len,
3400 3399 resp_buf->im_bufs_cl_data_len,
3401 3400 tnf_opaque, unpacked_payload_len, *length);
3402 3401 } else {
3403 3402
3404 3403 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1,
3405 3404 ibmf_saa_impl_prepare_response_err,
3406 3405 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s,"
3407 3406 "attr_id = 0x%x, attr_offset = %d, packed_payload_len = %d,"
3408 3407 "status = %d\n",
3409 3408 tnf_string, msg, "Could not unpack payload",
3410 3409 tnf_opaque, attr_id, attr_id,
3411 3410 tnf_int, attr_offset, attr_offset,
3412 3411 tnf_int, packed_payload_len,
3413 3412 resp_buf->im_bufs_cl_data_len,
3414 3413 tnf_int, status, *status);
3415 3414 }
3416 3415 exit:
3417 3416 if (sa_hdr != NULL)
3418 3417 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3419 3418
3420 3419 ibmf_saa_impl_free_msg(ibmf_handle, msgp);
3421 3420
3422 3421 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3423 3422 ibmf_saa_impl_prepare_response_end,
3424 3423 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() exit,"
3425 3424 " status = 0x%d\n", tnf_int, status, *status);
3426 3425 }
3427 3426
3428 3427
3429 3428 /*
3430 3429 * ibmf_saa_impl_check_sa_support:
3431 3430 * Checks the capability mask (returned from the SA classportinfo response) to
3432 3431 * determine whether the sa supports the specified attribute ID.
3433 3432 *
3434 3433 * Input Arguments
3435 3434 * cap_mask 16-bit capability mask returned in SA's classportinfo
3436 3435 * attr_id attribute ID of current request
3437 3436 *
3438 3437 * Returns
3439 3438 * IBMF_NOT_SUPPORTED if capability mask indicates SA does not support attribute
3440 3439 * IBMF_SUCCESS otherwise
3441 3440 */
3442 3441 static int
3443 3442 ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id)
3444 3443 {
3445 3444 boolean_t attr_supported = B_TRUE;
3446 3445
3447 3446 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
3448 3447 ibmf_saa_impl_check_sa_support, IBMF_TNF_TRACE, "",
3449 3448 "ibmf_saa_impl_check_sa_support: cap_mask = 0x%x, "
3450 3449 "attr_id = 0x%x\n", tnf_opaque, cap_mask, cap_mask,
3451 3450 tnf_opaque, attr_id, attr_id);
3452 3451
3453 3452 switch (attr_id) {
3454 3453
3455 3454 case SA_SWITCHINFORECORD_ATTRID:
3456 3455 case SA_LINEARFDBRECORD_ATTRID:
3457 3456 case SA_RANDOMFDBRECORD_ATTRID:
3458 3457 case SA_MULTICASTFDBRECORD_ATTRID:
3459 3458 case SA_SMINFORECORD_ATTRID:
3460 3459 case SA_INFORMINFORECORD_ATTRID:
3461 3460 case SA_LINKRECORD_ATTRID:
3462 3461 case SA_GUIDINFORECORD_ATTRID:
3463 3462 case SA_TRACERECORD_ATTRID:
3464 3463 case SA_SERVICEASSNRECORD_ATTRID:
3465 3464
3466 3465 if ((cap_mask &
3467 3466 SA_CAPMASK_OPT_RECORDS_SUPPORTED) == 0) {
3468 3467
3469 3468 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1,
3470 3469 ibmf_saa_impl_check_sa_support,
3471 3470 IBMF_TNF_ERROR, "",
3472 3471 "ibmf_saa_impl_check_sa_support: %s, "
3473 3472 "cap_mask = 0x%x\n", tnf_string, msg,
3474 3473 "SA does not support optional records",
3475 3474 tnf_opaque, cap_mask, cap_mask,
3476 3475 tnf_opaque, attr_id, attr_id);
3477 3476
3478 3477 attr_supported = B_FALSE;
3479 3478 }
3480 3479 break;
3481 3480
3482 3481 case SA_MULTIPATHRECORD_ATTRID:
3483 3482
3484 3483 if ((cap_mask & SA_CAPMASK_MULTIPATH_SUPPORTED) == 0) {
3485 3484
3486 3485 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
3487 3486 ibmf_saa_impl_check_sa_support,
3488 3487 IBMF_TNF_ERROR, "",
3489 3488 "ibmf_saa_impl_check_sa_support: %s, "
3490 3489 "cap_mask = 0x%x\n", tnf_string, msg,
3491 3490 "SA does not support multipath records",
3492 3491 tnf_opaque, cap_mask, cap_mask);
3493 3492
3494 3493 attr_supported = B_FALSE;
3495 3494 }
3496 3495 break;
3497 3496
3498 3497 case SA_MCMEMBERRECORD_ATTRID:
3499 3498
3500 3499 if ((cap_mask & SA_CAPMASK_UD_MCAST_SUPPORTED) == 0) {
3501 3500
3502 3501 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
3503 3502 ibmf_saa_impl_check_sa_support,
3504 3503 IBMF_TNF_ERROR, "",
3505 3504 "ibmf_saa_impl_check_sa_support: %s, "
3506 3505 "cap_mask = 0x%x\n", tnf_string, msg,
3507 3506 "SA does not support ud multicast",
3508 3507 tnf_opaque, cap_mask, cap_mask);
3509 3508
3510 3509 attr_supported = B_FALSE;
3511 3510 }
3512 3511 break;
3513 3512
3514 3513 default:
3515 3514 break;
3516 3515 } /* switch */
3517 3516
3518 3517 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3519 3518 ibmf_saa_impl_check_sa_support_end, IBMF_TNF_TRACE, "",
3520 3519 "ibmf_saa_impl_check_sa_support() exiting, attr_supported = %d\n",
3521 3520 tnf_opaque, attr_supported, attr_supported);
3522 3521
3523 3522 if (attr_supported == B_FALSE)
3524 3523 return (IBMF_UNSUPP_METHOD_ATTR);
3525 3524 else
3526 3525 return (IBMF_SUCCESS);
3527 3526 }
3528 3527
3529 3528 /*
3530 3529 * ibmf_saa_impl_get_attr_id_length:
3531 3530 *
3532 3531 * Returns the host size of the specified sa record. Returns 0 for unknown
3533 3532 * attributes. multipath record size is a dynamic value given as a parameter
3534 3533 * specified with the ibmf_sa_access() call.
3535 3534 */
3536 3535 static uint_t
3537 3536 ibmf_saa_impl_get_attr_id_length(uint16_t attr_id)
3538 3537 {
3539 3538 uint_t attr_length;
3540 3539
3541 3540 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3542 3541 ibmf_saa_impl_get_attr_id_length_start,
3543 3542 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length() enter\n");
3544 3543
3545 3544 /* this function should not be used for multipath record */
3546 3545 ASSERT(attr_id != SA_MULTIPATHRECORD_ATTRID);
3547 3546
3548 3547 switch (attr_id) {
3549 3548 case SA_CLASSPORTINFO_ATTRID:
3550 3549 attr_length = sizeof (ib_mad_classportinfo_t);
3551 3550 break;
3552 3551 case SA_NOTICE_ATTRID:
3553 3552 attr_length = sizeof (ib_mad_notice_t);
3554 3553 break;
3555 3554 case SA_INFORMINFO_ATTRID:
3556 3555 attr_length = sizeof (ib_mad_informinfo_t);
3557 3556 break;
3558 3557 case SA_NODERECORD_ATTRID:
3559 3558 attr_length = sizeof (sa_node_record_t);
3560 3559 break;
3561 3560 case SA_PORTINFORECORD_ATTRID:
3562 3561 attr_length = sizeof (sa_portinfo_record_t);
3563 3562 break;
3564 3563 case SA_SLTOVLRECORD_ATTRID:
3565 3564 attr_length = sizeof (sa_SLtoVLmapping_record_t);
3566 3565 break;
3567 3566 case SA_SWITCHINFORECORD_ATTRID:
3568 3567 attr_length = sizeof (sa_switchinfo_record_t);
3569 3568 break;
3570 3569 case SA_LINEARFDBRECORD_ATTRID:
3571 3570 attr_length = sizeof (sa_linearft_record_t);
3572 3571 break;
3573 3572 case SA_RANDOMFDBRECORD_ATTRID:
3574 3573 attr_length = sizeof (sa_randomft_record_t);
3575 3574 break;
3576 3575 case SA_MULTICASTFDBRECORD_ATTRID:
3577 3576 attr_length = sizeof (sa_multicastft_record_t);
3578 3577 break;
3579 3578 case SA_SMINFORECORD_ATTRID:
3580 3579 attr_length = sizeof (sa_sminfo_record_t);
3581 3580 break;
3582 3581 case SA_INFORMINFORECORD_ATTRID:
3583 3582 attr_length = sizeof (sa_informinfo_record_t);
3584 3583 break;
3585 3584 case SA_LINKRECORD_ATTRID:
3586 3585 attr_length = sizeof (sa_link_record_t);
3587 3586 break;
3588 3587 case SA_GUIDINFORECORD_ATTRID:
3589 3588 attr_length = sizeof (sa_guidinfo_record_t);
3590 3589 break;
3591 3590 case SA_SERVICERECORD_ATTRID:
3592 3591 attr_length = sizeof (sa_service_record_t);
3593 3592 break;
3594 3593 case SA_PARTITIONRECORD_ATTRID:
3595 3594 attr_length = sizeof (sa_pkey_table_record_t);
3596 3595 break;
3597 3596 case SA_PATHRECORD_ATTRID:
3598 3597 attr_length = sizeof (sa_path_record_t);
3599 3598 break;
3600 3599 case SA_VLARBRECORD_ATTRID:
3601 3600 attr_length = sizeof (sa_VLarb_table_record_t);
3602 3601 break;
3603 3602 case SA_MCMEMBERRECORD_ATTRID:
3604 3603 attr_length = sizeof (sa_mcmember_record_t);
3605 3604 break;
3606 3605 case SA_TRACERECORD_ATTRID:
3607 3606 attr_length = sizeof (sa_trace_record_t);
3608 3607 break;
3609 3608 case SA_SERVICEASSNRECORD_ATTRID:
3610 3609 attr_length = sizeof (sa_service_assn_record_t);
3611 3610 break;
3612 3611 default:
3613 3612 /* should only get the above type of packets */
3614 3613 attr_length = 0;
3615 3614 break;
3616 3615 }
3617 3616
3618 3617 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
3619 3618 ibmf_saa_impl_get_attr_id_length_end,
3620 3619 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length():"
3621 3620 " attr_id: 0x%x size %d\n",
3622 3621 tnf_opaque, attr_id, attr_id, tnf_uint, attr_length, attr_length);
3623 3622
3624 3623 return (attr_length);
3625 3624 }
3626 3625
3627 3626 /*
3628 3627 * ibmf_saa_impl_free_msg:
3629 3628 * Takes a completed message and free memory associated with the message,
3630 3629 * including the individual fields of the im_msgbufs_send.
3631 3630 * ibmf_free_msg, called at the end of this function, takes a pointer to the
3632 3631 * message pointer so that it can set the message pointer to NULL. This
3633 3632 * function takes just the message pointer so the msgp will not be NULL after
3634 3633 * this function returns.
3635 3634 *
3636 3635 * Input Arguments
3637 3636 * ibmf_hdl ibmf handle used in ibmf_msg_alloc
3638 3637 * msgp pointer to ibmf_msg_t to free
3639 3638 *
3640 3639 * Returns
3641 3640 * void
3642 3641 */
3643 3642 static void
3644 3643 ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp)
3645 3644 {
3646 3645 int res;
3647 3646
3648 3647 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3649 3648 ibmf_saa_impl_free_msg_start,
3650 3649 IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() enter: msg %p\n",
3651 3650 tnf_opaque, msg, msgp);
3652 3651
3653 3652 ASSERT(msgp != NULL);
3654 3653
3655 3654 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr,
3656 3655 sizeof (ib_mad_hdr_t));
3657 3656
3658 3657 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_hdr,
3659 3658 msgp->im_msgbufs_send.im_bufs_cl_hdr_len);
3660 3659
3661 3660 if (msgp->im_msgbufs_send.im_bufs_cl_data_len > 0)
3662 3661 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_data,
3663 3662 msgp->im_msgbufs_send.im_bufs_cl_data_len);
3664 3663
3665 3664 res = ibmf_free_msg(ibmf_hdl, &msgp);
3666 3665 ASSERT(res == IBMF_SUCCESS);
3667 3666
3668 3667 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3669 3668 ibmf_saa_impl_free_msg_end,
3670 3669 IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() exit\n");
3671 3670 }
3672 3671
3673 3672 /*
3674 3673 * ibmf_saa_impl_get_port_guid:
3675 3674 */
3676 3675 static int
3677 3676 ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop,
3678 3677 ib_guid_t *guid_ret)
3679 3678 {
3680 3679 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3681 3680 ibmf_saa_impl_get_port_guid_start,
3682 3681 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_port_guid() enter\n");
3683 3682
3684 3683 if (ibt_portinfop->p_linkstate != IBT_PORT_ACTIVE) {
3685 3684
3686 3685 return (IBMF_BAD_PORT_STATE);
3687 3686 }
3688 3687
3689 3688 if (ibt_portinfop->p_sgid_tbl_sz == 0) {
3690 3689
3691 3690 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L2,
3692 3691 ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "",
3693 3692 "ibmf_saa_impl_get_port_guid: %s\n", tnf_string, msg,
3694 3693 "portinfo sgid table size is 0. Exiting.\n");
3695 3694
3696 3695 return (IBMF_TRANSPORT_FAILURE);
3697 3696 }
3698 3697
3699 3698 *guid_ret = ibt_portinfop->p_sgid_tbl[0].gid_guid;
3700 3699
3701 3700 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3702 3701 ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "",
3703 3702 "ibmf_saa_impl_get_port_guid: Returning port_guid %016" PRIx64 "\n",
3704 3703 tnf_opaque, port_guid, *guid_ret);
3705 3704
3706 3705 return (IBMF_SUCCESS);
3707 3706 }
3708 3707
3709 3708 /*
3710 3709 * ibmf_saa_impl_set_transaction_params:
3711 3710 */
3712 3711 static void
3713 3712 ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp,
3714 3713 ibt_hca_portinfo_t *portinfop)
3715 3714 {
3716 3715 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3717 3716 ibmf_saa_impl_set_transaction_params_start,
3718 3717 IBMF_TNF_TRACE, "",
3719 3718 "ibmf_saa_impl_set_transaction_params() enter\n");
3720 3719
3721 3720 _NOTE(ASSUMING_PROTECTED(*saa_portp))
3722 3721
3723 3722 saa_portp->saa_pt_ibmf_retrans.retrans_retries =
3724 3723 ibmf_saa_retrans_retries;
3725 3724 /*
3726 3725 * For the first transaction (generally getting the
3727 3726 * classportinfo) have ibmf pick our timeouts. It should be using the
3728 3727 * default IB spec values.
3729 3728 * Once we get the classportinfo we'll update the correct response time
3730 3729 * value (rtv) and round-trip time (rttv). ibmf should always calculate
3731 3730 * trans_to since it depends on the particular transaction's number of
3732 3731 * packets.
3733 3732 */
3734 3733 saa_portp->saa_pt_ibmf_retrans.retrans_rtv = 0;
3735 3734 saa_portp->saa_pt_ibmf_retrans.retrans_rttv = 0;
3736 3735 saa_portp->saa_pt_ibmf_retrans.retrans_trans_to = 0;
3737 3736
3738 3737 /*
3739 3738 * Assume that the SA supports all optional records. If it
3740 3739 * does not, the request will get returned with ERR_NOT_SUPP. When
3741 3740 * the classportinfo response comes back we will update the cap mask
3742 3741 * to prevent unnecessary unsupported requests.
3743 3742 */
3744 3743 saa_portp->saa_pt_sa_cap_mask = 0xFFFF;
3745 3744
3746 3745 saa_portp->saa_pt_ibmf_msg_flags = 0;
3747 3746 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1;
3748 3747 saa_portp->saa_pt_ibmf_addr_info.ia_p_key =
3749 3748 IB_PKEY_DEFAULT_LIMITED;
3750 3749 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY;
3751 3750
3752 3751 /*
3753 3752 * fill out addr information for MADs that will be sent
3754 3753 * to SA on this port
3755 3754 */
3756 3755 saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = portinfop->p_base_lid;
3757 3756 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = portinfop->p_sm_lid;
3758 3757 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = portinfop->p_sm_sl;
3759 3758
3760 3759 /* place upper bound on subnet timeout in case of faulty SM */
3761 3760 saa_portp->saa_pt_timeout = portinfop->p_subnet_timeout;
3762 3761
3763 3762 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout)
3764 3763 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout;
3765 3764
3766 3765 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
3767 3766 ibmf_saa_impl_set_transaction_params,
3768 3767 IBMF_TNF_TRACE, "",
3769 3768 "ibmf_saa_impl_set_transaction_params: local_lid = 0x%x, "
3770 3769 "sm_lid = 0x%x, sm_sl = 0x%x, sn_timeout = 0x%x\n",
3771 3770 tnf_opaque, local_lid, portinfop->p_base_lid,
3772 3771 tnf_opaque, sm_lid, portinfop->p_sm_lid,
3773 3772 tnf_opaque, sm_sl, portinfop->p_sm_sl,
3774 3773 tnf_opaque, subnet_timeout, portinfop->p_subnet_timeout);
3775 3774
3776 3775 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3777 3776 ibmf_saa_impl_set_transaction_params_end,
3778 3777 IBMF_TNF_TRACE, "",
3779 3778 "ibmf_saa_impl_set_transaction_params() exit\n");
3780 3779 }
3781 3780
3782 3781
3783 3782 /*
3784 3783 * ibmf_saa_impl_update_sa_address_info
3785 3784 */
3786 3785 static void
3787 3786 ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp, ibmf_msg_t *msgp)
3788 3787 {
3789 3788 void *result;
3790 3789 ib_sa_hdr_t *sa_hdr;
3791 3790 int rv;
3792 3791 size_t length;
3793 3792 uint16_t attr_id;
3794 3793 ib_mad_classportinfo_t *cpi;
3795 3794 ibmf_global_addr_info_t *gaddrp = &saa_portp->saa_pt_ibmf_global_addr;
3796 3795 ibt_hca_portinfo_t *ibt_pinfo;
3797 3796 uint_t nports, size;
3798 3797 ibt_status_t ibt_status;
3799 3798
3800 3799 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3801 3800 ibmf_saa_impl_update_sa_address_info,
3802 3801 IBMF_TNF_TRACE, "",
3803 3802 "ibmf_saa_impl_update_sa_address_info() enter\n");
3804 3803
3805 3804 /*
3806 3805 * decode the respons of msgp as a classportinfo attribute
3807 3806 */
3808 3807 rv = ibmf_saa_utils_unpack_sa_hdr(msgp->im_msgbufs_recv.im_bufs_cl_hdr,
3809 3808 msgp->im_msgbufs_recv.im_bufs_cl_hdr_len, &sa_hdr, KM_NOSLEEP);
3810 3809 if (rv != IBMF_SUCCESS) {
3811 3810
3812 3811 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3813 3812 ibmf_saa_impl_update_sa_address_err,
3814 3813 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: "
3815 3814 "%s, ibmf_status = %d\n", tnf_string, msg,
3816 3815 "Could not unpack sa hdr", tnf_int, ibmf_status, rv);
3817 3816
3818 3817 return;
3819 3818 }
3820 3819
3821 3820 attr_id = b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID);
3822 3821 if (attr_id != MAD_ATTR_ID_CLASSPORTINFO) {
3823 3822 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3824 3823 ibmf_saa_impl_update_sa_address_info_err,
3825 3824 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: "
3826 3825 "%s, attrID = %x\n", tnf_string, msg,
3827 3826 "Wrong attribute ID", tnf_int, ibmf_status, attr_id);
3828 3827
3829 3828 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3830 3829 return;
3831 3830 }
3832 3831 rv = ibmf_saa_utils_unpack_payload(
3833 3832 msgp->im_msgbufs_recv.im_bufs_cl_data,
3834 3833 msgp->im_msgbufs_recv.im_bufs_cl_data_len, attr_id, &result,
3835 3834 &length, sa_hdr->AttributeOffset, B_TRUE, KM_NOSLEEP);
3836 3835 if (rv != IBMF_SUCCESS) {
3837 3836
3838 3837 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3839 3838 ibmf_saa_impl_update_sa_address_err,
3840 3839 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: "
3841 3840 "%s, ibmf_status = %d\n", tnf_string, msg,
3842 3841 "Could not unpack payload", tnf_int, ibmf_status, rv);
3843 3842
3844 3843 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3845 3844 return;
3846 3845 }
3847 3846
3848 3847 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3849 3848
3850 3849 /*
3851 3850 * Use the classportinfo contents to update the SA address info
3852 3851 */
3853 3852 cpi = (ib_mad_classportinfo_t *)result;
3854 3853 mutex_enter(&saa_portp->saa_pt_mutex);
3855 3854 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = cpi->RedirectLID;
3856 3855 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = cpi->RedirectQP;
3857 3856 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = cpi->RedirectP_Key;
3858 3857 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = cpi->RedirectQ_Key;
3859 3858 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = cpi->RedirectSL;
3860 3859
3861 3860 saa_portp->saa_pt_redirect_active = B_TRUE;
3862 3861
3863 3862 if ((cpi->RedirectGID_hi != 0) || (cpi->RedirectGID_lo != 0)) {
3864 3863
3865 3864 mutex_exit(&saa_portp->saa_pt_mutex);
3866 3865 ibt_status = ibt_query_hca_ports_byguid(
3867 3866 saa_portp->saa_pt_node_guid, saa_portp->saa_pt_port_num,
3868 3867 &ibt_pinfo, &nports, &size);
3869 3868 if (ibt_status != IBT_SUCCESS) {
3870 3869
3871 3870 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3872 3871 ibmf_saa_impl_update_sa_address_err, IBMF_TNF_TRACE,
3873 3872 "", "ibmf_saa_impl_update_sa_address_info: "
3874 3873 "%s, ibt_status = %d\n", tnf_string, msg,
3875 3874 "Could not query hca port",
3876 3875 tnf_int, ibt_status, ibt_status);
3877 3876
3878 3877 kmem_free(result, length);
3879 3878 return;
3880 3879 }
3881 3880
3882 3881 mutex_enter(&saa_portp->saa_pt_mutex);
3883 3882 /*
3884 3883 * Fill in global address info parameters
3885 3884 *
3886 3885 * NOTE: The HopLimit value is not specified through the
3887 3886 * contents of ClassPortInfo. It may be possible to find
3888 3887 * out the proper value to use even for SA beeing redirected
3889 3888 * to another subnet. But we do only support redirect within
3890 3889 * our local subnet
3891 3890 */
3892 3891 gaddrp->ig_sender_gid.gid_prefix =
3893 3892 ibt_pinfo->p_sgid_tbl[0].gid_prefix;
3894 3893 gaddrp->ig_sender_gid.gid_guid = saa_portp->saa_pt_port_guid;
3895 3894 gaddrp->ig_recver_gid.gid_prefix = cpi->RedirectGID_hi;
3896 3895 gaddrp->ig_recver_gid.gid_guid = cpi->RedirectGID_lo;
3897 3896 gaddrp->ig_flow_label = cpi->RedirectFL;
3898 3897 gaddrp->ig_tclass = cpi->RedirectTC;
3899 3898 gaddrp->ig_hop_limit = 0;
3900 3899
3901 3900 saa_portp->saa_pt_ibmf_msg_flags =
3902 3901 IBMF_MSG_FLAGS_GLOBAL_ADDRESS;
3903 3902
3904 3903 mutex_exit(&saa_portp->saa_pt_mutex);
3905 3904 ibt_free_portinfo(ibt_pinfo, size);
3906 3905 } else {
3907 3906 saa_portp->saa_pt_ibmf_msg_flags = 0;
3908 3907 mutex_exit(&saa_portp->saa_pt_mutex);
3909 3908 }
3910 3909 kmem_free(result, length);
3911 3910
3912 3911 /*
3913 3912 * Update the address info of msgp with the new address parameters
3914 3913 */
3915 3914 mutex_enter(&saa_portp->saa_pt_mutex);
3916 3915 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr,
3917 3916 sizeof (ibmf_addr_info_t));
3918 3917 if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) {
3919 3918
3920 3919 msgp->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS;
3921 3920
3922 3921 bcopy(&saa_portp->saa_pt_ibmf_global_addr,
3923 3922 &msgp->im_global_addr, sizeof (ibmf_global_addr_info_t));
3924 3923 } else {
3925 3924 msgp->im_msg_flags = 0;
3926 3925 }
3927 3926 mutex_exit(&saa_portp->saa_pt_mutex);
3928 3927 }
3929 3928
3930 3929 /*
3931 3930 * ibmf_saa_impl_ibmf_unreg:
3932 3931 */
3933 3932 static int
3934 3933 ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp)
3935 3934 {
3936 3935 int ibmf_status;
3937 3936
3938 3937 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_start,
3939 3938 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() enter\n");
3940 3939
3941 3940 /* teardown async cb */
3942 3941 ibmf_status = ibmf_tear_down_async_cb(saa_portp->saa_pt_ibmf_handle,
3943 3942 saa_portp->saa_pt_qp_handle, 0);
3944 3943 if (ibmf_status != IBMF_SUCCESS) {
3945 3944
3946 3945 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3947 3946 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "",
3948 3947 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n",
3949 3948 tnf_string, msg, "Could not tear down async cb",
3950 3949 tnf_int, ibmf_status, ibmf_status);
3951 3950
3952 3951 goto bail;
3953 3952 }
3954 3953
3955 3954 /* free qp */
3956 3955 ibmf_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle,
3957 3956 &saa_portp->saa_pt_qp_handle, 0);
3958 3957
3959 3958 if (ibmf_status != IBMF_SUCCESS) {
3960 3959
3961 3960 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3962 3961 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "",
3963 3962 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n",
3964 3963 tnf_string, msg, "Could not free queue pair",
3965 3964 tnf_int, ibmf_status, ibmf_status);
3966 3965
3967 3966 (void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 1);
3968 3967 goto bail;
3969 3968 }
3970 3969
3971 3970 ibmf_status = ibmf_unregister(&saa_portp->saa_pt_ibmf_handle, 0);
3972 3971
3973 3972 if (ibmf_status != IBMF_SUCCESS) {
3974 3973
3975 3974 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3976 3975 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "",
3977 3976 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n",
3978 3977 tnf_string, msg, "ibmf_unregister() failed",
3979 3978 tnf_int, ibmf_status, ibmf_status);
3980 3979
3981 3980 (void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0);
3982 3981 }
3983 3982
3984 3983 bail:
3985 3984 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_end,
3986 3985 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() exit\n");
3987 3986
3988 3987 return (ibmf_status);
3989 3988 }
|
↓ open down ↓ |
2412 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX