1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 #include <sys/ib/mgt/ibmf/ibmf_saa_impl.h>
  27 #include <sys/ib/mgt/ibmf/ibmf_saa_utils.h>
  28 
  29 /* Global sa_access State Pointer */
  30 saa_state_t *saa_statep;
  31 _NOTE(READ_ONLY_DATA(saa_statep))
  32 
  33 extern  int     ibmf_trace_level;
  34 
  35 extern  int     ibmf_taskq_max_tasks;
  36 
  37 static int
  38 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
  39     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags);
  40 static int
  41 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp,
  42     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags);
  43 static int
  44 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
  45     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg,
  46     hrtime_t trans_send_time, int transport_flags);
  47 static int ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info,
  48     boolean_t sleep_flag, ibmf_msg_t **msgp, uint32_t *transport_flagsp,
  49     ibmf_retrans_t *ibmf_retransp);
  50 static int ibmf_saa_must_purge(saa_port_t *saa_portp);
  51 static void ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp);
  52 static void ibmf_saa_impl_destroy_port(saa_port_t *saa_portp);
  53 static void ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp);
  54 static void ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer,
  55     int status);
  56 static void ibmf_saa_impl_async_event_cb(ibmf_handle_t ibmf_handle,
  57     void *clnt_private, ibmf_async_event_t event_type);
  58 static void ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num);
  59 static void ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num);
  60 static void ibmf_saa_impl_port_chg(ibt_async_event_t *event);
  61 static void ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num);
  62 static void ibmf_saa_impl_hca_detach(saa_port_t *saa_removed);
  63 static void ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle,
  64     ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result,
  65     size_t *length, boolean_t sleep_flag);
  66 static int ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id);
  67 static uint_t ibmf_saa_impl_get_attr_id_length(uint16_t attr_id);
  68 static void ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp);
  69 static int ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop,
  70     ib_guid_t *guid_ret);
  71 static void ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp,
  72     ibt_hca_portinfo_t *portinfop);
  73 static void ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp,
  74     ibmf_msg_t *msgp);
  75 static int ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp);
  76 
  77 int     ibmf_saa_max_wait_time = IBMF_SAA_MAX_WAIT_TIME_IN_SECS;
  78 int     ibmf_saa_trans_wait_time = IBMF_SAA_TRANS_WAIT_TIME_IN_SECS;
  79 int     ibmf_saa_max_resp_time = IBMF_SAA_MAX_RESP_TIME;
  80 int     ibmf_saa_max_subnet_timeout = IBMF_SAA_MAX_SUBNET_TIMEOUT;
  81 int     ibmf_saa_retrans_retries = IBMF_SAA_RETRANS_RETRIES;
  82 
  83 /*
  84  * ibmf_saa_impl_init:
  85  * Allocates memory for the ibmf_saa state structure and initializes the taskq.
  86  * Called from the modules init() routine.
  87  *
  88  * Input Arguments
  89  * none
  90  *
  91  * Output Arguments
  92  * none
  93  *
  94  * Returns
  95  * IBMF_NO_RESOURCES if taskq could not be created.
  96  * IBMF_SUCCESS on success
  97  *
  98  */
  99 int
 100 ibmf_saa_impl_init()
 101 {
 102         int             res;
 103 
 104         /* CONSTCOND */
 105         ASSERT(NO_COMPETING_THREADS);
 106 
 107         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_init_start,
 108             IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() enter\n");
 109 
 110         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_statep))
 111 
 112         saa_statep = kmem_zalloc(sizeof (saa_state_t), KM_SLEEP);
 113 
 114         /* create taskq for notifying event subscribers */
 115         saa_statep->saa_event_taskq = taskq_create(
 116             "ibmf_saa_event_taskq", IBMF_TASKQ_NTHREADS,
 117             MINCLSYSPRI, 1, ibmf_taskq_max_tasks, TASKQ_DYNAMIC |
 118             TASKQ_PREPOPULATE);
 119         if (saa_statep->saa_event_taskq == NULL) {
 120 
 121                 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L4,
 122                     ibmf_saa_impl_init_end_err,
 123                     IBMF_TNF_TRACE, "", "ibmf_saa_impl_init(): %s\n",
 124                     tnf_string, msg, "event taskq create failed");
 125 
 126                 kmem_free(saa_statep, sizeof (saa_state_t));
 127 
 128                 res = IBMF_NO_RESOURCES;
 129 
 130                 goto bail;
 131         }
 132 
 133         mutex_init(&saa_statep->saa_port_list_mutex, NULL, MUTEX_DRIVER,
 134             NULL);
 135 
 136         _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_statep))
 137 
 138         res = IBMF_SUCCESS;
 139 bail:
 140 
 141         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_init_end,
 142             IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() exit: status = %d\n",
 143             tnf_int, res, res);
 144 
 145         return (res);
 146 }
 147 
 148 /*
 149  * ibmf_saa_impl_fini:
 150  * If there are no registered clients, cleans up all memory associated with the
 151  * state, including each of the port list entries.
 152  * Called from the modules fini() routine.
 153  *
 154  * Input Arguments
 155  * none
 156  *
 157  * Output Arguments
 158  * none
 159  *
 160  * Returns
 161  * EBUSY if there are outstanding transactions or registered clients
 162  * 0 if cleanup was sucessfull
 163  *
 164  */
 165 int
 166 ibmf_saa_impl_fini()
 167 {
 168         int             ret = 0;
 169         saa_port_t      *saa_portp;
 170         saa_port_t      *next;
 171 
 172         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_start,
 173             IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() enter\n");
 174 
 175         /* make sure there are no registered clients */
 176         mutex_enter(&saa_statep->saa_port_list_mutex);
 177 
 178         saa_portp = saa_statep->saa_port_list;
 179         while (saa_portp != NULL) {
 180 
 181                 mutex_enter(&saa_portp->saa_pt_mutex);
 182 
 183                 if (saa_portp->saa_pt_reference_count > 0) {
 184 
 185                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
 186                             ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "",
 187                             "ibmf_saa_impl_fini: %s, port %016" PRIx64 "\n",
 188                             tnf_string, msg,
 189                             "cannot unload ibmf_saa. Client on port still"
 190                             " registered", tnf_opaque, port,
 191                             saa_portp->saa_pt_port_guid);
 192 
 193                         mutex_exit(&saa_portp->saa_pt_mutex);
 194 
 195                         mutex_exit(&saa_statep->saa_port_list_mutex);
 196 
 197                         ret = EBUSY;
 198                         goto bail;
 199                 }
 200 
 201                 /* make sure there are no outstanding transactions */
 202 
 203                 if (saa_portp->saa_pt_num_outstanding_trans > 0) {
 204 
 205                         IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1,
 206                             ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "",
 207                             "ibmf_saa_impl_fini: %s, port = %016" PRIx64
 208                             ", num transactions = %d\n",
 209                             tnf_string, msg, "Cannot unload ibmf_saa."
 210                             "  Outstanding transactions on port.",
 211                             tnf_opaque, port,
 212                             saa_portp->saa_pt_port_guid,
 213                             tnf_uint, outstanding_transactions,
 214                             saa_portp->saa_pt_num_outstanding_trans);
 215 
 216                         mutex_exit(&saa_portp->saa_pt_mutex);
 217 
 218                         mutex_exit(&saa_statep->saa_port_list_mutex);
 219 
 220                         ret = EBUSY;
 221                         goto bail;
 222                 }
 223 
 224                 mutex_exit(&saa_portp->saa_pt_mutex);
 225 
 226                 saa_portp = saa_portp->next;
 227         }
 228 
 229         mutex_exit(&saa_statep->saa_port_list_mutex);
 230 
 231         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(saa_statep->saa_port_list,
 232             *saa_portp))
 233 
 234         /*
 235          * no more clients nor pending transaction:
 236          * unregister ibmf and destroy port entries
 237          */
 238         while (saa_statep->saa_port_list != NULL) {
 239 
 240                 saa_portp = saa_statep->saa_port_list;
 241                 next = saa_portp->next;
 242 
 243                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
 244                     ibmf_saa_impl_fini, IBMF_TNF_TRACE, "",
 245                     "ibmf_saa_impl_fini: %s, prefix = %016" PRIx64 "\n",
 246                     tnf_string, msg, "deinitializing port",
 247                     tnf_opaque, port_guid, saa_portp->saa_pt_port_guid);
 248 
 249                 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp))
 250 
 251                 mutex_enter(&saa_portp->saa_pt_mutex);
 252 
 253                 /* unregister from ibmf */
 254                 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) {
 255 
 256                         mutex_exit(&saa_portp->saa_pt_mutex);
 257 
 258                         if (ibmf_saa_impl_ibmf_unreg(saa_portp)
 259                             != IBMF_SUCCESS) {
 260                                 ret = EBUSY;
 261                                 goto bail;
 262                         }
 263                 } else
 264                         mutex_exit(&saa_portp->saa_pt_mutex);
 265 
 266                 ibmf_saa_impl_destroy_port(saa_portp);
 267 
 268                 saa_statep->saa_port_list = next;
 269         }
 270 
 271         taskq_destroy(saa_statep->saa_event_taskq);
 272 
 273         mutex_destroy(&saa_statep->saa_port_list_mutex);
 274 
 275         kmem_free(saa_statep, sizeof (saa_state_t));
 276 
 277 bail:
 278         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_end,
 279             IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() exit\n");
 280 
 281         return (ret);
 282 }
 283 
 284 /*
 285  * ibmf_saa_is_valid
 286  * Returns true the entry is valid.
 287  *
 288  * Input Arguments
 289  * saa_portp            pointer to state structure
 290  * add_ref              if B_TRUE ref count is incremented on a valid portp
 291  *
 292  * Output Arguments
 293  * none
 294  *
 295  * Returns
 296  * B_TRUE if entry was in a valid state, B_FALSE otherwise
 297  */
 298 boolean_t
 299 ibmf_saa_is_valid(saa_port_t *saa_portp, int add_ref)
 300 {
 301         boolean_t is_valid = B_TRUE;
 302 
 303         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_start,
 304             IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() enter\n");
 305 
 306         mutex_enter(&saa_portp->saa_pt_mutex);
 307 
 308         if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID ||
 309             saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_PURGING) {
 310 
 311                 is_valid = B_FALSE;
 312 
 313         } else if (add_ref == B_TRUE) {
 314                 /*
 315                  * increment reference count here to ensure that
 316                  * entry does not get purged behind our backs
 317                  */
 318                 saa_portp->saa_pt_reference_count++;
 319         }
 320         mutex_exit(&saa_portp->saa_pt_mutex);
 321 
 322         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_end,
 323             IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() exit\n");
 324 
 325         return (is_valid);
 326 }
 327 
 328 /*
 329  * ibmf_saa_must_purge
 330  * Determines if we can purge a portp (remove it from the list) based on the
 331  * state and number of clients
 332  *
 333  * Input Arguments
 334  * saa_portp            pointer to state structure
 335  *
 336  * Output Arguments
 337  * none
 338  *
 339  * Returns
 340  * B_TRUE if the entry can be removed, B_FALSE otherwise
 341  */
 342 static int
 343 ibmf_saa_must_purge(saa_port_t *saa_portp)
 344 {
 345         int must_purge = B_FALSE;
 346 
 347         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_start,
 348             IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() enter\n");
 349 
 350         mutex_enter(&saa_portp->saa_pt_mutex);
 351 
 352         if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID &&
 353             saa_portp->saa_pt_reference_count == 0) {
 354 
 355                 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_PURGING;
 356                 must_purge = B_TRUE;
 357         }
 358 
 359         mutex_exit(&saa_portp->saa_pt_mutex);
 360 
 361         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_end,
 362             IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() exit\n");
 363 
 364         return (must_purge);
 365 }
 366 
 367 
 368 /*
 369  * ibmf_saa_impl_purge:
 370  * Removes invalid port state entries from the list
 371  *
 372  * Input Arguments
 373  * none
 374  *
 375  * Output Arguments
 376  * none
 377  *
 378  * Returns
 379  * void
 380  */
 381 void
 382 ibmf_saa_impl_purge()
 383 {
 384         saa_port_t *cur_portp  = NULL;
 385         saa_port_t *prev_portp = NULL;
 386         saa_port_t *rem_portp  = NULL;
 387 
 388         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_start,
 389             IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() enter\n");
 390 
 391         mutex_enter(&saa_statep->saa_port_list_mutex);
 392 
 393         cur_portp = saa_statep->saa_port_list;
 394         prev_portp = cur_portp;
 395 
 396         while (cur_portp != NULL) {
 397 
 398                 if (ibmf_saa_must_purge(cur_portp) == B_TRUE) {
 399 
 400                         rem_portp = cur_portp;
 401 
 402                         /* unlink entry */
 403                         if (cur_portp == saa_statep->saa_port_list) {
 404 
 405                                 saa_statep->saa_port_list = cur_portp->next;
 406                                 cur_portp = saa_statep->saa_port_list;
 407                                 prev_portp = cur_portp;
 408 
 409                         } else {
 410 
 411                                 prev_portp->next = cur_portp->next;
 412                                 cur_portp = cur_portp->next;
 413                         }
 414 
 415                         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*rem_portp))
 416 
 417                         /* destroy entry */
 418                         ASSERT(rem_portp != NULL);
 419                         ibmf_saa_impl_destroy_port(rem_portp);
 420 
 421                 } else {
 422 
 423                         prev_portp = cur_portp;
 424                         cur_portp = cur_portp->next;
 425                 }
 426         }
 427 
 428         mutex_exit(&saa_statep->saa_port_list_mutex);
 429 
 430         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_end,
 431             IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() exit\n");
 432 }
 433 
 434 /*
 435  * saa_impl_add_client:
 436  * Adds a client for a particular portp.  Reference count has been incremented
 437  * before this call.  It is decremented by saa_impl_add_client() if the call
 438  * fails.
 439  *
 440  * Input Arguments
 441  * none
 442  *
 443  * Output Arguments
 444  * none
 445  *
 446  * Returns
 447  * IBMF_BUSY if there are already too many clients registered,
 448  * IBMF_BAD_PORT_STATE if the port is invalid (generally because a previous
 449  * client failed during registration for this port)
 450  * IBMF_SUCCESS otherwise
 451  */
 452 int
 453 ibmf_saa_impl_add_client(saa_port_t *saa_portp)
 454 {
 455         int status = IBMF_SUCCESS;
 456 
 457         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_add_client_start,
 458             IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client() enter\n");
 459 
 460         mutex_enter(&saa_portp->saa_pt_mutex);
 461 
 462         /*
 463          * check that we don't exceed max clients
 464          */
 465         if (saa_portp->saa_pt_reference_count >
 466             SAA_MAX_CLIENTS_PER_PORT) {
 467 
 468                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
 469                     ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, "",
 470                     "ibmf_saa_impl_add_client: %s, num_reg_clients %d\n",
 471                     tnf_string, msg, "too many clients registered for"
 472                     " port", tnf_uint, num_reg_clients,
 473                     saa_portp->saa_pt_reference_count);
 474 
 475                 status = IBMF_BUSY;
 476                 goto bail;
 477         }
 478 
 479         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
 480             ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "",
 481             "ibmf_saa_impl_add_client: num_registered_clients %d\n",
 482             tnf_uint, num_registered_clients,
 483             saa_portp->saa_pt_reference_count);
 484 
 485         /*
 486          * wait until anyone who is currently registering
 487          * this port with ibmf is done
 488          */
 489         while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) {
 490 
 491                 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
 492                     ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "",
 493                     "ibmf_saa_impl_add_client: %s\n",
 494                     tnf_string, msg, "someone is registering. waiting"
 495                     " for them to finish");
 496 
 497                 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv,
 498                     &saa_portp->saa_pt_mutex);
 499 
 500                 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
 501                     ibmf_saa_impl_add_client,
 502                     IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client: %s\n",
 503                     tnf_string, msg, "done waiting");
 504         }
 505 
 506         /*
 507          * if port isn't ready here, fail.
 508          */
 509         if (saa_portp->saa_pt_state != IBMF_SAA_PORT_STATE_READY) {
 510 
 511                 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
 512                     ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR,
 513                     "", "ibmf_saa_impl_add_client: %s\n",
 514                     tnf_string, msg, "port state not ready,"
 515                     " removing client.");
 516 
 517                 status = IBMF_BAD_PORT_STATE;
 518                 goto bail;
 519         }
 520 
 521 bail:
 522         mutex_exit(&saa_portp->saa_pt_mutex);
 523 
 524         if (status != IBMF_SUCCESS) {
 525 
 526                 mutex_enter(&saa_portp->saa_pt_kstat_mutex);
 527 
 528                 IBMF_SAA_ADD32_KSTATS(saa_portp,
 529                     clients_reg_failed, 1);
 530 
 531                 mutex_exit(&saa_portp->saa_pt_kstat_mutex);
 532 
 533                 /* decrementing refcount is last thing we do on entry */
 534 
 535                 mutex_enter(&saa_portp->saa_pt_mutex);
 536 
 537                 ASSERT(saa_portp->saa_pt_reference_count > 0);
 538                 saa_portp->saa_pt_reference_count--;
 539 
 540                 mutex_exit(&saa_portp->saa_pt_mutex);
 541         }
 542 
 543         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 544             ibmf_saa_impl_add_client_end, IBMF_TNF_TRACE, "",
 545             "ibmf_saa_impl_add_client() exit\n");
 546 
 547         return (status);
 548 }
 549 
 550 /*
 551  * ibmf_saa_impl_create_port()
 552  * Create port entry with mimimal inits because
 553  * we're holding the list mutex: NO BLOCKING CALLS HERE, please.
 554  *
 555  * Initialize port state to "registering", so that clients accessing
 556  * same port concurrently will wait for the end of the ibmf registration.
 557  * Note: this thread will access port members without locking mutex.
 558  *
 559  * Input Arguments
 560  * pt_guid              guid of port
 561  *
 562  * Output Arguments
 563  * saa_portpp           pointer to new saa_portp structure
 564  *
 565  * Returns
 566  * IBMF_NO_MEMORY if memory could not be allocated
 567  * IBMF_SUCCESS otherwise
 568  */
 569 int
 570 ibmf_saa_impl_create_port(ib_guid_t pt_guid, saa_port_t **saa_portpp)
 571 {
 572         int             status          = IBMF_SUCCESS;
 573         saa_port_t      *saa_portp      = NULL;
 574 
 575         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_create_port_start,
 576             IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port:"
 577             " guid %016" PRIx64 "\n",
 578             tnf_opaque, port_guid, pt_guid);
 579 
 580         ASSERT(MUTEX_HELD(&saa_statep->saa_port_list_mutex));
 581 
 582         /* create & initialize new port */
 583         saa_portp = kmem_zalloc(sizeof (saa_port_t), KM_NOSLEEP);
 584 
 585         if (saa_portp == NULL) {
 586 
 587                 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
 588                     ibmf_sa_session_open_err, IBMF_TNF_ERROR, "",
 589                     "ibmf_saa_impl_create_port: %s\n",
 590                     tnf_string, msg, "could not allocate memory for "
 591                     "new port");
 592 
 593                 status = IBMF_NO_MEMORY;
 594                 goto bail;
 595         }
 596 
 597         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_sa_session_open,
 598             IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port: %s\n",
 599             tnf_string, msg, "first client registering, initializing");
 600 
 601         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp))
 602 
 603         /* tell everyone that kstats are not initialized */
 604         saa_portp->saa_pt_kstatp = NULL;
 605 
 606         /*
 607          * set up mutexe and state variable to indicate to
 608          * other clients that were currently in the process of
 609          * setting up the port data.  This will prevent a subsequent
 610          * client from trying to to register with ibmf before the
 611          * port data has been initialized.
 612          */
 613         mutex_init(&saa_portp->saa_pt_mutex, NULL, MUTEX_DRIVER, NULL);
 614         cv_init(&saa_portp->saa_pt_ibmf_reg_cv, NULL, CV_DRIVER, NULL);
 615 
 616         saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_REGISTERING;
 617 
 618         /* create other mutexes */
 619         mutex_init(&saa_portp->saa_pt_kstat_mutex, NULL, MUTEX_DRIVER, NULL);
 620 
 621         mutex_init(&saa_portp->saa_pt_event_sub_mutex, NULL, MUTEX_DRIVER,
 622             NULL);
 623 
 624         /*
 625          * clients assume all arrive; set mask to this so we only notify
 626          * if something failed
 627          */
 628         saa_portp->saa_pt_event_sub_last_success_mask =
 629             IBMF_SAA_PORT_EVENT_SUB_ALL_ARRIVE;
 630 
 631         /*
 632          * set port_guid now so any immediately subsequent clients
 633          * registering on this port, guid will know we're already here
 634          */
 635         saa_portp->saa_pt_port_guid = pt_guid;
 636         saa_portp->saa_pt_reference_count = 1;
 637         saa_portp->saa_pt_current_tid = pt_guid << 32;
 638 
 639         saa_portp->saa_pt_redirect_active = B_FALSE;
 640 
 641         /* set sa_uptime now in case we never receive anything from SA */
 642         saa_portp->saa_pt_sa_uptime = gethrtime();
 643 
 644         _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_portp))
 645 
 646         /* Set new pointer in caller's */
 647         *saa_portpp = saa_portp;
 648 
 649 bail:
 650         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_create_port_end,
 651             IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port() exit\n");
 652 
 653         return (status);
 654 }
 655 
 656 /*
 657  * ibmf_saa_impl_invalidate_port:
 658  * invalidates port entry (assumes exist) and deletes kstat object
 659  * kstat object is destroyed in order to allow creating port entry
 660  * even if this entry is not purged
 661  */
 662 static void
 663 ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp)
 664 {
 665         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 666             ibmf_saa_impl_invalidate_port_start,
 667             IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() enter\n");
 668 
 669         ASSERT(saa_portp != NULL);
 670         ASSERT(MUTEX_HELD(&saa_portp->saa_pt_mutex));
 671 
 672         saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_INVALID;
 673         ibmf_saa_impl_uninit_kstats(saa_portp);
 674 
 675         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 676             ibmf_saa_impl_invalidate_port_end,
 677             IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() exit\n");
 678 }
 679 
 680 /*
 681  * ibmf_saa_impl_destroy_port:
 682  * Frees the resources associated with an saa_portp structure.  Assumes the
 683  * saa_portp exists
 684  *
 685  * Input Arguments
 686  * saa_portp            pointer to saa_portp structure
 687  *
 688  * Output Arguments
 689  * none
 690  *
 691  * Returns
 692  * void
 693  */
 694 static void
 695 ibmf_saa_impl_destroy_port(saa_port_t *saa_portp)
 696 {
 697         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_start,
 698             IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() enter\n");
 699 
 700         ASSERT(saa_portp != NULL);
 701 
 702         _NOTE(ASSUMING_PROTECTED(*saa_portp))
 703 
 704         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
 705             ibmf_saa_impl_destroy, IBMF_TNF_TRACE, "",
 706             "ibmf_saa_impl_destroy(): destroying port_guid %016" PRIx64 "\n",
 707             tnf_opaque, port_guid, saa_portp->saa_pt_port_guid);
 708 
 709         ibmf_saa_impl_uninit_kstats(saa_portp);
 710 
 711         /* uninit synchronization variables used for registration */
 712         mutex_destroy(&saa_portp->saa_pt_mutex);
 713         cv_destroy(&saa_portp->saa_pt_ibmf_reg_cv);
 714 
 715         mutex_destroy(&saa_portp->saa_pt_event_sub_mutex);
 716         mutex_destroy(&saa_portp->saa_pt_kstat_mutex);
 717 
 718         kmem_free(saa_portp, sizeof (saa_port_t));
 719 
 720         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_end,
 721             IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() exit\n");
 722 }
 723 
 724 /*
 725  * ibmf_saa_impl_init_kstats:
 726  * Create kstats structure.  Should be called when memory is alloced for a new
 727  * port entry.
 728  */
 729 int
 730 ibmf_saa_impl_init_kstats(saa_port_t *saa_portp)
 731 {
 732         char                    buf[128];
 733         ibmf_saa_kstat_t        *ksp;
 734 
 735         _NOTE(ASSUMING_PROTECTED(saa_portp->saa_pt_kstatp))
 736 
 737         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 738             ibmf_saa_impl_init_kstats_start,
 739             IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() enter\n");
 740 
 741         /* set up kstats structure */
 742         (void) sprintf(buf, "ibmf_saa_%016" PRIx64 "_stat",
 743             saa_portp->saa_pt_port_guid);
 744 
 745         saa_portp->saa_pt_kstatp = kstat_create("ibmf_saa",
 746             0, buf, "misc", KSTAT_TYPE_NAMED,
 747             sizeof (ibmf_saa_kstat_t) / sizeof (kstat_named_t),
 748             KSTAT_FLAG_WRITABLE);
 749 
 750         if (saa_portp->saa_pt_kstatp == NULL)
 751                 return (IBMF_NO_RESOURCES);
 752 
 753         ksp = (ibmf_saa_kstat_t *)saa_portp->saa_pt_kstatp->ks_data;
 754 
 755         kstat_named_init(&ksp->clients_registered,
 756             "clients_registered", KSTAT_DATA_UINT32);
 757 
 758         kstat_named_init(&ksp->clients_reg_failed,
 759             "clients_reg_failed", KSTAT_DATA_UINT32);
 760 
 761         kstat_named_init(&ksp->outstanding_requests,
 762             "outstanding_requests", KSTAT_DATA_UINT32);
 763 
 764         kstat_named_init(&ksp->total_requests,
 765             "total_requests", KSTAT_DATA_UINT32);
 766 
 767         kstat_named_init(&ksp->failed_requests,
 768             "failed_requests", KSTAT_DATA_UINT32);
 769 
 770         kstat_named_init(&ksp->requests_timedout,
 771             "requests_timedout", KSTAT_DATA_UINT32);
 772 
 773         kstat_install(saa_portp->saa_pt_kstatp);
 774 
 775         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 776             ibmf_saa_impl_init_kstats_end,
 777             IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() exit\n");
 778 
 779         return (IBMF_SUCCESS);
 780 }
 781 
 782 /*
 783  * ibmf_saa_impl_uninit_kstats:
 784  * Free kstats context.  Should be called when port is either destroyed
 785  * or invalidated.
 786  */
 787 static void
 788 ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp)
 789 {
 790         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 791             ibmf_saa_impl_uninit_kstats_start,
 792             IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() enter\n");
 793 
 794         mutex_enter(&saa_portp->saa_pt_kstat_mutex);
 795 
 796         if (saa_portp->saa_pt_kstatp != NULL) {
 797                 kstat_delete(saa_portp->saa_pt_kstatp);
 798         }
 799         saa_portp->saa_pt_kstatp = NULL;
 800 
 801         mutex_exit(&saa_portp->saa_pt_kstat_mutex);
 802 
 803         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 804             ibmf_saa_impl_uninit_kstats_end,
 805             IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() exit\n");
 806 }
 807 
 808 /*
 809  * ibmf_saa_impl_register_failed:
 810  * invalidate entry and kick waiters
 811  */
 812 void
 813 ibmf_saa_impl_register_failed(saa_port_t *saa_portp)
 814 {
 815         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 816             ibmf_saa_impl_register_failed_start,
 817             IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() enter\n");
 818 
 819         mutex_enter(&saa_portp->saa_pt_mutex);
 820 
 821         ibmf_saa_impl_invalidate_port(saa_portp);
 822 
 823         cv_broadcast(&saa_portp->saa_pt_ibmf_reg_cv);
 824 
 825         /* decrementing refcount is last thing we do on entry */
 826 
 827         ASSERT(saa_portp->saa_pt_reference_count > 0);
 828         saa_portp->saa_pt_reference_count--;
 829 
 830         mutex_exit(&saa_portp->saa_pt_mutex);
 831 
 832         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 833             ibmf_saa_impl_register_failed_end,
 834             IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() exit\n");
 835 }
 836 
 837 static int
 838 ibmf_saa_impl_setup_qp_async_cb(saa_port_t *saa_portp, int setup_async_cb_only)
 839 {
 840         int             status;
 841         int             unreg_status;
 842         ib_pkey_t       p_key;
 843         ib_qkey_t       q_key;
 844         uint8_t         portnum;
 845         boolean_t       qp_alloced = B_FALSE;
 846 
 847         if (setup_async_cb_only == 0) {
 848 
 849                 /* allocate a qp through ibmf */
 850                 status = ibmf_alloc_qp(saa_portp->saa_pt_ibmf_handle,
 851                     IB_PKEY_DEFAULT_LIMITED, IB_GSI_QKEY,
 852                     IBMF_ALT_QP_MAD_RMPP, &saa_portp->saa_pt_qp_handle);
 853 
 854                 if (status != IBMF_SUCCESS) {
 855 
 856                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
 857                             ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
 858                             "ibmf_saa_impl_setup_qp_async_cb: %s, "
 859                             "ibmf_status = %d\n",
 860                             tnf_string, msg, "Cannot alloc qp with ibmf",
 861                             tnf_int, status, status);
 862 
 863                         return (status);
 864                 }
 865 
 866                 qp_alloced = B_TRUE;
 867 
 868                 /*
 869                  * query the queue pair number; we will need it to unsubscribe
 870                  * from notice reports
 871                  */
 872                 status = ibmf_query_qp(saa_portp->saa_pt_ibmf_handle,
 873                     saa_portp->saa_pt_qp_handle, &saa_portp->saa_pt_qpn,
 874                     &p_key, &q_key, &portnum, 0);
 875 
 876                 if (status != IBMF_SUCCESS) {
 877 
 878                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
 879                             ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
 880                             "ibmf_saa_impl_setup_qp_async_cb: %s, "
 881                             "ibmf_status = %d\n",
 882                             tnf_string, msg,
 883                             "Cannot query alt qp to get qp num",
 884                             tnf_int, status, status);
 885 
 886                         goto bail;
 887                 }
 888         }
 889 
 890         /*
 891          * core ibmf is taking advantage of the fact that saa_portp is our
 892          * callback arg. If this changes, the code in ibmf_recv would need to
 893          * change as well
 894          */
 895         status = ibmf_setup_async_cb(saa_portp->saa_pt_ibmf_handle,
 896             saa_portp->saa_pt_qp_handle, ibmf_saa_report_cb, saa_portp, 0);
 897         if (status != IBMF_SUCCESS) {
 898 
 899                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
 900                     ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
 901                     "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status = %d\n",
 902                     tnf_string, msg, "Cannot register async cb with ibmf",
 903                     tnf_int, status, status);
 904 
 905                 goto bail;
 906         }
 907 
 908         return (IBMF_SUCCESS);
 909 
 910 bail:
 911         if (qp_alloced == B_TRUE) {
 912                 /* free alternate qp */
 913                 unreg_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle,
 914                     &saa_portp->saa_pt_qp_handle, 0);
 915                 if (unreg_status != IBMF_SUCCESS) {
 916 
 917                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
 918                             ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
 919                             "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status ="
 920                             " %d\n", tnf_string, msg,
 921                             "Cannot free alternate queue pair with ibmf",
 922                             tnf_int, unreg_status, unreg_status);
 923                 }
 924         }
 925 
 926         return (status);
 927 }
 928 
 929 /*
 930  * ibmf_saa_impl_register_port:
 931  */
 932 int
 933 ibmf_saa_impl_register_port(
 934         saa_port_t *saa_portp)
 935 {
 936         uint_t          hca_count       = 0;
 937         ib_guid_t       *hca_list       = NULL;
 938         int             status          = IBMF_SUCCESS;
 939         int             unreg_status    = IBMF_SUCCESS;
 940         int             ibt_status      = IBT_SUCCESS;
 941         ibt_hca_portinfo_t *port_info_list = NULL;
 942         uint_t          port_count      = 0;
 943         uint_t          port_size       = 0;
 944         int             ihca, iport;
 945         ib_guid_t       port_guid;
 946         boolean_t       ibmf_reg = B_FALSE;
 947 
 948         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 949             ibmf_saa_impl_register_port_start, IBMF_TNF_TRACE, "",
 950             "ibmf_saa_impl_register_port() enter\n");
 951 
 952         ASSERT(saa_portp != NULL);
 953 
 954         _NOTE(ASSUMING_PROTECTED(*saa_portp))
 955 
 956         /* get the HCA list */
 957 
 958         hca_count = ibt_get_hca_list(&hca_list);
 959 
 960         if (hca_count == 0) {
 961 
 962                 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
 963                     ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
 964                     "ibmf_saa_impl_register_port: %s\n",
 965                     tnf_string, msg, "cannot register port (no HCAs).\n");
 966 
 967                 status = IBMF_BAD_PORT;
 968                 goto bail;
 969         }
 970 
 971         /* lookup requested port guid in hca list */
 972         for (ihca = 0; ihca != hca_count; ihca++) {
 973 
 974                 ibt_status = ibt_query_hca_ports_byguid(hca_list[ihca],
 975                     0 /* all ports */, &port_info_list,
 976                     &port_count, &port_size);
 977 
 978                 if (ibt_status != IBT_SUCCESS) {
 979 
 980                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
 981                             ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
 982                             "ibmf_saa_impl_register_port: %s, %016" PRIx64 "\n",
 983                             tnf_string, msg, "Could not query hca.  Exiting.",
 984                             tnf_opaque, guid, hca_list[ihca]);
 985 
 986                         status = IBMF_TRANSPORT_FAILURE;
 987                         break;
 988                 }
 989 
 990                 for (iport = 0; iport < port_count; iport++) {
 991 
 992                         /* get port guid associated with hca guid, port num */
 993                         if (ibmf_saa_impl_get_port_guid(
 994                             port_info_list + iport, &port_guid) != IBMF_SUCCESS)
 995                                 continue;
 996 
 997                         if (saa_portp->saa_pt_port_guid != port_guid)
 998                                 continue;
 999 
1000                         IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
1001                             ibmf_saa_impl_register_port,
1002                             IBMF_TNF_TRACE, "",
1003                             "ibmf_saa_impl_register_port: %s, hca_guid = %016"
1004                             PRIx64 ", port_guid = %016" PRIx64
1005                             ", number = %d\n",
1006                             tnf_string, msg, "found port",
1007                             tnf_opaque, hca_guid, hca_list[ihca],
1008                             tnf_opaque, port_guid, port_guid,
1009                             tnf_uint,   port, iport + 1);
1010 
1011                         /*
1012                          * we're here? then we found our port:
1013                          * fill in ibmf registration info
1014                          * and address parameters from the portinfo
1015                          */
1016 
1017                         saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid
1018                             = hca_list[ihca];
1019                         saa_portp->saa_pt_ibmf_reginfo.ir_port_num = iport+1;
1020                         saa_portp->saa_pt_ibmf_reginfo.ir_client_class
1021                             = SUBN_ADM_MANAGER;
1022 
1023                         saa_portp->saa_pt_node_guid = hca_list[ihca];
1024                         saa_portp->saa_pt_port_num = iport + 1;
1025 
1026                         ibmf_saa_impl_set_transaction_params(
1027                             saa_portp, port_info_list + iport);
1028                         break;
1029                 }
1030 
1031                 ibt_free_portinfo(port_info_list, port_size);
1032 
1033                 if (iport != port_count)
1034                         break;  /* found our port */
1035         }
1036 
1037         ibt_free_hca_list(hca_list, hca_count);
1038 
1039         if (ihca == hca_count) {
1040 
1041                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1042                     ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
1043                     "ibmf_saa_impl_register_port: %s, port_guid %016"
1044                     PRIx64 "\n",
1045                     tnf_string, msg, "Could not find port,  exiting",
1046                     tnf_opaque, port_guid, saa_portp->saa_pt_port_guid);
1047 
1048                 status = IBMF_BAD_PORT;
1049         }
1050 
1051         if (status != IBMF_SUCCESS) {
1052 
1053                 goto bail;
1054         }
1055 
1056         /*
1057          * Now we found the port we searched for,
1058          * and open an ibmf session on that port.
1059          */
1060 
1061         IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1062             ibmf_saa_impl_register_port, IBMF_TNF_TRACE, "",
1063             "ibmf_saa_impl_register_port: %s, port_guid = %016" PRIx64
1064             ", port = %d\n", tnf_string, msg, "Registering with ibmf",
1065             tnf_opaque, port_guid, saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid,
1066             tnf_uint, port, saa_portp->saa_pt_ibmf_reginfo.ir_port_num);
1067 
1068         status = ibmf_register(&saa_portp->saa_pt_ibmf_reginfo,
1069             IBMF_VERSION, IBMF_REG_FLAG_RMPP,
1070             ibmf_saa_impl_async_event_cb, saa_portp,
1071             &saa_portp->saa_pt_ibmf_handle,
1072             &saa_portp->saa_pt_ibmf_impl_features);
1073 
1074         if (status != IBMF_SUCCESS) {
1075 
1076                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1077                     ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
1078                     "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n",
1079                     tnf_string, msg, "Could not register with ibmf",
1080                     tnf_int, status, status);
1081 
1082                 goto bail;
1083         }
1084 
1085         ibmf_reg = B_TRUE;
1086 
1087         if (ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0) == IBMF_SUCCESS)
1088                 return (IBMF_SUCCESS);
1089 
1090 bail:
1091         if (ibmf_reg == B_TRUE) {
1092                 /* unregister from ibmf */
1093                 unreg_status = ibmf_unregister(
1094                     &saa_portp->saa_pt_ibmf_handle, 0);
1095 
1096                 if (unreg_status != IBMF_SUCCESS) {
1097 
1098                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1099                             ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
1100                             "ibmf_saa_impl_register_port: %s, ibmf_status ="
1101                             " %d\n", tnf_string, msg,
1102                             "Cannot unregister from ibmf",
1103                             tnf_int, unreg_status, unreg_status);
1104                 }
1105         }
1106 
1107         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_register_port_end,
1108             IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_port() exit\n");
1109 
1110         return (status);
1111 }
1112 
1113 /*
1114  * ibmf_saa_impl_getclassportinfo:
1115  */
1116 void
1117 ibmf_saa_impl_get_classportinfo(saa_port_t *saa_portp)
1118 {
1119         int                     res;
1120         saa_impl_trans_info_t   *trans_info;
1121 
1122         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1123             ibmf_saa_impl_get_classportinfo_start,
1124             IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() enter\n");
1125 
1126         /*
1127          * allocate memory for trans_info; send_request's callback will free up
1128          * memory since request is asynchronous
1129          */
1130         trans_info = kmem_zalloc(sizeof (saa_impl_trans_info_t), KM_NOSLEEP);
1131         if (trans_info == NULL) {
1132 
1133                 mutex_enter(&saa_portp->saa_pt_mutex);
1134 
1135                 /* cpi transaction is handled as a client, decrement refcount */
1136                 ASSERT(saa_portp->saa_pt_reference_count > 0);
1137                 saa_portp->saa_pt_reference_count--;
1138 
1139                 mutex_exit(&saa_portp->saa_pt_mutex);
1140 
1141                 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
1142                     ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_ERROR, "",
1143                     "ibmf_saa_impl_get_classportinfo: %s\n", tnf_string, msg,
1144                     "Could not allocate memory for classportinfo trans_info");
1145 
1146                 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1147                     ibmf_saa_impl_get_classportinfo_end, IBMF_TNF_TRACE, "",
1148                     "ibmf_saa_impl_get_classportinfo() exiting\n");
1149 
1150                 return;
1151         }
1152 
1153         /* no specific client associated with this transaction */
1154         trans_info->si_trans_client_data = NULL;
1155         trans_info->si_trans_port     = saa_portp;
1156         trans_info->si_trans_method   = SA_SUBN_ADM_GET;
1157         trans_info->si_trans_attr_id  = MAD_ATTR_ID_CLASSPORTINFO;
1158 
1159         trans_info->si_trans_callback = ibmf_saa_impl_get_cpi_cb;
1160         trans_info->si_trans_callback_arg = saa_portp;
1161 
1162         mutex_enter(&saa_portp->saa_pt_kstat_mutex);
1163 
1164         IBMF_SAA_ADD32_KSTATS(saa_portp, outstanding_requests, 1);
1165         IBMF_SAA_ADD32_KSTATS(saa_portp, total_requests, 1);
1166 
1167         mutex_exit(&saa_portp->saa_pt_kstat_mutex);
1168 
1169         res = ibmf_saa_impl_send_request(trans_info);
1170 
1171         if (res != IBMF_SUCCESS) {
1172 
1173                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1174                     ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_TRACE, "",
1175                     "ibmf_saa_impl_get_classportinfo: %s, res = 0x%x\n",
1176                     tnf_string, msg, "ibmf_saa_impl_send_request failed",
1177                     tnf_opaque, res, res);
1178 
1179                 mutex_enter(&saa_portp->saa_pt_kstat_mutex);
1180 
1181                 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1);
1182                 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1);
1183 
1184                 mutex_exit(&saa_portp->saa_pt_kstat_mutex);
1185 
1186                 mutex_enter(&saa_portp->saa_pt_mutex);
1187 
1188                 /* cpi transaction is handled as a client, decrement refcount */
1189                 ASSERT(saa_portp->saa_pt_reference_count > 0);
1190                 saa_portp->saa_pt_reference_count--;
1191 
1192                 mutex_exit(&saa_portp->saa_pt_mutex);
1193 
1194         }
1195 
1196         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1197             ibmf_saa_impl_get_classportinfo_end,
1198             IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() exit\n");
1199 }
1200 
1201 /*
1202  * ibmf_saa_impl_get_cpi_cb:
1203  *
1204  * Called when the asynchronous getportinfo request receives its response.
1205  * Checks the status.  If success, updates the times in the port's
1206  * ibmf_retrans structure that is used in ibmf_msg_transport calls.  If failure,
1207  * just use default values.
1208  *
1209  * Input Arguments
1210  * arg          user-specified pointer (points to the current port data)
1211  * length       length of payload returned (should be size of classportinfo_t)
1212  * buffer       pointer to classportinfo returned (should not be null)
1213  * status       status of sa access request
1214  *
1215  * Output Arguments
1216  * none
1217  *
1218  * Returns void
1219  */
1220 static void
1221 ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, int status)
1222 {
1223         saa_port_t              *saa_portp;
1224         uint64_t                base_time, resp_timeout, rttv_timeout;
1225         ib_mad_classportinfo_t  *classportinfo;
1226         int                     resp_time_value;
1227         uint16_t                sa_cap_mask;
1228 
1229         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_start,
1230             IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() enter\n");
1231 
1232         /*
1233          * access port entry: note that it may have become invalid
1234          * but we hold a ref count for cpi and the interactions on
1235          * the entry are harmless
1236          */
1237         saa_portp = (saa_port_t *)arg;
1238 
1239         /* process response */
1240 
1241         if ((status != IBMF_SUCCESS) || (buffer == NULL)) {
1242 
1243                 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1,
1244                     ibmf_saa_impl_get_cpi_cb, IBMF_TNF_ERROR, "",
1245                     "ibmf_saa_impl_get_cpi_cb: %s, status = %d, buffer = "
1246                     " 0x%p, length = %d\n", tnf_string, msg,
1247                     "could not get classportinfo.  Check node and path to sm"
1248                     " lid", tnf_int, status, status,
1249                     tnf_opaque, buffer, buffer, tnf_uint, length, length);
1250 
1251                 /*
1252                  * IB spec (C13-13) indicates 20 can be used as default or
1253                  * intial value for classportinfo->resptimeout value
1254                  */
1255                 resp_time_value = 20;
1256 
1257                 sa_cap_mask = 0xFFFF;
1258 
1259         } else if (buffer != NULL) {
1260 
1261                 classportinfo = (ib_mad_classportinfo_t *)buffer;
1262 
1263                 resp_time_value = classportinfo->RespTimeValue & 0x1f;
1264 
1265                 /*
1266                  * Because some subnet managers might not provide sane
1267                  * value for "resp_time_value", we limit it here.  In
1268                  * case this limit is too restrictive (very large fabric),
1269                  * we allow the limit to be raised (/etc/system).
1270                  */
1271                 if (resp_time_value > ibmf_saa_max_resp_time) {
1272                         cmn_err(CE_CONT, "!ibmf_saa_max_resp_time (%d) "
1273                             "exceeded.", ibmf_saa_max_resp_time);
1274                         cmn_err(CE_CONT, "!Reducing subnet administrator "
1275                             "resp_time value from %d to %d.",
1276                             resp_time_value, ibmf_saa_max_resp_time);
1277                         resp_time_value = ibmf_saa_max_resp_time;
1278                 }
1279 
1280                 sa_cap_mask = classportinfo->CapabilityMask;
1281 
1282                 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1283                     ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "",
1284                     "ibmf_saa_impl_get_cpi_cb: %s, timeout = 0x%x,"
1285                     " cap_mask = 0x%x\n",
1286                     tnf_string, msg, "got classportinfo",
1287                     tnf_opaque, timeout, resp_time_value,
1288                     tnf_opaque, cap_mask, sa_cap_mask);
1289 
1290                 kmem_free(buffer, length);
1291         }
1292 
1293         /*
1294          * using IB spec calculation from 13.4.6.2
1295          * use bit shifting for 2^x.
1296          */
1297         base_time = (1 << resp_time_value);
1298 
1299         resp_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000;
1300 
1301         mutex_enter(&saa_portp->saa_pt_mutex);
1302 
1303         base_time = 2 * (1 << saa_portp->saa_pt_timeout);
1304 
1305         rttv_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000;
1306 
1307         saa_portp->saa_pt_ibmf_retrans.retrans_rtv = resp_timeout;
1308         saa_portp->saa_pt_ibmf_retrans.retrans_rttv = rttv_timeout;
1309         saa_portp->saa_pt_sa_cap_mask = sa_cap_mask;
1310 
1311         /*
1312          * cpi transaction is handled as a client,
1313          * decrement refcount; make sure it's the last
1314          * thing we do on this entry
1315          */
1316         ASSERT(saa_portp->saa_pt_reference_count > 0);
1317         saa_portp->saa_pt_reference_count--;
1318 
1319         IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1320             ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "",
1321             "ibmf_saa_impl_get_cpi_cb: %s, subnet_timeout = 0x%x, "
1322             "resp_time_value = 0x%x\n",
1323             tnf_string, msg, "updated resp timeout",
1324             tnf_opaque, subnet_timeout, saa_portp->saa_pt_timeout,
1325             tnf_opaque, resp_time_value, resp_time_value);
1326 
1327         mutex_exit(&saa_portp->saa_pt_mutex);
1328 
1329         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_end,
1330             IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() exit\n");
1331 }
1332 
1333 /*
1334  * ibmf_saa_impl_send_request:
1335  * Sends a request to the sa.  Can be used for both classportinfo and record
1336  * requests.  Will set up all data structures for using the multi-packet
1337  * protocol, create the mad, and send it.  Returns SA_SUCCESS if msg transport
1338  * worked, meaning succesful send for the async case and a succesful send and
1339  * recv for the sync case.
1340  */
1341 int
1342 ibmf_saa_impl_send_request(saa_impl_trans_info_t *trans_info)
1343 {
1344         uint16_t                attr_id;
1345         saa_client_data_t       *client_data;
1346         saa_port_t              *saa_portp;
1347         uint32_t                transport_flags;
1348         ibmf_msg_cb_t           ibmf_callback;
1349         void                    *ibmf_callback_arg;
1350         ibmf_msg_t              *msgp;
1351         ibmf_retrans_t          ibmf_retrans;
1352         uint16_t                sa_cap_mask;
1353         boolean_t               sleep_flag;
1354         int                     ibmf_status = IBMF_SUCCESS;
1355         int                     retry_count;
1356         uint16_t                mad_status;
1357         boolean_t               sa_is_redirected = B_FALSE;
1358 
1359         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1360             ibmf_saa_impl_send_request_start,
1361             IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() enter\n");
1362 
1363         attr_id = trans_info->si_trans_attr_id;
1364         client_data = trans_info->si_trans_client_data;
1365         saa_portp   = trans_info->si_trans_port;
1366 
1367         /*
1368          * don't send on invalid entry
1369          * Note that there is a window where it could become
1370          * invalid after this test is done, but we'd rely on ibmf errors...
1371          */
1372         if (ibmf_saa_is_valid(saa_portp, B_FALSE) == B_FALSE) {
1373 
1374                 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1,
1375                     ibmf_saa_impl_send_request,
1376                     IBMF_TNF_ERROR, "",
1377                     "ibmf_saa_impl_send_request: %s, hca_guid = %016"
1378                     PRIx64 ", port_guid = %016" PRIx64
1379                     ", number = %d\n",
1380                     tnf_string, msg, "sending on invalid port",
1381                     tnf_opaque, hca_guid,
1382                     saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid,
1383                     tnf_opaque, port_guid,
1384                     saa_portp->saa_pt_port_guid,
1385                     tnf_uint,   port,
1386                     saa_portp->saa_pt_ibmf_reginfo.ir_port_num);
1387 
1388                 ibmf_status = IBMF_REQ_INVALID;
1389                 goto bail;
1390         }
1391 
1392         /* check whether SA supports this attribute */
1393         mutex_enter(&saa_portp->saa_pt_mutex);
1394 
1395         sa_cap_mask = saa_portp->saa_pt_sa_cap_mask;
1396         sa_is_redirected = saa_portp->saa_pt_redirect_active;
1397 
1398         mutex_exit(&saa_portp->saa_pt_mutex);
1399 
1400         ibmf_status = ibmf_saa_impl_check_sa_support(sa_cap_mask, attr_id);
1401 
1402         if (ibmf_status != IBMF_SUCCESS) {
1403 
1404                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1405                     ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "",
1406                     "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n",
1407                     tnf_string, msg, "SA does not support attribute",
1408                     tnf_int, ibmf_status, ibmf_status);
1409 
1410                 goto bail;
1411         }
1412 
1413         /* make only non-blocking calls if this is an async request */
1414         if ((trans_info->si_trans_callback == NULL) &&
1415             (trans_info->si_trans_sub_callback == NULL)) {
1416                 ibmf_callback = NULL;
1417                 ibmf_callback_arg = NULL;
1418                 sleep_flag = B_TRUE;
1419         } else {
1420                 ibmf_callback = ibmf_saa_async_cb;
1421                 ibmf_callback_arg = (void *)trans_info;
1422                 sleep_flag = B_FALSE;
1423         }
1424 
1425         ibmf_status = ibmf_saa_impl_init_msg(trans_info, sleep_flag, &msgp,
1426             &transport_flags, &ibmf_retrans);
1427         if (ibmf_status != IBMF_SUCCESS) {
1428 
1429                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1430                     ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "",
1431                     "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n",
1432                     tnf_string, msg, "init_msg() failed",
1433                     tnf_int, ibmf_status, ibmf_status);
1434 
1435                 goto bail;
1436         }
1437 
1438         mutex_enter(&saa_portp->saa_pt_mutex);
1439 
1440         saa_portp->saa_pt_num_outstanding_trans++;
1441 
1442         mutex_exit(&saa_portp->saa_pt_mutex);
1443 
1444         /*
1445          * increment the number of outstanding transaction so
1446          * ibmf_close_sa_session() will wait.  classportinfo requests
1447          * don't have associated clients so check for valid clientp
1448          */
1449         if (client_data != NULL) {
1450 
1451                 mutex_enter(&client_data->saa_client_mutex);
1452 
1453                 client_data->saa_client_num_pending_trans++;
1454 
1455                 mutex_exit(&client_data->saa_client_mutex);
1456         }
1457 
1458         /*
1459          * make the call to msg_transport.  If synchronous and success,
1460          * check that the response mad isn't status busy.  If so, repeat the
1461          * call
1462          */
1463         retry_count = 0;
1464 
1465         /*
1466          * set the send time here. We only set this once at the beginning of
1467          * the transaction.  Retrying because of busys or mastersmlid changes
1468          * does not change the original send time.  It is meant to be an
1469          * absolute time out value and will only be used if there are other
1470          * problems (i.e. a buggy SA)
1471          */
1472         trans_info->si_trans_send_time = gethrtime();
1473 
1474         for (;;) {
1475 
1476                 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle,
1477                     saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
1478                     ibmf_callback, ibmf_callback_arg, transport_flags);
1479 
1480                 if (ibmf_callback != NULL)
1481                         break;
1482 
1483                 /*
1484                  * stop here for non-sequenced transactions since they wouldn't
1485                  * receive a timeout or busy response
1486                  */
1487                 if (!(transport_flags & IBMF_MSG_TRANS_FLAG_SEQ))
1488                         break;
1489 
1490                 /*
1491                  * if the transaction timed out and this was a synchronous
1492                  * request there's a possiblity we were talking to the wrong
1493                  * master smlid or that the SA has stopped responding on the
1494                  * redirected desination (if redirect is active).
1495                  * Check this and retry if necessary.
1496                  */
1497                 if ((ibmf_status == IBMF_TRANS_TIMEOUT) &&
1498                     (sleep_flag == B_TRUE)) {
1499                         if (sa_is_redirected == B_TRUE) {
1500                                 ibmf_status = ibmf_saa_impl_revert_to_qp1(
1501                                     saa_portp, msgp, ibmf_callback,
1502                                     ibmf_callback_arg, transport_flags);
1503                         } else {
1504                                 ibmf_status = ibmf_saa_impl_new_smlid_retry(
1505                                     saa_portp, msgp, ibmf_callback,
1506                                     ibmf_callback_arg, transport_flags);
1507                         }
1508                 }
1509 
1510                 /*
1511                  * if the transaction timed out (and retrying with a new SM LID
1512                  * didn't help) check how long it's been since we received an SA
1513                  * packet.  If it hasn't been max_wait_time then retry the
1514                  * request.
1515                  */
1516                 if ((ibmf_status == IBMF_TRANS_TIMEOUT) &&
1517                     (sleep_flag == B_TRUE)) {
1518 
1519                         ibmf_status = ibmf_saa_check_sa_and_retry(
1520                             saa_portp, msgp, ibmf_callback, ibmf_callback_arg,
1521                             trans_info->si_trans_send_time, transport_flags);
1522                 }
1523 
1524                 if (ibmf_status != IBMF_SUCCESS)
1525                         break;
1526 
1527                 if (retry_count >= IBMF_SAA_MAX_BUSY_RETRY_COUNT)
1528                         break;
1529 
1530                 /* sync transaction with status SUCCESS should have response */
1531                 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL);
1532 
1533                 mad_status = b2h16(msgp->im_msgbufs_recv.
1534                     im_bufs_mad_hdr->Status);
1535 
1536                 if ((mad_status != MAD_STATUS_BUSY) &&
1537                     (mad_status != MAD_STATUS_REDIRECT_REQUIRED))
1538                         break;
1539 
1540                 if (mad_status == MAD_STATUS_REDIRECT_REQUIRED) {
1541 
1542                         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1543                             ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1544                             "ibmf_saa_impl_send_request: %s, retry_count %d\n",
1545                             tnf_string, msg,
1546                             "response returned redirect status",
1547                             tnf_int, retry_count, retry_count);
1548 
1549                         /* update address info and copy it into msgp */
1550                         ibmf_saa_impl_update_sa_address_info(saa_portp, msgp);
1551                 } else {
1552                         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1553                             ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1554                             "ibmf_saa_impl_send_request: %s, retry_count %d\n",
1555                             tnf_string, msg, "response returned busy status",
1556                             tnf_int, retry_count, retry_count);
1557                 }
1558 
1559                 retry_count++;
1560 
1561                 /*
1562                  * since this is a blocking call, sleep for some time
1563                  * to allow SA to transition from busy state (if busy)
1564                  */
1565                 if (mad_status == MAD_STATUS_BUSY)
1566                         delay(drv_sectohz(IBMF_SAA_BUSY_RETRY_SLEEP_SECS));
1567         }
1568 
1569         if (ibmf_status != IBMF_SUCCESS) {
1570 
1571                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1572                     ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1573                     "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n",
1574                     tnf_string, msg, "ibmf_msg_transport() failed",
1575                     tnf_int, ibmf_status, ibmf_status);
1576 
1577                 ibmf_saa_impl_free_msg(saa_portp->saa_pt_ibmf_handle, msgp);
1578 
1579                 mutex_enter(&saa_portp->saa_pt_mutex);
1580 
1581                 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0);
1582                 saa_portp->saa_pt_num_outstanding_trans--;
1583 
1584                 mutex_exit(&saa_portp->saa_pt_mutex);
1585 
1586                 if (client_data != NULL) {
1587 
1588                         mutex_enter(&client_data->saa_client_mutex);
1589 
1590                         ASSERT(client_data->saa_client_num_pending_trans > 0);
1591                         client_data->saa_client_num_pending_trans--;
1592 
1593                         if ((client_data->saa_client_num_pending_trans == 0) &&
1594                             (client_data->saa_client_state ==
1595                             SAA_CLIENT_STATE_WAITING))
1596                                 cv_signal(&client_data->saa_client_state_cv);
1597 
1598                         mutex_exit(&client_data->saa_client_mutex);
1599                 }
1600 
1601         } else if (sleep_flag == B_TRUE) {
1602 
1603                 mutex_enter(&saa_portp->saa_pt_mutex);
1604 
1605                 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0);
1606                 saa_portp->saa_pt_num_outstanding_trans--;
1607 
1608                 mutex_exit(&saa_portp->saa_pt_mutex);
1609 
1610                 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1611                     ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1612                     "ibmf_saa_impl_send_request: %s\n",
1613                     tnf_string, msg, "Message sent and received successfully");
1614 
1615                 /* fill in response values and free the message */
1616                 ibmf_saa_impl_prepare_response(saa_portp->saa_pt_ibmf_handle,
1617                     msgp, B_FALSE, &trans_info->si_trans_status,
1618                     &trans_info->si_trans_result,
1619                     &trans_info->si_trans_length, sleep_flag);
1620 
1621                 if (client_data != NULL) {
1622                         mutex_enter(&client_data->saa_client_mutex);
1623 
1624                         ASSERT(client_data->saa_client_num_pending_trans > 0);
1625                         client_data->saa_client_num_pending_trans--;
1626 
1627                         if ((client_data->saa_client_num_pending_trans == 0) &&
1628                             (client_data->saa_client_state ==
1629                             SAA_CLIENT_STATE_WAITING))
1630                                 cv_signal(&client_data->saa_client_state_cv);
1631 
1632                         mutex_exit(&client_data->saa_client_mutex);
1633                 }
1634         } else {
1635 
1636                 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1637                     ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1638                     "ibmf_saa_impl_send_request: %s\n",
1639                     tnf_string, msg, "Message sent successfully");
1640         }
1641 
1642 bail:
1643         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1644             ibmf_saa_impl_send_request_end,
1645             IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() exiting"
1646             " ibmf_status = %d\n", tnf_int, result, ibmf_status);
1647 
1648         return (ibmf_status);
1649 }
1650 
1651 /*
1652  * ibmf_saa_impl_init_msg:
1653  * Allocates an ibmf message and fills out the header fields and formatted data
1654  * fields.  Also sets up the correct transport_flags and retrans argument for
1655  * the message transport call based on the request information.
1656  *
1657  * Input Arguments
1658  * trans_info           saa_trans_info structure passed to send_request
1659  * sleep_flag           B_TRUE if init_msg can sleep in function calls
1660  *
1661  * Output Arguments
1662  * msgp                 ibmf message that should be given to msg_transport
1663  * transport_flagsp     transport flags that should be given to msg_transport
1664  * ibmf_retrans_t       retrans parameter that should be given to msg_transport
1665  *
1666  * Returns
1667  * ibmf_status
1668  */
1669 static int
1670 ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, boolean_t sleep_flag,
1671     ibmf_msg_t **msgp, uint32_t *transport_flagsp,
1672     ibmf_retrans_t *ibmf_retransp)
1673 {
1674         int                     ibmf_status;
1675         ibmf_msg_bufs_t         *req_mad;
1676         ib_mad_hdr_t            *mad_hdr;
1677         int                     ibmf_sleep_flag, km_sleep_flag;
1678         int                     free_res;
1679         ib_sa_hdr_t             sa_hdr;
1680         ibmf_msg_t              *ibmf_msg;
1681         uint16_t                attr_id, pack_attr_id;
1682         uint8_t                 method;
1683         saa_client_data_t       *client_data;
1684         saa_port_t              *saa_portp;
1685         sa_multipath_record_t   *multipath_template;
1686         size_t                  payload_length;
1687         uint32_t                transport_flags;
1688 
1689         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1690             ibmf_saa_impl_init_msg_start,
1691             IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() entering\n");
1692 
1693         attr_id = trans_info->si_trans_attr_id;
1694         method = trans_info->si_trans_method;
1695         client_data = trans_info->si_trans_client_data;
1696         saa_portp   = trans_info->si_trans_port;
1697 
1698         if (sleep_flag == B_TRUE) {
1699                 ibmf_sleep_flag = IBMF_ALLOC_SLEEP;
1700                 km_sleep_flag = KM_SLEEP;
1701         } else {
1702                 ibmf_sleep_flag = IBMF_ALLOC_NOSLEEP;
1703                 km_sleep_flag = KM_NOSLEEP;
1704         }
1705 
1706         ibmf_status = ibmf_alloc_msg(saa_portp->saa_pt_ibmf_handle,
1707             ibmf_sleep_flag, &ibmf_msg);
1708         if (ibmf_status != IBMF_SUCCESS) {
1709 
1710                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1711                     ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1712                     "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n",
1713                     tnf_string, msg, "Cannot allocate msg_buf.",
1714                     tnf_int, ibmf_status, ibmf_status);
1715 
1716                 goto bail;
1717         }
1718 
1719         req_mad = &ibmf_msg->im_msgbufs_send;
1720 
1721         /* create a template (SA MAD) */
1722         mad_hdr = kmem_zalloc(sizeof (ib_mad_hdr_t), km_sleep_flag);
1723 
1724         if (mad_hdr == NULL) {
1725 
1726                 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
1727                     ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1728                     "ibmf_saa_impl_init_msg: %s\n",
1729                     tnf_string, msg, "Cannot allocate mad header.");
1730 
1731                 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1732                     &ibmf_msg);
1733                 ASSERT(free_res == IBMF_SUCCESS);
1734 
1735                 ibmf_status = IBMF_NO_MEMORY;
1736                 goto bail;
1737         }
1738 
1739         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mad_hdr,
1740             *req_mad))
1741 
1742         bzero(mad_hdr, sizeof (ib_mad_hdr_t));
1743         mad_hdr->BaseVersion = SAA_MAD_BASE_VERSION;
1744         mad_hdr->MgmtClass = MAD_MGMT_CLASS_SUBN_ADM;
1745         mad_hdr->ClassVersion = SAA_MAD_CLASS_VERSION;
1746         mad_hdr->R_Method = method;
1747         mad_hdr->AttributeID = h2b16(attr_id);
1748 
1749         /* attribute modifier is all Fs since RIDs are no longer used */
1750         mad_hdr->AttributeModifier = h2b32(0xffffffff);
1751 
1752         IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
1753             ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "",
1754             "ibmf_saa_impl_init_msg: %s, class = 0x%x, method = 0x%x,"
1755             " attr_id = 0x%x\n", tnf_string, msg, "Sending MAD",
1756             tnf_opaque, class, mad_hdr->MgmtClass,
1757             tnf_opaque, method, mad_hdr->R_Method,
1758             tnf_opaque, attr_id, attr_id);
1759 
1760         bzero(&sa_hdr, sizeof (ib_sa_hdr_t));
1761         sa_hdr.ComponentMask = trans_info->si_trans_component_mask;
1762 
1763         if (client_data != NULL)
1764                 sa_hdr.SM_KEY = client_data->saa_client_sm_key;
1765 
1766         /*
1767          * pack data for IB wire format; req_mad will have different pointers to
1768          * sa header and payload, mad_hdr will be the same
1769          */
1770         req_mad->im_bufs_mad_hdr = mad_hdr;
1771 
1772         ibmf_status = ibmf_saa_utils_pack_sa_hdr(&sa_hdr,
1773             &req_mad->im_bufs_cl_hdr, &req_mad->im_bufs_cl_hdr_len,
1774             km_sleep_flag);
1775 
1776         if (ibmf_status != IBMF_SUCCESS) {
1777 
1778                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1779                     ibmf_saa_impl_init_msg, IBMF_TNF_ERROR, "",
1780                     "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n",
1781                     tnf_string, msg, "ibmf_saa_utils_pack_sa_hdr() failed",
1782                     tnf_int, ibmf_status, ibmf_status);
1783 
1784                 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t));
1785 
1786                 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1787                     &ibmf_msg);
1788                 ASSERT(free_res == IBMF_SUCCESS);
1789 
1790                 goto bail;
1791         }
1792 
1793         if (attr_id == SA_MULTIPATHRECORD_ATTRID) {
1794 
1795                 multipath_template =
1796                     (sa_multipath_record_t *)trans_info->si_trans_template;
1797 
1798                 payload_length = sizeof (sa_multipath_record_t) +
1799                     ((multipath_template->SGIDCount +
1800                     multipath_template->DGIDCount) * sizeof (ib_gid_t));
1801 
1802                 pack_attr_id = attr_id;
1803         } else {
1804 
1805                 /* trace record template is a path record */
1806                 pack_attr_id = (attr_id == SA_TRACERECORD_ATTRID) ?
1807                     SA_PATHRECORD_ATTRID : attr_id;
1808 
1809                 payload_length = ibmf_saa_impl_get_attr_id_length(pack_attr_id);
1810 
1811                 if (payload_length == 0) {
1812                         payload_length = trans_info->si_trans_template_length;
1813 
1814                         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
1815                             ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "",
1816                             "ibmf_saa_impl_init_msg: %s, length = %d\n",
1817                             tnf_string, msg,
1818                             "Unknown attribute.  Using user-defined length.",
1819                             tnf_uint, length, payload_length)
1820                 }
1821         }
1822 
1823         /* transport type depends on method */
1824         switch (method) {
1825 
1826                 case SA_SUBN_ADM_GET:
1827                 case SA_SUBN_ADM_DELETE:
1828                 case SA_SUBN_ADM_GET_TABLE:
1829                 case SA_SUBN_ADM_GET_TRACE_TABLE:
1830                         transport_flags = IBMF_MSG_TRANS_FLAG_SEQ;
1831                         break;
1832                 case SA_SUBN_ADM_SET:
1833                         /* unsubscribes can be sequenced or unsequenced */
1834                         if (trans_info->si_trans_unseq_unsubscribe == B_TRUE) {
1835                                 transport_flags = 0;
1836                         } else {
1837                                 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ;
1838                         }
1839                         break;
1840                 case SA_SUBN_ADM_GET_MULTI:
1841                         transport_flags = IBMF_MSG_TRANS_FLAG_SEQ |
1842                             IBMF_MSG_TRANS_FLAG_RMPP;
1843                         break;
1844                 default :
1845                         ibmf_status = IBMF_UNSUPP_METHOD;
1846                         goto bail;
1847         }
1848 
1849         trans_info->si_trans_transport_flags = transport_flags;
1850 
1851         if (trans_info->si_trans_template != NULL) {
1852 
1853                 ibmf_status = ibmf_saa_utils_pack_payload(
1854                     trans_info->si_trans_template, payload_length, pack_attr_id,
1855                     &req_mad->im_bufs_cl_data, &req_mad->im_bufs_cl_data_len,
1856                     km_sleep_flag);
1857                 if (ibmf_status != IBMF_SUCCESS) {
1858 
1859                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1860                             ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1861                             "ibmf_saa_impl_init_msg: %s, ibmf_status ="
1862                             " %d\n", tnf_string, msg,
1863                             "ibmf_saa_utils_pack_payload() failed",
1864                             tnf_int, ibmf_status, ibmf_status);
1865 
1866                         kmem_free(mad_hdr, sizeof (ib_mad_hdr_t));
1867 
1868                         kmem_free(req_mad->im_bufs_cl_hdr,
1869                             req_mad->im_bufs_cl_hdr_len);
1870 
1871                         free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1872                             &ibmf_msg);
1873                         ASSERT(free_res == IBMF_SUCCESS);
1874 
1875                         goto bail;
1876                 }
1877 
1878                 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1879                     ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "",
1880                     "ibmf_saa_impl_init_msg: %s, attr_id = 0x%x, length ="
1881                     " %d\n", tnf_string, msg, "Packed payload successfully",
1882                     tnf_opaque, attr_id, attr_id,
1883                     tnf_uint, length, req_mad->im_bufs_cl_data_len);
1884 
1885                 /* non-RMPP transactions have template size limit */
1886                 if (((transport_flags & IBMF_MSG_TRANS_FLAG_RMPP) == 0) &&
1887                     ((req_mad->im_bufs_cl_data_len + req_mad->im_bufs_cl_hdr_len
1888                     + sizeof (ib_mad_hdr_t)) > IBMF_MAD_SIZE)) {
1889 
1890                         IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
1891                             ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1892                             "ibmf_saa_impl_init_msg: %s\n", tnf_string, msg,
1893                             "Template too large to fit in single packet");
1894 
1895                         kmem_free(mad_hdr, sizeof (ib_mad_hdr_t));
1896 
1897                         kmem_free(req_mad->im_bufs_cl_hdr,
1898                             req_mad->im_bufs_cl_hdr_len);
1899 
1900                         kmem_free(req_mad->im_bufs_cl_data,
1901                             req_mad->im_bufs_cl_data_len);
1902 
1903                         free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1904                             &ibmf_msg);
1905                         ASSERT(free_res == IBMF_SUCCESS);
1906 
1907                         ibmf_status = IBMF_REQ_INVALID;
1908                         goto bail;
1909                 }
1910         }
1911 
1912         mutex_enter(&saa_portp->saa_pt_mutex);
1913 
1914         mad_hdr->TransactionID = h2b64(saa_portp->saa_pt_current_tid++);
1915 
1916         bcopy(&saa_portp->saa_pt_ibmf_retrans, ibmf_retransp,
1917             sizeof (ibmf_retrans_t));
1918 
1919         /* copy local addressing information to message */
1920         bcopy(&saa_portp->saa_pt_ibmf_addr_info, &ibmf_msg->im_local_addr,
1921             sizeof (ibmf_addr_info_t));
1922 
1923         /* copy global addressing information to message if in use */
1924         if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) {
1925 
1926                 ibmf_msg->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS;
1927 
1928                 bcopy(&saa_portp->saa_pt_ibmf_global_addr,
1929                     &ibmf_msg->im_global_addr,
1930                     sizeof (ibmf_global_addr_info_t));
1931         } else {
1932                 ibmf_msg->im_msg_flags = 0;
1933         }
1934 
1935         mutex_exit(&saa_portp->saa_pt_mutex);
1936 
1937         *msgp = ibmf_msg;
1938         *transport_flagsp = transport_flags;
1939 bail:
1940         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1941             ibmf_saa_impl_init_msg_end,
1942             IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() exiting"
1943             " ibmf_status = %d\n", tnf_int, result, ibmf_status);
1944 
1945         return (ibmf_status);
1946 
1947 }
1948 
1949 /*
1950  * ibmf_saa_impl_new_smlid_retry:
1951  *
1952  * It's possible for the MasterSMLID to change while ibmf_saa is running.  The
1953  * MasterSMLID is set when we first register with ibmf_saa.  If a request
1954  * timesout, this function should be called to check whether the SM LID changed.
1955  * If so, it will call msg_transport again with the request.
1956  *
1957  * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the
1958  * same values passed to the original ibmf_msg_transport that timedout.  The
1959  * ibmf_retrans parameter will be re-retrieved from the saa_portp structure.
1960  *
1961  * If the lid did not change then this function returns IBMF_TRANS_TIMEOUT.
1962  * That way, callers can simply return the result of this function.
1963  *
1964  * Input Arguments
1965  * saa_portp            pointer to saa_port structure
1966  * msgp                 ibmf message that timedout
1967  * ibmf_callback        callback that should be called by msg_transport
1968  * ibmf_callback_arg    args for ibmf_callback
1969  * transport_flags      flags for ibmf_msg_transport
1970  *
1971  * Output Arguments
1972  * none
1973  *
1974  * Returns
1975  * IBMF_SUCCESS if lid changed and request was resent successfully,
1976  * IBMF_TRANS_TIMEOUT if lid did not change,
1977  * same values as ibmf_msg_transport() if lid changed but request could not be
1978  * resent.
1979  */
1980 static int
1981 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
1982     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags)
1983 {
1984         ibt_hca_portinfo_t      *ibt_portinfop;
1985         ib_lid_t                master_sm_lid;
1986         int                     subnet_timeout;
1987         uint_t                  nports, size;
1988         ibmf_retrans_t          ibmf_retrans;
1989         int                     ibmf_status;
1990         ibt_status_t            ibt_status;
1991 
1992         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1993             ibmf_saa_impl_new_smlid_retry_start,
1994             IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() enter\n");
1995 
1996         _NOTE(ASSUMING_PROTECTED(*msgp))
1997         _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr))
1998 
1999         /* first query the portinfo to see if the lid changed */
2000         ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid,
2001             saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size);
2002 
2003         if (ibt_status != IBT_SUCCESS)  {
2004 
2005                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2006                     ibmf_saa_impl_new_smlid_retry_err, IBMF_TNF_ERROR, "",
2007                     "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status ="
2008                     " %d\n", tnf_string, msg,
2009                     "ibt_query_hca_ports_byguid() failed",
2010                     tnf_int, ibt_status, ibt_status);
2011 
2012                 ibmf_status = IBMF_TRANSPORT_FAILURE;
2013 
2014                 goto bail;
2015         }
2016 
2017         master_sm_lid = ibt_portinfop->p_sm_lid;
2018         subnet_timeout = ibt_portinfop->p_subnet_timeout;
2019 
2020         ibt_free_portinfo(ibt_portinfop, size);
2021 
2022         /* if master smlid is different than the remote lid we sent to */
2023         if (master_sm_lid != msgp->im_local_addr.ia_remote_lid) {
2024 
2025                 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L2,
2026                     ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "",
2027                     "ibmf_saa_impl_new_smlid_retry: %s, new_lid 0x%x,"
2028                     " old_lid 0x%x\n", tnf_string, msg,
2029                     "master smlid has changed.  retrying msg_transport",
2030                     tnf_opaque, new_lid, master_sm_lid,
2031                     tnf_opaque, old_lid, msgp->im_local_addr.ia_remote_lid);
2032 
2033                 mutex_enter(&saa_portp->saa_pt_mutex);
2034 
2035                 /* update the master sm lid value in ibmf_saa */
2036                 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2037                     master_sm_lid;
2038 
2039                 /* new tid needed */
2040                 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID =
2041                     h2b64(saa_portp->saa_pt_current_tid++);
2042 
2043                 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2044                     sizeof (ibmf_retrans_t));
2045 
2046                 /* update the subnet timeout since this may be a new sm/sa */
2047                 saa_portp->saa_pt_timeout = subnet_timeout;
2048 
2049                 /* place upper bound on subnet timeout in case of faulty SM */
2050                 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout)
2051                         saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout;
2052 
2053                 /* increment the reference count to account for the cpi call */
2054                 saa_portp->saa_pt_reference_count++;
2055 
2056                 mutex_exit(&saa_portp->saa_pt_mutex);
2057 
2058                 /* update the remote lid for this particular message */
2059                 msgp->im_local_addr.ia_remote_lid = master_sm_lid;
2060 
2061                 /* get the classportinfo again since this may be a new sm/sa */
2062                 ibmf_saa_impl_get_classportinfo(saa_portp);
2063 
2064                 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle,
2065                     saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2066                     ibmf_callback, ibmf_callback_arg, transport_flags);
2067 
2068                 if (ibmf_status != IBMF_SUCCESS) {
2069 
2070                         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
2071                             ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "",
2072                             "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status = "
2073                             "%d\n", tnf_string, msg,
2074                             "ibmf_msg_transport() failed",
2075                             tnf_int, ibmf_status, ibmf_status);
2076                 }
2077 
2078                 goto bail;
2079         }
2080 
2081         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
2082             ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "",
2083             "ibmf_saa_impl_new_smlid_retry: %s, master_smlid = 0x%x\n",
2084             tnf_string, msg,
2085             "master smlid did not change.  returning failure",
2086             tnf_opaque, master_smlid, master_sm_lid);
2087 
2088         /* mark status as timeout since that was original failure */
2089         ibmf_status = IBMF_TRANS_TIMEOUT;
2090 
2091 bail:
2092         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2093             ibmf_saa_impl_new_smlid_retry_end,
2094             IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() exiting"
2095             " ibmf_status = %d\n", tnf_int, result, ibmf_status);
2096 
2097         return (ibmf_status);
2098 }
2099 
2100 /*
2101  * ibmf_saa_impl_revert_to_qp1()
2102  *
2103  * The SA that we had contact with via redirect may fail to respond. If this
2104  * occurs SA should revert back to qp1 and the SMLID set in the port.
2105  * msg_transport for the message that timed out will be retried with
2106  * these new parameters.
2107  *
2108  * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the
2109  * same values passed to the original ibmf_msg_transport that timedout.  The
2110  * ibmf_retrans parameter will be re-retrieved from the saa_portp structure.
2111  *
2112  * Input Arguments
2113  * saa_portp            pointer to saa_port structure
2114  * msgp                 ibmf message that timedout
2115  * ibmf_callback        callback that should be called by msg_transport
2116  * ibmf_callback_arg    args for ibmf_callback
2117  * transport_flags      flags for ibmf_msg_transport
2118  *
2119  * Output Arguments
2120  * none
2121  *
2122  * Returns
2123  * none
2124  */
2125 static int
2126 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp,
2127     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags)
2128 {
2129         ibt_hca_portinfo_t      *ibt_portinfop;
2130         ib_lid_t                master_sm_lid, base_lid;
2131         uint8_t                 sm_sl;
2132         int                     subnet_timeout;
2133         uint_t                  nports, size;
2134         ibmf_retrans_t          ibmf_retrans;
2135         int                     ibmf_status;
2136         ibt_status_t            ibt_status;
2137 
2138         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
2139             ibmf_saa_impl_revert_to_qp1_start,
2140             IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() enter\n");
2141 
2142         _NOTE(ASSUMING_PROTECTED(*msgp))
2143         _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr))
2144 
2145         /* first query the portinfo to see if the lid changed */
2146         ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid,
2147             saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size);
2148 
2149         if (ibt_status != IBT_SUCCESS)  {
2150 
2151                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2152                     ibmf_saa_impl_revert_to_qp1_err, IBMF_TNF_ERROR, "",
2153                     "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status ="
2154                     " %d\n", tnf_string, msg,
2155                     "ibt_query_hca_ports_byguid() failed",
2156                     tnf_int, ibt_status, ibt_status);
2157 
2158                 ibmf_status = IBMF_TRANSPORT_FAILURE;
2159 
2160                 goto bail;
2161         }
2162 
2163         master_sm_lid = ibt_portinfop->p_sm_lid;
2164         base_lid = ibt_portinfop->p_base_lid;
2165         sm_sl = ibt_portinfop->p_sm_sl;
2166         subnet_timeout = ibt_portinfop->p_subnet_timeout;
2167 
2168         ibt_free_portinfo(ibt_portinfop, size);
2169 
2170 
2171         mutex_enter(&saa_portp->saa_pt_mutex);
2172 
2173         saa_portp->saa_pt_redirect_active = B_FALSE;
2174 
2175         /* update the address info in ibmf_saa */
2176         saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = base_lid;
2177         saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = master_sm_lid;
2178         saa_portp->saa_pt_ibmf_addr_info.ia_service_level = sm_sl;
2179         saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1;
2180         saa_portp->saa_pt_ibmf_addr_info.ia_p_key = IB_PKEY_DEFAULT_LIMITED;
2181         saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY;
2182         saa_portp->saa_pt_ibmf_msg_flags = 0;
2183 
2184         /* new tid needed */
2185         msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID =
2186             h2b64(saa_portp->saa_pt_current_tid++);
2187 
2188         bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2189             sizeof (ibmf_retrans_t));
2190 
2191         /* update the subnet timeout since this may be a new sm/sa */
2192         saa_portp->saa_pt_timeout = subnet_timeout;
2193 
2194         /* place upper bound on subnet timeout in case of faulty SM */
2195         if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout)
2196                 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout;
2197 
2198         /* increment the reference count to account for the cpi call */
2199         saa_portp->saa_pt_reference_count++;
2200 
2201         mutex_exit(&saa_portp->saa_pt_mutex);
2202 
2203         /* update the address info for this particular message */
2204         bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr,
2205             sizeof (ibmf_addr_info_t));
2206         msgp->im_msg_flags = 0; /* No GRH */
2207 
2208         /* get the classportinfo again since this may be a new sm/sa */
2209         ibmf_saa_impl_get_classportinfo(saa_portp);
2210 
2211         ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle,
2212             saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2213             ibmf_callback, ibmf_callback_args, transport_flags);
2214 
2215         if (ibmf_status != IBMF_SUCCESS) {
2216 
2217                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
2218                     ibmf_saa_impl_revert_to_qp1, IBMF_TNF_TRACE, "",
2219                     "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status = "
2220                     "%d\n", tnf_string, msg,
2221                     "ibmf_msg_transport() failed",
2222                     tnf_int, ibmf_status, ibmf_status);
2223         }
2224 
2225 bail:
2226 
2227         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2228             ibmf_saa_impl_revert_to_qp1_end,
2229             IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() exiting"
2230             " ibmf_status = %d\n", tnf_int, result, ibmf_status);
2231 
2232         return (ibmf_status);
2233 }
2234 
2235 /*
2236  * ibmf_saa_impl_async_event_cb:
2237  *      ibmf event callback, argument to ibmf_register
2238  *      ibmf_handle is unused
2239  */
2240 /*  ARGSUSED */
2241 static void
2242 ibmf_saa_impl_async_event_cb(
2243         ibmf_handle_t           ibmf_handle,
2244         void                    *clnt_private,
2245         ibmf_async_event_t      event_type)
2246 {
2247         saa_port_t              *saa_portp;
2248 
2249         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2250             ibmf_saa_impl_async_event_cb_start, IBMF_TNF_TRACE, "",
2251             "ibmf_saa_impl_async_event_cb: Handling event type 0x%x\n",
2252             tnf_opaque, event_type, event_type);
2253 
2254         saa_portp = (saa_port_t *)clnt_private;
2255         ASSERT(saa_portp != NULL);
2256 
2257         switch (event_type) {
2258 
2259         case IBMF_CI_OFFLINE:
2260                 ibmf_saa_impl_hca_detach(saa_portp);
2261                 break;
2262         default:
2263                 break;
2264         }
2265 
2266         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3,
2267             ibmf_saa_impl_async_event_cb_end,
2268             IBMF_TNF_TRACE, "", "ibmf_saa_impl_async_event_cb() exit\n");
2269 }
2270 
2271 
2272 /*
2273  * ibmf_saa_impl_ibt_async_handler:
2274  * MUST NOT BE STATIC (referred from within IBMF)
2275  */
2276 void
2277 ibmf_saa_impl_ibt_async_handler(ibt_async_code_t code, ibt_async_event_t *event)
2278 {
2279         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2280             ibmf_saa_impl_ibt_async_handler_start, IBMF_TNF_TRACE, "",
2281             "ibmf_saa_impl_ibt_async_handler: Handling event code 0x%x\n",
2282             tnf_opaque, code, code);
2283 
2284         switch (code) {
2285 
2286         case IBT_EVENT_PORT_UP:
2287                 ibmf_saa_impl_port_up(event->ev_hca_guid, event->ev_port);
2288                 break;
2289         case IBT_ERROR_PORT_DOWN:
2290                 ibmf_saa_impl_port_down(event->ev_hca_guid, event->ev_port);
2291                 break;
2292         case IBT_PORT_CHANGE_EVENT:
2293                 ibmf_saa_impl_port_chg(event);
2294                 break;
2295         case IBT_CLNT_REREG_EVENT:
2296                 ibmf_saa_impl_client_rereg(event->ev_hca_guid, event->ev_port);
2297                 break;
2298         default:
2299                 break;
2300         }
2301 
2302         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_async_handler_end,
2303             IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibt_async_handler() exit\n");
2304 }
2305 
2306 /*
2307  * ibmf_saa_impl_port_chg:
2308  */
2309 static void
2310 ibmf_saa_impl_port_chg(ibt_async_event_t *event)
2311 {
2312         saa_port_t              *saa_portp      = NULL;
2313         boolean_t               is_ready = B_FALSE;
2314         ibt_hca_portinfo_t      *ibt_portinfop;
2315         uint_t                  nports, size;
2316         ibt_status_t            ibt_status;
2317         ib_guid_t               ci_guid;
2318         int                     port_num;
2319 
2320         ci_guid = event->ev_hca_guid;
2321         port_num = event->ev_port;
2322 
2323         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_chg_start,
2324             IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg: Handling port chg"
2325             " guid %016" PRIx64 " port %d\n",
2326             tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2327 
2328         /* Get classportinfo of corresponding entry */
2329         mutex_enter(&saa_statep->saa_port_list_mutex);
2330 
2331         saa_portp = saa_statep->saa_port_list;
2332         while (saa_portp != NULL) {
2333                 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
2334                     saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
2335                         mutex_enter(&saa_portp->saa_pt_mutex);
2336 
2337                         is_ready = (saa_portp->saa_pt_state
2338                             == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
2339 
2340                         /*
2341                          * increment reference count to account for cpi and
2342                          * informinfos.  All 4 informinfo's sent are treated as
2343                          * one port client reference
2344                          */
2345                         if (is_ready)
2346                                 saa_portp->saa_pt_reference_count ++;
2347 
2348                         mutex_exit(&saa_portp->saa_pt_mutex);
2349 
2350                         if (is_ready)
2351                                 break; /* normally, only 1 port entry */
2352                 }
2353                 saa_portp = saa_portp->next;
2354         }
2355 
2356         mutex_exit(&saa_statep->saa_port_list_mutex);
2357 
2358         if (saa_portp != NULL) {
2359                 /* first query the portinfo to see if the lid changed */
2360                 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
2361                     &ibt_portinfop, &nports, &size);
2362 
2363                 if (ibt_status != IBT_SUCCESS) {
2364 
2365                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2366                             ibmf_saa_impl_port_chg_err, IBMF_TNF_ERROR, "",
2367                             "ibmf_saa_impl_port_chg: %s, ibmf_status ="
2368                             " %d\n", tnf_string, msg,
2369                             "ibt_query_hca_ports_byguid() failed",
2370                             tnf_int, ibt_status, ibt_status);
2371 
2372                         goto bail;
2373                 }
2374 
2375                 mutex_enter(&saa_portp->saa_pt_mutex);
2376                 if (event->ev_port_flags & IBT_PORT_CHANGE_SM_LID) {
2377                         /* update the Master SM Lid value in ibmf_saa */
2378                         saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2379                             ibt_portinfop->p_sm_lid;
2380                 }
2381                 if (event->ev_port_flags & IBT_PORT_CHANGE_SM_SL) {
2382                         /* update the Master SM SL value in ibmf_saa */
2383                         saa_portp->saa_pt_ibmf_addr_info.ia_service_level =
2384                             ibt_portinfop->p_sm_sl;
2385                 }
2386                 if (event->ev_port_flags & IBT_PORT_CHANGE_SUB_TIMEOUT) {
2387                         /* update the Subnet timeout value in ibmf_saa */
2388                         saa_portp->saa_pt_timeout =
2389                             ibt_portinfop->p_subnet_timeout;
2390                 }
2391                 mutex_exit(&saa_portp->saa_pt_mutex);
2392 
2393                 ibt_free_portinfo(ibt_portinfop, size);
2394 
2395                 /* get the classportinfo again */
2396                 ibmf_saa_impl_get_classportinfo(saa_portp);
2397         }
2398 bail:
2399 
2400         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_chg_end,
2401             IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg() exit\n");
2402 }
2403 /*
2404  * ibmf_saa_impl_client_rereg:
2405  */
2406 static void
2407 ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num)
2408 {
2409         saa_port_t              *saa_portp      = NULL;
2410         boolean_t               is_ready = B_FALSE;
2411         ibt_hca_portinfo_t      *ibt_portinfop;
2412         ib_lid_t                master_sm_lid;
2413         uint_t                  nports, size;
2414         ibt_status_t            ibt_status;
2415         boolean_t               event_subs = B_FALSE;
2416 
2417         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_rereg_start,
2418             IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg: Handling clnt "
2419             "rereg guid %016" PRIx64 " port %d\n",
2420             tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2421 
2422         /* Get classportinfo of corresponding entry */
2423         mutex_enter(&saa_statep->saa_port_list_mutex);
2424 
2425         saa_portp = saa_statep->saa_port_list;
2426         while (saa_portp != NULL) {
2427 
2428                 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
2429                     saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
2430 
2431                         mutex_enter(&saa_portp->saa_pt_mutex);
2432 
2433                         is_ready = (saa_portp->saa_pt_state
2434                             == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
2435 
2436                         /*
2437                          * increment reference count to account for cpi and
2438                          * informinfos.  All 4 informinfo's sent are treated as
2439                          * one port client reference
2440                          */
2441                         if (is_ready)
2442                                 saa_portp->saa_pt_reference_count += 2;
2443 
2444                         mutex_exit(&saa_portp->saa_pt_mutex);
2445 
2446                         if (is_ready)
2447                                 break; /* normally, only 1 port entry */
2448                 }
2449                 saa_portp = saa_portp->next;
2450         }
2451 
2452         mutex_exit(&saa_statep->saa_port_list_mutex);
2453 
2454         if (saa_portp != NULL && is_ready == B_TRUE) {
2455 
2456                 /* verify whether master sm lid changed */
2457 
2458                 /* first query the portinfo to see if the lid changed */
2459                 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
2460                     &ibt_portinfop, &nports, &size);
2461 
2462                 if (ibt_status != IBT_SUCCESS) {
2463 
2464                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2465                             ibmf_saa_impl_port_rereg_err, IBMF_TNF_ERROR, "",
2466                             "ibmf_saa_impl_client_rereg: %s, ibmf_status ="
2467                             " %d\n", tnf_string, msg,
2468                             "ibt_query_hca_ports_byguid() failed",
2469                             tnf_int, ibt_status, ibt_status);
2470 
2471                         goto bail;
2472                 }
2473 
2474                 master_sm_lid = ibt_portinfop->p_sm_lid;
2475 
2476                 ibt_free_portinfo(ibt_portinfop, size);
2477 
2478                 /* check whether we need to subscribe for events */
2479                 mutex_enter(&saa_portp->saa_pt_event_sub_mutex);
2480 
2481                 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ?
2482                     B_TRUE : B_FALSE;
2483 
2484                 mutex_exit(&saa_portp->saa_pt_event_sub_mutex);
2485 
2486                 /* update the master smlid */
2487                 mutex_enter(&saa_portp->saa_pt_mutex);
2488 
2489                 /* update the master sm lid value in ibmf_saa */
2490                 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2491                     master_sm_lid;
2492 
2493                 /* if we're not subscribed for events, dec reference count */
2494                 if (event_subs == B_FALSE)
2495                         saa_portp->saa_pt_reference_count--;
2496 
2497                 mutex_exit(&saa_portp->saa_pt_mutex);
2498 
2499                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
2500                     ibmf_saa_impl_port_rereg, IBMF_TNF_TRACE, "",
2501                     "ibmf_saa_impl_client_rereg: %s, master_sm_lid = 0x%x\n",
2502                     tnf_string, msg,
2503                     "port is up.  Sending classportinfo request",
2504                     tnf_opaque, master_sm_lid, master_sm_lid);
2505 
2506                 /* get the classportinfo again */
2507                 ibmf_saa_impl_get_classportinfo(saa_portp);
2508 
2509                 /*
2510                  * resubscribe to events if there are subscribers since SA may
2511                  * have removed our subscription records when the port went down
2512                  */
2513                 if (event_subs == B_TRUE)
2514                         ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE);
2515         }
2516 
2517 bail:
2518 
2519         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_rereg_end,
2520             IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg() exit\n");
2521 }
2522 /*
2523  * ibmf_saa_impl_port_up:
2524  */
2525 static void
2526 ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num)
2527 {
2528         saa_port_t              *saa_portp      = NULL;
2529         int                     is_ready;
2530         ibt_hca_portinfo_t      *ibt_portinfop;
2531         ib_lid_t                master_sm_lid;
2532         uint_t                  nports, size;
2533         ibt_status_t            ibt_status;
2534         boolean_t               event_subs = B_FALSE;
2535 
2536         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_up_start,
2537             IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up: Handling port up"
2538             " guid %016" PRIx64 " port %d\n",
2539             tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2540 
2541         /* Get classportinfo of corresponding entry */
2542         mutex_enter(&saa_statep->saa_port_list_mutex);
2543 
2544         saa_portp = saa_statep->saa_port_list;
2545         while (saa_portp != NULL) {
2546 
2547                 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
2548                     saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
2549 
2550                         mutex_enter(&saa_portp->saa_pt_mutex);
2551 
2552                         is_ready = (saa_portp->saa_pt_state
2553                             == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
2554 
2555                         /*
2556                          * increment reference count to account for cpi and
2557                          * informinfos.  All 4 informinfo's sent are treated as
2558                          * one port client reference
2559                          */
2560                         if (is_ready == B_TRUE)
2561                                 saa_portp->saa_pt_reference_count += 2;
2562 
2563                         mutex_exit(&saa_portp->saa_pt_mutex);
2564 
2565                         if (is_ready == B_TRUE)
2566                                 break; /* normally, only 1 port entry */
2567                 }
2568                 saa_portp = saa_portp->next;
2569         }
2570 
2571         mutex_exit(&saa_statep->saa_port_list_mutex);
2572 
2573         if (saa_portp != NULL && is_ready == B_TRUE) {
2574 
2575                 /* verify whether master sm lid changed */
2576 
2577                 /* first query the portinfo to see if the lid changed */
2578                 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
2579                     &ibt_portinfop, &nports, &size);
2580 
2581                 if (ibt_status != IBT_SUCCESS) {
2582 
2583                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2584                             ibmf_saa_impl_port_up_err, IBMF_TNF_ERROR, "",
2585                             "ibmf_saa_impl_port_up: %s, ibmf_status ="
2586                             " %d\n", tnf_string, msg,
2587                             "ibt_query_hca_ports_byguid() failed",
2588                             tnf_int, ibt_status, ibt_status);
2589 
2590                         goto bail;
2591                 }
2592 
2593                 master_sm_lid = ibt_portinfop->p_sm_lid;
2594 
2595                 ibt_free_portinfo(ibt_portinfop, size);
2596 
2597                 /* check whether we need to subscribe for events */
2598                 mutex_enter(&saa_portp->saa_pt_event_sub_mutex);
2599 
2600                 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ?
2601                     B_TRUE : B_FALSE;
2602 
2603                 mutex_exit(&saa_portp->saa_pt_event_sub_mutex);
2604 
2605                 /* update the master smlid */
2606                 mutex_enter(&saa_portp->saa_pt_mutex);
2607 
2608                 /* update the master sm lid value in ibmf_saa */
2609                 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2610                     master_sm_lid;
2611 
2612                 /* if we're not subscribed for events, dec reference count */
2613                 if (event_subs == B_FALSE)
2614                         saa_portp->saa_pt_reference_count--;
2615 
2616                 mutex_exit(&saa_portp->saa_pt_mutex);
2617 
2618                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
2619                     ibmf_saa_impl_port_up, IBMF_TNF_TRACE, "",
2620                     "ibmf_saa_impl_port_up: %s, master_sm_lid = 0x%x\n",
2621                     tnf_string, msg,
2622                     "port is up.  Sending classportinfo request",
2623                     tnf_opaque, master_sm_lid, master_sm_lid);
2624 
2625                 /* get the classportinfo again */
2626                 ibmf_saa_impl_get_classportinfo(saa_portp);
2627 
2628                 /*
2629                  * resubscribe to events if there are subscribers since SA may
2630                  * have removed our subscription records when the port went down
2631                  */
2632                 if (event_subs == B_TRUE)
2633                         ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE);
2634         }
2635 
2636 bail:
2637 
2638         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_up_end,
2639             IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up() exit\n");
2640 }
2641 
2642 /*
2643  * ibmf_saa_impl_port_down:
2644  */
2645 static void
2646 ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num)
2647 {
2648 
2649         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_down_start,
2650             IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down: Handling port down"
2651             " guid %016" PRIx64 " port %d\n",
2652             tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2653 
2654 
2655         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_down_end,
2656             IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down() exit\n");
2657 }
2658 
2659 /*
2660  * ibmf_saa_impl_hca_detach:
2661  * find entry, unregister if there are no clients
2662  * have to unregister since ibmf needs to close the hca and will only do this if
2663  * no clients are registered
2664  */
2665 static void
2666 ibmf_saa_impl_hca_detach(saa_port_t *saa_removed)
2667 {
2668         saa_port_t      *saa_portp;
2669         boolean_t       must_unreg, must_unsub;
2670 
2671         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_start,
2672             IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: Detaching"
2673             " entry %016" PRIx64 "\n", tnf_opaque, entry, saa_removed);
2674 
2675         /* find this entry */
2676         mutex_enter(&saa_statep->saa_port_list_mutex);
2677 
2678         saa_portp = saa_statep->saa_port_list;
2679         while (saa_portp != NULL) {
2680 
2681                 if (saa_portp == saa_removed)
2682                         break;
2683 
2684                 saa_portp = saa_portp->next;
2685         }
2686         mutex_exit(&saa_statep->saa_port_list_mutex);
2687 
2688         ASSERT(saa_portp != NULL);
2689 
2690         if (saa_portp == NULL) {
2691 
2692                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
2693                     ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "",
2694                     "ibmf_saa_impl_hca_detach: %s, entry %016"
2695                     PRIx64 "\n",
2696                     tnf_string, msg,
2697                     "Port entry NOT found",
2698                     tnf_opaque, entryp, saa_removed);
2699 
2700                 goto bail;
2701         }
2702 
2703         /* if there are clients expecting Reports(), unsusbscribe */
2704         mutex_enter(&saa_portp->saa_pt_event_sub_mutex);
2705 
2706         must_unsub = (saa_portp->saa_pt_event_sub_client_list != NULL) ?
2707             B_TRUE : B_FALSE;
2708 
2709         mutex_exit(&saa_portp->saa_pt_event_sub_mutex);
2710 
2711         /* fail if outstanding transactions */
2712         mutex_enter(&saa_portp->saa_pt_mutex);
2713 
2714         if (saa_portp->saa_pt_num_outstanding_trans > 0) {
2715 
2716                 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L1,
2717                     ibmf_saa_impl_fini_err, IBMF_TNF_TRACE, "",
2718                     "ibmf_saa_impl_fini: %s, port = %016" PRIx64
2719                     ", num transactions = %d\n",
2720                     tnf_string, msg, "Detaching HCA."
2721                     "  Outstanding transactions on port.",
2722                     tnf_opaque, port,
2723                     saa_portp->saa_pt_port_guid,
2724                     tnf_uint, outstanding_transactions,
2725                     saa_portp->saa_pt_num_outstanding_trans);
2726 
2727                 mutex_exit(&saa_portp->saa_pt_mutex);
2728 
2729                 goto bail;
2730         }
2731 
2732 
2733         /*
2734          * increment reference count by one to account for unsubscribe requests
2735          * that are about to be sent.  All four informinfo's are treated as one
2736          * port client reference.  The count will be decremented by
2737          * subscribe_events() before the call returns.
2738          */
2739         if (must_unsub == B_TRUE)
2740                 saa_portp->saa_pt_reference_count++;
2741 
2742         mutex_exit(&saa_portp->saa_pt_mutex);
2743 
2744         /*
2745          * try and unsubscribe from SA.  Generate synchronous, unsequenced
2746          * unsubscribe requests.
2747          */
2748         if (must_unsub == B_TRUE)
2749                 ibmf_saa_subscribe_events(saa_portp, B_FALSE, B_TRUE);
2750 
2751         /* warning if registered clients */
2752         mutex_enter(&saa_portp->saa_pt_mutex);
2753 
2754         if (saa_portp->saa_pt_reference_count > 0) {
2755 
2756                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
2757                     ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "",
2758                     "ibmf_saa_impl_hca_detach: %s, port %016"
2759                     PRIx64 "\n",
2760                     tnf_string, msg,
2761                     "Detaching HCA for port with clients still"
2762                     " registered", tnf_opaque, port,
2763                     saa_portp->saa_pt_port_guid);
2764         }
2765 
2766         /* synchronize on end of registration */
2767         while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) {
2768 
2769                 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1,
2770                     ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "",
2771                     "ibmf_saa_impl_hca_detach: %s\n",
2772                     tnf_string, msg, "someone is registering. waiting"
2773                     " for them to finish");
2774 
2775                 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv,
2776                     &saa_portp->saa_pt_mutex);
2777 
2778                 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1,
2779                     ibmf_saa_impl_hca_detach,
2780                     IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: %s\n",
2781                     tnf_string, msg, "done waiting");
2782         }
2783 
2784         /* unregister from ibmf */
2785         if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) {
2786                 must_unreg = B_TRUE;
2787         } else
2788                 must_unreg = B_FALSE;
2789 
2790         ibmf_saa_impl_invalidate_port(saa_portp);
2791 
2792         mutex_exit(&saa_portp->saa_pt_mutex);
2793 
2794         if (must_unreg == B_TRUE) {
2795                 if (ibmf_saa_impl_ibmf_unreg(saa_portp) != IBMF_SUCCESS) {
2796                         mutex_enter(&saa_portp->saa_pt_mutex);
2797                         mutex_enter(&saa_portp->saa_pt_kstat_mutex);
2798                         (void) ibmf_saa_impl_init_kstats(saa_portp);
2799                         mutex_exit(&saa_portp->saa_pt_kstat_mutex);
2800                         saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_READY;
2801                         if (must_unsub == B_TRUE)
2802                                 saa_portp->saa_pt_reference_count++;
2803                         mutex_exit(&saa_portp->saa_pt_mutex);
2804 
2805                         if (must_unsub == B_TRUE) {
2806                                 ibmf_saa_subscribe_events(saa_portp, B_TRUE,
2807                                     B_FALSE);
2808                         }
2809                 }
2810         }
2811 bail:
2812         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_end,
2813             IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach() exit\n");
2814 }
2815 
2816 /* ARGSUSED */
2817 void
2818 ibmf_saa_async_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, void *args)
2819 {
2820         saa_impl_trans_info_t   *trans_info;
2821         int                     status;
2822         size_t                  length;
2823         void                    *result;
2824         saa_port_t              *saa_portp;
2825         saa_client_data_t       *client_data;
2826         int                     ibmf_status;
2827         boolean_t               ignore_data;
2828         ibmf_retrans_t          ibmf_retrans;
2829         boolean_t               sa_is_redirected = B_FALSE;
2830 
2831         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_start,
2832             IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() enter\n");
2833 
2834         trans_info = (saa_impl_trans_info_t *)args;
2835 
2836         client_data = trans_info->si_trans_client_data;
2837         saa_portp   = trans_info->si_trans_port;
2838 
2839         mutex_enter(&saa_portp->saa_pt_mutex);
2840         sa_is_redirected = saa_portp->saa_pt_redirect_active;
2841         mutex_exit(&saa_portp->saa_pt_mutex);
2842 
2843         if ((msgp->im_msg_status == IBMF_TRANS_TIMEOUT) &&
2844             (sa_is_redirected == B_TRUE)) {
2845 
2846                 /*
2847                  * We should retry the request using SM_LID and QP1 if we
2848                  * have been using redirect up until now
2849                  */
2850                 ibmf_status = ibmf_saa_impl_revert_to_qp1(
2851                     saa_portp, msgp, ibmf_saa_async_cb, args,
2852                     trans_info->si_trans_transport_flags);
2853 
2854                 /*
2855                  * If revert_to_qp1 returns success msg was resent.
2856                  * Otherwise msg could not be resent. Continue normally
2857                  */
2858                 if (ibmf_status == IBMF_SUCCESS)
2859                         goto bail;
2860 
2861         } else if (msgp->im_msg_status == IBMF_TRANS_TIMEOUT) {
2862 
2863 
2864                 ibmf_status = ibmf_saa_impl_new_smlid_retry(saa_portp, msgp,
2865                     ibmf_saa_async_cb, args,
2866                     trans_info->si_trans_transport_flags);
2867 
2868                 /*
2869                  * if smlid_retry() returns success sm lid changed and msg
2870                  * was resent.  Otherwise, lid did not change or msg could not
2871                  * be resent.  Continue normally.
2872                  */
2873                 if (ibmf_status == IBMF_SUCCESS)
2874                         goto bail;
2875 
2876                 /*
2877                  * check whether we've received anything from the SA in a while.
2878                  * If we have, this function will retry and return success.  If
2879                  * we haven't continue normally so that we return a timeout to
2880                  * the client
2881                  */
2882                 ibmf_status = ibmf_saa_check_sa_and_retry(
2883                     saa_portp, msgp, ibmf_saa_async_cb, args,
2884                     trans_info->si_trans_send_time,
2885                     trans_info->si_trans_transport_flags);
2886 
2887                 if (ibmf_status == IBMF_SUCCESS)
2888                         goto bail;
2889         }
2890 
2891         /*
2892          * If SA returned success but mad status is busy, retry a few times.
2893          * If SA returned success but mad status says redirect is required,
2894          * update the address info and retry the request to the new SA address
2895          */
2896         if (msgp->im_msg_status == IBMF_SUCCESS) {
2897 
2898                 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL);
2899 
2900                 if ((b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status) ==
2901                     MAD_STATUS_BUSY) &&
2902                     (trans_info->si_trans_retry_busy_count <
2903                     IBMF_SAA_MAX_BUSY_RETRY_COUNT)) {
2904 
2905                         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
2906                             ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
2907                             "ibmf_saa_async_cb: %s, retry_count = %d\n",
2908                             tnf_string, msg,
2909                             "async response returned busy status",
2910                             tnf_int, retry_count,
2911                             trans_info->si_trans_retry_busy_count);
2912 
2913                         trans_info->si_trans_retry_busy_count++;
2914 
2915                         bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2916                             sizeof (ibmf_retrans_t));
2917 
2918                         ibmf_status = ibmf_msg_transport(
2919                             saa_portp->saa_pt_ibmf_handle,
2920                             saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2921                             ibmf_saa_async_cb, args,
2922                             trans_info->si_trans_transport_flags);
2923 
2924                         /*
2925                          * if retry is successful, quit here since async_cb will
2926                          * get called again; otherwise, let this function call
2927                          * handle the cleanup
2928                          */
2929                         if (ibmf_status == IBMF_SUCCESS)
2930                                 goto bail;
2931                 } else if (b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status)
2932                     == MAD_STATUS_REDIRECT_REQUIRED) {
2933 
2934                         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L2,
2935                             ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
2936                             "ibmf_saa_async_cb: "
2937                             "async response returned redirect status\n");
2938 
2939                         /* update address info and copy it into msgp */
2940                         ibmf_saa_impl_update_sa_address_info(saa_portp, msgp);
2941 
2942                         /* retry with new address info */
2943                         bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2944                             sizeof (ibmf_retrans_t));
2945 
2946                         ibmf_status = ibmf_msg_transport(
2947                             saa_portp->saa_pt_ibmf_handle,
2948                             saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2949                             ibmf_saa_async_cb, args,
2950                             trans_info->si_trans_transport_flags);
2951 
2952                         /*
2953                          * if retry is successful, quit here since async_cb will
2954                          * get called again; otherwise, let this function call
2955                          * handle the cleanup
2956                          */
2957                         if (ibmf_status == IBMF_SUCCESS)
2958                                 goto bail;
2959                 }
2960         }
2961 
2962         mutex_enter(&saa_portp->saa_pt_mutex);
2963 
2964         ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0);
2965         saa_portp->saa_pt_num_outstanding_trans--;
2966 
2967         mutex_exit(&saa_portp->saa_pt_mutex);
2968 
2969         if ((trans_info->si_trans_callback == NULL) &&
2970             (trans_info->si_trans_sub_callback == NULL))
2971                 ignore_data = B_TRUE;
2972         else
2973                 ignore_data = B_FALSE;
2974 
2975         ibmf_saa_impl_prepare_response(ibmf_handle, msgp, ignore_data, &status,
2976             &result, &length, B_FALSE);
2977 
2978         mutex_enter(&saa_portp->saa_pt_kstat_mutex);
2979 
2980         IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1);
2981 
2982         if (status != IBMF_SUCCESS)
2983                 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1);
2984 
2985         if (status == IBMF_TRANS_TIMEOUT)
2986                 IBMF_SAA_ADD32_KSTATS(saa_portp, requests_timedout, 1);
2987 
2988         mutex_exit(&saa_portp->saa_pt_kstat_mutex);
2989 
2990         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2991             ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
2992             "ibmf_saa_async_cb: %s\n", tnf_string, msg,
2993             "Calling ibmf_saa client's callback");
2994 
2995         /*
2996          * there are three classes or trans_info users: ibmf_saa clients and
2997          * classportinfo requests; informinfo subscribe requests, and report
2998          * responses.  For the first two, call the correct callback.  For report
2999          * responses there's no need to notify anyone.
3000          */
3001         if (trans_info->si_trans_callback != NULL) {
3002                 /* ibmf_saa client or classportinfo request */
3003                 trans_info->si_trans_callback(trans_info->si_trans_callback_arg,
3004                     length, result, status);
3005         } else if (trans_info->si_trans_sub_callback != NULL) {
3006                 /* informinfo subscribe request */
3007                 trans_info->si_trans_sub_callback(
3008                     trans_info->si_trans_callback_arg, length, result, status,
3009                     trans_info->si_trans_sub_producer_type);
3010         }
3011 
3012         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3013             ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
3014             "ibmf_saa_async_cb: %s\n", tnf_string, msg,
3015             "Returned from callback");
3016 
3017         if (client_data != NULL) {
3018                 mutex_enter(&client_data->saa_client_mutex);
3019 
3020                 ASSERT(client_data->saa_client_num_pending_trans > 0);
3021                 client_data->saa_client_num_pending_trans--;
3022 
3023                 if ((client_data->saa_client_num_pending_trans == 0) &&
3024                     (client_data->saa_client_state == SAA_CLIENT_STATE_WAITING))
3025                         cv_signal(&client_data->saa_client_state_cv);
3026 
3027                 mutex_exit(&client_data->saa_client_mutex);
3028         }
3029 
3030         kmem_free(trans_info, sizeof (saa_impl_trans_info_t));
3031 
3032 bail:
3033 
3034         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_end,
3035             IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() exit\n");
3036 }
3037 
3038 /*
3039  * ibmf_saa_check_sa_and_retry:
3040  *
3041  * If a particular transaction times out, we don't want to give up if we know
3042  * the SA is responding.  Check the time since we last received a response. If
3043  * it's less than ibmf_saa_max_wait_time retry the request.
3044  *
3045  * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the
3046  * same values passed to the original ibmf_msg_transport that timed out.  The
3047  * ibmf_retrans parameter will be re-retrieved from the saa_portp structure.
3048  *
3049  * If max_wait_time seconds have passed, this function returns IBMF_TIMEOUT.
3050  * That way, callers can simply return the result of this function.
3051  *
3052  * Input Arguments
3053  * saa_portp            pointer to saa_port structure
3054  * msgp                 ibmf message that timedout
3055  * ibmf_callback        callback that should be called by msg_transport
3056  * ibmf_callback_arg    args for ibmf_callback
3057  * transport_flags      flags for ibmf_msg_transport
3058  *
3059  * Output Arguments
3060  * none
3061  *
3062  * Returns
3063  * IBMF_SUCCESS if we've recently received data from the SA and request was
3064  * resent.
3065  * IBMF_TRANS_TIMEOUT if no data has been received from the SA in max_wait_time
3066  * same values as ibmf_msg_transport() if data has been received but request
3067  * could not be resent.
3068  */
3069 static int
3070 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
3071     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg,
3072     hrtime_t trans_send_time, int transport_flags)
3073 {
3074         hrtime_t                curr_time, sa_uptime;
3075         ibmf_retrans_t          ibmf_retrans;
3076         int                     ibmf_status;
3077 
3078         do {
3079 
3080                 mutex_enter(&saa_portp->saa_pt_mutex);
3081 
3082                 sa_uptime = saa_portp->saa_pt_sa_uptime;
3083 
3084                 /* if nothing received from SA since we sent */
3085                 curr_time = gethrtime();
3086 
3087                 /*
3088                  * check if it's been a very long time since this
3089                  * particular transaction was sent
3090                  */
3091                 if (((curr_time - trans_send_time) / 1000000000) >
3092                     ibmf_saa_trans_wait_time) {
3093 
3094                         mutex_exit(&saa_portp->saa_pt_mutex);
3095 
3096                         IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1,
3097                             ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "",
3098                             "ibmf_saa_check_sa_and_retry: %s, msgp = "
3099                             "%p sa_uptime = %" PRIu64 ", trans send time = %"
3100                             PRIu64 ", curr_time = %" PRIu64 "\n",
3101                             tnf_string, msg,
3102                             "Nothing received for this transaction",
3103                             tnf_opaque, msgp, msgp,
3104                             tnf_long, sa_uptime, sa_uptime,
3105                             tnf_long, trans_send_time, trans_send_time,
3106                             tnf_long, curr_time, curr_time);
3107 
3108                         ibmf_status = IBMF_TRANS_TIMEOUT;
3109 
3110                         break;
3111                 }
3112 
3113                 /*
3114                  * check time since we received something,
3115                  * and make sure that it hasn't been an extra long
3116                  * time for this particular transaction
3117                  */
3118                 if (((curr_time - sa_uptime) / 1000000000) <
3119                     ibmf_saa_max_wait_time) {
3120 
3121                         IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L2,
3122                             ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "",
3123                             "ibmf_saa_check_sa_and_retry: %s, msgp = "
3124                             "%p sa_uptime = %" PRIu64 " trans_send_time = %"
3125                             PRIu64 " curr_time = %" PRIu64 "\n",
3126                             tnf_string, msg, "Something received.  Retrying",
3127                             tnf_opaque, msgp, msgp,
3128                             tnf_long, sa_uptime, sa_uptime,
3129                             tnf_long, trans_send_time, trans_send_time,
3130                             tnf_long, curr_time, curr_time);
3131 
3132                         /*
3133                          * something received in WAIT_TIME_IN_SECS;
3134                          * resend request
3135                          */
3136 
3137                         /* new tid needed */
3138                         msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID =
3139                             h2b64(saa_portp->saa_pt_current_tid++);
3140 
3141                         /*
3142                          * We are going to retry the access to the SM but
3143                          * Master SMLID could have changed due to a port change
3144                          * event. So update the remote_lid of the message with
3145                          * the SMLID from saa_portp for this port before the
3146                          * retry.
3147                          */
3148                         msgp->im_local_addr.ia_remote_lid =
3149                             saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid;
3150 
3151                         bcopy(&saa_portp->saa_pt_ibmf_retrans,
3152                             &ibmf_retrans, sizeof (ibmf_retrans_t));
3153 
3154                         mutex_exit(&saa_portp->saa_pt_mutex);
3155 
3156                         ibmf_status = ibmf_msg_transport(
3157                             saa_portp->saa_pt_ibmf_handle,
3158                             saa_portp->saa_pt_qp_handle, msgp,
3159                             &ibmf_retrans, ibmf_callback, ibmf_callback_arg,
3160                             transport_flags);
3161 
3162                         if (ibmf_status == IBMF_SUCCESS)
3163                                 goto bail;
3164 
3165                         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
3166                             ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "",
3167                             "ibmf_saa_check_sa_and_retry: %s, ibmf_status = "
3168                             "%d\n", tnf_string, msg,
3169                             "ibmf_msg_transport() failed",
3170                             tnf_int, ibmf_status, ibmf_status);
3171                 } else {
3172 
3173                         mutex_exit(&saa_portp->saa_pt_mutex);
3174 
3175                         IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L1,
3176                             ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "",
3177                             "ibmf_saa_check_sa_and_retry: %s, msgp = "
3178                             "%p sa_uptime = %" PRIu64 " curr_time = %"
3179                             PRIu64 "\n", tnf_string, msg,
3180                             "Nothing received.  Timing out",
3181                             tnf_opaque, msgp, msgp,
3182                             tnf_long, sa_uptime, sa_uptime,
3183                             tnf_long, curr_time, curr_time);
3184 
3185                         ibmf_status = IBMF_TRANS_TIMEOUT;
3186 
3187                         break;
3188                 }
3189         } while (ibmf_status == IBMF_TRANS_TIMEOUT);
3190 
3191 bail:
3192         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3193             ibmf_saa_check_sa_and_retry_end,
3194             IBMF_TNF_TRACE, "", "ibmf_saa_check_sa_and_retry() exiting"
3195             " ibmf_status = %d\n", tnf_int, result, ibmf_status);
3196 
3197         return (ibmf_status);
3198 }
3199 
3200 
3201 /*
3202  * ibmf_saa_impl_prepare_response:
3203  */
3204 static void
3205 ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle,
3206     ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result,
3207     size_t *length, boolean_t sleep_flag)
3208 {
3209         ibmf_msg_bufs_t *resp_buf;
3210         uint16_t        attr_id;
3211         uint8_t         method;
3212         boolean_t       is_get_resp;
3213         uint16_t        mad_status;
3214         uint16_t        attr_offset;
3215         ib_sa_hdr_t     *sa_hdr;
3216 
3217         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3218             ibmf_saa_impl_prepare_response_start,
3219             IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() enter\n");
3220 
3221         _NOTE(ASSUMING_PROTECTED(*msgp))
3222 
3223         *result = NULL;
3224         *length = 0;
3225         sa_hdr = NULL;
3226 
3227         resp_buf = &msgp->im_msgbufs_recv;
3228 
3229         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*resp_buf))
3230 
3231         if (msgp->im_msg_status != IBMF_SUCCESS) {
3232 
3233                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
3234                     ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3235                     "ibmf_saa_impl_prepare_response: %s, msg_status = %d\n",
3236                     tnf_string, msg, "Bad ibmf status",
3237                     tnf_int, msg_status, msgp->im_msg_status);
3238 
3239                 *status = msgp->im_msg_status;
3240 
3241                 goto exit;
3242         }
3243 
3244         if (resp_buf->im_bufs_mad_hdr == NULL) {
3245 
3246                 /*
3247                  * this was an unsequenced transaction (from an unsubscribe for
3248                  * following a CI_OFFLINE event)
3249                  */
3250                 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3251                     ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3252                     "ibmf_saa_impl_prepare_response: %s\n",
3253                     tnf_string, msg, "Unsequenced transaction callback");
3254 
3255                 goto exit;
3256         }
3257 
3258         if ((mad_status = b2h16(resp_buf->im_bufs_mad_hdr->Status)) !=
3259             MAD_STATUS_NO_INVALID_FIELDS) {
3260 
3261                 /* convert mad packet status to IBMF status */
3262                 switch (mad_status) {
3263 
3264                         case SA_STATUS_ERR_NO_RESOURCES:
3265                                 *status = IBMF_NO_RESOURCES;
3266                                 break;
3267                         case SA_STATUS_ERR_REQ_INVALID:
3268                                 *status = IBMF_REQ_INVALID;
3269                                 break;
3270                         case SA_STATUS_ERR_NO_RECORDS:
3271                                 *status = IBMF_NO_RECORDS;
3272                                 break;
3273                         case SA_STATUS_ERR_TOO_MANY_RECORDS:
3274                                 *status = IBMF_TOO_MANY_RECORDS;
3275                                 break;
3276                         case SA_STATUS_ERR_REQ_INVALID_GID:
3277                                 *status = IBMF_INVALID_GID;
3278                                 break;
3279                         case SA_STATUS_ERR_REQ_INSUFFICIENT_COMPONENTS:
3280                                 *status = IBMF_INSUFF_COMPS;
3281                                 break;
3282                         case MAD_STATUS_UNSUPP_METHOD:
3283                                 *status = IBMF_UNSUPP_METHOD;
3284                                 break;
3285                         case MAD_STATUS_UNSUPP_METHOD_ATTR:
3286                                 *status = IBMF_UNSUPP_METHOD_ATTR;
3287                                 break;
3288                         case MAD_STATUS_INVALID_FIELD:
3289                                 *status = IBMF_INVALID_FIELD;
3290                                 break;
3291                         default:
3292                                 *status = IBMF_REQ_INVALID;
3293                                 break;
3294                 }
3295 
3296                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
3297                     ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3298                     "ibmf_saa_impl_prepare_response: %s, mad_status = %x\n",
3299                     tnf_string, msg, "Bad MAD status",
3300                     tnf_int, mad_status, mad_status);
3301 
3302                 goto exit;
3303         }
3304 
3305         attr_id = b2h16(resp_buf->im_bufs_mad_hdr->AttributeID);
3306         method = resp_buf->im_bufs_mad_hdr->R_Method;
3307 
3308         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
3309             ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3310             "ibmf_saa_impl_prepare_response: attr_id = 0x%x, method = "
3311             "0x%x\n",
3312             tnf_opaque, attr_id, attr_id,
3313             tnf_opaque, method, method);
3314 
3315         /*
3316          * ignore any data from deleteresp since there's no way to know whether
3317          * real data was returned; also ignore data if this was a Report
3318          * response
3319          */
3320         if (method == SA_SUBN_ADM_DELETE_RESP) {
3321 
3322                 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3323                     ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3324                     "impf_saa_impl_prepare_response: %s\n",
3325                     tnf_string, msg,
3326                     "DeleteResp or NoticeResp returned.  "
3327                     "Ignoring response data");
3328 
3329                 *status = IBMF_SUCCESS;
3330 
3331                 *length = 0;
3332                 *result = NULL;
3333 
3334                 goto exit;
3335         }
3336 
3337         if (attr_id == SA_MULTIPATHRECORD_ATTRID) {
3338 
3339                 /*
3340                  * getmulti is only for requests; attribute should not
3341                  * be returned from SA
3342                  */
3343                 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
3344                     ibmf_saa_impl_prepare_response_err, IBMF_TNF_ERROR,
3345                     "", "ibmf_saa_impl_prepare_response: %s\n",
3346                     tnf_string, msg, "SA returned getmulti record");
3347 
3348                 *status = IBMF_REQ_INVALID;
3349 
3350                 goto exit;
3351         }
3352 
3353         /* if we are supposed to ignore data, stop here */
3354         if (ignore_data == B_TRUE) {
3355 
3356                 *status = IBMF_SUCCESS;
3357 
3358                 goto exit;
3359         }
3360 
3361         is_get_resp = resp_buf->im_bufs_mad_hdr->R_Method ==
3362             SA_SUBN_ADM_GET_RESP ? B_TRUE: B_FALSE;
3363 
3364         /* unpack the sa header to get the attribute offset */
3365         *status = ibmf_saa_utils_unpack_sa_hdr(resp_buf->im_bufs_cl_hdr,
3366             resp_buf->im_bufs_cl_hdr_len, &sa_hdr, sleep_flag);
3367         if (*status != IBMF_SUCCESS) {
3368 
3369                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3370                     ibmf_saa_impl_prepare_response_err,
3371                     IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s,"
3372                     " ibmf_status = %d\n", tnf_string, msg,
3373                     "Could not unpack sa hdr", tnf_int, ibmf_status, *status);
3374 
3375                 goto exit;
3376         }
3377 
3378         attr_offset = sa_hdr->AttributeOffset;
3379 
3380         /*
3381          * unpack data payload; if unpack function doesn't return success
3382          * (because it could not allocate memory) forward this status to waiting
3383          * client
3384          */
3385         *status = ibmf_saa_utils_unpack_payload(resp_buf->im_bufs_cl_data,
3386             resp_buf->im_bufs_cl_data_len, attr_id, result, length,
3387             attr_offset, is_get_resp, sleep_flag);
3388         if (*status == IBMF_SUCCESS) {
3389 
3390                 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
3391                     ibmf_saa_impl_prepare_response,
3392                     IBMF_TNF_TRACE, "",
3393                     "ibmf_saa_impl_prepare_response: attr_id = "
3394                     "0x%x, attr_offset = %d, packed_payload_len = %d, "
3395                     "unpacked_payload_len = %d\n",
3396                     tnf_opaque, attr_id, attr_id,
3397                     tnf_opaque, attr_offset, attr_offset,
3398                     tnf_opaque, packed_payload_len,
3399                     resp_buf->im_bufs_cl_data_len,
3400                     tnf_opaque, unpacked_payload_len, *length);
3401         } else {
3402 
3403                 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1,
3404                     ibmf_saa_impl_prepare_response_err,
3405                     IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s,"
3406                     "attr_id = 0x%x, attr_offset = %d, packed_payload_len = %d,"
3407                     "status = %d\n",
3408                     tnf_string, msg, "Could not unpack payload",
3409                     tnf_opaque, attr_id, attr_id,
3410                     tnf_int, attr_offset, attr_offset,
3411                     tnf_int, packed_payload_len,
3412                     resp_buf->im_bufs_cl_data_len,
3413                     tnf_int, status, *status);
3414         }
3415 exit:
3416         if (sa_hdr != NULL)
3417                 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3418 
3419         ibmf_saa_impl_free_msg(ibmf_handle, msgp);
3420 
3421         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3422             ibmf_saa_impl_prepare_response_end,
3423             IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() exit,"
3424             " status = 0x%d\n", tnf_int, status, *status);
3425 }
3426 
3427 
3428 /*
3429  * ibmf_saa_impl_check_sa_support:
3430  * Checks the capability mask (returned from the SA classportinfo response) to
3431  * determine whether the sa supports the specified attribute ID.
3432  *
3433  * Input Arguments
3434  * cap_mask     16-bit capability mask returned in SA's classportinfo
3435  * attr_id      attribute ID of current request
3436  *
3437  * Returns
3438  * IBMF_NOT_SUPPORTED if capability mask indicates SA does not support attribute
3439  * IBMF_SUCCESS otherwise
3440  */
3441 static int
3442 ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id)
3443 {
3444         boolean_t       attr_supported = B_TRUE;
3445 
3446         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
3447             ibmf_saa_impl_check_sa_support, IBMF_TNF_TRACE, "",
3448             "ibmf_saa_impl_check_sa_support: cap_mask = 0x%x, "
3449             "attr_id = 0x%x\n", tnf_opaque, cap_mask, cap_mask,
3450             tnf_opaque, attr_id, attr_id);
3451 
3452         switch (attr_id) {
3453 
3454                 case SA_SWITCHINFORECORD_ATTRID:
3455                 case SA_LINEARFDBRECORD_ATTRID:
3456                 case SA_RANDOMFDBRECORD_ATTRID:
3457                 case SA_MULTICASTFDBRECORD_ATTRID:
3458                 case SA_SMINFORECORD_ATTRID:
3459                 case SA_INFORMINFORECORD_ATTRID:
3460                 case SA_LINKRECORD_ATTRID:
3461                 case SA_GUIDINFORECORD_ATTRID:
3462                 case SA_TRACERECORD_ATTRID:
3463                 case SA_SERVICEASSNRECORD_ATTRID:
3464 
3465                         if ((cap_mask &
3466                             SA_CAPMASK_OPT_RECORDS_SUPPORTED) == 0) {
3467 
3468                                 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1,
3469                                     ibmf_saa_impl_check_sa_support,
3470                                     IBMF_TNF_ERROR, "",
3471                                     "ibmf_saa_impl_check_sa_support: %s, "
3472                                     "cap_mask = 0x%x\n", tnf_string, msg,
3473                                     "SA does not support optional records",
3474                                     tnf_opaque, cap_mask, cap_mask,
3475                                     tnf_opaque, attr_id, attr_id);
3476 
3477                                 attr_supported = B_FALSE;
3478                         }
3479                         break;
3480 
3481                 case SA_MULTIPATHRECORD_ATTRID:
3482 
3483                         if ((cap_mask & SA_CAPMASK_MULTIPATH_SUPPORTED) == 0) {
3484 
3485                                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
3486                                     ibmf_saa_impl_check_sa_support,
3487                                     IBMF_TNF_ERROR, "",
3488                                     "ibmf_saa_impl_check_sa_support: %s, "
3489                                     "cap_mask = 0x%x\n", tnf_string, msg,
3490                                     "SA does not support multipath records",
3491                                     tnf_opaque, cap_mask, cap_mask);
3492 
3493                                 attr_supported = B_FALSE;
3494                         }
3495                         break;
3496 
3497                 case SA_MCMEMBERRECORD_ATTRID:
3498 
3499                         if ((cap_mask & SA_CAPMASK_UD_MCAST_SUPPORTED) == 0) {
3500 
3501                                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
3502                                     ibmf_saa_impl_check_sa_support,
3503                                     IBMF_TNF_ERROR, "",
3504                                     "ibmf_saa_impl_check_sa_support: %s, "
3505                                     "cap_mask = 0x%x\n", tnf_string, msg,
3506                                     "SA does not support ud multicast",
3507                                     tnf_opaque, cap_mask, cap_mask);
3508 
3509                                 attr_supported = B_FALSE;
3510                         }
3511                         break;
3512 
3513                 default:
3514                         break;
3515         } /* switch */
3516 
3517         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3518             ibmf_saa_impl_check_sa_support_end, IBMF_TNF_TRACE, "",
3519             "ibmf_saa_impl_check_sa_support() exiting, attr_supported = %d\n",
3520             tnf_opaque, attr_supported, attr_supported);
3521 
3522         if (attr_supported == B_FALSE)
3523                 return (IBMF_UNSUPP_METHOD_ATTR);
3524         else
3525                 return (IBMF_SUCCESS);
3526 }
3527 
3528 /*
3529  * ibmf_saa_impl_get_attr_id_length:
3530  *
3531  * Returns the host size of the specified sa record.  Returns 0 for unknown
3532  * attributes.  multipath record size is a dynamic value given as a parameter
3533  * specified with the ibmf_sa_access() call.
3534  */
3535 static uint_t
3536 ibmf_saa_impl_get_attr_id_length(uint16_t attr_id)
3537 {
3538         uint_t  attr_length;
3539 
3540         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3541             ibmf_saa_impl_get_attr_id_length_start,
3542             IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length() enter\n");
3543 
3544         /* this function should not be used for multipath record */
3545         ASSERT(attr_id != SA_MULTIPATHRECORD_ATTRID);
3546 
3547         switch (attr_id) {
3548                 case SA_CLASSPORTINFO_ATTRID:
3549                         attr_length = sizeof (ib_mad_classportinfo_t);
3550                         break;
3551                 case SA_NOTICE_ATTRID:
3552                         attr_length = sizeof (ib_mad_notice_t);
3553                         break;
3554                 case SA_INFORMINFO_ATTRID:
3555                         attr_length = sizeof (ib_mad_informinfo_t);
3556                         break;
3557                 case SA_NODERECORD_ATTRID:
3558                         attr_length = sizeof (sa_node_record_t);
3559                         break;
3560                 case SA_PORTINFORECORD_ATTRID:
3561                         attr_length = sizeof (sa_portinfo_record_t);
3562                         break;
3563                 case SA_SLTOVLRECORD_ATTRID:
3564                         attr_length = sizeof (sa_SLtoVLmapping_record_t);
3565                         break;
3566                 case SA_SWITCHINFORECORD_ATTRID:
3567                         attr_length = sizeof (sa_switchinfo_record_t);
3568                         break;
3569                 case SA_LINEARFDBRECORD_ATTRID:
3570                         attr_length = sizeof (sa_linearft_record_t);
3571                         break;
3572                 case SA_RANDOMFDBRECORD_ATTRID:
3573                         attr_length = sizeof (sa_randomft_record_t);
3574                         break;
3575                 case SA_MULTICASTFDBRECORD_ATTRID:
3576                         attr_length = sizeof (sa_multicastft_record_t);
3577                         break;
3578                 case SA_SMINFORECORD_ATTRID:
3579                         attr_length = sizeof (sa_sminfo_record_t);
3580                         break;
3581                 case SA_INFORMINFORECORD_ATTRID:
3582                         attr_length = sizeof (sa_informinfo_record_t);
3583                         break;
3584                 case SA_LINKRECORD_ATTRID:
3585                         attr_length = sizeof (sa_link_record_t);
3586                         break;
3587                 case SA_GUIDINFORECORD_ATTRID:
3588                         attr_length = sizeof (sa_guidinfo_record_t);
3589                         break;
3590                 case SA_SERVICERECORD_ATTRID:
3591                         attr_length = sizeof (sa_service_record_t);
3592                         break;
3593                 case SA_PARTITIONRECORD_ATTRID:
3594                         attr_length = sizeof (sa_pkey_table_record_t);
3595                         break;
3596                 case SA_PATHRECORD_ATTRID:
3597                         attr_length = sizeof (sa_path_record_t);
3598                         break;
3599                 case SA_VLARBRECORD_ATTRID:
3600                         attr_length = sizeof (sa_VLarb_table_record_t);
3601                         break;
3602                 case SA_MCMEMBERRECORD_ATTRID:
3603                         attr_length = sizeof (sa_mcmember_record_t);
3604                         break;
3605                 case SA_TRACERECORD_ATTRID:
3606                         attr_length = sizeof (sa_trace_record_t);
3607                         break;
3608                 case SA_SERVICEASSNRECORD_ATTRID:
3609                         attr_length = sizeof (sa_service_assn_record_t);
3610                         break;
3611                 default:
3612                         /* should only get the above type of packets */
3613                         attr_length = 0;
3614                         break;
3615         }
3616 
3617         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
3618             ibmf_saa_impl_get_attr_id_length_end,
3619             IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length():"
3620             " attr_id: 0x%x size %d\n",
3621             tnf_opaque, attr_id, attr_id, tnf_uint, attr_length, attr_length);
3622 
3623         return (attr_length);
3624 }
3625 
3626 /*
3627  * ibmf_saa_impl_free_msg:
3628  * Takes a completed message and free memory associated with the message,
3629  * including the individual fields of the im_msgbufs_send.
3630  * ibmf_free_msg, called at the end of this function, takes a pointer to the
3631  * message pointer so that it can set the message pointer to NULL.  This
3632  * function takes just the message pointer so the msgp will not be NULL after
3633  * this function returns.
3634  *
3635  * Input Arguments
3636  * ibmf_hdl     ibmf handle used in ibmf_msg_alloc
3637  * msgp         pointer to ibmf_msg_t to free
3638  *
3639  * Returns
3640  * void
3641  */
3642 static void
3643 ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp)
3644 {
3645         int     res;
3646 
3647         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3648             ibmf_saa_impl_free_msg_start,
3649             IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() enter: msg %p\n",
3650             tnf_opaque, msg, msgp);
3651 
3652         ASSERT(msgp != NULL);
3653 
3654         kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr,
3655             sizeof (ib_mad_hdr_t));
3656 
3657         kmem_free(msgp->im_msgbufs_send.im_bufs_cl_hdr,
3658             msgp->im_msgbufs_send.im_bufs_cl_hdr_len);
3659 
3660         if (msgp->im_msgbufs_send.im_bufs_cl_data_len > 0)
3661                 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_data,
3662                     msgp->im_msgbufs_send.im_bufs_cl_data_len);
3663 
3664         res = ibmf_free_msg(ibmf_hdl, &msgp);
3665         ASSERT(res == IBMF_SUCCESS);
3666 
3667         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3668             ibmf_saa_impl_free_msg_end,
3669             IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() exit\n");
3670 }
3671 
3672 /*
3673  * ibmf_saa_impl_get_port_guid:
3674  */
3675 static int
3676 ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop,
3677     ib_guid_t *guid_ret)
3678 {
3679         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3680             ibmf_saa_impl_get_port_guid_start,
3681             IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_port_guid() enter\n");
3682 
3683         if (ibt_portinfop->p_linkstate != IBT_PORT_ACTIVE) {
3684 
3685                 return (IBMF_BAD_PORT_STATE);
3686         }
3687 
3688         if (ibt_portinfop->p_sgid_tbl_sz == 0) {
3689 
3690                 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L2,
3691                     ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "",
3692                     "ibmf_saa_impl_get_port_guid: %s\n", tnf_string, msg,
3693                     "portinfo sgid table size is 0. Exiting.\n");
3694 
3695                 return (IBMF_TRANSPORT_FAILURE);
3696         }
3697 
3698         *guid_ret = ibt_portinfop->p_sgid_tbl[0].gid_guid;
3699 
3700         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3701             ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "",
3702             "ibmf_saa_impl_get_port_guid: Returning port_guid %016" PRIx64 "\n",
3703             tnf_opaque, port_guid, *guid_ret);
3704 
3705         return (IBMF_SUCCESS);
3706 }
3707 
3708 /*
3709  * ibmf_saa_impl_set_transaction_params:
3710  */
3711 static void
3712 ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp,
3713     ibt_hca_portinfo_t *portinfop)
3714 {
3715         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3716             ibmf_saa_impl_set_transaction_params_start,
3717             IBMF_TNF_TRACE, "",
3718             "ibmf_saa_impl_set_transaction_params() enter\n");
3719 
3720         _NOTE(ASSUMING_PROTECTED(*saa_portp))
3721 
3722         saa_portp->saa_pt_ibmf_retrans.retrans_retries =
3723             ibmf_saa_retrans_retries;
3724         /*
3725          * For the first transaction (generally getting the
3726          * classportinfo) have ibmf pick our timeouts.  It should be using the
3727          * default IB spec values.
3728          * Once we get the classportinfo we'll update the correct response time
3729          * value (rtv) and round-trip time (rttv).  ibmf should always calculate
3730          * trans_to since it depends on the particular transaction's number of
3731          * packets.
3732          */
3733         saa_portp->saa_pt_ibmf_retrans.retrans_rtv = 0;
3734         saa_portp->saa_pt_ibmf_retrans.retrans_rttv = 0;
3735         saa_portp->saa_pt_ibmf_retrans.retrans_trans_to = 0;
3736 
3737         /*
3738          * Assume that the SA supports all optional records. If it
3739          * does not, the request will get returned with ERR_NOT_SUPP.  When
3740          * the classportinfo response comes back we will update the cap mask
3741          * to prevent unnecessary unsupported requests.
3742          */
3743         saa_portp->saa_pt_sa_cap_mask = 0xFFFF;
3744 
3745         saa_portp->saa_pt_ibmf_msg_flags = 0;
3746         saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno       = 1;
3747         saa_portp->saa_pt_ibmf_addr_info.ia_p_key    =
3748             IB_PKEY_DEFAULT_LIMITED;
3749         saa_portp->saa_pt_ibmf_addr_info.ia_q_key    = IB_GSI_QKEY;
3750 
3751         /*
3752          * fill out addr information for MADs that will be sent
3753          * to SA on this port
3754          */
3755         saa_portp->saa_pt_ibmf_addr_info.ia_local_lid        = portinfop->p_base_lid;
3756         saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid       = portinfop->p_sm_lid;
3757         saa_portp->saa_pt_ibmf_addr_info.ia_service_level = portinfop->p_sm_sl;
3758 
3759         /* place upper bound on subnet timeout in case of faulty SM */
3760         saa_portp->saa_pt_timeout = portinfop->p_subnet_timeout;
3761 
3762         if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout)
3763                 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout;
3764 
3765         IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
3766             ibmf_saa_impl_set_transaction_params,
3767             IBMF_TNF_TRACE, "",
3768             "ibmf_saa_impl_set_transaction_params: local_lid = 0x%x, "
3769             "sm_lid = 0x%x, sm_sl = 0x%x, sn_timeout = 0x%x\n",
3770             tnf_opaque, local_lid, portinfop->p_base_lid,
3771             tnf_opaque, sm_lid, portinfop->p_sm_lid,
3772             tnf_opaque, sm_sl, portinfop->p_sm_sl,
3773             tnf_opaque, subnet_timeout, portinfop->p_subnet_timeout);
3774 
3775         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3776             ibmf_saa_impl_set_transaction_params_end,
3777             IBMF_TNF_TRACE, "",
3778             "ibmf_saa_impl_set_transaction_params() exit\n");
3779 }
3780 
3781 
3782 /*
3783  * ibmf_saa_impl_update_sa_address_info
3784  */
3785 static void
3786 ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp, ibmf_msg_t *msgp)
3787 {
3788         void                    *result;
3789         ib_sa_hdr_t             *sa_hdr;
3790         int                     rv;
3791         size_t                  length;
3792         uint16_t                attr_id;
3793         ib_mad_classportinfo_t  *cpi;
3794         ibmf_global_addr_info_t *gaddrp = &saa_portp->saa_pt_ibmf_global_addr;
3795         ibt_hca_portinfo_t      *ibt_pinfo;
3796         uint_t                  nports, size;
3797         ibt_status_t            ibt_status;
3798 
3799         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3800             ibmf_saa_impl_update_sa_address_info,
3801             IBMF_TNF_TRACE, "",
3802             "ibmf_saa_impl_update_sa_address_info() enter\n");
3803 
3804         /*
3805          * decode the respons of msgp as a classportinfo attribute
3806          */
3807         rv = ibmf_saa_utils_unpack_sa_hdr(msgp->im_msgbufs_recv.im_bufs_cl_hdr,
3808             msgp->im_msgbufs_recv.im_bufs_cl_hdr_len, &sa_hdr, KM_NOSLEEP);
3809         if (rv != IBMF_SUCCESS) {
3810 
3811                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3812                     ibmf_saa_impl_update_sa_address_err,
3813                     IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: "
3814                     "%s, ibmf_status = %d\n", tnf_string, msg,
3815                     "Could not unpack sa hdr", tnf_int, ibmf_status, rv);
3816 
3817                 return;
3818         }
3819 
3820         attr_id = b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID);
3821         if (attr_id != MAD_ATTR_ID_CLASSPORTINFO) {
3822                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3823                     ibmf_saa_impl_update_sa_address_info_err,
3824                     IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: "
3825                     "%s, attrID = %x\n", tnf_string, msg,
3826                     "Wrong attribute ID", tnf_int, ibmf_status, attr_id);
3827 
3828                 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3829                 return;
3830         }
3831         rv = ibmf_saa_utils_unpack_payload(
3832             msgp->im_msgbufs_recv.im_bufs_cl_data,
3833             msgp->im_msgbufs_recv.im_bufs_cl_data_len, attr_id, &result,
3834             &length, sa_hdr->AttributeOffset, B_TRUE, KM_NOSLEEP);
3835         if (rv != IBMF_SUCCESS) {
3836 
3837                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3838                     ibmf_saa_impl_update_sa_address_err,
3839                     IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: "
3840                     "%s, ibmf_status = %d\n", tnf_string, msg,
3841                     "Could not unpack payload", tnf_int, ibmf_status, rv);
3842 
3843                 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3844                 return;
3845         }
3846 
3847         kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3848 
3849         /*
3850          * Use the classportinfo contents to update the SA address info
3851          */
3852         cpi = (ib_mad_classportinfo_t *)result;
3853         mutex_enter(&saa_portp->saa_pt_mutex);
3854         saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid       = cpi->RedirectLID;
3855         saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno       = cpi->RedirectQP;
3856         saa_portp->saa_pt_ibmf_addr_info.ia_p_key    = cpi->RedirectP_Key;
3857         saa_portp->saa_pt_ibmf_addr_info.ia_q_key    = cpi->RedirectQ_Key;
3858         saa_portp->saa_pt_ibmf_addr_info.ia_service_level = cpi->RedirectSL;
3859 
3860         saa_portp->saa_pt_redirect_active = B_TRUE;
3861 
3862         if ((cpi->RedirectGID_hi != 0) || (cpi->RedirectGID_lo != 0)) {
3863 
3864                 mutex_exit(&saa_portp->saa_pt_mutex);
3865                 ibt_status = ibt_query_hca_ports_byguid(
3866                     saa_portp->saa_pt_node_guid, saa_portp->saa_pt_port_num,
3867                     &ibt_pinfo, &nports, &size);
3868                 if (ibt_status != IBT_SUCCESS) {
3869 
3870                         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3871                             ibmf_saa_impl_update_sa_address_err, IBMF_TNF_TRACE,
3872                             "", "ibmf_saa_impl_update_sa_address_info: "
3873                             "%s, ibt_status = %d\n", tnf_string, msg,
3874                             "Could not query hca port",
3875                             tnf_int, ibt_status, ibt_status);
3876 
3877                         kmem_free(result, length);
3878                         return;
3879                 }
3880 
3881                 mutex_enter(&saa_portp->saa_pt_mutex);
3882                 /*
3883                  * Fill in global address info parameters
3884                  *
3885                  * NOTE: The HopLimit value is not specified through the
3886                  * contents of ClassPortInfo. It may be possible to find
3887                  * out the proper value to use even for SA beeing redirected
3888                  * to another subnet. But we do only support redirect within
3889                  * our local subnet
3890                  */
3891                 gaddrp->ig_sender_gid.gid_prefix =
3892                     ibt_pinfo->p_sgid_tbl[0].gid_prefix;
3893                 gaddrp->ig_sender_gid.gid_guid = saa_portp->saa_pt_port_guid;
3894                 gaddrp->ig_recver_gid.gid_prefix = cpi->RedirectGID_hi;
3895                 gaddrp->ig_recver_gid.gid_guid = cpi->RedirectGID_lo;
3896                 gaddrp->ig_flow_label = cpi->RedirectFL;
3897                 gaddrp->ig_tclass = cpi->RedirectTC;
3898                 gaddrp->ig_hop_limit = 0;
3899 
3900                 saa_portp->saa_pt_ibmf_msg_flags =
3901                     IBMF_MSG_FLAGS_GLOBAL_ADDRESS;
3902 
3903                 mutex_exit(&saa_portp->saa_pt_mutex);
3904                 ibt_free_portinfo(ibt_pinfo, size);
3905         } else {
3906                 saa_portp->saa_pt_ibmf_msg_flags = 0;
3907                 mutex_exit(&saa_portp->saa_pt_mutex);
3908         }
3909         kmem_free(result, length);
3910 
3911         /*
3912          * Update the address info of msgp with the new address parameters
3913          */
3914         mutex_enter(&saa_portp->saa_pt_mutex);
3915         bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr,
3916             sizeof (ibmf_addr_info_t));
3917         if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) {
3918 
3919                 msgp->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS;
3920 
3921                 bcopy(&saa_portp->saa_pt_ibmf_global_addr,
3922                     &msgp->im_global_addr, sizeof (ibmf_global_addr_info_t));
3923         } else {
3924                 msgp->im_msg_flags = 0;
3925         }
3926         mutex_exit(&saa_portp->saa_pt_mutex);
3927 }
3928 
3929 /*
3930  * ibmf_saa_impl_ibmf_unreg:
3931  */
3932 static int
3933 ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp)
3934 {
3935         int     ibmf_status;
3936 
3937         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_start,
3938             IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() enter\n");
3939 
3940         /* teardown async cb */
3941         ibmf_status = ibmf_tear_down_async_cb(saa_portp->saa_pt_ibmf_handle,
3942             saa_portp->saa_pt_qp_handle, 0);
3943         if (ibmf_status != IBMF_SUCCESS) {
3944 
3945                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3946                     ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "",
3947                     "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n",
3948                     tnf_string, msg, "Could not tear down async cb",
3949                     tnf_int, ibmf_status, ibmf_status);
3950 
3951                 goto bail;
3952         }
3953 
3954         /* free qp */
3955         ibmf_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle,
3956             &saa_portp->saa_pt_qp_handle, 0);
3957 
3958         if (ibmf_status != IBMF_SUCCESS) {
3959 
3960                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3961                     ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "",
3962                     "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n",
3963                     tnf_string, msg, "Could not free queue pair",
3964                     tnf_int, ibmf_status, ibmf_status);
3965 
3966                 (void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 1);
3967                 goto bail;
3968         }
3969 
3970         ibmf_status = ibmf_unregister(&saa_portp->saa_pt_ibmf_handle, 0);
3971 
3972         if (ibmf_status != IBMF_SUCCESS) {
3973 
3974                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3975                     ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "",
3976                     "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n",
3977                     tnf_string, msg, "ibmf_unregister() failed",
3978                     tnf_int, ibmf_status, ibmf_status);
3979 
3980                 (void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0);
3981         }
3982 
3983 bail:
3984         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_end,
3985             IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() exit\n");
3986 
3987         return (ibmf_status);
3988 }