1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 #include <sys/ib/mgt/ibmf/ibmf_saa_impl.h>
  27 #include <sys/ib/mgt/ibmf/ibmf_saa_utils.h>
  28 
  29 /* Global sa_access State Pointer */
  30 saa_state_t *saa_statep;
  31 _NOTE(READ_ONLY_DATA(saa_statep))
  32 
  33 extern  int     ibmf_trace_level;
  34 
  35 extern  int     ibmf_taskq_max_tasks;
  36 
  37 static int
  38 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
  39     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags);
  40 static int
  41 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp,
  42     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags);
  43 static int
  44 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
  45     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg,
  46     hrtime_t trans_send_time, int transport_flags);
  47 static int ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info,
  48     boolean_t sleep_flag, ibmf_msg_t **msgp, uint32_t *transport_flagsp,
  49     ibmf_retrans_t *ibmf_retransp);
  50 static int ibmf_saa_must_purge(saa_port_t *saa_portp);
  51 static void ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp);
  52 static void ibmf_saa_impl_destroy_port(saa_port_t *saa_portp);
  53 static void ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp);
  54 static void ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer,
  55     int status);
  56 static void ibmf_saa_impl_async_event_cb(ibmf_handle_t ibmf_handle,
  57     void *clnt_private, ibmf_async_event_t event_type);
  58 static void ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num);
  59 static void ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num);
  60 static void ibmf_saa_impl_port_chg(ibt_async_event_t *event);
  61 static void ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num);
  62 static void ibmf_saa_impl_hca_detach(saa_port_t *saa_removed);
  63 static void ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle,
  64     ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result,
  65     size_t *length, boolean_t sleep_flag);
  66 static int ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id);
  67 static uint_t ibmf_saa_impl_get_attr_id_length(uint16_t attr_id);
  68 static void ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp);
  69 static int ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop,
  70     ib_guid_t *guid_ret);
  71 static void ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp,
  72     ibt_hca_portinfo_t *portinfop);
  73 static void ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp,
  74     ibmf_msg_t *msgp);
  75 static int ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp);
  76 
  77 int     ibmf_saa_max_wait_time = IBMF_SAA_MAX_WAIT_TIME_IN_SECS;
  78 int     ibmf_saa_trans_wait_time = IBMF_SAA_TRANS_WAIT_TIME_IN_SECS;
  79 int     ibmf_saa_max_resp_time = IBMF_SAA_MAX_RESP_TIME;
  80 int     ibmf_saa_max_subnet_timeout = IBMF_SAA_MAX_SUBNET_TIMEOUT;
  81 int     ibmf_saa_retrans_retries = IBMF_SAA_RETRANS_RETRIES;
  82 
  83 /*
  84  * ibmf_saa_impl_init:
  85  * Allocates memory for the ibmf_saa state structure and initializes the taskq.
  86  * Called from the modules init() routine.
  87  *
  88  * Input Arguments
  89  * none
  90  *
  91  * Output Arguments
  92  * none
  93  *
  94  * Returns
  95  * IBMF_NO_RESOURCES if taskq could not be created.
  96  * IBMF_SUCCESS on success
  97  *
  98  */
  99 int
 100 ibmf_saa_impl_init()
 101 {
 102         int             res;
 103 
 104         /* CONSTCOND */
 105         ASSERT(NO_COMPETING_THREADS);
 106 
 107         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_init_start,
 108             IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() enter\n");
 109 
 110         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_statep))
 111 
 112         saa_statep = kmem_zalloc(sizeof (saa_state_t), KM_SLEEP);
 113 
 114         /* create taskq for notifying event subscribers */
 115         saa_statep->saa_event_taskq = taskq_create(
 116             "ibmf_saa_event_taskq", IBMF_TASKQ_NTHREADS,
 117             MINCLSYSPRI, 1, ibmf_taskq_max_tasks, TASKQ_DYNAMIC |
 118             TASKQ_PREPOPULATE);
 119         if (saa_statep->saa_event_taskq == NULL) {
 120 
 121                 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L4,
 122                     ibmf_saa_impl_init_end_err,
 123                     IBMF_TNF_TRACE, "", "ibmf_saa_impl_init(): %s\n",
 124                     tnf_string, msg, "event taskq create failed");
 125 
 126                 kmem_free(saa_statep, sizeof (saa_state_t));
 127 
 128                 res = IBMF_NO_RESOURCES;
 129 
 130                 goto bail;
 131         }
 132 
 133         mutex_init(&saa_statep->saa_port_list_mutex, NULL, MUTEX_DRIVER,
 134             NULL);
 135 
 136         _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_statep))
 137 
 138         res = IBMF_SUCCESS;
 139 bail:
 140 
 141         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_init_end,
 142             IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() exit: status = %d\n",
 143             tnf_int, res, res);
 144 
 145         return (res);
 146 }
 147 
 148 /*
 149  * ibmf_saa_impl_fini:
 150  * If there are no registered clients, cleans up all memory associated with the
 151  * state, including each of the port list entries.
 152  * Called from the modules fini() routine.
 153  *
 154  * Input Arguments
 155  * none
 156  *
 157  * Output Arguments
 158  * none
 159  *
 160  * Returns
 161  * EBUSY if there are outstanding transactions or registered clients
 162  * 0 if cleanup was sucessfull
 163  *
 164  */
 165 int
 166 ibmf_saa_impl_fini()
 167 {
 168         int             ret = 0;
 169         saa_port_t      *saa_portp;
 170         saa_port_t      *next;
 171 
 172         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_start,
 173             IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() enter\n");
 174 
 175         /* make sure there are no registered clients */
 176         mutex_enter(&saa_statep->saa_port_list_mutex);
 177 
 178         saa_portp = saa_statep->saa_port_list;
 179         while (saa_portp != NULL) {
 180 
 181                 mutex_enter(&saa_portp->saa_pt_mutex);
 182 
 183                 if (saa_portp->saa_pt_reference_count > 0) {
 184 
 185                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
 186                             ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "",
 187                             "ibmf_saa_impl_fini: %s, port %016" PRIx64 "\n",
 188                             tnf_string, msg,
 189                             "cannot unload ibmf_saa. Client on port still"
 190                             " registered", tnf_opaque, port,
 191                             saa_portp->saa_pt_port_guid);
 192 
 193                         mutex_exit(&saa_portp->saa_pt_mutex);
 194 
 195                         mutex_exit(&saa_statep->saa_port_list_mutex);
 196 
 197                         ret = EBUSY;
 198                         goto bail;
 199                 }
 200 
 201                 /* make sure there are no outstanding transactions */
 202 
 203                 if (saa_portp->saa_pt_num_outstanding_trans > 0) {
 204 
 205                         IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1,
 206                             ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "",
 207                             "ibmf_saa_impl_fini: %s, port = %016" PRIx64
 208                             ", num transactions = %d\n",
 209                             tnf_string, msg, "Cannot unload ibmf_saa."
 210                             "  Outstanding transactions on port.",
 211                             tnf_opaque, port,
 212                             saa_portp->saa_pt_port_guid,
 213                             tnf_uint, outstanding_transactions,
 214                             saa_portp->saa_pt_num_outstanding_trans);
 215 
 216                         mutex_exit(&saa_portp->saa_pt_mutex);
 217 
 218                         mutex_exit(&saa_statep->saa_port_list_mutex);
 219 
 220                         ret = EBUSY;
 221                         goto bail;
 222                 }
 223 
 224                 mutex_exit(&saa_portp->saa_pt_mutex);
 225 
 226                 saa_portp = saa_portp->next;
 227         }
 228 
 229         mutex_exit(&saa_statep->saa_port_list_mutex);
 230 
 231         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(saa_statep->saa_port_list,
 232             *saa_portp))
 233 
 234         /*
 235          * no more clients nor pending transaction:
 236          * unregister ibmf and destroy port entries
 237          */
 238         while (saa_statep->saa_port_list != NULL) {
 239 
 240                 saa_portp = saa_statep->saa_port_list;
 241                 next = saa_portp->next;
 242 
 243                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
 244                     ibmf_saa_impl_fini, IBMF_TNF_TRACE, "",
 245                     "ibmf_saa_impl_fini: %s, prefix = %016" PRIx64 "\n",
 246                     tnf_string, msg, "deinitializing port",
 247                     tnf_opaque, port_guid, saa_portp->saa_pt_port_guid);
 248 
 249                 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp))
 250 
 251                 mutex_enter(&saa_portp->saa_pt_mutex);
 252 
 253                 /* unregister from ibmf */
 254                 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) {
 255 
 256                         mutex_exit(&saa_portp->saa_pt_mutex);
 257 
 258                         if (ibmf_saa_impl_ibmf_unreg(saa_portp)
 259                             != IBMF_SUCCESS) {
 260                                 ret = EBUSY;
 261                                 goto bail;
 262                         }
 263                 } else
 264                         mutex_exit(&saa_portp->saa_pt_mutex);
 265 
 266                 ibmf_saa_impl_destroy_port(saa_portp);
 267 
 268                 saa_statep->saa_port_list = next;
 269         }
 270 
 271         taskq_destroy(saa_statep->saa_event_taskq);
 272 
 273         mutex_destroy(&saa_statep->saa_port_list_mutex);
 274 
 275         kmem_free(saa_statep, sizeof (saa_state_t));
 276 
 277 bail:
 278         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_end,
 279             IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() exit\n");
 280 
 281         return (ret);
 282 }
 283 
 284 /*
 285  * ibmf_saa_is_valid
 286  * Returns true the entry is valid.
 287  *
 288  * Input Arguments
 289  * saa_portp            pointer to state structure
 290  * add_ref              if B_TRUE ref count is incremented on a valid portp
 291  *
 292  * Output Arguments
 293  * none
 294  *
 295  * Returns
 296  * B_TRUE if entry was in a valid state, B_FALSE otherwise
 297  */
 298 boolean_t
 299 ibmf_saa_is_valid(saa_port_t *saa_portp, int add_ref)
 300 {
 301         boolean_t is_valid = B_TRUE;
 302 
 303         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_start,
 304             IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() enter\n");
 305 
 306         mutex_enter(&saa_portp->saa_pt_mutex);
 307 
 308         if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID ||
 309             saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_PURGING) {
 310 
 311                 is_valid = B_FALSE;
 312 
 313         } else if (add_ref == B_TRUE) {
 314                 /*
 315                  * increment reference count here to ensure that
 316                  * entry does not get purged behind our backs
 317                  */
 318                 saa_portp->saa_pt_reference_count++;
 319         }
 320         mutex_exit(&saa_portp->saa_pt_mutex);
 321 
 322         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_end,
 323             IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() exit\n");
 324 
 325         return (is_valid);
 326 }
 327 
 328 /*
 329  * ibmf_saa_must_purge
 330  * Determines if we can purge a portp (remove it from the list) based on the
 331  * state and number of clients
 332  *
 333  * Input Arguments
 334  * saa_portp            pointer to state structure
 335  *
 336  * Output Arguments
 337  * none
 338  *
 339  * Returns
 340  * B_TRUE if the entry can be removed, B_FALSE otherwise
 341  */
 342 static int
 343 ibmf_saa_must_purge(saa_port_t *saa_portp)
 344 {
 345         int must_purge = B_FALSE;
 346 
 347         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_start,
 348             IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() enter\n");
 349 
 350         mutex_enter(&saa_portp->saa_pt_mutex);
 351 
 352         if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID &&
 353             saa_portp->saa_pt_reference_count == 0) {
 354 
 355                 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_PURGING;
 356                 must_purge = B_TRUE;
 357         }
 358 
 359         mutex_exit(&saa_portp->saa_pt_mutex);
 360 
 361         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_end,
 362             IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() exit\n");
 363 
 364         return (must_purge);
 365 }
 366 
 367 
 368 /*
 369  * ibmf_saa_impl_purge:
 370  * Removes invalid port state entries from the list
 371  *
 372  * Input Arguments
 373  * none
 374  *
 375  * Output Arguments
 376  * none
 377  *
 378  * Returns
 379  * void
 380  */
 381 void
 382 ibmf_saa_impl_purge()
 383 {
 384         saa_port_t *cur_portp  = NULL;
 385         saa_port_t *prev_portp = NULL;
 386         saa_port_t *rem_portp  = NULL;
 387 
 388         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_start,
 389             IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() enter\n");
 390 
 391         mutex_enter(&saa_statep->saa_port_list_mutex);
 392 
 393         cur_portp = saa_statep->saa_port_list;
 394         prev_portp = cur_portp;
 395 
 396         while (cur_portp != NULL) {
 397 
 398                 if (ibmf_saa_must_purge(cur_portp) == B_TRUE) {
 399 
 400                         rem_portp = cur_portp;
 401 
 402                         /* unlink entry */
 403                         if (cur_portp == saa_statep->saa_port_list) {
 404 
 405                                 saa_statep->saa_port_list = cur_portp->next;
 406                                 cur_portp = saa_statep->saa_port_list;
 407                                 prev_portp = cur_portp;
 408 
 409                         } else {
 410 
 411                                 prev_portp->next = cur_portp->next;
 412                                 cur_portp = cur_portp->next;
 413                         }
 414 
 415                         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*rem_portp))
 416 
 417                         /* destroy entry */
 418                         ASSERT(rem_portp != NULL);
 419                         ibmf_saa_impl_destroy_port(rem_portp);
 420 
 421                 } else {
 422 
 423                         prev_portp = cur_portp;
 424                         cur_portp = cur_portp->next;
 425                 }
 426         }
 427 
 428         mutex_exit(&saa_statep->saa_port_list_mutex);
 429 
 430         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_end,
 431             IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() exit\n");
 432 }
 433 
 434 /*
 435  * saa_impl_add_client:
 436  * Adds a client for a particular portp.  Reference count has been incremented
 437  * before this call.  It is decremented by saa_impl_add_client() if the call
 438  * fails.
 439  *
 440  * Input Arguments
 441  * none
 442  *
 443  * Output Arguments
 444  * none
 445  *
 446  * Returns
 447  * IBMF_BUSY if there are already too many clients registered,
 448  * IBMF_BAD_PORT_STATE if the port is invalid (generally because a previous
 449  * client failed during registration for this port)
 450  * IBMF_SUCCESS otherwise
 451  */
 452 int
 453 ibmf_saa_impl_add_client(saa_port_t *saa_portp)
 454 {
 455         int status = IBMF_SUCCESS;
 456 
 457         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_add_client_start,
 458             IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client() enter\n");
 459 
 460         mutex_enter(&saa_portp->saa_pt_mutex);
 461 
 462         /*
 463          * check that we don't exceed max clients
 464          */
 465         if (saa_portp->saa_pt_reference_count >
 466             SAA_MAX_CLIENTS_PER_PORT) {
 467 
 468                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
 469                     ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, "",
 470                     "ibmf_saa_impl_add_client: %s, num_reg_clients %d\n",
 471                     tnf_string, msg, "too many clients registered for"
 472                     " port", tnf_uint, num_reg_clients,
 473                     saa_portp->saa_pt_reference_count);
 474 
 475                 status = IBMF_BUSY;
 476                 goto bail;
 477         }
 478 
 479         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
 480             ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "",
 481             "ibmf_saa_impl_add_client: num_registered_clients %d\n",
 482             tnf_uint, num_registered_clients,
 483             saa_portp->saa_pt_reference_count);
 484 
 485         /*
 486          * wait until anyone who is currently registering
 487          * this port with ibmf is done
 488          */
 489         while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) {
 490 
 491                 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
 492                     ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "",
 493                     "ibmf_saa_impl_add_client: %s\n",
 494                     tnf_string, msg, "someone is registering. waiting"
 495                     " for them to finish");
 496 
 497                 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv,
 498                     &saa_portp->saa_pt_mutex);
 499 
 500                 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
 501                     ibmf_saa_impl_add_client,
 502                     IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client: %s\n",
 503                     tnf_string, msg, "done waiting");
 504         }
 505 
 506         /*
 507          * if port isn't ready here, fail.
 508          */
 509         if (saa_portp->saa_pt_state != IBMF_SAA_PORT_STATE_READY) {
 510 
 511                 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
 512                     ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR,
 513                     "", "ibmf_saa_impl_add_client: %s\n",
 514                     tnf_string, msg, "port state not ready,"
 515                     " removing client.");
 516 
 517                 status = IBMF_BAD_PORT_STATE;
 518                 goto bail;
 519         }
 520 
 521 bail:
 522         mutex_exit(&saa_portp->saa_pt_mutex);
 523 
 524         if (status != IBMF_SUCCESS) {
 525 
 526                 mutex_enter(&saa_portp->saa_pt_kstat_mutex);
 527 
 528                 IBMF_SAA_ADD32_KSTATS(saa_portp,
 529                     clients_reg_failed, 1);
 530 
 531                 mutex_exit(&saa_portp->saa_pt_kstat_mutex);
 532 
 533                 /* decrementing refcount is last thing we do on entry */
 534 
 535                 mutex_enter(&saa_portp->saa_pt_mutex);
 536 
 537                 ASSERT(saa_portp->saa_pt_reference_count > 0);
 538                 saa_portp->saa_pt_reference_count--;
 539 
 540                 mutex_exit(&saa_portp->saa_pt_mutex);
 541         }
 542 
 543         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 544             ibmf_saa_impl_add_client_end, IBMF_TNF_TRACE, "",
 545             "ibmf_saa_impl_add_client() exit\n");
 546 
 547         return (status);
 548 }
 549 
 550 /*
 551  * ibmf_saa_impl_create_port()
 552  * Create port entry with mimimal inits because
 553  * we're holding the list mutex: NO BLOCKING CALLS HERE, please.
 554  *
 555  * Initialize port state to "registering", so that clients accessing
 556  * same port concurrently will wait for the end of the ibmf registration.
 557  * Note: this thread will access port members without locking mutex.
 558  *
 559  * Input Arguments
 560  * pt_guid              guid of port
 561  *
 562  * Output Arguments
 563  * saa_portpp           pointer to new saa_portp structure
 564  *
 565  * Returns
 566  * IBMF_NO_MEMORY if memory could not be allocated
 567  * IBMF_SUCCESS otherwise
 568  */
 569 int
 570 ibmf_saa_impl_create_port(ib_guid_t pt_guid, saa_port_t **saa_portpp)
 571 {
 572         int             status          = IBMF_SUCCESS;
 573         saa_port_t      *saa_portp      = NULL;
 574 
 575         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_create_port_start,
 576             IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port:"
 577             " guid %016" PRIx64 "\n",
 578             tnf_opaque, port_guid, pt_guid);
 579 
 580         ASSERT(MUTEX_HELD(&saa_statep->saa_port_list_mutex));
 581 
 582         /* create & initialize new port */
 583         saa_portp = kmem_zalloc(sizeof (saa_port_t), KM_NOSLEEP);
 584 
 585         if (saa_portp == NULL) {
 586 
 587                 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
 588                     ibmf_sa_session_open_err, IBMF_TNF_ERROR, "",
 589                     "ibmf_saa_impl_create_port: %s\n",
 590                     tnf_string, msg, "could not allocate memory for "
 591                     "new port");
 592 
 593                 status = IBMF_NO_MEMORY;
 594                 goto bail;
 595         }
 596 
 597         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_sa_session_open,
 598             IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port: %s\n",
 599             tnf_string, msg, "first client registering, initializing");
 600 
 601         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp))
 602 
 603         /* tell everyone that kstats are not initialized */
 604         saa_portp->saa_pt_kstatp = NULL;
 605 
 606         /*
 607          * set up mutexe and state variable to indicate to
 608          * other clients that were currently in the process of
 609          * setting up the port data.  This will prevent a subsequent
 610          * client from trying to to register with ibmf before the
 611          * port data has been initialized.
 612          */
 613         mutex_init(&saa_portp->saa_pt_mutex, NULL, MUTEX_DRIVER, NULL);
 614         cv_init(&saa_portp->saa_pt_ibmf_reg_cv, NULL, CV_DRIVER, NULL);
 615 
 616         saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_REGISTERING;
 617 
 618         /* create other mutexes */
 619         mutex_init(&saa_portp->saa_pt_kstat_mutex, NULL, MUTEX_DRIVER, NULL);
 620 
 621         mutex_init(&saa_portp->saa_pt_event_sub_mutex, NULL, MUTEX_DRIVER,
 622             NULL);
 623 
 624         /*
 625          * clients assume all arrive; set mask to this so we only notify
 626          * if something failed
 627          */
 628         saa_portp->saa_pt_event_sub_last_success_mask =
 629             IBMF_SAA_PORT_EVENT_SUB_ALL_ARRIVE;
 630 
 631         /*
 632          * set port_guid now so any immediately subsequent clients
 633          * registering on this port, guid will know we're already here
 634          */
 635         saa_portp->saa_pt_port_guid = pt_guid;
 636         saa_portp->saa_pt_reference_count = 1;
 637         saa_portp->saa_pt_current_tid = pt_guid << 32;
 638 
 639         saa_portp->saa_pt_redirect_active = B_FALSE;
 640 
 641         /* set sa_uptime now in case we never receive anything from SA */
 642         saa_portp->saa_pt_sa_uptime = gethrtime();
 643 
 644         _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_portp))
 645 
 646         /* Set new pointer in caller's */
 647         *saa_portpp = saa_portp;
 648 
 649 bail:
 650         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_create_port_end,
 651             IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port() exit\n");
 652 
 653         return (status);
 654 }
 655 
 656 /*
 657  * ibmf_saa_impl_invalidate_port:
 658  * invalidates port entry (assumes exist) and deletes kstat object
 659  * kstat object is destroyed in order to allow creating port entry
 660  * even if this entry is not purged
 661  */
 662 static void
 663 ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp)
 664 {
 665         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 666             ibmf_saa_impl_invalidate_port_start,
 667             IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() enter\n");
 668 
 669         ASSERT(saa_portp != NULL);
 670         ASSERT(MUTEX_HELD(&saa_portp->saa_pt_mutex));
 671 
 672         saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_INVALID;
 673         ibmf_saa_impl_uninit_kstats(saa_portp);
 674 
 675         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 676             ibmf_saa_impl_invalidate_port_end,
 677             IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() exit\n");
 678 }
 679 
 680 /*
 681  * ibmf_saa_impl_destroy_port:
 682  * Frees the resources associated with an saa_portp structure.  Assumes the
 683  * saa_portp exists
 684  *
 685  * Input Arguments
 686  * saa_portp            pointer to saa_portp structure
 687  *
 688  * Output Arguments
 689  * none
 690  *
 691  * Returns
 692  * void
 693  */
 694 static void
 695 ibmf_saa_impl_destroy_port(saa_port_t *saa_portp)
 696 {
 697         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_start,
 698             IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() enter\n");
 699 
 700         ASSERT(saa_portp != NULL);
 701 
 702         _NOTE(ASSUMING_PROTECTED(*saa_portp))
 703 
 704         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
 705             ibmf_saa_impl_destroy, IBMF_TNF_TRACE, "",
 706             "ibmf_saa_impl_destroy(): destroying port_guid %016" PRIx64 "\n",
 707             tnf_opaque, port_guid, saa_portp->saa_pt_port_guid);
 708 
 709         ibmf_saa_impl_uninit_kstats(saa_portp);
 710 
 711         /* uninit synchronization variables used for registration */
 712         mutex_destroy(&saa_portp->saa_pt_mutex);
 713         cv_destroy(&saa_portp->saa_pt_ibmf_reg_cv);
 714 
 715         mutex_destroy(&saa_portp->saa_pt_event_sub_mutex);
 716         mutex_destroy(&saa_portp->saa_pt_kstat_mutex);
 717 
 718         kmem_free(saa_portp, sizeof (saa_port_t));
 719 
 720         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_end,
 721             IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() exit\n");
 722 }
 723 
 724 /*
 725  * ibmf_saa_impl_init_kstats:
 726  * Create kstats structure.  Should be called when memory is alloced for a new
 727  * port entry.
 728  */
 729 int
 730 ibmf_saa_impl_init_kstats(saa_port_t *saa_portp)
 731 {
 732         char                    buf[128];
 733         ibmf_saa_kstat_t        *ksp;
 734 
 735         _NOTE(ASSUMING_PROTECTED(saa_portp->saa_pt_kstatp))
 736 
 737         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 738             ibmf_saa_impl_init_kstats_start,
 739             IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() enter\n");
 740 
 741         /* set up kstats structure */
 742         (void) sprintf(buf, "ibmf_saa_%016" PRIx64 "_stat",
 743             saa_portp->saa_pt_port_guid);
 744 
 745         saa_portp->saa_pt_kstatp = kstat_create("ibmf_saa",
 746             0, buf, "misc", KSTAT_TYPE_NAMED,
 747             sizeof (ibmf_saa_kstat_t) / sizeof (kstat_named_t),
 748             KSTAT_FLAG_WRITABLE);
 749 
 750         if (saa_portp->saa_pt_kstatp == NULL)
 751                 return (IBMF_NO_RESOURCES);
 752 
 753         ksp = (ibmf_saa_kstat_t *)saa_portp->saa_pt_kstatp->ks_data;
 754 
 755         kstat_named_init(&ksp->clients_registered,
 756             "clients_registered", KSTAT_DATA_UINT32);
 757 
 758         kstat_named_init(&ksp->clients_reg_failed,
 759             "clients_reg_failed", KSTAT_DATA_UINT32);
 760 
 761         kstat_named_init(&ksp->outstanding_requests,
 762             "outstanding_requests", KSTAT_DATA_UINT32);
 763 
 764         kstat_named_init(&ksp->total_requests,
 765             "total_requests", KSTAT_DATA_UINT32);
 766 
 767         kstat_named_init(&ksp->failed_requests,
 768             "failed_requests", KSTAT_DATA_UINT32);
 769 
 770         kstat_named_init(&ksp->requests_timedout,
 771             "requests_timedout", KSTAT_DATA_UINT32);
 772 
 773         kstat_install(saa_portp->saa_pt_kstatp);
 774 
 775         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 776             ibmf_saa_impl_init_kstats_end,
 777             IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() exit\n");
 778 
 779         return (IBMF_SUCCESS);
 780 }
 781 
 782 /*
 783  * ibmf_saa_impl_uninit_kstats:
 784  * Free kstats context.  Should be called when port is either destroyed
 785  * or invalidated.
 786  */
 787 static void
 788 ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp)
 789 {
 790         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 791             ibmf_saa_impl_uninit_kstats_start,
 792             IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() enter\n");
 793 
 794         mutex_enter(&saa_portp->saa_pt_kstat_mutex);
 795 
 796         if (saa_portp->saa_pt_kstatp != NULL) {
 797                 kstat_delete(saa_portp->saa_pt_kstatp);
 798         }
 799         saa_portp->saa_pt_kstatp = NULL;
 800 
 801         mutex_exit(&saa_portp->saa_pt_kstat_mutex);
 802 
 803         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 804             ibmf_saa_impl_uninit_kstats_end,
 805             IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() exit\n");
 806 }
 807 
 808 /*
 809  * ibmf_saa_impl_register_failed:
 810  * invalidate entry and kick waiters
 811  */
 812 void
 813 ibmf_saa_impl_register_failed(saa_port_t *saa_portp)
 814 {
 815         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 816             ibmf_saa_impl_register_failed_start,
 817             IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() enter\n");
 818 
 819         mutex_enter(&saa_portp->saa_pt_mutex);
 820 
 821         ibmf_saa_impl_invalidate_port(saa_portp);
 822 
 823         cv_broadcast(&saa_portp->saa_pt_ibmf_reg_cv);
 824 
 825         /* decrementing refcount is last thing we do on entry */
 826 
 827         ASSERT(saa_portp->saa_pt_reference_count > 0);
 828         saa_portp->saa_pt_reference_count--;
 829 
 830         mutex_exit(&saa_portp->saa_pt_mutex);
 831 
 832         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 833             ibmf_saa_impl_register_failed_end,
 834             IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() exit\n");
 835 }
 836 
 837 static int
 838 ibmf_saa_impl_setup_qp_async_cb(saa_port_t *saa_portp, int setup_async_cb_only)
 839 {
 840         int             status;
 841         int             unreg_status;
 842         ib_pkey_t       p_key;
 843         ib_qkey_t       q_key;
 844         uint8_t         portnum;
 845         boolean_t       qp_alloced = B_FALSE;
 846 
 847         if (setup_async_cb_only == 0) {
 848 
 849                 /* allocate a qp through ibmf */
 850                 status = ibmf_alloc_qp(saa_portp->saa_pt_ibmf_handle,
 851                     IB_PKEY_DEFAULT_LIMITED, IB_GSI_QKEY,
 852                     IBMF_ALT_QP_MAD_RMPP, &saa_portp->saa_pt_qp_handle);
 853 
 854                 if (status != IBMF_SUCCESS) {
 855 
 856                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
 857                             ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
 858                             "ibmf_saa_impl_setup_qp_async_cb: %s, "
 859                             "ibmf_status = %d\n",
 860                             tnf_string, msg, "Cannot alloc qp with ibmf",
 861                             tnf_int, status, status);
 862 
 863                         return (status);
 864                 }
 865 
 866                 qp_alloced = B_TRUE;
 867 
 868                 /*
 869                  * query the queue pair number; we will need it to unsubscribe
 870                  * from notice reports
 871                  */
 872                 status = ibmf_query_qp(saa_portp->saa_pt_ibmf_handle,
 873                     saa_portp->saa_pt_qp_handle, &saa_portp->saa_pt_qpn,
 874                     &p_key, &q_key, &portnum, 0);
 875 
 876                 if (status != IBMF_SUCCESS) {
 877 
 878                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
 879                             ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
 880                             "ibmf_saa_impl_setup_qp_async_cb: %s, "
 881                             "ibmf_status = %d\n",
 882                             tnf_string, msg,
 883                             "Cannot query alt qp to get qp num",
 884                             tnf_int, status, status);
 885 
 886                         goto bail;
 887                 }
 888         }
 889 
 890         /*
 891          * core ibmf is taking advantage of the fact that saa_portp is our
 892          * callback arg. If this changes, the code in ibmf_recv would need to
 893          * change as well
 894          */
 895         status = ibmf_setup_async_cb(saa_portp->saa_pt_ibmf_handle,
 896             saa_portp->saa_pt_qp_handle, ibmf_saa_report_cb, saa_portp, 0);
 897         if (status != IBMF_SUCCESS) {
 898 
 899                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
 900                     ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
 901                     "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status = %d\n",
 902                     tnf_string, msg, "Cannot register async cb with ibmf",
 903                     tnf_int, status, status);
 904 
 905                 goto bail;
 906         }
 907 
 908         return (IBMF_SUCCESS);
 909 
 910 bail:
 911         if (qp_alloced == B_TRUE) {
 912                 /* free alternate qp */
 913                 unreg_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle,
 914                     &saa_portp->saa_pt_qp_handle, 0);
 915                 if (unreg_status != IBMF_SUCCESS) {
 916 
 917                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
 918                             ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
 919                             "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status ="
 920                             " %d\n", tnf_string, msg,
 921                             "Cannot free alternate queue pair with ibmf",
 922                             tnf_int, unreg_status, unreg_status);
 923                 }
 924         }
 925 
 926         return (status);
 927 }
 928 
 929 /*
 930  * ibmf_saa_impl_register_port:
 931  */
 932 int
 933 ibmf_saa_impl_register_port(
 934         saa_port_t *saa_portp)
 935 {
 936         uint_t          hca_count       = 0;
 937         ib_guid_t       *hca_list       = NULL;
 938         int             status          = IBMF_SUCCESS;
 939         int             unreg_status    = IBMF_SUCCESS;
 940         int             ibt_status      = IBT_SUCCESS;
 941         ibt_hca_portinfo_t *port_info_list = NULL;
 942         uint_t          port_count      = 0;
 943         uint_t          port_size       = 0;
 944         int             ihca, iport;
 945         ib_guid_t       port_guid;
 946         boolean_t       ibmf_reg = B_FALSE;
 947 
 948         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 949             ibmf_saa_impl_register_port_start, IBMF_TNF_TRACE, "",
 950             "ibmf_saa_impl_register_port() enter\n");
 951 
 952         ASSERT(saa_portp != NULL);
 953 
 954         _NOTE(ASSUMING_PROTECTED(*saa_portp))
 955 
 956         /* get the HCA list */
 957 
 958         hca_count = ibt_get_hca_list(&hca_list);
 959 
 960         if (hca_count == 0) {
 961 
 962                 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
 963                     ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
 964                     "ibmf_saa_impl_register_port: %s\n",
 965                     tnf_string, msg, "cannot register port (no HCAs).\n");
 966 
 967                 status = IBMF_BAD_PORT;
 968                 goto bail;
 969         }
 970 
 971         /* lookup requested port guid in hca list */
 972         for (ihca = 0; ihca != hca_count; ihca++) {
 973 
 974                 ibt_status = ibt_query_hca_ports_byguid(hca_list[ihca],
 975                     0 /* all ports */, &port_info_list,
 976                     &port_count, &port_size);
 977 
 978                 if (ibt_status != IBT_SUCCESS) {
 979 
 980                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
 981                             ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
 982                             "ibmf_saa_impl_register_port: %s, %016" PRIx64 "\n",
 983                             tnf_string, msg, "Could not query hca.  Exiting.",
 984                             tnf_opaque, guid, hca_list[ihca]);
 985 
 986                         status = IBMF_TRANSPORT_FAILURE;
 987                         break;
 988                 }
 989 
 990                 for (iport = 0; iport < port_count; iport++) {
 991 
 992                         /* get port guid associated with hca guid, port num */
 993                         if (ibmf_saa_impl_get_port_guid(
 994                             port_info_list + iport, &port_guid) != IBMF_SUCCESS)
 995                                 continue;
 996 
 997                         if (saa_portp->saa_pt_port_guid != port_guid)
 998                                 continue;
 999 
1000                         IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
1001                             ibmf_saa_impl_register_port,
1002                             IBMF_TNF_TRACE, "",
1003                             "ibmf_saa_impl_register_port: %s, hca_guid = %016"
1004                             PRIx64 ", port_guid = %016" PRIx64
1005                             ", number = %d\n",
1006                             tnf_string, msg, "found port",
1007                             tnf_opaque, hca_guid, hca_list[ihca],
1008                             tnf_opaque, port_guid, port_guid,
1009                             tnf_uint,   port, iport + 1);
1010 
1011                         /*
1012                          * we're here? then we found our port:
1013                          * fill in ibmf registration info
1014                          * and address parameters from the portinfo
1015                          */
1016 
1017                         saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid
1018                             = hca_list[ihca];
1019                         saa_portp->saa_pt_ibmf_reginfo.ir_port_num = iport+1;
1020                         saa_portp->saa_pt_ibmf_reginfo.ir_client_class
1021                             = SUBN_ADM_MANAGER;
1022 
1023                         saa_portp->saa_pt_node_guid = hca_list[ihca];
1024                         saa_portp->saa_pt_port_num = iport + 1;
1025 
1026                         ibmf_saa_impl_set_transaction_params(
1027                             saa_portp, port_info_list + iport);
1028                         break;
1029                 }
1030 
1031                 ibt_free_portinfo(port_info_list, port_size);
1032 
1033                 if (iport != port_count)
1034                         break;  /* found our port */
1035         }
1036 
1037         ibt_free_hca_list(hca_list, hca_count);
1038 
1039         if (ihca == hca_count) {
1040 
1041                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1042                     ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
1043                     "ibmf_saa_impl_register_port: %s, port_guid %016"
1044                     PRIx64 "\n",
1045                     tnf_string, msg, "Could not find port,  exiting",
1046                     tnf_opaque, port_guid, saa_portp->saa_pt_port_guid);
1047 
1048                 status = IBMF_BAD_PORT;
1049         }
1050 
1051         if (status != IBMF_SUCCESS) {
1052 
1053                 goto bail;
1054         }
1055 
1056         /*
1057          * Now we found the port we searched for,
1058          * and open an ibmf session on that port.
1059          */
1060 
1061         IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1062             ibmf_saa_impl_register_port, IBMF_TNF_TRACE, "",
1063             "ibmf_saa_impl_register_port: %s, port_guid = %016" PRIx64
1064             ", port = %d\n", tnf_string, msg, "Registering with ibmf",
1065             tnf_opaque, port_guid, saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid,
1066             tnf_uint, port, saa_portp->saa_pt_ibmf_reginfo.ir_port_num);
1067 
1068         status = ibmf_register(&saa_portp->saa_pt_ibmf_reginfo,
1069             IBMF_VERSION, IBMF_REG_FLAG_RMPP,
1070             ibmf_saa_impl_async_event_cb, saa_portp,
1071             &saa_portp->saa_pt_ibmf_handle,
1072             &saa_portp->saa_pt_ibmf_impl_features);
1073 
1074         if (status != IBMF_SUCCESS) {
1075 
1076                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1077                     ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
1078                     "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n",
1079                     tnf_string, msg, "Could not register with ibmf",
1080                     tnf_int, status, status);
1081 
1082                 goto bail;
1083         }
1084 
1085         ibmf_reg = B_TRUE;
1086 
1087         if (ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0) == IBMF_SUCCESS)
1088                 return (IBMF_SUCCESS);
1089 
1090 bail:
1091         if (ibmf_reg == B_TRUE) {
1092                 /* unregister from ibmf */
1093                 unreg_status = ibmf_unregister(
1094                     &saa_portp->saa_pt_ibmf_handle, 0);
1095 
1096                 if (unreg_status != IBMF_SUCCESS) {
1097 
1098                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1099                             ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
1100                             "ibmf_saa_impl_register_port: %s, ibmf_status ="
1101                             " %d\n", tnf_string, msg,
1102                             "Cannot unregister from ibmf",
1103                             tnf_int, unreg_status, unreg_status);
1104                 }
1105         }
1106 
1107         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_register_port_end,
1108             IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_port() exit\n");
1109 
1110         return (status);
1111 }
1112 
1113 /*
1114  * ibmf_saa_impl_getclassportinfo:
1115  */
1116 void
1117 ibmf_saa_impl_get_classportinfo(saa_port_t *saa_portp)
1118 {
1119         int                     res;
1120         saa_impl_trans_info_t   *trans_info;
1121 
1122         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1123             ibmf_saa_impl_get_classportinfo_start,
1124             IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() enter\n");
1125 
1126         /*
1127          * allocate memory for trans_info; send_request's callback will free up
1128          * memory since request is asynchronous
1129          */
1130         trans_info = kmem_zalloc(sizeof (saa_impl_trans_info_t), KM_NOSLEEP);
1131         if (trans_info == NULL) {
1132 
1133                 mutex_enter(&saa_portp->saa_pt_mutex);
1134 
1135                 /* cpi transaction is handled as a client, decrement refcount */
1136                 ASSERT(saa_portp->saa_pt_reference_count > 0);
1137                 saa_portp->saa_pt_reference_count--;
1138 
1139                 mutex_exit(&saa_portp->saa_pt_mutex);
1140 
1141                 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
1142                     ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_ERROR, "",
1143                     "ibmf_saa_impl_get_classportinfo: %s\n", tnf_string, msg,
1144                     "Could not allocate memory for classportinfo trans_info");
1145 
1146                 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1147                     ibmf_saa_impl_get_classportinfo_end, IBMF_TNF_TRACE, "",
1148                     "ibmf_saa_impl_get_classportinfo() exiting\n");
1149 
1150                 return;
1151         }
1152 
1153         /* no specific client associated with this transaction */
1154         trans_info->si_trans_client_data = NULL;
1155         trans_info->si_trans_port     = saa_portp;
1156         trans_info->si_trans_method   = SA_SUBN_ADM_GET;
1157         trans_info->si_trans_attr_id  = MAD_ATTR_ID_CLASSPORTINFO;
1158 
1159         trans_info->si_trans_callback = ibmf_saa_impl_get_cpi_cb;
1160         trans_info->si_trans_callback_arg = saa_portp;
1161 
1162         mutex_enter(&saa_portp->saa_pt_kstat_mutex);
1163 
1164         IBMF_SAA_ADD32_KSTATS(saa_portp, outstanding_requests, 1);
1165         IBMF_SAA_ADD32_KSTATS(saa_portp, total_requests, 1);
1166 
1167         mutex_exit(&saa_portp->saa_pt_kstat_mutex);
1168 
1169         res = ibmf_saa_impl_send_request(trans_info);
1170 
1171         if (res != IBMF_SUCCESS) {
1172 
1173                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1174                     ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_TRACE, "",
1175                     "ibmf_saa_impl_get_classportinfo: %s, res = 0x%x\n",
1176                     tnf_string, msg, "ibmf_saa_impl_send_request failed",
1177                     tnf_opaque, res, res);
1178 
1179                 mutex_enter(&saa_portp->saa_pt_kstat_mutex);
1180 
1181                 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1);
1182                 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1);
1183 
1184                 mutex_exit(&saa_portp->saa_pt_kstat_mutex);
1185 
1186                 mutex_enter(&saa_portp->saa_pt_mutex);
1187 
1188                 /* cpi transaction is handled as a client, decrement refcount */
1189                 ASSERT(saa_portp->saa_pt_reference_count > 0);
1190                 saa_portp->saa_pt_reference_count--;
1191 
1192                 mutex_exit(&saa_portp->saa_pt_mutex);
1193 
1194         }
1195 
1196         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1197             ibmf_saa_impl_get_classportinfo_end,
1198             IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() exit\n");
1199 }
1200 
1201 /*
1202  * ibmf_saa_impl_get_cpi_cb:
1203  *
1204  * Called when the asynchronous getportinfo request receives its response.
1205  * Checks the status.  If success, updates the times in the port's
1206  * ibmf_retrans structure that is used in ibmf_msg_transport calls.  If failure,
1207  * just use default values.
1208  *
1209  * Input Arguments
1210  * arg          user-specified pointer (points to the current port data)
1211  * length       length of payload returned (should be size of classportinfo_t)
1212  * buffer       pointer to classportinfo returned (should not be null)
1213  * status       status of sa access request
1214  *
1215  * Output Arguments
1216  * none
1217  *
1218  * Returns void
1219  */
1220 static void
1221 ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, int status)
1222 {
1223         saa_port_t              *saa_portp;
1224         uint64_t                base_time, resp_timeout, rttv_timeout;
1225         ib_mad_classportinfo_t  *classportinfo;
1226         int                     resp_time_value;
1227         uint16_t                sa_cap_mask;
1228 
1229         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_start,
1230             IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() enter\n");
1231 
1232         /*
1233          * access port entry: note that it may have become invalid
1234          * but we hold a ref count for cpi and the interactions on
1235          * the entry are harmless
1236          */
1237         saa_portp = (saa_port_t *)arg;
1238 
1239         /* process response */
1240 
1241         if ((status != IBMF_SUCCESS) || (buffer == NULL)) {
1242 
1243                 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1,
1244                     ibmf_saa_impl_get_cpi_cb, IBMF_TNF_ERROR, "",
1245                     "ibmf_saa_impl_get_cpi_cb: %s, status = %d, buffer = "
1246                     " 0x%p, length = %d\n", tnf_string, msg,
1247                     "could not get classportinfo.  Check node and path to sm"
1248                     " lid", tnf_int, status, status,
1249                     tnf_opaque, buffer, buffer, tnf_uint, length, length);
1250 
1251                 /*
1252                  * IB spec (C13-13) indicates 20 can be used as default or
1253                  * intial value for classportinfo->resptimeout value
1254                  */
1255                 resp_time_value = 20;
1256 
1257                 sa_cap_mask = 0xFFFF;
1258 
1259         } else if (buffer != NULL) {
1260 
1261                 classportinfo = (ib_mad_classportinfo_t *)buffer;
1262 
1263                 resp_time_value = classportinfo->RespTimeValue & 0x1f;
1264 
1265                 /*
1266                  * Because some subnet managers might not provide sane
1267                  * value for "resp_time_value", we limit it here.  In
1268                  * case this limit is too restrictive (very large fabric),
1269                  * we allow the limit to be raised (/etc/system).
1270                  */
1271                 if (resp_time_value > ibmf_saa_max_resp_time) {
1272                         cmn_err(CE_CONT, "!ibmf_saa_max_resp_time (%d) "
1273                             "exceeded.", ibmf_saa_max_resp_time);
1274                         cmn_err(CE_CONT, "!Reducing subnet administrator "
1275                             "resp_time value from %d to %d.",
1276                             resp_time_value, ibmf_saa_max_resp_time);
1277                         resp_time_value = ibmf_saa_max_resp_time;
1278                 }
1279 
1280                 sa_cap_mask = classportinfo->CapabilityMask;
1281 
1282                 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1283                     ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "",
1284                     "ibmf_saa_impl_get_cpi_cb: %s, timeout = 0x%x,"
1285                     " cap_mask = 0x%x\n",
1286                     tnf_string, msg, "got classportinfo",
1287                     tnf_opaque, timeout, resp_time_value,
1288                     tnf_opaque, cap_mask, sa_cap_mask);
1289 
1290                 kmem_free(buffer, length);
1291         }
1292 
1293         /*
1294          * using IB spec calculation from 13.4.6.2
1295          * use bit shifting for 2^x.
1296          */
1297         base_time = (1 << resp_time_value);
1298 
1299         resp_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000;
1300 
1301         mutex_enter(&saa_portp->saa_pt_mutex);
1302 
1303         base_time = 2 * (1 << saa_portp->saa_pt_timeout);
1304 
1305         rttv_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000;
1306 
1307         saa_portp->saa_pt_ibmf_retrans.retrans_rtv = resp_timeout;
1308         saa_portp->saa_pt_ibmf_retrans.retrans_rttv = rttv_timeout;
1309         saa_portp->saa_pt_sa_cap_mask = sa_cap_mask;
1310 
1311         /*
1312          * cpi transaction is handled as a client,
1313          * decrement refcount; make sure it's the last
1314          * thing we do on this entry
1315          */
1316         ASSERT(saa_portp->saa_pt_reference_count > 0);
1317         saa_portp->saa_pt_reference_count--;
1318 
1319         IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1320             ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "",
1321             "ibmf_saa_impl_get_cpi_cb: %s, subnet_timeout = 0x%x, "
1322             "resp_time_value = 0x%x\n",
1323             tnf_string, msg, "updated resp timeout",
1324             tnf_opaque, subnet_timeout, saa_portp->saa_pt_timeout,
1325             tnf_opaque, resp_time_value, resp_time_value);
1326 
1327         mutex_exit(&saa_portp->saa_pt_mutex);
1328 
1329         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_end,
1330             IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() exit\n");
1331 }
1332 
1333 /*
1334  * ibmf_saa_impl_send_request:
1335  * Sends a request to the sa.  Can be used for both classportinfo and record
1336  * requests.  Will set up all data structures for using the multi-packet
1337  * protocol, create the mad, and send it.  Returns SA_SUCCESS if msg transport
1338  * worked, meaning succesful send for the async case and a succesful send and
1339  * recv for the sync case.
1340  */
1341 int
1342 ibmf_saa_impl_send_request(saa_impl_trans_info_t *trans_info)
1343 {
1344         uint16_t                attr_id;
1345         saa_client_data_t       *client_data;
1346         saa_port_t              *saa_portp;
1347         uint32_t                transport_flags;
1348         ibmf_msg_cb_t           ibmf_callback;
1349         void                    *ibmf_callback_arg;
1350         ibmf_msg_t              *msgp;
1351         ibmf_retrans_t          ibmf_retrans;
1352         uint16_t                sa_cap_mask;
1353         boolean_t               sleep_flag;
1354         int                     ibmf_status = IBMF_SUCCESS;
1355         int                     retry_count;
1356         uint16_t                mad_status;
1357         boolean_t               sa_is_redirected = B_FALSE;
1358 
1359         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1360             ibmf_saa_impl_send_request_start,
1361             IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() enter\n");
1362 
1363         attr_id = trans_info->si_trans_attr_id;
1364         client_data = trans_info->si_trans_client_data;
1365         saa_portp   = trans_info->si_trans_port;
1366 
1367         /*
1368          * don't send on invalid entry
1369          * Note that there is a window where it could become
1370          * invalid after this test is done, but we'd rely on ibmf errors...
1371          */
1372         if (ibmf_saa_is_valid(saa_portp, B_FALSE) == B_FALSE) {
1373 
1374                 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1,
1375                     ibmf_saa_impl_send_request,
1376                     IBMF_TNF_ERROR, "",
1377                     "ibmf_saa_impl_send_request: %s, hca_guid = %016"
1378                     PRIx64 ", port_guid = %016" PRIx64
1379                     ", number = %d\n",
1380                     tnf_string, msg, "sending on invalid port",
1381                     tnf_opaque, hca_guid,
1382                     saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid,
1383                     tnf_opaque, port_guid,
1384                     saa_portp->saa_pt_port_guid,
1385                     tnf_uint,   port,
1386                     saa_portp->saa_pt_ibmf_reginfo.ir_port_num);
1387 
1388                 ibmf_status = IBMF_REQ_INVALID;
1389                 goto bail;
1390         }
1391 
1392         /* check whether SA supports this attribute */
1393         mutex_enter(&saa_portp->saa_pt_mutex);
1394 
1395         sa_cap_mask = saa_portp->saa_pt_sa_cap_mask;
1396         sa_is_redirected = saa_portp->saa_pt_redirect_active;
1397 
1398         mutex_exit(&saa_portp->saa_pt_mutex);
1399 
1400         ibmf_status = ibmf_saa_impl_check_sa_support(sa_cap_mask, attr_id);
1401 
1402         if (ibmf_status != IBMF_SUCCESS) {
1403 
1404                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1405                     ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "",
1406                     "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n",
1407                     tnf_string, msg, "SA does not support attribute",
1408                     tnf_int, ibmf_status, ibmf_status);
1409 
1410                 goto bail;
1411         }
1412 
1413         /* make only non-blocking calls if this is an async request */
1414         if ((trans_info->si_trans_callback == NULL) &&
1415             (trans_info->si_trans_sub_callback == NULL)) {
1416                 ibmf_callback = NULL;
1417                 ibmf_callback_arg = NULL;
1418                 sleep_flag = B_TRUE;
1419         } else {
1420                 ibmf_callback = ibmf_saa_async_cb;
1421                 ibmf_callback_arg = (void *)trans_info;
1422                 sleep_flag = B_FALSE;
1423         }
1424 
1425         ibmf_status = ibmf_saa_impl_init_msg(trans_info, sleep_flag, &msgp,
1426             &transport_flags, &ibmf_retrans);
1427         if (ibmf_status != IBMF_SUCCESS) {
1428 
1429                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1430                     ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "",
1431                     "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n",
1432                     tnf_string, msg, "init_msg() failed",
1433                     tnf_int, ibmf_status, ibmf_status);
1434 
1435                 goto bail;
1436         }
1437 
1438         mutex_enter(&saa_portp->saa_pt_mutex);
1439 
1440         saa_portp->saa_pt_num_outstanding_trans++;
1441 
1442         mutex_exit(&saa_portp->saa_pt_mutex);
1443 
1444         /*
1445          * increment the number of outstanding transaction so
1446          * ibmf_close_sa_session() will wait.  classportinfo requests
1447          * don't have associated clients so check for valid clientp
1448          */
1449         if (client_data != NULL) {
1450 
1451                 mutex_enter(&client_data->saa_client_mutex);
1452 
1453                 client_data->saa_client_num_pending_trans++;
1454 
1455                 mutex_exit(&client_data->saa_client_mutex);
1456         }
1457 
1458         /*
1459          * make the call to msg_transport.  If synchronous and success,
1460          * check that the response mad isn't status busy.  If so, repeat the
1461          * call
1462          */
1463         retry_count = 0;
1464 
1465         /*
1466          * set the send time here. We only set this once at the beginning of
1467          * the transaction.  Retrying because of busys or mastersmlid changes
1468          * does not change the original send time.  It is meant to be an
1469          * absolute time out value and will only be used if there are other
1470          * problems (i.e. a buggy SA)
1471          */
1472         trans_info->si_trans_send_time = gethrtime();
1473 
1474         for (;;) {
1475 
1476                 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle,
1477                     saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
1478                     ibmf_callback, ibmf_callback_arg, transport_flags);
1479 
1480                 if (ibmf_callback != NULL)
1481                         break;
1482 
1483                 /*
1484                  * stop here for non-sequenced transactions since they wouldn't
1485                  * receive a timeout or busy response
1486                  */
1487                 if (!(transport_flags & IBMF_MSG_TRANS_FLAG_SEQ))
1488                         break;
1489 
1490                 /*
1491                  * if the transaction timed out and this was a synchronous
1492                  * request there's a possiblity we were talking to the wrong
1493                  * master smlid or that the SA has stopped responding on the
1494                  * redirected desination (if redirect is active).
1495                  * Check this and retry if necessary.
1496                  */
1497                 if ((ibmf_status == IBMF_TRANS_TIMEOUT) &&
1498                     (sleep_flag == B_TRUE)) {
1499                         if (sa_is_redirected == B_TRUE) {
1500                                 ibmf_status = ibmf_saa_impl_revert_to_qp1(
1501                                     saa_portp, msgp, ibmf_callback,
1502                                     ibmf_callback_arg, transport_flags);
1503                         } else {
1504                                 ibmf_status = ibmf_saa_impl_new_smlid_retry(
1505                                     saa_portp, msgp, ibmf_callback,
1506                                     ibmf_callback_arg, transport_flags);
1507                         }
1508                 }
1509 
1510                 /*
1511                  * if the transaction timed out (and retrying with a new SM LID
1512                  * didn't help) check how long it's been since we received an SA
1513                  * packet.  If it hasn't been max_wait_time then retry the
1514                  * request.
1515                  */
1516                 if ((ibmf_status == IBMF_TRANS_TIMEOUT) &&
1517                     (sleep_flag == B_TRUE)) {
1518 
1519                         ibmf_status = ibmf_saa_check_sa_and_retry(
1520                             saa_portp, msgp, ibmf_callback, ibmf_callback_arg,
1521                             trans_info->si_trans_send_time, transport_flags);
1522                 }
1523 
1524                 if (ibmf_status != IBMF_SUCCESS)
1525                         break;
1526 
1527                 if (retry_count >= IBMF_SAA_MAX_BUSY_RETRY_COUNT)
1528                         break;
1529 
1530                 /* sync transaction with status SUCCESS should have response */
1531                 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL);
1532 
1533                 mad_status = b2h16(msgp->im_msgbufs_recv.
1534                     im_bufs_mad_hdr->Status);
1535 
1536                 if ((mad_status != MAD_STATUS_BUSY) &&
1537                     (mad_status != MAD_STATUS_REDIRECT_REQUIRED))
1538                         break;
1539 
1540                 if (mad_status == MAD_STATUS_REDIRECT_REQUIRED) {
1541 
1542                         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1543                             ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1544                             "ibmf_saa_impl_send_request: %s, retry_count %d\n",
1545                             tnf_string, msg,
1546                             "response returned redirect status",
1547                             tnf_int, retry_count, retry_count);
1548 
1549                         /* update address info and copy it into msgp */
1550                         ibmf_saa_impl_update_sa_address_info(saa_portp, msgp);
1551                 } else {
1552                         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1553                             ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1554                             "ibmf_saa_impl_send_request: %s, retry_count %d\n",
1555                             tnf_string, msg, "response returned busy status",
1556                             tnf_int, retry_count, retry_count);
1557                 }
1558 
1559                 retry_count++;
1560 
1561                 /*
1562                  * since this is a blocking call, sleep for some time
1563                  * to allow SA to transition from busy state (if busy)
1564                  */
1565                 if (mad_status == MAD_STATUS_BUSY)
1566                         delay(drv_usectohz(
1567                             IBMF_SAA_BUSY_RETRY_SLEEP_SECS * 1000000));
1568         }
1569 
1570         if (ibmf_status != IBMF_SUCCESS) {
1571 
1572                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1573                     ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1574                     "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n",
1575                     tnf_string, msg, "ibmf_msg_transport() failed",
1576                     tnf_int, ibmf_status, ibmf_status);
1577 
1578                 ibmf_saa_impl_free_msg(saa_portp->saa_pt_ibmf_handle, msgp);
1579 
1580                 mutex_enter(&saa_portp->saa_pt_mutex);
1581 
1582                 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0);
1583                 saa_portp->saa_pt_num_outstanding_trans--;
1584 
1585                 mutex_exit(&saa_portp->saa_pt_mutex);
1586 
1587                 if (client_data != NULL) {
1588 
1589                         mutex_enter(&client_data->saa_client_mutex);
1590 
1591                         ASSERT(client_data->saa_client_num_pending_trans > 0);
1592                         client_data->saa_client_num_pending_trans--;
1593 
1594                         if ((client_data->saa_client_num_pending_trans == 0) &&
1595                             (client_data->saa_client_state ==
1596                             SAA_CLIENT_STATE_WAITING))
1597                                 cv_signal(&client_data->saa_client_state_cv);
1598 
1599                         mutex_exit(&client_data->saa_client_mutex);
1600                 }
1601 
1602         } else if (sleep_flag == B_TRUE) {
1603 
1604                 mutex_enter(&saa_portp->saa_pt_mutex);
1605 
1606                 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0);
1607                 saa_portp->saa_pt_num_outstanding_trans--;
1608 
1609                 mutex_exit(&saa_portp->saa_pt_mutex);
1610 
1611                 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1612                     ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1613                     "ibmf_saa_impl_send_request: %s\n",
1614                     tnf_string, msg, "Message sent and received successfully");
1615 
1616                 /* fill in response values and free the message */
1617                 ibmf_saa_impl_prepare_response(saa_portp->saa_pt_ibmf_handle,
1618                     msgp, B_FALSE, &trans_info->si_trans_status,
1619                     &trans_info->si_trans_result,
1620                     &trans_info->si_trans_length, sleep_flag);
1621 
1622                 if (client_data != NULL) {
1623                         mutex_enter(&client_data->saa_client_mutex);
1624 
1625                         ASSERT(client_data->saa_client_num_pending_trans > 0);
1626                         client_data->saa_client_num_pending_trans--;
1627 
1628                         if ((client_data->saa_client_num_pending_trans == 0) &&
1629                             (client_data->saa_client_state ==
1630                             SAA_CLIENT_STATE_WAITING))
1631                                 cv_signal(&client_data->saa_client_state_cv);
1632 
1633                         mutex_exit(&client_data->saa_client_mutex);
1634                 }
1635         } else {
1636 
1637                 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1638                     ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1639                     "ibmf_saa_impl_send_request: %s\n",
1640                     tnf_string, msg, "Message sent successfully");
1641         }
1642 
1643 bail:
1644         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1645             ibmf_saa_impl_send_request_end,
1646             IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() exiting"
1647             " ibmf_status = %d\n", tnf_int, result, ibmf_status);
1648 
1649         return (ibmf_status);
1650 }
1651 
1652 /*
1653  * ibmf_saa_impl_init_msg:
1654  * Allocates an ibmf message and fills out the header fields and formatted data
1655  * fields.  Also sets up the correct transport_flags and retrans argument for
1656  * the message transport call based on the request information.
1657  *
1658  * Input Arguments
1659  * trans_info           saa_trans_info structure passed to send_request
1660  * sleep_flag           B_TRUE if init_msg can sleep in function calls
1661  *
1662  * Output Arguments
1663  * msgp                 ibmf message that should be given to msg_transport
1664  * transport_flagsp     transport flags that should be given to msg_transport
1665  * ibmf_retrans_t       retrans parameter that should be given to msg_transport
1666  *
1667  * Returns
1668  * ibmf_status
1669  */
1670 static int
1671 ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, boolean_t sleep_flag,
1672     ibmf_msg_t **msgp, uint32_t *transport_flagsp,
1673     ibmf_retrans_t *ibmf_retransp)
1674 {
1675         int                     ibmf_status;
1676         ibmf_msg_bufs_t         *req_mad;
1677         ib_mad_hdr_t            *mad_hdr;
1678         int                     ibmf_sleep_flag, km_sleep_flag;
1679         int                     free_res;
1680         ib_sa_hdr_t             sa_hdr;
1681         ibmf_msg_t              *ibmf_msg;
1682         uint16_t                attr_id, pack_attr_id;
1683         uint8_t                 method;
1684         saa_client_data_t       *client_data;
1685         saa_port_t              *saa_portp;
1686         sa_multipath_record_t   *multipath_template;
1687         size_t                  payload_length;
1688         uint32_t                transport_flags;
1689 
1690         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1691             ibmf_saa_impl_init_msg_start,
1692             IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() entering\n");
1693 
1694         attr_id = trans_info->si_trans_attr_id;
1695         method = trans_info->si_trans_method;
1696         client_data = trans_info->si_trans_client_data;
1697         saa_portp   = trans_info->si_trans_port;
1698 
1699         if (sleep_flag == B_TRUE) {
1700                 ibmf_sleep_flag = IBMF_ALLOC_SLEEP;
1701                 km_sleep_flag = KM_SLEEP;
1702         } else {
1703                 ibmf_sleep_flag = IBMF_ALLOC_NOSLEEP;
1704                 km_sleep_flag = KM_NOSLEEP;
1705         }
1706 
1707         ibmf_status = ibmf_alloc_msg(saa_portp->saa_pt_ibmf_handle,
1708             ibmf_sleep_flag, &ibmf_msg);
1709         if (ibmf_status != IBMF_SUCCESS) {
1710 
1711                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1712                     ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1713                     "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n",
1714                     tnf_string, msg, "Cannot allocate msg_buf.",
1715                     tnf_int, ibmf_status, ibmf_status);
1716 
1717                 goto bail;
1718         }
1719 
1720         req_mad = &ibmf_msg->im_msgbufs_send;
1721 
1722         /* create a template (SA MAD) */
1723         mad_hdr = kmem_zalloc(sizeof (ib_mad_hdr_t), km_sleep_flag);
1724 
1725         if (mad_hdr == NULL) {
1726 
1727                 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
1728                     ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1729                     "ibmf_saa_impl_init_msg: %s\n",
1730                     tnf_string, msg, "Cannot allocate mad header.");
1731 
1732                 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1733                     &ibmf_msg);
1734                 ASSERT(free_res == IBMF_SUCCESS);
1735 
1736                 ibmf_status = IBMF_NO_MEMORY;
1737                 goto bail;
1738         }
1739 
1740         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mad_hdr,
1741             *req_mad))
1742 
1743         bzero(mad_hdr, sizeof (ib_mad_hdr_t));
1744         mad_hdr->BaseVersion = SAA_MAD_BASE_VERSION;
1745         mad_hdr->MgmtClass = MAD_MGMT_CLASS_SUBN_ADM;
1746         mad_hdr->ClassVersion = SAA_MAD_CLASS_VERSION;
1747         mad_hdr->R_Method = method;
1748         mad_hdr->AttributeID = h2b16(attr_id);
1749 
1750         /* attribute modifier is all Fs since RIDs are no longer used */
1751         mad_hdr->AttributeModifier = h2b32(0xffffffff);
1752 
1753         IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
1754             ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "",
1755             "ibmf_saa_impl_init_msg: %s, class = 0x%x, method = 0x%x,"
1756             " attr_id = 0x%x\n", tnf_string, msg, "Sending MAD",
1757             tnf_opaque, class, mad_hdr->MgmtClass,
1758             tnf_opaque, method, mad_hdr->R_Method,
1759             tnf_opaque, attr_id, attr_id);
1760 
1761         bzero(&sa_hdr, sizeof (ib_sa_hdr_t));
1762         sa_hdr.ComponentMask = trans_info->si_trans_component_mask;
1763 
1764         if (client_data != NULL)
1765                 sa_hdr.SM_KEY = client_data->saa_client_sm_key;
1766 
1767         /*
1768          * pack data for IB wire format; req_mad will have different pointers to
1769          * sa header and payload, mad_hdr will be the same
1770          */
1771         req_mad->im_bufs_mad_hdr = mad_hdr;
1772 
1773         ibmf_status = ibmf_saa_utils_pack_sa_hdr(&sa_hdr,
1774             &req_mad->im_bufs_cl_hdr, &req_mad->im_bufs_cl_hdr_len,
1775             km_sleep_flag);
1776 
1777         if (ibmf_status != IBMF_SUCCESS) {
1778 
1779                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1780                     ibmf_saa_impl_init_msg, IBMF_TNF_ERROR, "",
1781                     "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n",
1782                     tnf_string, msg, "ibmf_saa_utils_pack_sa_hdr() failed",
1783                     tnf_int, ibmf_status, ibmf_status);
1784 
1785                 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t));
1786 
1787                 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1788                     &ibmf_msg);
1789                 ASSERT(free_res == IBMF_SUCCESS);
1790 
1791                 goto bail;
1792         }
1793 
1794         if (attr_id == SA_MULTIPATHRECORD_ATTRID) {
1795 
1796                 multipath_template =
1797                     (sa_multipath_record_t *)trans_info->si_trans_template;
1798 
1799                 payload_length = sizeof (sa_multipath_record_t) +
1800                     ((multipath_template->SGIDCount +
1801                     multipath_template->DGIDCount) * sizeof (ib_gid_t));
1802 
1803                 pack_attr_id = attr_id;
1804         } else {
1805 
1806                 /* trace record template is a path record */
1807                 pack_attr_id = (attr_id == SA_TRACERECORD_ATTRID) ?
1808                     SA_PATHRECORD_ATTRID : attr_id;
1809 
1810                 payload_length = ibmf_saa_impl_get_attr_id_length(pack_attr_id);
1811 
1812                 if (payload_length == 0) {
1813                         payload_length = trans_info->si_trans_template_length;
1814 
1815                         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
1816                             ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "",
1817                             "ibmf_saa_impl_init_msg: %s, length = %d\n",
1818                             tnf_string, msg,
1819                             "Unknown attribute.  Using user-defined length.",
1820                             tnf_uint, length, payload_length)
1821                 }
1822         }
1823 
1824         /* transport type depends on method */
1825         switch (method) {
1826 
1827                 case SA_SUBN_ADM_GET:
1828                 case SA_SUBN_ADM_DELETE:
1829                 case SA_SUBN_ADM_GET_TABLE:
1830                 case SA_SUBN_ADM_GET_TRACE_TABLE:
1831                         transport_flags = IBMF_MSG_TRANS_FLAG_SEQ;
1832                         break;
1833                 case SA_SUBN_ADM_SET:
1834                         /* unsubscribes can be sequenced or unsequenced */
1835                         if (trans_info->si_trans_unseq_unsubscribe == B_TRUE) {
1836                                 transport_flags = 0;
1837                         } else {
1838                                 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ;
1839                         }
1840                         break;
1841                 case SA_SUBN_ADM_GET_MULTI:
1842                         transport_flags = IBMF_MSG_TRANS_FLAG_SEQ |
1843                             IBMF_MSG_TRANS_FLAG_RMPP;
1844                         break;
1845                 default :
1846                         ibmf_status = IBMF_UNSUPP_METHOD;
1847                         goto bail;
1848         }
1849 
1850         trans_info->si_trans_transport_flags = transport_flags;
1851 
1852         if (trans_info->si_trans_template != NULL) {
1853 
1854                 ibmf_status = ibmf_saa_utils_pack_payload(
1855                     trans_info->si_trans_template, payload_length, pack_attr_id,
1856                     &req_mad->im_bufs_cl_data, &req_mad->im_bufs_cl_data_len,
1857                     km_sleep_flag);
1858                 if (ibmf_status != IBMF_SUCCESS) {
1859 
1860                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1861                             ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1862                             "ibmf_saa_impl_init_msg: %s, ibmf_status ="
1863                             " %d\n", tnf_string, msg,
1864                             "ibmf_saa_utils_pack_payload() failed",
1865                             tnf_int, ibmf_status, ibmf_status);
1866 
1867                         kmem_free(mad_hdr, sizeof (ib_mad_hdr_t));
1868 
1869                         kmem_free(req_mad->im_bufs_cl_hdr,
1870                             req_mad->im_bufs_cl_hdr_len);
1871 
1872                         free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1873                             &ibmf_msg);
1874                         ASSERT(free_res == IBMF_SUCCESS);
1875 
1876                         goto bail;
1877                 }
1878 
1879                 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1880                     ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "",
1881                     "ibmf_saa_impl_init_msg: %s, attr_id = 0x%x, length ="
1882                     " %d\n", tnf_string, msg, "Packed payload successfully",
1883                     tnf_opaque, attr_id, attr_id,
1884                     tnf_uint, length, req_mad->im_bufs_cl_data_len);
1885 
1886                 /* non-RMPP transactions have template size limit */
1887                 if (((transport_flags & IBMF_MSG_TRANS_FLAG_RMPP) == 0) &&
1888                     ((req_mad->im_bufs_cl_data_len + req_mad->im_bufs_cl_hdr_len
1889                     + sizeof (ib_mad_hdr_t)) > IBMF_MAD_SIZE)) {
1890 
1891                         IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
1892                             ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1893                             "ibmf_saa_impl_init_msg: %s\n", tnf_string, msg,
1894                             "Template too large to fit in single packet");
1895 
1896                         kmem_free(mad_hdr, sizeof (ib_mad_hdr_t));
1897 
1898                         kmem_free(req_mad->im_bufs_cl_hdr,
1899                             req_mad->im_bufs_cl_hdr_len);
1900 
1901                         kmem_free(req_mad->im_bufs_cl_data,
1902                             req_mad->im_bufs_cl_data_len);
1903 
1904                         free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1905                             &ibmf_msg);
1906                         ASSERT(free_res == IBMF_SUCCESS);
1907 
1908                         ibmf_status = IBMF_REQ_INVALID;
1909                         goto bail;
1910                 }
1911         }
1912 
1913         mutex_enter(&saa_portp->saa_pt_mutex);
1914 
1915         mad_hdr->TransactionID = h2b64(saa_portp->saa_pt_current_tid++);
1916 
1917         bcopy(&saa_portp->saa_pt_ibmf_retrans, ibmf_retransp,
1918             sizeof (ibmf_retrans_t));
1919 
1920         /* copy local addressing information to message */
1921         bcopy(&saa_portp->saa_pt_ibmf_addr_info, &ibmf_msg->im_local_addr,
1922             sizeof (ibmf_addr_info_t));
1923 
1924         /* copy global addressing information to message if in use */
1925         if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) {
1926 
1927                 ibmf_msg->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS;
1928 
1929                 bcopy(&saa_portp->saa_pt_ibmf_global_addr,
1930                     &ibmf_msg->im_global_addr,
1931                     sizeof (ibmf_global_addr_info_t));
1932         } else {
1933                 ibmf_msg->im_msg_flags = 0;
1934         }
1935 
1936         mutex_exit(&saa_portp->saa_pt_mutex);
1937 
1938         *msgp = ibmf_msg;
1939         *transport_flagsp = transport_flags;
1940 bail:
1941         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1942             ibmf_saa_impl_init_msg_end,
1943             IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() exiting"
1944             " ibmf_status = %d\n", tnf_int, result, ibmf_status);
1945 
1946         return (ibmf_status);
1947 
1948 }
1949 
1950 /*
1951  * ibmf_saa_impl_new_smlid_retry:
1952  *
1953  * It's possible for the MasterSMLID to change while ibmf_saa is running.  The
1954  * MasterSMLID is set when we first register with ibmf_saa.  If a request
1955  * timesout, this function should be called to check whether the SM LID changed.
1956  * If so, it will call msg_transport again with the request.
1957  *
1958  * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the
1959  * same values passed to the original ibmf_msg_transport that timedout.  The
1960  * ibmf_retrans parameter will be re-retrieved from the saa_portp structure.
1961  *
1962  * If the lid did not change then this function returns IBMF_TRANS_TIMEOUT.
1963  * That way, callers can simply return the result of this function.
1964  *
1965  * Input Arguments
1966  * saa_portp            pointer to saa_port structure
1967  * msgp                 ibmf message that timedout
1968  * ibmf_callback        callback that should be called by msg_transport
1969  * ibmf_callback_arg    args for ibmf_callback
1970  * transport_flags      flags for ibmf_msg_transport
1971  *
1972  * Output Arguments
1973  * none
1974  *
1975  * Returns
1976  * IBMF_SUCCESS if lid changed and request was resent successfully,
1977  * IBMF_TRANS_TIMEOUT if lid did not change,
1978  * same values as ibmf_msg_transport() if lid changed but request could not be
1979  * resent.
1980  */
1981 static int
1982 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
1983     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags)
1984 {
1985         ibt_hca_portinfo_t      *ibt_portinfop;
1986         ib_lid_t                master_sm_lid;
1987         int                     subnet_timeout;
1988         uint_t                  nports, size;
1989         ibmf_retrans_t          ibmf_retrans;
1990         int                     ibmf_status;
1991         ibt_status_t            ibt_status;
1992 
1993         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1994             ibmf_saa_impl_new_smlid_retry_start,
1995             IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() enter\n");
1996 
1997         _NOTE(ASSUMING_PROTECTED(*msgp))
1998         _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr))
1999 
2000         /* first query the portinfo to see if the lid changed */
2001         ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid,
2002             saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size);
2003 
2004         if (ibt_status != IBT_SUCCESS)  {
2005 
2006                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2007                     ibmf_saa_impl_new_smlid_retry_err, IBMF_TNF_ERROR, "",
2008                     "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status ="
2009                     " %d\n", tnf_string, msg,
2010                     "ibt_query_hca_ports_byguid() failed",
2011                     tnf_int, ibt_status, ibt_status);
2012 
2013                 ibmf_status = IBMF_TRANSPORT_FAILURE;
2014 
2015                 goto bail;
2016         }
2017 
2018         master_sm_lid = ibt_portinfop->p_sm_lid;
2019         subnet_timeout = ibt_portinfop->p_subnet_timeout;
2020 
2021         ibt_free_portinfo(ibt_portinfop, size);
2022 
2023         /* if master smlid is different than the remote lid we sent to */
2024         if (master_sm_lid != msgp->im_local_addr.ia_remote_lid) {
2025 
2026                 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L2,
2027                     ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "",
2028                     "ibmf_saa_impl_new_smlid_retry: %s, new_lid 0x%x,"
2029                     " old_lid 0x%x\n", tnf_string, msg,
2030                     "master smlid has changed.  retrying msg_transport",
2031                     tnf_opaque, new_lid, master_sm_lid,
2032                     tnf_opaque, old_lid, msgp->im_local_addr.ia_remote_lid);
2033 
2034                 mutex_enter(&saa_portp->saa_pt_mutex);
2035 
2036                 /* update the master sm lid value in ibmf_saa */
2037                 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2038                     master_sm_lid;
2039 
2040                 /* new tid needed */
2041                 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID =
2042                     h2b64(saa_portp->saa_pt_current_tid++);
2043 
2044                 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2045                     sizeof (ibmf_retrans_t));
2046 
2047                 /* update the subnet timeout since this may be a new sm/sa */
2048                 saa_portp->saa_pt_timeout = subnet_timeout;
2049 
2050                 /* place upper bound on subnet timeout in case of faulty SM */
2051                 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout)
2052                         saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout;
2053 
2054                 /* increment the reference count to account for the cpi call */
2055                 saa_portp->saa_pt_reference_count++;
2056 
2057                 mutex_exit(&saa_portp->saa_pt_mutex);
2058 
2059                 /* update the remote lid for this particular message */
2060                 msgp->im_local_addr.ia_remote_lid = master_sm_lid;
2061 
2062                 /* get the classportinfo again since this may be a new sm/sa */
2063                 ibmf_saa_impl_get_classportinfo(saa_portp);
2064 
2065                 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle,
2066                     saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2067                     ibmf_callback, ibmf_callback_arg, transport_flags);
2068 
2069                 if (ibmf_status != IBMF_SUCCESS) {
2070 
2071                         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
2072                             ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "",
2073                             "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status = "
2074                             "%d\n", tnf_string, msg,
2075                             "ibmf_msg_transport() failed",
2076                             tnf_int, ibmf_status, ibmf_status);
2077                 }
2078 
2079                 goto bail;
2080         }
2081 
2082         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
2083             ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "",
2084             "ibmf_saa_impl_new_smlid_retry: %s, master_smlid = 0x%x\n",
2085             tnf_string, msg,
2086             "master smlid did not change.  returning failure",
2087             tnf_opaque, master_smlid, master_sm_lid);
2088 
2089         /* mark status as timeout since that was original failure */
2090         ibmf_status = IBMF_TRANS_TIMEOUT;
2091 
2092 bail:
2093         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2094             ibmf_saa_impl_new_smlid_retry_end,
2095             IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() exiting"
2096             " ibmf_status = %d\n", tnf_int, result, ibmf_status);
2097 
2098         return (ibmf_status);
2099 }
2100 
2101 /*
2102  * ibmf_saa_impl_revert_to_qp1()
2103  *
2104  * The SA that we had contact with via redirect may fail to respond. If this
2105  * occurs SA should revert back to qp1 and the SMLID set in the port.
2106  * msg_transport for the message that timed out will be retried with
2107  * these new parameters.
2108  *
2109  * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the
2110  * same values passed to the original ibmf_msg_transport that timedout.  The
2111  * ibmf_retrans parameter will be re-retrieved from the saa_portp structure.
2112  *
2113  * Input Arguments
2114  * saa_portp            pointer to saa_port structure
2115  * msgp                 ibmf message that timedout
2116  * ibmf_callback        callback that should be called by msg_transport
2117  * ibmf_callback_arg    args for ibmf_callback
2118  * transport_flags      flags for ibmf_msg_transport
2119  *
2120  * Output Arguments
2121  * none
2122  *
2123  * Returns
2124  * none
2125  */
2126 static int
2127 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp,
2128     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags)
2129 {
2130         ibt_hca_portinfo_t      *ibt_portinfop;
2131         ib_lid_t                master_sm_lid, base_lid;
2132         uint8_t                 sm_sl;
2133         int                     subnet_timeout;
2134         uint_t                  nports, size;
2135         ibmf_retrans_t          ibmf_retrans;
2136         int                     ibmf_status;
2137         ibt_status_t            ibt_status;
2138 
2139         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
2140             ibmf_saa_impl_revert_to_qp1_start,
2141             IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() enter\n");
2142 
2143         _NOTE(ASSUMING_PROTECTED(*msgp))
2144         _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr))
2145 
2146         /* first query the portinfo to see if the lid changed */
2147         ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid,
2148             saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size);
2149 
2150         if (ibt_status != IBT_SUCCESS)  {
2151 
2152                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2153                     ibmf_saa_impl_revert_to_qp1_err, IBMF_TNF_ERROR, "",
2154                     "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status ="
2155                     " %d\n", tnf_string, msg,
2156                     "ibt_query_hca_ports_byguid() failed",
2157                     tnf_int, ibt_status, ibt_status);
2158 
2159                 ibmf_status = IBMF_TRANSPORT_FAILURE;
2160 
2161                 goto bail;
2162         }
2163 
2164         master_sm_lid = ibt_portinfop->p_sm_lid;
2165         base_lid = ibt_portinfop->p_base_lid;
2166         sm_sl = ibt_portinfop->p_sm_sl;
2167         subnet_timeout = ibt_portinfop->p_subnet_timeout;
2168 
2169         ibt_free_portinfo(ibt_portinfop, size);
2170 
2171 
2172         mutex_enter(&saa_portp->saa_pt_mutex);
2173 
2174         saa_portp->saa_pt_redirect_active = B_FALSE;
2175 
2176         /* update the address info in ibmf_saa */
2177         saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = base_lid;
2178         saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = master_sm_lid;
2179         saa_portp->saa_pt_ibmf_addr_info.ia_service_level = sm_sl;
2180         saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1;
2181         saa_portp->saa_pt_ibmf_addr_info.ia_p_key = IB_PKEY_DEFAULT_LIMITED;
2182         saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY;
2183         saa_portp->saa_pt_ibmf_msg_flags = 0;
2184 
2185         /* new tid needed */
2186         msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID =
2187             h2b64(saa_portp->saa_pt_current_tid++);
2188 
2189         bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2190             sizeof (ibmf_retrans_t));
2191 
2192         /* update the subnet timeout since this may be a new sm/sa */
2193         saa_portp->saa_pt_timeout = subnet_timeout;
2194 
2195         /* place upper bound on subnet timeout in case of faulty SM */
2196         if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout)
2197                 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout;
2198 
2199         /* increment the reference count to account for the cpi call */
2200         saa_portp->saa_pt_reference_count++;
2201 
2202         mutex_exit(&saa_portp->saa_pt_mutex);
2203 
2204         /* update the address info for this particular message */
2205         bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr,
2206             sizeof (ibmf_addr_info_t));
2207         msgp->im_msg_flags = 0; /* No GRH */
2208 
2209         /* get the classportinfo again since this may be a new sm/sa */
2210         ibmf_saa_impl_get_classportinfo(saa_portp);
2211 
2212         ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle,
2213             saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2214             ibmf_callback, ibmf_callback_args, transport_flags);
2215 
2216         if (ibmf_status != IBMF_SUCCESS) {
2217 
2218                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
2219                     ibmf_saa_impl_revert_to_qp1, IBMF_TNF_TRACE, "",
2220                     "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status = "
2221                     "%d\n", tnf_string, msg,
2222                     "ibmf_msg_transport() failed",
2223                     tnf_int, ibmf_status, ibmf_status);
2224         }
2225 
2226 bail:
2227 
2228         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2229             ibmf_saa_impl_revert_to_qp1_end,
2230             IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() exiting"
2231             " ibmf_status = %d\n", tnf_int, result, ibmf_status);
2232 
2233         return (ibmf_status);
2234 }
2235 
2236 /*
2237  * ibmf_saa_impl_async_event_cb:
2238  *      ibmf event callback, argument to ibmf_register
2239  *      ibmf_handle is unused
2240  */
2241 /*  ARGSUSED */
2242 static void
2243 ibmf_saa_impl_async_event_cb(
2244         ibmf_handle_t           ibmf_handle,
2245         void                    *clnt_private,
2246         ibmf_async_event_t      event_type)
2247 {
2248         saa_port_t              *saa_portp;
2249 
2250         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2251             ibmf_saa_impl_async_event_cb_start, IBMF_TNF_TRACE, "",
2252             "ibmf_saa_impl_async_event_cb: Handling event type 0x%x\n",
2253             tnf_opaque, event_type, event_type);
2254 
2255         saa_portp = (saa_port_t *)clnt_private;
2256         ASSERT(saa_portp != NULL);
2257 
2258         switch (event_type) {
2259 
2260         case IBMF_CI_OFFLINE:
2261                 ibmf_saa_impl_hca_detach(saa_portp);
2262                 break;
2263         default:
2264                 break;
2265         }
2266 
2267         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3,
2268             ibmf_saa_impl_async_event_cb_end,
2269             IBMF_TNF_TRACE, "", "ibmf_saa_impl_async_event_cb() exit\n");
2270 }
2271 
2272 
2273 /*
2274  * ibmf_saa_impl_ibt_async_handler:
2275  * MUST NOT BE STATIC (referred from within IBMF)
2276  */
2277 void
2278 ibmf_saa_impl_ibt_async_handler(ibt_async_code_t code, ibt_async_event_t *event)
2279 {
2280         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2281             ibmf_saa_impl_ibt_async_handler_start, IBMF_TNF_TRACE, "",
2282             "ibmf_saa_impl_ibt_async_handler: Handling event code 0x%x\n",
2283             tnf_opaque, code, code);
2284 
2285         switch (code) {
2286 
2287         case IBT_EVENT_PORT_UP:
2288                 ibmf_saa_impl_port_up(event->ev_hca_guid, event->ev_port);
2289                 break;
2290         case IBT_ERROR_PORT_DOWN:
2291                 ibmf_saa_impl_port_down(event->ev_hca_guid, event->ev_port);
2292                 break;
2293         case IBT_PORT_CHANGE_EVENT:
2294                 ibmf_saa_impl_port_chg(event);
2295                 break;
2296         case IBT_CLNT_REREG_EVENT:
2297                 ibmf_saa_impl_client_rereg(event->ev_hca_guid, event->ev_port);
2298                 break;
2299         default:
2300                 break;
2301         }
2302 
2303         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_async_handler_end,
2304             IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibt_async_handler() exit\n");
2305 }
2306 
2307 /*
2308  * ibmf_saa_impl_port_chg:
2309  */
2310 static void
2311 ibmf_saa_impl_port_chg(ibt_async_event_t *event)
2312 {
2313         saa_port_t              *saa_portp      = NULL;
2314         boolean_t               is_ready = B_FALSE;
2315         ibt_hca_portinfo_t      *ibt_portinfop;
2316         uint_t                  nports, size;
2317         ibt_status_t            ibt_status;
2318         ib_guid_t               ci_guid;
2319         int                     port_num;
2320 
2321         ci_guid = event->ev_hca_guid;
2322         port_num = event->ev_port;
2323 
2324         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_chg_start,
2325             IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg: Handling port chg"
2326             " guid %016" PRIx64 " port %d\n",
2327             tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2328 
2329         /* Get classportinfo of corresponding entry */
2330         mutex_enter(&saa_statep->saa_port_list_mutex);
2331 
2332         saa_portp = saa_statep->saa_port_list;
2333         while (saa_portp != NULL) {
2334                 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
2335                     saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
2336                         mutex_enter(&saa_portp->saa_pt_mutex);
2337 
2338                         is_ready = (saa_portp->saa_pt_state
2339                             == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
2340 
2341                         /*
2342                          * increment reference count to account for cpi and
2343                          * informinfos.  All 4 informinfo's sent are treated as
2344                          * one port client reference
2345                          */
2346                         if (is_ready)
2347                                 saa_portp->saa_pt_reference_count ++;
2348 
2349                         mutex_exit(&saa_portp->saa_pt_mutex);
2350 
2351                         if (is_ready)
2352                                 break; /* normally, only 1 port entry */
2353                 }
2354                 saa_portp = saa_portp->next;
2355         }
2356 
2357         mutex_exit(&saa_statep->saa_port_list_mutex);
2358 
2359         if (saa_portp != NULL) {
2360                 /* first query the portinfo to see if the lid changed */
2361                 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
2362                     &ibt_portinfop, &nports, &size);
2363 
2364                 if (ibt_status != IBT_SUCCESS) {
2365 
2366                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2367                             ibmf_saa_impl_port_chg_err, IBMF_TNF_ERROR, "",
2368                             "ibmf_saa_impl_port_chg: %s, ibmf_status ="
2369                             " %d\n", tnf_string, msg,
2370                             "ibt_query_hca_ports_byguid() failed",
2371                             tnf_int, ibt_status, ibt_status);
2372 
2373                         goto bail;
2374                 }
2375 
2376                 mutex_enter(&saa_portp->saa_pt_mutex);
2377                 if (event->ev_port_flags & IBT_PORT_CHANGE_SM_LID) {
2378                         /* update the Master SM Lid value in ibmf_saa */
2379                         saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2380                             ibt_portinfop->p_sm_lid;
2381                 }
2382                 if (event->ev_port_flags & IBT_PORT_CHANGE_SM_SL) {
2383                         /* update the Master SM SL value in ibmf_saa */
2384                         saa_portp->saa_pt_ibmf_addr_info.ia_service_level =
2385                             ibt_portinfop->p_sm_sl;
2386                 }
2387                 if (event->ev_port_flags & IBT_PORT_CHANGE_SUB_TIMEOUT) {
2388                         /* update the Subnet timeout value in ibmf_saa */
2389                         saa_portp->saa_pt_timeout =
2390                             ibt_portinfop->p_subnet_timeout;
2391                 }
2392                 mutex_exit(&saa_portp->saa_pt_mutex);
2393 
2394                 ibt_free_portinfo(ibt_portinfop, size);
2395 
2396                 /* get the classportinfo again */
2397                 ibmf_saa_impl_get_classportinfo(saa_portp);
2398         }
2399 bail:
2400 
2401         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_chg_end,
2402             IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg() exit\n");
2403 }
2404 /*
2405  * ibmf_saa_impl_client_rereg:
2406  */
2407 static void
2408 ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num)
2409 {
2410         saa_port_t              *saa_portp      = NULL;
2411         boolean_t               is_ready = B_FALSE;
2412         ibt_hca_portinfo_t      *ibt_portinfop;
2413         ib_lid_t                master_sm_lid;
2414         uint_t                  nports, size;
2415         ibt_status_t            ibt_status;
2416         boolean_t               event_subs = B_FALSE;
2417 
2418         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_rereg_start,
2419             IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg: Handling clnt "
2420             "rereg guid %016" PRIx64 " port %d\n",
2421             tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2422 
2423         /* Get classportinfo of corresponding entry */
2424         mutex_enter(&saa_statep->saa_port_list_mutex);
2425 
2426         saa_portp = saa_statep->saa_port_list;
2427         while (saa_portp != NULL) {
2428 
2429                 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
2430                     saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
2431 
2432                         mutex_enter(&saa_portp->saa_pt_mutex);
2433 
2434                         is_ready = (saa_portp->saa_pt_state
2435                             == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
2436 
2437                         /*
2438                          * increment reference count to account for cpi and
2439                          * informinfos.  All 4 informinfo's sent are treated as
2440                          * one port client reference
2441                          */
2442                         if (is_ready)
2443                                 saa_portp->saa_pt_reference_count += 2;
2444 
2445                         mutex_exit(&saa_portp->saa_pt_mutex);
2446 
2447                         if (is_ready)
2448                                 break; /* normally, only 1 port entry */
2449                 }
2450                 saa_portp = saa_portp->next;
2451         }
2452 
2453         mutex_exit(&saa_statep->saa_port_list_mutex);
2454 
2455         if (saa_portp != NULL && is_ready == B_TRUE) {
2456 
2457                 /* verify whether master sm lid changed */
2458 
2459                 /* first query the portinfo to see if the lid changed */
2460                 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
2461                     &ibt_portinfop, &nports, &size);
2462 
2463                 if (ibt_status != IBT_SUCCESS) {
2464 
2465                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2466                             ibmf_saa_impl_port_rereg_err, IBMF_TNF_ERROR, "",
2467                             "ibmf_saa_impl_client_rereg: %s, ibmf_status ="
2468                             " %d\n", tnf_string, msg,
2469                             "ibt_query_hca_ports_byguid() failed",
2470                             tnf_int, ibt_status, ibt_status);
2471 
2472                         goto bail;
2473                 }
2474 
2475                 master_sm_lid = ibt_portinfop->p_sm_lid;
2476 
2477                 ibt_free_portinfo(ibt_portinfop, size);
2478 
2479                 /* check whether we need to subscribe for events */
2480                 mutex_enter(&saa_portp->saa_pt_event_sub_mutex);
2481 
2482                 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ?
2483                     B_TRUE : B_FALSE;
2484 
2485                 mutex_exit(&saa_portp->saa_pt_event_sub_mutex);
2486 
2487                 /* update the master smlid */
2488                 mutex_enter(&saa_portp->saa_pt_mutex);
2489 
2490                 /* update the master sm lid value in ibmf_saa */
2491                 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2492                     master_sm_lid;
2493 
2494                 /* if we're not subscribed for events, dec reference count */
2495                 if (event_subs == B_FALSE)
2496                         saa_portp->saa_pt_reference_count--;
2497 
2498                 mutex_exit(&saa_portp->saa_pt_mutex);
2499 
2500                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
2501                     ibmf_saa_impl_port_rereg, IBMF_TNF_TRACE, "",
2502                     "ibmf_saa_impl_client_rereg: %s, master_sm_lid = 0x%x\n",
2503                     tnf_string, msg,
2504                     "port is up.  Sending classportinfo request",
2505                     tnf_opaque, master_sm_lid, master_sm_lid);
2506 
2507                 /* get the classportinfo again */
2508                 ibmf_saa_impl_get_classportinfo(saa_portp);
2509 
2510                 /*
2511                  * resubscribe to events if there are subscribers since SA may
2512                  * have removed our subscription records when the port went down
2513                  */
2514                 if (event_subs == B_TRUE)
2515                         ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE);
2516         }
2517 
2518 bail:
2519 
2520         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_rereg_end,
2521             IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg() exit\n");
2522 }
2523 /*
2524  * ibmf_saa_impl_port_up:
2525  */
2526 static void
2527 ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num)
2528 {
2529         saa_port_t              *saa_portp      = NULL;
2530         int                     is_ready;
2531         ibt_hca_portinfo_t      *ibt_portinfop;
2532         ib_lid_t                master_sm_lid;
2533         uint_t                  nports, size;
2534         ibt_status_t            ibt_status;
2535         boolean_t               event_subs = B_FALSE;
2536 
2537         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_up_start,
2538             IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up: Handling port up"
2539             " guid %016" PRIx64 " port %d\n",
2540             tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2541 
2542         /* Get classportinfo of corresponding entry */
2543         mutex_enter(&saa_statep->saa_port_list_mutex);
2544 
2545         saa_portp = saa_statep->saa_port_list;
2546         while (saa_portp != NULL) {
2547 
2548                 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
2549                     saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
2550 
2551                         mutex_enter(&saa_portp->saa_pt_mutex);
2552 
2553                         is_ready = (saa_portp->saa_pt_state
2554                             == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
2555 
2556                         /*
2557                          * increment reference count to account for cpi and
2558                          * informinfos.  All 4 informinfo's sent are treated as
2559                          * one port client reference
2560                          */
2561                         if (is_ready == B_TRUE)
2562                                 saa_portp->saa_pt_reference_count += 2;
2563 
2564                         mutex_exit(&saa_portp->saa_pt_mutex);
2565 
2566                         if (is_ready == B_TRUE)
2567                                 break; /* normally, only 1 port entry */
2568                 }
2569                 saa_portp = saa_portp->next;
2570         }
2571 
2572         mutex_exit(&saa_statep->saa_port_list_mutex);
2573 
2574         if (saa_portp != NULL && is_ready == B_TRUE) {
2575 
2576                 /* verify whether master sm lid changed */
2577 
2578                 /* first query the portinfo to see if the lid changed */
2579                 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
2580                     &ibt_portinfop, &nports, &size);
2581 
2582                 if (ibt_status != IBT_SUCCESS) {
2583 
2584                         IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2585                             ibmf_saa_impl_port_up_err, IBMF_TNF_ERROR, "",
2586                             "ibmf_saa_impl_port_up: %s, ibmf_status ="
2587                             " %d\n", tnf_string, msg,
2588                             "ibt_query_hca_ports_byguid() failed",
2589                             tnf_int, ibt_status, ibt_status);
2590 
2591                         goto bail;
2592                 }
2593 
2594                 master_sm_lid = ibt_portinfop->p_sm_lid;
2595 
2596                 ibt_free_portinfo(ibt_portinfop, size);
2597 
2598                 /* check whether we need to subscribe for events */
2599                 mutex_enter(&saa_portp->saa_pt_event_sub_mutex);
2600 
2601                 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ?
2602                     B_TRUE : B_FALSE;
2603 
2604                 mutex_exit(&saa_portp->saa_pt_event_sub_mutex);
2605 
2606                 /* update the master smlid */
2607                 mutex_enter(&saa_portp->saa_pt_mutex);
2608 
2609                 /* update the master sm lid value in ibmf_saa */
2610                 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2611                     master_sm_lid;
2612 
2613                 /* if we're not subscribed for events, dec reference count */
2614                 if (event_subs == B_FALSE)
2615                         saa_portp->saa_pt_reference_count--;
2616 
2617                 mutex_exit(&saa_portp->saa_pt_mutex);
2618 
2619                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
2620                     ibmf_saa_impl_port_up, IBMF_TNF_TRACE, "",
2621                     "ibmf_saa_impl_port_up: %s, master_sm_lid = 0x%x\n",
2622                     tnf_string, msg,
2623                     "port is up.  Sending classportinfo request",
2624                     tnf_opaque, master_sm_lid, master_sm_lid);
2625 
2626                 /* get the classportinfo again */
2627                 ibmf_saa_impl_get_classportinfo(saa_portp);
2628 
2629                 /*
2630                  * resubscribe to events if there are subscribers since SA may
2631                  * have removed our subscription records when the port went down
2632                  */
2633                 if (event_subs == B_TRUE)
2634                         ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE);
2635         }
2636 
2637 bail:
2638 
2639         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_up_end,
2640             IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up() exit\n");
2641 }
2642 
2643 /*
2644  * ibmf_saa_impl_port_down:
2645  */
2646 static void
2647 ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num)
2648 {
2649 
2650         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_down_start,
2651             IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down: Handling port down"
2652             " guid %016" PRIx64 " port %d\n",
2653             tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2654 
2655 
2656         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_down_end,
2657             IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down() exit\n");
2658 }
2659 
2660 /*
2661  * ibmf_saa_impl_hca_detach:
2662  * find entry, unregister if there are no clients
2663  * have to unregister since ibmf needs to close the hca and will only do this if
2664  * no clients are registered
2665  */
2666 static void
2667 ibmf_saa_impl_hca_detach(saa_port_t *saa_removed)
2668 {
2669         saa_port_t      *saa_portp;
2670         boolean_t       must_unreg, must_unsub;
2671 
2672         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_start,
2673             IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: Detaching"
2674             " entry %016" PRIx64 "\n", tnf_opaque, entry, saa_removed);
2675 
2676         /* find this entry */
2677         mutex_enter(&saa_statep->saa_port_list_mutex);
2678 
2679         saa_portp = saa_statep->saa_port_list;
2680         while (saa_portp != NULL) {
2681 
2682                 if (saa_portp == saa_removed)
2683                         break;
2684 
2685                 saa_portp = saa_portp->next;
2686         }
2687         mutex_exit(&saa_statep->saa_port_list_mutex);
2688 
2689         ASSERT(saa_portp != NULL);
2690 
2691         if (saa_portp == NULL) {
2692 
2693                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
2694                     ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "",
2695                     "ibmf_saa_impl_hca_detach: %s, entry %016"
2696                     PRIx64 "\n",
2697                     tnf_string, msg,
2698                     "Port entry NOT found",
2699                     tnf_opaque, entryp, saa_removed);
2700 
2701                 goto bail;
2702         }
2703 
2704         /* if there are clients expecting Reports(), unsusbscribe */
2705         mutex_enter(&saa_portp->saa_pt_event_sub_mutex);
2706 
2707         must_unsub = (saa_portp->saa_pt_event_sub_client_list != NULL) ?
2708             B_TRUE : B_FALSE;
2709 
2710         mutex_exit(&saa_portp->saa_pt_event_sub_mutex);
2711 
2712         /* fail if outstanding transactions */
2713         mutex_enter(&saa_portp->saa_pt_mutex);
2714 
2715         if (saa_portp->saa_pt_num_outstanding_trans > 0) {
2716 
2717                 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L1,
2718                     ibmf_saa_impl_fini_err, IBMF_TNF_TRACE, "",
2719                     "ibmf_saa_impl_fini: %s, port = %016" PRIx64
2720                     ", num transactions = %d\n",
2721                     tnf_string, msg, "Detaching HCA."
2722                     "  Outstanding transactions on port.",
2723                     tnf_opaque, port,
2724                     saa_portp->saa_pt_port_guid,
2725                     tnf_uint, outstanding_transactions,
2726                     saa_portp->saa_pt_num_outstanding_trans);
2727 
2728                 mutex_exit(&saa_portp->saa_pt_mutex);
2729 
2730                 goto bail;
2731         }
2732 
2733 
2734         /*
2735          * increment reference count by one to account for unsubscribe requests
2736          * that are about to be sent.  All four informinfo's are treated as one
2737          * port client reference.  The count will be decremented by
2738          * subscribe_events() before the call returns.
2739          */
2740         if (must_unsub == B_TRUE)
2741                 saa_portp->saa_pt_reference_count++;
2742 
2743         mutex_exit(&saa_portp->saa_pt_mutex);
2744 
2745         /*
2746          * try and unsubscribe from SA.  Generate synchronous, unsequenced
2747          * unsubscribe requests.
2748          */
2749         if (must_unsub == B_TRUE)
2750                 ibmf_saa_subscribe_events(saa_portp, B_FALSE, B_TRUE);
2751 
2752         /* warning if registered clients */
2753         mutex_enter(&saa_portp->saa_pt_mutex);
2754 
2755         if (saa_portp->saa_pt_reference_count > 0) {
2756 
2757                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
2758                     ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "",
2759                     "ibmf_saa_impl_hca_detach: %s, port %016"
2760                     PRIx64 "\n",
2761                     tnf_string, msg,
2762                     "Detaching HCA for port with clients still"
2763                     " registered", tnf_opaque, port,
2764                     saa_portp->saa_pt_port_guid);
2765         }
2766 
2767         /* synchronize on end of registration */
2768         while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) {
2769 
2770                 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1,
2771                     ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "",
2772                     "ibmf_saa_impl_hca_detach: %s\n",
2773                     tnf_string, msg, "someone is registering. waiting"
2774                     " for them to finish");
2775 
2776                 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv,
2777                     &saa_portp->saa_pt_mutex);
2778 
2779                 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1,
2780                     ibmf_saa_impl_hca_detach,
2781                     IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: %s\n",
2782                     tnf_string, msg, "done waiting");
2783         }
2784 
2785         /* unregister from ibmf */
2786         if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) {
2787                 must_unreg = B_TRUE;
2788         } else
2789                 must_unreg = B_FALSE;
2790 
2791         ibmf_saa_impl_invalidate_port(saa_portp);
2792 
2793         mutex_exit(&saa_portp->saa_pt_mutex);
2794 
2795         if (must_unreg == B_TRUE) {
2796                 if (ibmf_saa_impl_ibmf_unreg(saa_portp) != IBMF_SUCCESS) {
2797                         mutex_enter(&saa_portp->saa_pt_mutex);
2798                         mutex_enter(&saa_portp->saa_pt_kstat_mutex);
2799                         (void) ibmf_saa_impl_init_kstats(saa_portp);
2800                         mutex_exit(&saa_portp->saa_pt_kstat_mutex);
2801                         saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_READY;
2802                         if (must_unsub == B_TRUE)
2803                                 saa_portp->saa_pt_reference_count++;
2804                         mutex_exit(&saa_portp->saa_pt_mutex);
2805 
2806                         if (must_unsub == B_TRUE) {
2807                                 ibmf_saa_subscribe_events(saa_portp, B_TRUE,
2808                                     B_FALSE);
2809                         }
2810                 }
2811         }
2812 bail:
2813         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_end,
2814             IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach() exit\n");
2815 }
2816 
2817 /* ARGSUSED */
2818 void
2819 ibmf_saa_async_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, void *args)
2820 {
2821         saa_impl_trans_info_t   *trans_info;
2822         int                     status;
2823         size_t                  length;
2824         void                    *result;
2825         saa_port_t              *saa_portp;
2826         saa_client_data_t       *client_data;
2827         int                     ibmf_status;
2828         boolean_t               ignore_data;
2829         ibmf_retrans_t          ibmf_retrans;
2830         boolean_t               sa_is_redirected = B_FALSE;
2831 
2832         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_start,
2833             IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() enter\n");
2834 
2835         trans_info = (saa_impl_trans_info_t *)args;
2836 
2837         client_data = trans_info->si_trans_client_data;
2838         saa_portp   = trans_info->si_trans_port;
2839 
2840         mutex_enter(&saa_portp->saa_pt_mutex);
2841         sa_is_redirected = saa_portp->saa_pt_redirect_active;
2842         mutex_exit(&saa_portp->saa_pt_mutex);
2843 
2844         if ((msgp->im_msg_status == IBMF_TRANS_TIMEOUT) &&
2845             (sa_is_redirected == B_TRUE)) {
2846 
2847                 /*
2848                  * We should retry the request using SM_LID and QP1 if we
2849                  * have been using redirect up until now
2850                  */
2851                 ibmf_status = ibmf_saa_impl_revert_to_qp1(
2852                     saa_portp, msgp, ibmf_saa_async_cb, args,
2853                     trans_info->si_trans_transport_flags);
2854 
2855                 /*
2856                  * If revert_to_qp1 returns success msg was resent.
2857                  * Otherwise msg could not be resent. Continue normally
2858                  */
2859                 if (ibmf_status == IBMF_SUCCESS)
2860                         goto bail;
2861 
2862         } else if (msgp->im_msg_status == IBMF_TRANS_TIMEOUT) {
2863 
2864 
2865                 ibmf_status = ibmf_saa_impl_new_smlid_retry(saa_portp, msgp,
2866                     ibmf_saa_async_cb, args,
2867                     trans_info->si_trans_transport_flags);
2868 
2869                 /*
2870                  * if smlid_retry() returns success sm lid changed and msg
2871                  * was resent.  Otherwise, lid did not change or msg could not
2872                  * be resent.  Continue normally.
2873                  */
2874                 if (ibmf_status == IBMF_SUCCESS)
2875                         goto bail;
2876 
2877                 /*
2878                  * check whether we've received anything from the SA in a while.
2879                  * If we have, this function will retry and return success.  If
2880                  * we haven't continue normally so that we return a timeout to
2881                  * the client
2882                  */
2883                 ibmf_status = ibmf_saa_check_sa_and_retry(
2884                     saa_portp, msgp, ibmf_saa_async_cb, args,
2885                     trans_info->si_trans_send_time,
2886                     trans_info->si_trans_transport_flags);
2887 
2888                 if (ibmf_status == IBMF_SUCCESS)
2889                         goto bail;
2890         }
2891 
2892         /*
2893          * If SA returned success but mad status is busy, retry a few times.
2894          * If SA returned success but mad status says redirect is required,
2895          * update the address info and retry the request to the new SA address
2896          */
2897         if (msgp->im_msg_status == IBMF_SUCCESS) {
2898 
2899                 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL);
2900 
2901                 if ((b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status) ==
2902                     MAD_STATUS_BUSY) &&
2903                     (trans_info->si_trans_retry_busy_count <
2904                     IBMF_SAA_MAX_BUSY_RETRY_COUNT)) {
2905 
2906                         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
2907                             ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
2908                             "ibmf_saa_async_cb: %s, retry_count = %d\n",
2909                             tnf_string, msg,
2910                             "async response returned busy status",
2911                             tnf_int, retry_count,
2912                             trans_info->si_trans_retry_busy_count);
2913 
2914                         trans_info->si_trans_retry_busy_count++;
2915 
2916                         bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2917                             sizeof (ibmf_retrans_t));
2918 
2919                         ibmf_status = ibmf_msg_transport(
2920                             saa_portp->saa_pt_ibmf_handle,
2921                             saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2922                             ibmf_saa_async_cb, args,
2923                             trans_info->si_trans_transport_flags);
2924 
2925                         /*
2926                          * if retry is successful, quit here since async_cb will
2927                          * get called again; otherwise, let this function call
2928                          * handle the cleanup
2929                          */
2930                         if (ibmf_status == IBMF_SUCCESS)
2931                                 goto bail;
2932                 } else if (b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status)
2933                     == MAD_STATUS_REDIRECT_REQUIRED) {
2934 
2935                         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L2,
2936                             ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
2937                             "ibmf_saa_async_cb: "
2938                             "async response returned redirect status\n");
2939 
2940                         /* update address info and copy it into msgp */
2941                         ibmf_saa_impl_update_sa_address_info(saa_portp, msgp);
2942 
2943                         /* retry with new address info */
2944                         bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2945                             sizeof (ibmf_retrans_t));
2946 
2947                         ibmf_status = ibmf_msg_transport(
2948                             saa_portp->saa_pt_ibmf_handle,
2949                             saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2950                             ibmf_saa_async_cb, args,
2951                             trans_info->si_trans_transport_flags);
2952 
2953                         /*
2954                          * if retry is successful, quit here since async_cb will
2955                          * get called again; otherwise, let this function call
2956                          * handle the cleanup
2957                          */
2958                         if (ibmf_status == IBMF_SUCCESS)
2959                                 goto bail;
2960                 }
2961         }
2962 
2963         mutex_enter(&saa_portp->saa_pt_mutex);
2964 
2965         ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0);
2966         saa_portp->saa_pt_num_outstanding_trans--;
2967 
2968         mutex_exit(&saa_portp->saa_pt_mutex);
2969 
2970         if ((trans_info->si_trans_callback == NULL) &&
2971             (trans_info->si_trans_sub_callback == NULL))
2972                 ignore_data = B_TRUE;
2973         else
2974                 ignore_data = B_FALSE;
2975 
2976         ibmf_saa_impl_prepare_response(ibmf_handle, msgp, ignore_data, &status,
2977             &result, &length, B_FALSE);
2978 
2979         mutex_enter(&saa_portp->saa_pt_kstat_mutex);
2980 
2981         IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1);
2982 
2983         if (status != IBMF_SUCCESS)
2984                 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1);
2985 
2986         if (status == IBMF_TRANS_TIMEOUT)
2987                 IBMF_SAA_ADD32_KSTATS(saa_portp, requests_timedout, 1);
2988 
2989         mutex_exit(&saa_portp->saa_pt_kstat_mutex);
2990 
2991         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2992             ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
2993             "ibmf_saa_async_cb: %s\n", tnf_string, msg,
2994             "Calling ibmf_saa client's callback");
2995 
2996         /*
2997          * there are three classes or trans_info users: ibmf_saa clients and
2998          * classportinfo requests; informinfo subscribe requests, and report
2999          * responses.  For the first two, call the correct callback.  For report
3000          * responses there's no need to notify anyone.
3001          */
3002         if (trans_info->si_trans_callback != NULL) {
3003                 /* ibmf_saa client or classportinfo request */
3004                 trans_info->si_trans_callback(trans_info->si_trans_callback_arg,
3005                     length, result, status);
3006         } else if (trans_info->si_trans_sub_callback != NULL) {
3007                 /* informinfo subscribe request */
3008                 trans_info->si_trans_sub_callback(
3009                     trans_info->si_trans_callback_arg, length, result, status,
3010                     trans_info->si_trans_sub_producer_type);
3011         }
3012 
3013         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3014             ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
3015             "ibmf_saa_async_cb: %s\n", tnf_string, msg,
3016             "Returned from callback");
3017 
3018         if (client_data != NULL) {
3019                 mutex_enter(&client_data->saa_client_mutex);
3020 
3021                 ASSERT(client_data->saa_client_num_pending_trans > 0);
3022                 client_data->saa_client_num_pending_trans--;
3023 
3024                 if ((client_data->saa_client_num_pending_trans == 0) &&
3025                     (client_data->saa_client_state == SAA_CLIENT_STATE_WAITING))
3026                         cv_signal(&client_data->saa_client_state_cv);
3027 
3028                 mutex_exit(&client_data->saa_client_mutex);
3029         }
3030 
3031         kmem_free(trans_info, sizeof (saa_impl_trans_info_t));
3032 
3033 bail:
3034 
3035         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_end,
3036             IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() exit\n");
3037 }
3038 
3039 /*
3040  * ibmf_saa_check_sa_and_retry:
3041  *
3042  * If a particular transaction times out, we don't want to give up if we know
3043  * the SA is responding.  Check the time since we last received a response. If
3044  * it's less than ibmf_saa_max_wait_time retry the request.
3045  *
3046  * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the
3047  * same values passed to the original ibmf_msg_transport that timed out.  The
3048  * ibmf_retrans parameter will be re-retrieved from the saa_portp structure.
3049  *
3050  * If max_wait_time seconds have passed, this function returns IBMF_TIMEOUT.
3051  * That way, callers can simply return the result of this function.
3052  *
3053  * Input Arguments
3054  * saa_portp            pointer to saa_port structure
3055  * msgp                 ibmf message that timedout
3056  * ibmf_callback        callback that should be called by msg_transport
3057  * ibmf_callback_arg    args for ibmf_callback
3058  * transport_flags      flags for ibmf_msg_transport
3059  *
3060  * Output Arguments
3061  * none
3062  *
3063  * Returns
3064  * IBMF_SUCCESS if we've recently received data from the SA and request was
3065  * resent.
3066  * IBMF_TRANS_TIMEOUT if no data has been received from the SA in max_wait_time
3067  * same values as ibmf_msg_transport() if data has been received but request
3068  * could not be resent.
3069  */
3070 static int
3071 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
3072     ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg,
3073     hrtime_t trans_send_time, int transport_flags)
3074 {
3075         hrtime_t                curr_time, sa_uptime;
3076         ibmf_retrans_t          ibmf_retrans;
3077         int                     ibmf_status;
3078 
3079         do {
3080 
3081                 mutex_enter(&saa_portp->saa_pt_mutex);
3082 
3083                 sa_uptime = saa_portp->saa_pt_sa_uptime;
3084 
3085                 /* if nothing received from SA since we sent */
3086                 curr_time = gethrtime();
3087 
3088                 /*
3089                  * check if it's been a very long time since this
3090                  * particular transaction was sent
3091                  */
3092                 if (((curr_time - trans_send_time) / 1000000000) >
3093                     ibmf_saa_trans_wait_time) {
3094 
3095                         mutex_exit(&saa_portp->saa_pt_mutex);
3096 
3097                         IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1,
3098                             ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "",
3099                             "ibmf_saa_check_sa_and_retry: %s, msgp = "
3100                             "%p sa_uptime = %" PRIu64 ", trans send time = %"
3101                             PRIu64 ", curr_time = %" PRIu64 "\n",
3102                             tnf_string, msg,
3103                             "Nothing received for this transaction",
3104                             tnf_opaque, msgp, msgp,
3105                             tnf_long, sa_uptime, sa_uptime,
3106                             tnf_long, trans_send_time, trans_send_time,
3107                             tnf_long, curr_time, curr_time);
3108 
3109                         ibmf_status = IBMF_TRANS_TIMEOUT;
3110 
3111                         break;
3112                 }
3113 
3114                 /*
3115                  * check time since we received something,
3116                  * and make sure that it hasn't been an extra long
3117                  * time for this particular transaction
3118                  */
3119                 if (((curr_time - sa_uptime) / 1000000000) <
3120                     ibmf_saa_max_wait_time) {
3121 
3122                         IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L2,
3123                             ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "",
3124                             "ibmf_saa_check_sa_and_retry: %s, msgp = "
3125                             "%p sa_uptime = %" PRIu64 " trans_send_time = %"
3126                             PRIu64 " curr_time = %" PRIu64 "\n",
3127                             tnf_string, msg, "Something received.  Retrying",
3128                             tnf_opaque, msgp, msgp,
3129                             tnf_long, sa_uptime, sa_uptime,
3130                             tnf_long, trans_send_time, trans_send_time,
3131                             tnf_long, curr_time, curr_time);
3132 
3133                         /*
3134                          * something received in WAIT_TIME_IN_SECS;
3135                          * resend request
3136                          */
3137 
3138                         /* new tid needed */
3139                         msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID =
3140                             h2b64(saa_portp->saa_pt_current_tid++);
3141 
3142                         /*
3143                          * We are going to retry the access to the SM but
3144                          * Master SMLID could have changed due to a port change
3145                          * event. So update the remote_lid of the message with
3146                          * the SMLID from saa_portp for this port before the
3147                          * retry.
3148                          */
3149                         msgp->im_local_addr.ia_remote_lid =
3150                             saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid;
3151 
3152                         bcopy(&saa_portp->saa_pt_ibmf_retrans,
3153                             &ibmf_retrans, sizeof (ibmf_retrans_t));
3154 
3155                         mutex_exit(&saa_portp->saa_pt_mutex);
3156 
3157                         ibmf_status = ibmf_msg_transport(
3158                             saa_portp->saa_pt_ibmf_handle,
3159                             saa_portp->saa_pt_qp_handle, msgp,
3160                             &ibmf_retrans, ibmf_callback, ibmf_callback_arg,
3161                             transport_flags);
3162 
3163                         if (ibmf_status == IBMF_SUCCESS)
3164                                 goto bail;
3165 
3166                         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
3167                             ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "",
3168                             "ibmf_saa_check_sa_and_retry: %s, ibmf_status = "
3169                             "%d\n", tnf_string, msg,
3170                             "ibmf_msg_transport() failed",
3171                             tnf_int, ibmf_status, ibmf_status);
3172                 } else {
3173 
3174                         mutex_exit(&saa_portp->saa_pt_mutex);
3175 
3176                         IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L1,
3177                             ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "",
3178                             "ibmf_saa_check_sa_and_retry: %s, msgp = "
3179                             "%p sa_uptime = %" PRIu64 " curr_time = %"
3180                             PRIu64 "\n", tnf_string, msg,
3181                             "Nothing received.  Timing out",
3182                             tnf_opaque, msgp, msgp,
3183                             tnf_long, sa_uptime, sa_uptime,
3184                             tnf_long, curr_time, curr_time);
3185 
3186                         ibmf_status = IBMF_TRANS_TIMEOUT;
3187 
3188                         break;
3189                 }
3190         } while (ibmf_status == IBMF_TRANS_TIMEOUT);
3191 
3192 bail:
3193         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3194             ibmf_saa_check_sa_and_retry_end,
3195             IBMF_TNF_TRACE, "", "ibmf_saa_check_sa_and_retry() exiting"
3196             " ibmf_status = %d\n", tnf_int, result, ibmf_status);
3197 
3198         return (ibmf_status);
3199 }
3200 
3201 
3202 /*
3203  * ibmf_saa_impl_prepare_response:
3204  */
3205 static void
3206 ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle,
3207     ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result,
3208     size_t *length, boolean_t sleep_flag)
3209 {
3210         ibmf_msg_bufs_t *resp_buf;
3211         uint16_t        attr_id;
3212         uint8_t         method;
3213         boolean_t       is_get_resp;
3214         uint16_t        mad_status;
3215         uint16_t        attr_offset;
3216         ib_sa_hdr_t     *sa_hdr;
3217 
3218         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3219             ibmf_saa_impl_prepare_response_start,
3220             IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() enter\n");
3221 
3222         _NOTE(ASSUMING_PROTECTED(*msgp))
3223 
3224         *result = NULL;
3225         *length = 0;
3226         sa_hdr = NULL;
3227 
3228         resp_buf = &msgp->im_msgbufs_recv;
3229 
3230         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*resp_buf))
3231 
3232         if (msgp->im_msg_status != IBMF_SUCCESS) {
3233 
3234                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
3235                     ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3236                     "ibmf_saa_impl_prepare_response: %s, msg_status = %d\n",
3237                     tnf_string, msg, "Bad ibmf status",
3238                     tnf_int, msg_status, msgp->im_msg_status);
3239 
3240                 *status = msgp->im_msg_status;
3241 
3242                 goto exit;
3243         }
3244 
3245         if (resp_buf->im_bufs_mad_hdr == NULL) {
3246 
3247                 /*
3248                  * this was an unsequenced transaction (from an unsubscribe for
3249                  * following a CI_OFFLINE event)
3250                  */
3251                 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3252                     ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3253                     "ibmf_saa_impl_prepare_response: %s\n",
3254                     tnf_string, msg, "Unsequenced transaction callback");
3255 
3256                 goto exit;
3257         }
3258 
3259         if ((mad_status = b2h16(resp_buf->im_bufs_mad_hdr->Status)) !=
3260             MAD_STATUS_NO_INVALID_FIELDS) {
3261 
3262                 /* convert mad packet status to IBMF status */
3263                 switch (mad_status) {
3264 
3265                         case SA_STATUS_ERR_NO_RESOURCES:
3266                                 *status = IBMF_NO_RESOURCES;
3267                                 break;
3268                         case SA_STATUS_ERR_REQ_INVALID:
3269                                 *status = IBMF_REQ_INVALID;
3270                                 break;
3271                         case SA_STATUS_ERR_NO_RECORDS:
3272                                 *status = IBMF_NO_RECORDS;
3273                                 break;
3274                         case SA_STATUS_ERR_TOO_MANY_RECORDS:
3275                                 *status = IBMF_TOO_MANY_RECORDS;
3276                                 break;
3277                         case SA_STATUS_ERR_REQ_INVALID_GID:
3278                                 *status = IBMF_INVALID_GID;
3279                                 break;
3280                         case SA_STATUS_ERR_REQ_INSUFFICIENT_COMPONENTS:
3281                                 *status = IBMF_INSUFF_COMPS;
3282                                 break;
3283                         case MAD_STATUS_UNSUPP_METHOD:
3284                                 *status = IBMF_UNSUPP_METHOD;
3285                                 break;
3286                         case MAD_STATUS_UNSUPP_METHOD_ATTR:
3287                                 *status = IBMF_UNSUPP_METHOD_ATTR;
3288                                 break;
3289                         case MAD_STATUS_INVALID_FIELD:
3290                                 *status = IBMF_INVALID_FIELD;
3291                                 break;
3292                         default:
3293                                 *status = IBMF_REQ_INVALID;
3294                                 break;
3295                 }
3296 
3297                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
3298                     ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3299                     "ibmf_saa_impl_prepare_response: %s, mad_status = %x\n",
3300                     tnf_string, msg, "Bad MAD status",
3301                     tnf_int, mad_status, mad_status);
3302 
3303                 goto exit;
3304         }
3305 
3306         attr_id = b2h16(resp_buf->im_bufs_mad_hdr->AttributeID);
3307         method = resp_buf->im_bufs_mad_hdr->R_Method;
3308 
3309         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
3310             ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3311             "ibmf_saa_impl_prepare_response: attr_id = 0x%x, method = "
3312             "0x%x\n",
3313             tnf_opaque, attr_id, attr_id,
3314             tnf_opaque, method, method);
3315 
3316         /*
3317          * ignore any data from deleteresp since there's no way to know whether
3318          * real data was returned; also ignore data if this was a Report
3319          * response
3320          */
3321         if (method == SA_SUBN_ADM_DELETE_RESP) {
3322 
3323                 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3324                     ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3325                     "impf_saa_impl_prepare_response: %s\n",
3326                     tnf_string, msg,
3327                     "DeleteResp or NoticeResp returned.  "
3328                     "Ignoring response data");
3329 
3330                 *status = IBMF_SUCCESS;
3331 
3332                 *length = 0;
3333                 *result = NULL;
3334 
3335                 goto exit;
3336         }
3337 
3338         if (attr_id == SA_MULTIPATHRECORD_ATTRID) {
3339 
3340                 /*
3341                  * getmulti is only for requests; attribute should not
3342                  * be returned from SA
3343                  */
3344                 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
3345                     ibmf_saa_impl_prepare_response_err, IBMF_TNF_ERROR,
3346                     "", "ibmf_saa_impl_prepare_response: %s\n",
3347                     tnf_string, msg, "SA returned getmulti record");
3348 
3349                 *status = IBMF_REQ_INVALID;
3350 
3351                 goto exit;
3352         }
3353 
3354         /* if we are supposed to ignore data, stop here */
3355         if (ignore_data == B_TRUE) {
3356 
3357                 *status = IBMF_SUCCESS;
3358 
3359                 goto exit;
3360         }
3361 
3362         is_get_resp = resp_buf->im_bufs_mad_hdr->R_Method ==
3363             SA_SUBN_ADM_GET_RESP ? B_TRUE: B_FALSE;
3364 
3365         /* unpack the sa header to get the attribute offset */
3366         *status = ibmf_saa_utils_unpack_sa_hdr(resp_buf->im_bufs_cl_hdr,
3367             resp_buf->im_bufs_cl_hdr_len, &sa_hdr, sleep_flag);
3368         if (*status != IBMF_SUCCESS) {
3369 
3370                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3371                     ibmf_saa_impl_prepare_response_err,
3372                     IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s,"
3373                     " ibmf_status = %d\n", tnf_string, msg,
3374                     "Could not unpack sa hdr", tnf_int, ibmf_status, *status);
3375 
3376                 goto exit;
3377         }
3378 
3379         attr_offset = sa_hdr->AttributeOffset;
3380 
3381         /*
3382          * unpack data payload; if unpack function doesn't return success
3383          * (because it could not allocate memory) forward this status to waiting
3384          * client
3385          */
3386         *status = ibmf_saa_utils_unpack_payload(resp_buf->im_bufs_cl_data,
3387             resp_buf->im_bufs_cl_data_len, attr_id, result, length,
3388             attr_offset, is_get_resp, sleep_flag);
3389         if (*status == IBMF_SUCCESS) {
3390 
3391                 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
3392                     ibmf_saa_impl_prepare_response,
3393                     IBMF_TNF_TRACE, "",
3394                     "ibmf_saa_impl_prepare_response: attr_id = "
3395                     "0x%x, attr_offset = %d, packed_payload_len = %d, "
3396                     "unpacked_payload_len = %d\n",
3397                     tnf_opaque, attr_id, attr_id,
3398                     tnf_opaque, attr_offset, attr_offset,
3399                     tnf_opaque, packed_payload_len,
3400                     resp_buf->im_bufs_cl_data_len,
3401                     tnf_opaque, unpacked_payload_len, *length);
3402         } else {
3403 
3404                 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1,
3405                     ibmf_saa_impl_prepare_response_err,
3406                     IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s,"
3407                     "attr_id = 0x%x, attr_offset = %d, packed_payload_len = %d,"
3408                     "status = %d\n",
3409                     tnf_string, msg, "Could not unpack payload",
3410                     tnf_opaque, attr_id, attr_id,
3411                     tnf_int, attr_offset, attr_offset,
3412                     tnf_int, packed_payload_len,
3413                     resp_buf->im_bufs_cl_data_len,
3414                     tnf_int, status, *status);
3415         }
3416 exit:
3417         if (sa_hdr != NULL)
3418                 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3419 
3420         ibmf_saa_impl_free_msg(ibmf_handle, msgp);
3421 
3422         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3423             ibmf_saa_impl_prepare_response_end,
3424             IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() exit,"
3425             " status = 0x%d\n", tnf_int, status, *status);
3426 }
3427 
3428 
3429 /*
3430  * ibmf_saa_impl_check_sa_support:
3431  * Checks the capability mask (returned from the SA classportinfo response) to
3432  * determine whether the sa supports the specified attribute ID.
3433  *
3434  * Input Arguments
3435  * cap_mask     16-bit capability mask returned in SA's classportinfo
3436  * attr_id      attribute ID of current request
3437  *
3438  * Returns
3439  * IBMF_NOT_SUPPORTED if capability mask indicates SA does not support attribute
3440  * IBMF_SUCCESS otherwise
3441  */
3442 static int
3443 ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id)
3444 {
3445         boolean_t       attr_supported = B_TRUE;
3446 
3447         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
3448             ibmf_saa_impl_check_sa_support, IBMF_TNF_TRACE, "",
3449             "ibmf_saa_impl_check_sa_support: cap_mask = 0x%x, "
3450             "attr_id = 0x%x\n", tnf_opaque, cap_mask, cap_mask,
3451             tnf_opaque, attr_id, attr_id);
3452 
3453         switch (attr_id) {
3454 
3455                 case SA_SWITCHINFORECORD_ATTRID:
3456                 case SA_LINEARFDBRECORD_ATTRID:
3457                 case SA_RANDOMFDBRECORD_ATTRID:
3458                 case SA_MULTICASTFDBRECORD_ATTRID:
3459                 case SA_SMINFORECORD_ATTRID:
3460                 case SA_INFORMINFORECORD_ATTRID:
3461                 case SA_LINKRECORD_ATTRID:
3462                 case SA_GUIDINFORECORD_ATTRID:
3463                 case SA_TRACERECORD_ATTRID:
3464                 case SA_SERVICEASSNRECORD_ATTRID:
3465 
3466                         if ((cap_mask &
3467                             SA_CAPMASK_OPT_RECORDS_SUPPORTED) == 0) {
3468 
3469                                 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1,
3470                                     ibmf_saa_impl_check_sa_support,
3471                                     IBMF_TNF_ERROR, "",
3472                                     "ibmf_saa_impl_check_sa_support: %s, "
3473                                     "cap_mask = 0x%x\n", tnf_string, msg,
3474                                     "SA does not support optional records",
3475                                     tnf_opaque, cap_mask, cap_mask,
3476                                     tnf_opaque, attr_id, attr_id);
3477 
3478                                 attr_supported = B_FALSE;
3479                         }
3480                         break;
3481 
3482                 case SA_MULTIPATHRECORD_ATTRID:
3483 
3484                         if ((cap_mask & SA_CAPMASK_MULTIPATH_SUPPORTED) == 0) {
3485 
3486                                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
3487                                     ibmf_saa_impl_check_sa_support,
3488                                     IBMF_TNF_ERROR, "",
3489                                     "ibmf_saa_impl_check_sa_support: %s, "
3490                                     "cap_mask = 0x%x\n", tnf_string, msg,
3491                                     "SA does not support multipath records",
3492                                     tnf_opaque, cap_mask, cap_mask);
3493 
3494                                 attr_supported = B_FALSE;
3495                         }
3496                         break;
3497 
3498                 case SA_MCMEMBERRECORD_ATTRID:
3499 
3500                         if ((cap_mask & SA_CAPMASK_UD_MCAST_SUPPORTED) == 0) {
3501 
3502                                 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
3503                                     ibmf_saa_impl_check_sa_support,
3504                                     IBMF_TNF_ERROR, "",
3505                                     "ibmf_saa_impl_check_sa_support: %s, "
3506                                     "cap_mask = 0x%x\n", tnf_string, msg,
3507                                     "SA does not support ud multicast",
3508                                     tnf_opaque, cap_mask, cap_mask);
3509 
3510                                 attr_supported = B_FALSE;
3511                         }
3512                         break;
3513 
3514                 default:
3515                         break;
3516         } /* switch */
3517 
3518         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3519             ibmf_saa_impl_check_sa_support_end, IBMF_TNF_TRACE, "",
3520             "ibmf_saa_impl_check_sa_support() exiting, attr_supported = %d\n",
3521             tnf_opaque, attr_supported, attr_supported);
3522 
3523         if (attr_supported == B_FALSE)
3524                 return (IBMF_UNSUPP_METHOD_ATTR);
3525         else
3526                 return (IBMF_SUCCESS);
3527 }
3528 
3529 /*
3530  * ibmf_saa_impl_get_attr_id_length:
3531  *
3532  * Returns the host size of the specified sa record.  Returns 0 for unknown
3533  * attributes.  multipath record size is a dynamic value given as a parameter
3534  * specified with the ibmf_sa_access() call.
3535  */
3536 static uint_t
3537 ibmf_saa_impl_get_attr_id_length(uint16_t attr_id)
3538 {
3539         uint_t  attr_length;
3540 
3541         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3542             ibmf_saa_impl_get_attr_id_length_start,
3543             IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length() enter\n");
3544 
3545         /* this function should not be used for multipath record */
3546         ASSERT(attr_id != SA_MULTIPATHRECORD_ATTRID);
3547 
3548         switch (attr_id) {
3549                 case SA_CLASSPORTINFO_ATTRID:
3550                         attr_length = sizeof (ib_mad_classportinfo_t);
3551                         break;
3552                 case SA_NOTICE_ATTRID:
3553                         attr_length = sizeof (ib_mad_notice_t);
3554                         break;
3555                 case SA_INFORMINFO_ATTRID:
3556                         attr_length = sizeof (ib_mad_informinfo_t);
3557                         break;
3558                 case SA_NODERECORD_ATTRID:
3559                         attr_length = sizeof (sa_node_record_t);
3560                         break;
3561                 case SA_PORTINFORECORD_ATTRID:
3562                         attr_length = sizeof (sa_portinfo_record_t);
3563                         break;
3564                 case SA_SLTOVLRECORD_ATTRID:
3565                         attr_length = sizeof (sa_SLtoVLmapping_record_t);
3566                         break;
3567                 case SA_SWITCHINFORECORD_ATTRID:
3568                         attr_length = sizeof (sa_switchinfo_record_t);
3569                         break;
3570                 case SA_LINEARFDBRECORD_ATTRID:
3571                         attr_length = sizeof (sa_linearft_record_t);
3572                         break;
3573                 case SA_RANDOMFDBRECORD_ATTRID:
3574                         attr_length = sizeof (sa_randomft_record_t);
3575                         break;
3576                 case SA_MULTICASTFDBRECORD_ATTRID:
3577                         attr_length = sizeof (sa_multicastft_record_t);
3578                         break;
3579                 case SA_SMINFORECORD_ATTRID:
3580                         attr_length = sizeof (sa_sminfo_record_t);
3581                         break;
3582                 case SA_INFORMINFORECORD_ATTRID:
3583                         attr_length = sizeof (sa_informinfo_record_t);
3584                         break;
3585                 case SA_LINKRECORD_ATTRID:
3586                         attr_length = sizeof (sa_link_record_t);
3587                         break;
3588                 case SA_GUIDINFORECORD_ATTRID:
3589                         attr_length = sizeof (sa_guidinfo_record_t);
3590                         break;
3591                 case SA_SERVICERECORD_ATTRID:
3592                         attr_length = sizeof (sa_service_record_t);
3593                         break;
3594                 case SA_PARTITIONRECORD_ATTRID:
3595                         attr_length = sizeof (sa_pkey_table_record_t);
3596                         break;
3597                 case SA_PATHRECORD_ATTRID:
3598                         attr_length = sizeof (sa_path_record_t);
3599                         break;
3600                 case SA_VLARBRECORD_ATTRID:
3601                         attr_length = sizeof (sa_VLarb_table_record_t);
3602                         break;
3603                 case SA_MCMEMBERRECORD_ATTRID:
3604                         attr_length = sizeof (sa_mcmember_record_t);
3605                         break;
3606                 case SA_TRACERECORD_ATTRID:
3607                         attr_length = sizeof (sa_trace_record_t);
3608                         break;
3609                 case SA_SERVICEASSNRECORD_ATTRID:
3610                         attr_length = sizeof (sa_service_assn_record_t);
3611                         break;
3612                 default:
3613                         /* should only get the above type of packets */
3614                         attr_length = 0;
3615                         break;
3616         }
3617 
3618         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
3619             ibmf_saa_impl_get_attr_id_length_end,
3620             IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length():"
3621             " attr_id: 0x%x size %d\n",
3622             tnf_opaque, attr_id, attr_id, tnf_uint, attr_length, attr_length);
3623 
3624         return (attr_length);
3625 }
3626 
3627 /*
3628  * ibmf_saa_impl_free_msg:
3629  * Takes a completed message and free memory associated with the message,
3630  * including the individual fields of the im_msgbufs_send.
3631  * ibmf_free_msg, called at the end of this function, takes a pointer to the
3632  * message pointer so that it can set the message pointer to NULL.  This
3633  * function takes just the message pointer so the msgp will not be NULL after
3634  * this function returns.
3635  *
3636  * Input Arguments
3637  * ibmf_hdl     ibmf handle used in ibmf_msg_alloc
3638  * msgp         pointer to ibmf_msg_t to free
3639  *
3640  * Returns
3641  * void
3642  */
3643 static void
3644 ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp)
3645 {
3646         int     res;
3647 
3648         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3649             ibmf_saa_impl_free_msg_start,
3650             IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() enter: msg %p\n",
3651             tnf_opaque, msg, msgp);
3652 
3653         ASSERT(msgp != NULL);
3654 
3655         kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr,
3656             sizeof (ib_mad_hdr_t));
3657 
3658         kmem_free(msgp->im_msgbufs_send.im_bufs_cl_hdr,
3659             msgp->im_msgbufs_send.im_bufs_cl_hdr_len);
3660 
3661         if (msgp->im_msgbufs_send.im_bufs_cl_data_len > 0)
3662                 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_data,
3663                     msgp->im_msgbufs_send.im_bufs_cl_data_len);
3664 
3665         res = ibmf_free_msg(ibmf_hdl, &msgp);
3666         ASSERT(res == IBMF_SUCCESS);
3667 
3668         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3669             ibmf_saa_impl_free_msg_end,
3670             IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() exit\n");
3671 }
3672 
3673 /*
3674  * ibmf_saa_impl_get_port_guid:
3675  */
3676 static int
3677 ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop,
3678     ib_guid_t *guid_ret)
3679 {
3680         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3681             ibmf_saa_impl_get_port_guid_start,
3682             IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_port_guid() enter\n");
3683 
3684         if (ibt_portinfop->p_linkstate != IBT_PORT_ACTIVE) {
3685 
3686                 return (IBMF_BAD_PORT_STATE);
3687         }
3688 
3689         if (ibt_portinfop->p_sgid_tbl_sz == 0) {
3690 
3691                 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L2,
3692                     ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "",
3693                     "ibmf_saa_impl_get_port_guid: %s\n", tnf_string, msg,
3694                     "portinfo sgid table size is 0. Exiting.\n");
3695 
3696                 return (IBMF_TRANSPORT_FAILURE);
3697         }
3698 
3699         *guid_ret = ibt_portinfop->p_sgid_tbl[0].gid_guid;
3700 
3701         IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3702             ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "",
3703             "ibmf_saa_impl_get_port_guid: Returning port_guid %016" PRIx64 "\n",
3704             tnf_opaque, port_guid, *guid_ret);
3705 
3706         return (IBMF_SUCCESS);
3707 }
3708 
3709 /*
3710  * ibmf_saa_impl_set_transaction_params:
3711  */
3712 static void
3713 ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp,
3714     ibt_hca_portinfo_t *portinfop)
3715 {
3716         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3717             ibmf_saa_impl_set_transaction_params_start,
3718             IBMF_TNF_TRACE, "",
3719             "ibmf_saa_impl_set_transaction_params() enter\n");
3720 
3721         _NOTE(ASSUMING_PROTECTED(*saa_portp))
3722 
3723         saa_portp->saa_pt_ibmf_retrans.retrans_retries =
3724             ibmf_saa_retrans_retries;
3725         /*
3726          * For the first transaction (generally getting the
3727          * classportinfo) have ibmf pick our timeouts.  It should be using the
3728          * default IB spec values.
3729          * Once we get the classportinfo we'll update the correct response time
3730          * value (rtv) and round-trip time (rttv).  ibmf should always calculate
3731          * trans_to since it depends on the particular transaction's number of
3732          * packets.
3733          */
3734         saa_portp->saa_pt_ibmf_retrans.retrans_rtv = 0;
3735         saa_portp->saa_pt_ibmf_retrans.retrans_rttv = 0;
3736         saa_portp->saa_pt_ibmf_retrans.retrans_trans_to = 0;
3737 
3738         /*
3739          * Assume that the SA supports all optional records. If it
3740          * does not, the request will get returned with ERR_NOT_SUPP.  When
3741          * the classportinfo response comes back we will update the cap mask
3742          * to prevent unnecessary unsupported requests.
3743          */
3744         saa_portp->saa_pt_sa_cap_mask = 0xFFFF;
3745 
3746         saa_portp->saa_pt_ibmf_msg_flags = 0;
3747         saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno       = 1;
3748         saa_portp->saa_pt_ibmf_addr_info.ia_p_key    =
3749             IB_PKEY_DEFAULT_LIMITED;
3750         saa_portp->saa_pt_ibmf_addr_info.ia_q_key    = IB_GSI_QKEY;
3751 
3752         /*
3753          * fill out addr information for MADs that will be sent
3754          * to SA on this port
3755          */
3756         saa_portp->saa_pt_ibmf_addr_info.ia_local_lid        = portinfop->p_base_lid;
3757         saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid       = portinfop->p_sm_lid;
3758         saa_portp->saa_pt_ibmf_addr_info.ia_service_level = portinfop->p_sm_sl;
3759 
3760         /* place upper bound on subnet timeout in case of faulty SM */
3761         saa_portp->saa_pt_timeout = portinfop->p_subnet_timeout;
3762 
3763         if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout)
3764                 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout;
3765 
3766         IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
3767             ibmf_saa_impl_set_transaction_params,
3768             IBMF_TNF_TRACE, "",
3769             "ibmf_saa_impl_set_transaction_params: local_lid = 0x%x, "
3770             "sm_lid = 0x%x, sm_sl = 0x%x, sn_timeout = 0x%x\n",
3771             tnf_opaque, local_lid, portinfop->p_base_lid,
3772             tnf_opaque, sm_lid, portinfop->p_sm_lid,
3773             tnf_opaque, sm_sl, portinfop->p_sm_sl,
3774             tnf_opaque, subnet_timeout, portinfop->p_subnet_timeout);
3775 
3776         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3777             ibmf_saa_impl_set_transaction_params_end,
3778             IBMF_TNF_TRACE, "",
3779             "ibmf_saa_impl_set_transaction_params() exit\n");
3780 }
3781 
3782 
3783 /*
3784  * ibmf_saa_impl_update_sa_address_info
3785  */
3786 static void
3787 ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp, ibmf_msg_t *msgp)
3788 {
3789         void                    *result;
3790         ib_sa_hdr_t             *sa_hdr;
3791         int                     rv;
3792         size_t                  length;
3793         uint16_t                attr_id;
3794         ib_mad_classportinfo_t  *cpi;
3795         ibmf_global_addr_info_t *gaddrp = &saa_portp->saa_pt_ibmf_global_addr;
3796         ibt_hca_portinfo_t      *ibt_pinfo;
3797         uint_t                  nports, size;
3798         ibt_status_t            ibt_status;
3799 
3800         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3801             ibmf_saa_impl_update_sa_address_info,
3802             IBMF_TNF_TRACE, "",
3803             "ibmf_saa_impl_update_sa_address_info() enter\n");
3804 
3805         /*
3806          * decode the respons of msgp as a classportinfo attribute
3807          */
3808         rv = ibmf_saa_utils_unpack_sa_hdr(msgp->im_msgbufs_recv.im_bufs_cl_hdr,
3809             msgp->im_msgbufs_recv.im_bufs_cl_hdr_len, &sa_hdr, KM_NOSLEEP);
3810         if (rv != IBMF_SUCCESS) {
3811 
3812                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3813                     ibmf_saa_impl_update_sa_address_err,
3814                     IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: "
3815                     "%s, ibmf_status = %d\n", tnf_string, msg,
3816                     "Could not unpack sa hdr", tnf_int, ibmf_status, rv);
3817 
3818                 return;
3819         }
3820 
3821         attr_id = b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID);
3822         if (attr_id != MAD_ATTR_ID_CLASSPORTINFO) {
3823                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3824                     ibmf_saa_impl_update_sa_address_info_err,
3825                     IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: "
3826                     "%s, attrID = %x\n", tnf_string, msg,
3827                     "Wrong attribute ID", tnf_int, ibmf_status, attr_id);
3828 
3829                 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3830                 return;
3831         }
3832         rv = ibmf_saa_utils_unpack_payload(
3833             msgp->im_msgbufs_recv.im_bufs_cl_data,
3834             msgp->im_msgbufs_recv.im_bufs_cl_data_len, attr_id, &result,
3835             &length, sa_hdr->AttributeOffset, B_TRUE, KM_NOSLEEP);
3836         if (rv != IBMF_SUCCESS) {
3837 
3838                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3839                     ibmf_saa_impl_update_sa_address_err,
3840                     IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: "
3841                     "%s, ibmf_status = %d\n", tnf_string, msg,
3842                     "Could not unpack payload", tnf_int, ibmf_status, rv);
3843 
3844                 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3845                 return;
3846         }
3847 
3848         kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3849 
3850         /*
3851          * Use the classportinfo contents to update the SA address info
3852          */
3853         cpi = (ib_mad_classportinfo_t *)result;
3854         mutex_enter(&saa_portp->saa_pt_mutex);
3855         saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid       = cpi->RedirectLID;
3856         saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno       = cpi->RedirectQP;
3857         saa_portp->saa_pt_ibmf_addr_info.ia_p_key    = cpi->RedirectP_Key;
3858         saa_portp->saa_pt_ibmf_addr_info.ia_q_key    = cpi->RedirectQ_Key;
3859         saa_portp->saa_pt_ibmf_addr_info.ia_service_level = cpi->RedirectSL;
3860 
3861         saa_portp->saa_pt_redirect_active = B_TRUE;
3862 
3863         if ((cpi->RedirectGID_hi != 0) || (cpi->RedirectGID_lo != 0)) {
3864 
3865                 mutex_exit(&saa_portp->saa_pt_mutex);
3866                 ibt_status = ibt_query_hca_ports_byguid(
3867                     saa_portp->saa_pt_node_guid, saa_portp->saa_pt_port_num,
3868                     &ibt_pinfo, &nports, &size);
3869                 if (ibt_status != IBT_SUCCESS) {
3870 
3871                         IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3872                             ibmf_saa_impl_update_sa_address_err, IBMF_TNF_TRACE,
3873                             "", "ibmf_saa_impl_update_sa_address_info: "
3874                             "%s, ibt_status = %d\n", tnf_string, msg,
3875                             "Could not query hca port",
3876                             tnf_int, ibt_status, ibt_status);
3877 
3878                         kmem_free(result, length);
3879                         return;
3880                 }
3881 
3882                 mutex_enter(&saa_portp->saa_pt_mutex);
3883                 /*
3884                  * Fill in global address info parameters
3885                  *
3886                  * NOTE: The HopLimit value is not specified through the
3887                  * contents of ClassPortInfo. It may be possible to find
3888                  * out the proper value to use even for SA beeing redirected
3889                  * to another subnet. But we do only support redirect within
3890                  * our local subnet
3891                  */
3892                 gaddrp->ig_sender_gid.gid_prefix =
3893                     ibt_pinfo->p_sgid_tbl[0].gid_prefix;
3894                 gaddrp->ig_sender_gid.gid_guid = saa_portp->saa_pt_port_guid;
3895                 gaddrp->ig_recver_gid.gid_prefix = cpi->RedirectGID_hi;
3896                 gaddrp->ig_recver_gid.gid_guid = cpi->RedirectGID_lo;
3897                 gaddrp->ig_flow_label = cpi->RedirectFL;
3898                 gaddrp->ig_tclass = cpi->RedirectTC;
3899                 gaddrp->ig_hop_limit = 0;
3900 
3901                 saa_portp->saa_pt_ibmf_msg_flags =
3902                     IBMF_MSG_FLAGS_GLOBAL_ADDRESS;
3903 
3904                 mutex_exit(&saa_portp->saa_pt_mutex);
3905                 ibt_free_portinfo(ibt_pinfo, size);
3906         } else {
3907                 saa_portp->saa_pt_ibmf_msg_flags = 0;
3908                 mutex_exit(&saa_portp->saa_pt_mutex);
3909         }
3910         kmem_free(result, length);
3911 
3912         /*
3913          * Update the address info of msgp with the new address parameters
3914          */
3915         mutex_enter(&saa_portp->saa_pt_mutex);
3916         bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr,
3917             sizeof (ibmf_addr_info_t));
3918         if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) {
3919 
3920                 msgp->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS;
3921 
3922                 bcopy(&saa_portp->saa_pt_ibmf_global_addr,
3923                     &msgp->im_global_addr, sizeof (ibmf_global_addr_info_t));
3924         } else {
3925                 msgp->im_msg_flags = 0;
3926         }
3927         mutex_exit(&saa_portp->saa_pt_mutex);
3928 }
3929 
3930 /*
3931  * ibmf_saa_impl_ibmf_unreg:
3932  */
3933 static int
3934 ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp)
3935 {
3936         int     ibmf_status;
3937 
3938         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_start,
3939             IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() enter\n");
3940 
3941         /* teardown async cb */
3942         ibmf_status = ibmf_tear_down_async_cb(saa_portp->saa_pt_ibmf_handle,
3943             saa_portp->saa_pt_qp_handle, 0);
3944         if (ibmf_status != IBMF_SUCCESS) {
3945 
3946                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3947                     ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "",
3948                     "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n",
3949                     tnf_string, msg, "Could not tear down async cb",
3950                     tnf_int, ibmf_status, ibmf_status);
3951 
3952                 goto bail;
3953         }
3954 
3955         /* free qp */
3956         ibmf_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle,
3957             &saa_portp->saa_pt_qp_handle, 0);
3958 
3959         if (ibmf_status != IBMF_SUCCESS) {
3960 
3961                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3962                     ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "",
3963                     "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n",
3964                     tnf_string, msg, "Could not free queue pair",
3965                     tnf_int, ibmf_status, ibmf_status);
3966 
3967                 (void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 1);
3968                 goto bail;
3969         }
3970 
3971         ibmf_status = ibmf_unregister(&saa_portp->saa_pt_ibmf_handle, 0);
3972 
3973         if (ibmf_status != IBMF_SUCCESS) {
3974 
3975                 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3976                     ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "",
3977                     "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n",
3978                     tnf_string, msg, "ibmf_unregister() failed",
3979                     tnf_int, ibmf_status, ibmf_status);
3980 
3981                 (void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0);
3982         }
3983 
3984 bail:
3985         IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_end,
3986             IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() exit\n");
3987 
3988         return (ibmf_status);
3989 }