19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /*
26 * ibdm.c
27 *
28 * This file contains the InifiniBand Device Manager (IBDM) support functions.
29 * IB nexus driver will only be the client for the IBDM module.
30 *
31 * IBDM registers with IBTF for HCA arrival/removal notification.
32 * IBDM registers with SA access to send DM MADs to discover the IOC's behind
33 * the IOU's.
34 *
35 * IB nexus driver registers with IBDM to find the information about the
36 * HCA's and IOC's (behind the IOU) present on the IB fabric.
37 */
38
39 #include <sys/systm.h>
40 #include <sys/taskq.h>
41 #include <sys/ib/mgt/ibdm/ibdm_impl.h>
42 #include <sys/ib/mgt/ibmf/ibmf_impl.h>
43 #include <sys/ib/ibtl/impl/ibtl_ibnex.h>
44 #include <sys/modctl.h>
45
46 /* Function Prototype declarations */
47 static int ibdm_free_iou_info(ibdm_dp_gidinfo_t *, ibdm_iou_info_t **);
48 static int ibdm_fini(void);
49 static int ibdm_init(void);
50 static int ibdm_get_reachable_ports(ibdm_port_attr_t *,
51 ibdm_hca_list_t *);
52 static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t);
53 static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *);
54 static boolean_t ibdm_is_cisco(ib_guid_t);
55 static boolean_t ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *);
56 static void ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *);
57 static int ibdm_set_classportinfo(ibdm_dp_gidinfo_t *);
58 static int ibdm_send_classportinfo(ibdm_dp_gidinfo_t *);
954 (void) ibt_query_hca(hca_hdl, hca_attr);
955
956 IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x,"
957 " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id,
958 hca_attr->hca_version_id, hca_attr->hca_nports);
959
960 if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports,
961 &size)) != IBT_SUCCESS) {
962 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
963 "ibt_query_hca_ports failed, status 0x%x", status);
964 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
965 (void) ibt_close_hca(hca_hdl);
966 return;
967 }
968 hca_list = (ibdm_hca_list_t *)
969 kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP);
970 hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
971 (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP);
972 hca_list->hl_hca_guid = hca_attr->hca_node_guid;
973 hca_list->hl_nports = hca_attr->hca_nports;
974 hca_list->hl_attach_time = ddi_get_time();
975 hca_list->hl_hca_hdl = hca_hdl;
976
977 /*
978 * Init a dummy port attribute for the HCA node
979 * This is for Per-HCA Node. Initialize port_attr :
980 * hca_guid & port_guid -> hca_guid
981 * npkeys, pkey_tbl is NULL
982 * port_num, sn_prefix is 0
983 * vendorid, product_id, dev_version from HCA
984 * pa_state is IBT_PORT_ACTIVE
985 */
986 hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
987 sizeof (ibdm_port_attr_t), KM_SLEEP);
988 port_attr = hca_list->hl_hca_port_attr;
989 port_attr->pa_vendorid = hca_attr->hca_vendor_id;
990 port_attr->pa_productid = hca_attr->hca_device_id;
991 port_attr->pa_dev_version = hca_attr->hca_version_id;
992 port_attr->pa_hca_guid = hca_attr->hca_node_guid;
993 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl;
994 port_attr->pa_port_guid = hca_attr->hca_node_guid;
4674 mutex_exit(&ibdm.ibdm_ibnex_mutex);
4675 }
4676
4677
4678 /*
4679 * ibdm_ibnex_unregister_callbacks
4680 */
4681 void
4682 ibdm_ibnex_unregister_callback()
4683 {
4684 IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks");
4685 mutex_enter(&ibdm.ibdm_ibnex_mutex);
4686 ibdm.ibdm_ibnex_callback = NULL;
4687 mutex_exit(&ibdm.ibdm_ibnex_mutex);
4688 }
4689
4690 /*
4691 * ibdm_get_waittime()
4692 * Calculates the wait time based on the last HCA attach time
4693 */
4694 static time_t
4695 ibdm_get_waittime(ib_guid_t hca_guid, int dft_wait)
4696 {
4697 int ii;
4698 time_t temp, wait_time = 0;
4699 ibdm_hca_list_t *hca;
4700
4701 IBTF_DPRINTF_L4("ibdm", "\tget_waittime hcaguid:%llx"
4702 "\tport settling time %d", hca_guid, dft_wait);
4703
4704 ASSERT(mutex_owned(&ibdm.ibdm_hl_mutex));
4705
4706 hca = ibdm.ibdm_hca_list_head;
4707
4708 if (hca_guid) {
4709 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4710 if ((hca_guid == hca->hl_hca_guid) &&
4711 (hca->hl_nports != hca->hl_nports_active)) {
4712 wait_time =
4713 ddi_get_time() - hca->hl_attach_time;
4714 wait_time = ((wait_time >= dft_wait) ?
4715 0 : (dft_wait - wait_time));
4716 break;
4717 }
4718 hca = hca->hl_next;
4719 }
4720 IBTF_DPRINTF_L2("ibdm", "\tget_waittime: wait_time = %ld secs",
4721 (long)wait_time);
4722 return (wait_time);
4723 }
4724
4725 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4726 if (hca->hl_nports != hca->hl_nports_active) {
4727 temp = ddi_get_time() - hca->hl_attach_time;
4728 temp = ((temp >= dft_wait) ? 0 : (dft_wait - temp));
4729 wait_time = (temp > wait_time) ? temp : wait_time;
4730 }
4731 hca = hca->hl_next;
4732 }
4733 IBTF_DPRINTF_L2("ibdm", "\tget_waittime: wait_time = %ld secs",
4734 (long)wait_time);
4735 return (wait_time);
4736 }
4737
4738 void
4739 ibdm_ibnex_port_settle_wait(ib_guid_t hca_guid, int dft_wait)
4740 {
4741 time_t wait_time;
4742 clock_t delta;
4743
4744 mutex_enter(&ibdm.ibdm_hl_mutex);
4745
4746 while ((wait_time = ibdm_get_waittime(hca_guid, dft_wait)) > 0) {
4747 if (wait_time > dft_wait) {
4748 IBTF_DPRINTF_L1("ibdm",
4749 "\tibnex_port_settle_wait: wait_time = %ld secs; "
4750 "Resetting to %d secs",
4751 (long)wait_time, dft_wait);
4752 wait_time = dft_wait;
4753 }
4754 delta = drv_usectohz(wait_time * 1000000);
4755 (void) cv_reltimedwait(&ibdm.ibdm_port_settle_cv,
4756 &ibdm.ibdm_hl_mutex, delta, TR_CLOCK_TICK);
4757 }
4758
4759 mutex_exit(&ibdm.ibdm_hl_mutex);
4760 }
4761
4762
4763 /*
4764 * ibdm_ibnex_probe_hcaport
4765 * Probes the presence of HCA port (with HCA dip and port number)
4766 * Returns port attributes structure on SUCCESS
4767 */
4768 ibdm_port_attr_t *
4769 ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num)
4770 {
4771 int ii, jj;
4772 ibdm_hca_list_t *hca_list;
4773 ibdm_port_attr_t *port_attr;
4774
4775 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:");
4776
4777 mutex_enter(&ibdm.ibdm_hl_mutex);
|
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /*
26 * ibdm.c
27 *
28 * This file contains the InifiniBand Device Manager (IBDM) support functions.
29 * IB nexus driver will only be the client for the IBDM module.
30 *
31 * IBDM registers with IBTF for HCA arrival/removal notification.
32 * IBDM registers with SA access to send DM MADs to discover the IOC's behind
33 * the IOU's.
34 *
35 * IB nexus driver registers with IBDM to find the information about the
36 * HCA's and IOC's (behind the IOU) present on the IB fabric.
37 */
38
39 #include <sys/sysmacros.h>
40 #include <sys/systm.h>
41 #include <sys/taskq.h>
42 #include <sys/ib/mgt/ibdm/ibdm_impl.h>
43 #include <sys/ib/mgt/ibmf/ibmf_impl.h>
44 #include <sys/ib/ibtl/impl/ibtl_ibnex.h>
45 #include <sys/modctl.h>
46
47 /* Function Prototype declarations */
48 static int ibdm_free_iou_info(ibdm_dp_gidinfo_t *, ibdm_iou_info_t **);
49 static int ibdm_fini(void);
50 static int ibdm_init(void);
51 static int ibdm_get_reachable_ports(ibdm_port_attr_t *,
52 ibdm_hca_list_t *);
53 static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t);
54 static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *);
55 static boolean_t ibdm_is_cisco(ib_guid_t);
56 static boolean_t ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *);
57 static void ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *);
58 static int ibdm_set_classportinfo(ibdm_dp_gidinfo_t *);
59 static int ibdm_send_classportinfo(ibdm_dp_gidinfo_t *);
955 (void) ibt_query_hca(hca_hdl, hca_attr);
956
957 IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x,"
958 " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id,
959 hca_attr->hca_version_id, hca_attr->hca_nports);
960
961 if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports,
962 &size)) != IBT_SUCCESS) {
963 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
964 "ibt_query_hca_ports failed, status 0x%x", status);
965 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
966 (void) ibt_close_hca(hca_hdl);
967 return;
968 }
969 hca_list = (ibdm_hca_list_t *)
970 kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP);
971 hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
972 (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP);
973 hca_list->hl_hca_guid = hca_attr->hca_node_guid;
974 hca_list->hl_nports = hca_attr->hca_nports;
975 hca_list->hl_attach_time = gethrtime();
976 hca_list->hl_hca_hdl = hca_hdl;
977
978 /*
979 * Init a dummy port attribute for the HCA node
980 * This is for Per-HCA Node. Initialize port_attr :
981 * hca_guid & port_guid -> hca_guid
982 * npkeys, pkey_tbl is NULL
983 * port_num, sn_prefix is 0
984 * vendorid, product_id, dev_version from HCA
985 * pa_state is IBT_PORT_ACTIVE
986 */
987 hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
988 sizeof (ibdm_port_attr_t), KM_SLEEP);
989 port_attr = hca_list->hl_hca_port_attr;
990 port_attr->pa_vendorid = hca_attr->hca_vendor_id;
991 port_attr->pa_productid = hca_attr->hca_device_id;
992 port_attr->pa_dev_version = hca_attr->hca_version_id;
993 port_attr->pa_hca_guid = hca_attr->hca_node_guid;
994 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl;
995 port_attr->pa_port_guid = hca_attr->hca_node_guid;
4675 mutex_exit(&ibdm.ibdm_ibnex_mutex);
4676 }
4677
4678
4679 /*
4680 * ibdm_ibnex_unregister_callbacks
4681 */
4682 void
4683 ibdm_ibnex_unregister_callback()
4684 {
4685 IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks");
4686 mutex_enter(&ibdm.ibdm_ibnex_mutex);
4687 ibdm.ibdm_ibnex_callback = NULL;
4688 mutex_exit(&ibdm.ibdm_ibnex_mutex);
4689 }
4690
4691 /*
4692 * ibdm_get_waittime()
4693 * Calculates the wait time based on the last HCA attach time
4694 */
4695 static clock_t
4696 ibdm_get_waittime(ib_guid_t hca_guid, time_t dft_wait_sec)
4697 {
4698 const hrtime_t dft_wait = dft_wait_sec * NANOSEC;
4699 hrtime_t temp, wait_time = 0;
4700 clock_t usecs;
4701 int i;
4702 ibdm_hca_list_t *hca;
4703
4704 IBTF_DPRINTF_L4("ibdm", "\tget_waittime hcaguid:%llx"
4705 "\tport settling time %d", hca_guid, dft_wait);
4706
4707 ASSERT(mutex_owned(&ibdm.ibdm_hl_mutex));
4708
4709 hca = ibdm.ibdm_hca_list_head;
4710
4711 for (i = 0; i < ibdm.ibdm_hca_count; i++, hca = hca->hl_next) {
4712 if (hca->hl_nports == hca->hl_nports_active)
4713 continue;
4714
4715 if (hca_guid && (hca_guid != hca->hl_hca_guid))
4716 continue;
4717
4718 temp = gethrtime() - hca->hl_attach_time;
4719 temp = MAX(0, (dft_wait - temp));
4720
4721 if (hca_guid) {
4722 wait_time = temp;
4723 break;
4724 }
4725
4726 wait_time = MAX(temp, wait_time);
4727 }
4728
4729 /* convert to microseconds */
4730 usecs = MIN(wait_time, dft_wait) / (NANOSEC / MICROSEC);
4731
4732 IBTF_DPRINTF_L2("ibdm", "\tget_waittime: wait_time = %ld usecs",
4733 (long) usecs);
4734
4735 return (drv_usectohz(usecs));
4736 }
4737
4738 void
4739 ibdm_ibnex_port_settle_wait(ib_guid_t hca_guid, time_t dft_wait)
4740 {
4741 clock_t wait_time;
4742
4743 mutex_enter(&ibdm.ibdm_hl_mutex);
4744
4745 while ((wait_time = ibdm_get_waittime(hca_guid, dft_wait)) > 0)
4746 (void) cv_reltimedwait(&ibdm.ibdm_port_settle_cv,
4747 &ibdm.ibdm_hl_mutex, wait_time, TR_CLOCK_TICK);
4748
4749 mutex_exit(&ibdm.ibdm_hl_mutex);
4750 }
4751
4752
4753 /*
4754 * ibdm_ibnex_probe_hcaport
4755 * Probes the presence of HCA port (with HCA dip and port number)
4756 * Returns port attributes structure on SUCCESS
4757 */
4758 ibdm_port_attr_t *
4759 ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num)
4760 {
4761 int ii, jj;
4762 ibdm_hca_list_t *hca_list;
4763 ibdm_port_attr_t *port_attr;
4764
4765 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:");
4766
4767 mutex_enter(&ibdm.ibdm_hl_mutex);
|