1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2007-2012 Intel Corporation. All rights reserved.
  24  */
  25 
  26 /*
  27  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Copyright 2013, Nexenta Systems, Inc. All rights reserved.
  29  */
  30 
  31 #include "igb_sw.h"
  32 
  33 static char ident[] = "Intel 1Gb Ethernet";
  34 static char igb_version[] = "igb 2.3.8-ish";
  35 
  36 /*
  37  * Local function protoypes
  38  */
  39 static int igb_register_mac(igb_t *);
  40 static int igb_identify_hardware(igb_t *);
  41 static int igb_regs_map(igb_t *);
  42 static void igb_init_properties(igb_t *);
  43 static int igb_init_driver_settings(igb_t *);
  44 static void igb_init_locks(igb_t *);
  45 static void igb_destroy_locks(igb_t *);
  46 static int igb_init_mac_address(igb_t *);
  47 static int igb_init(igb_t *);
  48 static int igb_init_adapter(igb_t *);
  49 static void igb_stop_adapter(igb_t *);
  50 static int igb_reset(igb_t *);
  51 static void igb_tx_clean(igb_t *);
  52 static boolean_t igb_tx_drain(igb_t *);
  53 static boolean_t igb_rx_drain(igb_t *);
  54 static int igb_alloc_rings(igb_t *);
  55 static int igb_alloc_rx_data(igb_t *);
  56 static void igb_free_rx_data(igb_t *);
  57 static void igb_free_rings(igb_t *);
  58 static void igb_setup_rings(igb_t *);
  59 static void igb_setup_rx(igb_t *);
  60 static void igb_setup_tx(igb_t *);
  61 static void igb_setup_rx_ring(igb_rx_ring_t *);
  62 static void igb_setup_tx_ring(igb_tx_ring_t *);
  63 static void igb_setup_rss(igb_t *);
  64 static void igb_setup_mac_rss_classify(igb_t *);
  65 static void igb_setup_mac_classify(igb_t *);
  66 static void igb_init_unicst(igb_t *);
  67 static void igb_setup_multicst(igb_t *);
  68 static void igb_get_phy_state(igb_t *);
  69 static void igb_param_sync(igb_t *);
  70 static void igb_get_conf(igb_t *);
  71 static int igb_get_prop(igb_t *, char *, int, int, int);
  72 static boolean_t igb_is_link_up(igb_t *);
  73 static boolean_t igb_link_check(igb_t *);
  74 static void igb_local_timer(void *);
  75 static void igb_link_timer(void *);
  76 static void igb_arm_watchdog_timer(igb_t *);
  77 static void igb_start_watchdog_timer(igb_t *);
  78 static void igb_restart_watchdog_timer(igb_t *);
  79 static void igb_stop_watchdog_timer(igb_t *);
  80 static void igb_start_link_timer(igb_t *);
  81 static void igb_stop_link_timer(igb_t *);
  82 static void igb_disable_adapter_interrupts(igb_t *);
  83 static void igb_enable_adapter_interrupts_82575(igb_t *);
  84 static void igb_enable_adapter_interrupts_82576(igb_t *);
  85 static void igb_enable_adapter_interrupts_82580(igb_t *);
  86 static boolean_t is_valid_mac_addr(uint8_t *);
  87 static boolean_t igb_stall_check(igb_t *);
  88 static boolean_t igb_set_loopback_mode(igb_t *, uint32_t);
  89 static void igb_set_external_loopback(igb_t *);
  90 static void igb_set_internal_phy_loopback(igb_t *);
  91 static void igb_set_internal_serdes_loopback(igb_t *);
  92 static boolean_t igb_find_mac_address(igb_t *);
  93 static int igb_alloc_intrs(igb_t *);
  94 static int igb_alloc_intr_handles(igb_t *, int);
  95 static int igb_add_intr_handlers(igb_t *);
  96 static void igb_rem_intr_handlers(igb_t *);
  97 static void igb_rem_intrs(igb_t *);
  98 static int igb_enable_intrs(igb_t *);
  99 static int igb_disable_intrs(igb_t *);
 100 static void igb_setup_msix_82575(igb_t *);
 101 static void igb_setup_msix_82576(igb_t *);
 102 static void igb_setup_msix_82580(igb_t *);
 103 static uint_t igb_intr_legacy(void *, void *);
 104 static uint_t igb_intr_msi(void *, void *);
 105 static uint_t igb_intr_rx(void *, void *);
 106 static uint_t igb_intr_tx(void *, void *);
 107 static uint_t igb_intr_tx_other(void *, void *);
 108 static void igb_intr_rx_work(igb_rx_ring_t *);
 109 static void igb_intr_tx_work(igb_tx_ring_t *);
 110 static void igb_intr_link_work(igb_t *);
 111 static void igb_get_driver_control(struct e1000_hw *);
 112 static void igb_release_driver_control(struct e1000_hw *);
 113 
 114 static int igb_attach(dev_info_t *, ddi_attach_cmd_t);
 115 static int igb_detach(dev_info_t *, ddi_detach_cmd_t);
 116 static int igb_resume(dev_info_t *);
 117 static int igb_suspend(dev_info_t *);
 118 static int igb_quiesce(dev_info_t *);
 119 static void igb_unconfigure(dev_info_t *, igb_t *);
 120 static int igb_fm_error_cb(dev_info_t *, ddi_fm_error_t *,
 121     const void *);
 122 static void igb_fm_init(igb_t *);
 123 static void igb_fm_fini(igb_t *);
 124 static void igb_release_multicast(igb_t *);
 125 
 126 char *igb_priv_props[] = {
 127         "_eee_support",
 128         "_tx_copy_thresh",
 129         "_tx_recycle_thresh",
 130         "_tx_overload_thresh",
 131         "_tx_resched_thresh",
 132         "_rx_copy_thresh",
 133         "_rx_limit_per_intr",
 134         "_intr_throttling",
 135         "_adv_pause_cap",
 136         "_adv_asym_pause_cap",
 137         NULL
 138 };
 139 
 140 static struct cb_ops igb_cb_ops = {
 141         nulldev,                /* cb_open */
 142         nulldev,                /* cb_close */
 143         nodev,                  /* cb_strategy */
 144         nodev,                  /* cb_print */
 145         nodev,                  /* cb_dump */
 146         nodev,                  /* cb_read */
 147         nodev,                  /* cb_write */
 148         nodev,                  /* cb_ioctl */
 149         nodev,                  /* cb_devmap */
 150         nodev,                  /* cb_mmap */
 151         nodev,                  /* cb_segmap */
 152         nochpoll,               /* cb_chpoll */
 153         ddi_prop_op,            /* cb_prop_op */
 154         NULL,                   /* cb_stream */
 155         D_MP | D_HOTPLUG,       /* cb_flag */
 156         CB_REV,                 /* cb_rev */
 157         nodev,                  /* cb_aread */
 158         nodev                   /* cb_awrite */
 159 };
 160 
 161 static struct dev_ops igb_dev_ops = {
 162         DEVO_REV,               /* devo_rev */
 163         0,                      /* devo_refcnt */
 164         NULL,                   /* devo_getinfo */
 165         nulldev,                /* devo_identify */
 166         nulldev,                /* devo_probe */
 167         igb_attach,             /* devo_attach */
 168         igb_detach,             /* devo_detach */
 169         nodev,                  /* devo_reset */
 170         &igb_cb_ops,                /* devo_cb_ops */
 171         NULL,                   /* devo_bus_ops */
 172         ddi_power,              /* devo_power */
 173         igb_quiesce,    /* devo_quiesce */
 174 };
 175 
 176 static struct modldrv igb_modldrv = {
 177         &mod_driverops,             /* Type of module.  This one is a driver */
 178         ident,                  /* Discription string */
 179         &igb_dev_ops,               /* driver ops */
 180 };
 181 
 182 static struct modlinkage igb_modlinkage = {
 183         MODREV_1, &igb_modldrv, NULL
 184 };
 185 
 186 /* Access attributes for register mapping */
 187 ddi_device_acc_attr_t igb_regs_acc_attr = {
 188         DDI_DEVICE_ATTR_V1,
 189         DDI_STRUCTURE_LE_ACC,
 190         DDI_STRICTORDER_ACC,
 191         DDI_FLAGERR_ACC
 192 };
 193 
 194 #define IGB_M_CALLBACK_FLAGS \
 195         (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
 196 
 197 static mac_callbacks_t igb_m_callbacks = {
 198         IGB_M_CALLBACK_FLAGS,
 199         igb_m_stat,
 200         igb_m_start,
 201         igb_m_stop,
 202         igb_m_promisc,
 203         igb_m_multicst,
 204         NULL,
 205         NULL,
 206         NULL,
 207         igb_m_ioctl,
 208         igb_m_getcapab,
 209         NULL,
 210         NULL,
 211         igb_m_setprop,
 212         igb_m_getprop,
 213         igb_m_propinfo
 214 };
 215 
 216 /*
 217  * Initialize capabilities of each supported adapter type
 218  */
 219 static adapter_info_t igb_82575_cap = {
 220         /* limits */
 221         4,              /* maximum number of rx queues */
 222         1,              /* minimum number of rx queues */
 223         4,              /* default number of rx queues */
 224         4,              /* maximum number of tx queues */
 225         1,              /* minimum number of tx queues */
 226         4,              /* default number of tx queues */
 227         65535,          /* maximum interrupt throttle rate */
 228         0,              /* minimum interrupt throttle rate */
 229         200,            /* default interrupt throttle rate */
 230 
 231         /* function pointers */
 232         igb_enable_adapter_interrupts_82575,
 233         igb_setup_msix_82575,
 234 
 235         /* capabilities */
 236         (IGB_FLAG_HAS_DCA |     /* capability flags */
 237         IGB_FLAG_VMDQ_POOL),
 238 
 239         0xffc00000              /* mask for RXDCTL register */
 240 };
 241 
 242 static adapter_info_t igb_82576_cap = {
 243         /* limits */
 244         16,             /* maximum number of rx queues */
 245         1,              /* minimum number of rx queues */
 246         4,              /* default number of rx queues */
 247         16,             /* maximum number of tx queues */
 248         1,              /* minimum number of tx queues */
 249         4,              /* default number of tx queues */
 250         65535,          /* maximum interrupt throttle rate */
 251         0,              /* minimum interrupt throttle rate */
 252         200,            /* default interrupt throttle rate */
 253 
 254         /* function pointers */
 255         igb_enable_adapter_interrupts_82576,
 256         igb_setup_msix_82576,
 257 
 258         /* capabilities */
 259         (IGB_FLAG_HAS_DCA |     /* capability flags */
 260         IGB_FLAG_VMDQ_POOL |
 261         IGB_FLAG_NEED_CTX_IDX),
 262 
 263         0xffe00000              /* mask for RXDCTL register */
 264 };
 265 
 266 static adapter_info_t igb_82580_cap = {
 267         /* limits */
 268         8,              /* maximum number of rx queues */
 269         1,              /* minimum number of rx queues */
 270         4,              /* default number of rx queues */
 271         8,              /* maximum number of tx queues */
 272         1,              /* minimum number of tx queues */
 273         4,              /* default number of tx queues */
 274         65535,          /* maximum interrupt throttle rate */
 275         0,              /* minimum interrupt throttle rate */
 276         200,            /* default interrupt throttle rate */
 277 
 278         /* function pointers */
 279         igb_enable_adapter_interrupts_82580,
 280         igb_setup_msix_82580,
 281 
 282         /* capabilities */
 283         (IGB_FLAG_HAS_DCA |     /* capability flags */
 284         IGB_FLAG_VMDQ_POOL |
 285         IGB_FLAG_NEED_CTX_IDX),
 286 
 287         0xffe00000              /* mask for RXDCTL register */
 288 };
 289 
 290 static adapter_info_t igb_i350_cap = {
 291         /* limits */
 292         8,              /* maximum number of rx queues */
 293         1,              /* minimum number of rx queues */
 294         4,              /* default number of rx queues */
 295         8,              /* maximum number of tx queues */
 296         1,              /* minimum number of tx queues */
 297         4,              /* default number of tx queues */
 298         65535,          /* maximum interrupt throttle rate */
 299         0,              /* minimum interrupt throttle rate */
 300         200,            /* default interrupt throttle rate */
 301 
 302         /* function pointers */
 303         igb_enable_adapter_interrupts_82580,
 304         igb_setup_msix_82580,
 305 
 306         /* capabilities */
 307         (IGB_FLAG_HAS_DCA |     /* capability flags */
 308         IGB_FLAG_VMDQ_POOL |
 309         IGB_FLAG_NEED_CTX_IDX),
 310 
 311         0xffe00000              /* mask for RXDCTL register */
 312 };
 313 
 314 static adapter_info_t igb_i210_cap = {
 315         /* limits */
 316         4,              /* maximum number of rx queues */
 317         1,              /* minimum number of rx queues */
 318         4,              /* default number of rx queues */
 319         4,              /* maximum number of tx queues */
 320         1,              /* minimum number of tx queues */
 321         4,              /* default number of tx queues */
 322         65535,          /* maximum interrupt throttle rate */
 323         0,              /* minimum interrupt throttle rate */
 324         200,            /* default interrupt throttle rate */
 325 
 326         /* function pointers */
 327         igb_enable_adapter_interrupts_82580,
 328         igb_setup_msix_82580,
 329 
 330         /* capabilities */
 331         (IGB_FLAG_HAS_DCA |     /* capability flags */
 332         IGB_FLAG_VMDQ_POOL |
 333         IGB_FLAG_NEED_CTX_IDX),
 334 
 335         0xfff00000              /* mask for RXDCTL register */
 336 };
 337 
 338 static adapter_info_t igb_i354_cap = {
 339         /* limits */
 340         8,              /* maximum number of rx queues */
 341         1,              /* minimum number of rx queues */
 342         4,              /* default number of rx queues */
 343         8,              /* maximum number of tx queues */
 344         1,              /* minimum number of tx queues */
 345         4,              /* default number of tx queues */
 346         65535,          /* maximum interrupt throttle rate */
 347         0,              /* minimum interrupt throttle rate */
 348         200,            /* default interrupt throttle rate */
 349 
 350         /* function pointers */
 351         igb_enable_adapter_interrupts_82580,
 352         igb_setup_msix_82580,
 353 
 354         /* capabilities */
 355         (IGB_FLAG_HAS_DCA |     /* capability flags */
 356         IGB_FLAG_VMDQ_POOL |
 357         IGB_FLAG_NEED_CTX_IDX),
 358 
 359         0xfff00000              /* mask for RXDCTL register */
 360 };
 361 
 362 /*
 363  * Module Initialization Functions
 364  */
 365 
 366 int
 367 _init(void)
 368 {
 369         int status;
 370 
 371         mac_init_ops(&igb_dev_ops, MODULE_NAME);
 372 
 373         status = mod_install(&igb_modlinkage);
 374 
 375         if (status != DDI_SUCCESS) {
 376                 mac_fini_ops(&igb_dev_ops);
 377         }
 378 
 379         return (status);
 380 }
 381 
 382 int
 383 _fini(void)
 384 {
 385         int status;
 386 
 387         status = mod_remove(&igb_modlinkage);
 388 
 389         if (status == DDI_SUCCESS) {
 390                 mac_fini_ops(&igb_dev_ops);
 391         }
 392 
 393         return (status);
 394 
 395 }
 396 
 397 int
 398 _info(struct modinfo *modinfop)
 399 {
 400         int status;
 401 
 402         status = mod_info(&igb_modlinkage, modinfop);
 403 
 404         return (status);
 405 }
 406 
 407 /*
 408  * igb_attach - driver attach
 409  *
 410  * This function is the device specific initialization entry
 411  * point. This entry point is required and must be written.
 412  * The DDI_ATTACH command must be provided in the attach entry
 413  * point. When attach() is called with cmd set to DDI_ATTACH,
 414  * all normal kernel services (such as kmem_alloc(9F)) are
 415  * available for use by the driver.
 416  *
 417  * The attach() function will be called once for each instance
 418  * of  the  device  on  the  system with cmd set to DDI_ATTACH.
 419  * Until attach() succeeds, the only driver entry points which
 420  * may be called are open(9E) and getinfo(9E).
 421  */
 422 static int
 423 igb_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
 424 {
 425         igb_t *igb;
 426         struct igb_osdep *osdep;
 427         struct e1000_hw *hw;
 428         int instance;
 429 
 430         /*
 431          * Check the command and perform corresponding operations
 432          */
 433         switch (cmd) {
 434         default:
 435                 return (DDI_FAILURE);
 436 
 437         case DDI_RESUME:
 438                 return (igb_resume(devinfo));
 439 
 440         case DDI_ATTACH:
 441                 break;
 442         }
 443 
 444         /* Get the device instance */
 445         instance = ddi_get_instance(devinfo);
 446 
 447         /* Allocate memory for the instance data structure */
 448         igb = kmem_zalloc(sizeof (igb_t), KM_SLEEP);
 449 
 450         igb->dip = devinfo;
 451         igb->instance = instance;
 452 
 453         hw = &igb->hw;
 454         osdep = &igb->osdep;
 455         hw->back = osdep;
 456         osdep->igb = igb;
 457 
 458         /* Attach the instance pointer to the dev_info data structure */
 459         ddi_set_driver_private(devinfo, igb);
 460 
 461 
 462         /* Initialize for fma support */
 463         igb->fm_capabilities = igb_get_prop(igb, "fm-capable",
 464             0, 0x0f,
 465             DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
 466             DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
 467         igb_fm_init(igb);
 468         igb->attach_progress |= ATTACH_PROGRESS_FMINIT;
 469 
 470         /*
 471          * Map PCI config space registers
 472          */
 473         if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
 474                 igb_error(igb, "Failed to map PCI configurations");
 475                 goto attach_fail;
 476         }
 477         igb->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
 478 
 479         /*
 480          * Identify the chipset family
 481          */
 482         if (igb_identify_hardware(igb) != IGB_SUCCESS) {
 483                 igb_error(igb, "Failed to identify hardware");
 484                 goto attach_fail;
 485         }
 486 
 487         /*
 488          * Map device registers
 489          */
 490         if (igb_regs_map(igb) != IGB_SUCCESS) {
 491                 igb_error(igb, "Failed to map device registers");
 492                 goto attach_fail;
 493         }
 494         igb->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
 495 
 496         /*
 497          * Initialize driver parameters
 498          */
 499         igb_init_properties(igb);
 500         igb->attach_progress |= ATTACH_PROGRESS_PROPS;
 501 
 502         /*
 503          * Allocate interrupts
 504          */
 505         if (igb_alloc_intrs(igb) != IGB_SUCCESS) {
 506                 igb_error(igb, "Failed to allocate interrupts");
 507                 goto attach_fail;
 508         }
 509         igb->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
 510 
 511         /*
 512          * Allocate rx/tx rings based on the ring numbers.
 513          * The actual numbers of rx/tx rings are decided by the number of
 514          * allocated interrupt vectors, so we should allocate the rings after
 515          * interrupts are allocated.
 516          */
 517         if (igb_alloc_rings(igb) != IGB_SUCCESS) {
 518                 igb_error(igb, "Failed to allocate rx/tx rings or groups");
 519                 goto attach_fail;
 520         }
 521         igb->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
 522 
 523         /*
 524          * Add interrupt handlers
 525          */
 526         if (igb_add_intr_handlers(igb) != IGB_SUCCESS) {
 527                 igb_error(igb, "Failed to add interrupt handlers");
 528                 goto attach_fail;
 529         }
 530         igb->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
 531 
 532         /*
 533          * Initialize driver parameters
 534          */
 535         if (igb_init_driver_settings(igb) != IGB_SUCCESS) {
 536                 igb_error(igb, "Failed to initialize driver settings");
 537                 goto attach_fail;
 538         }
 539 
 540         if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) {
 541                 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
 542                 goto attach_fail;
 543         }
 544 
 545         /*
 546          * Initialize mutexes for this device.
 547          * Do this before enabling the interrupt handler and
 548          * register the softint to avoid the condition where
 549          * interrupt handler can try using uninitialized mutex
 550          */
 551         igb_init_locks(igb);
 552         igb->attach_progress |= ATTACH_PROGRESS_LOCKS;
 553 
 554         /*
 555          * Initialize the adapter
 556          */
 557         if (igb_init(igb) != IGB_SUCCESS) {
 558                 igb_error(igb, "Failed to initialize adapter");
 559                 goto attach_fail;
 560         }
 561         igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER;
 562 
 563         /*
 564          * Initialize statistics
 565          */
 566         if (igb_init_stats(igb) != IGB_SUCCESS) {
 567                 igb_error(igb, "Failed to initialize statistics");
 568                 goto attach_fail;
 569         }
 570         igb->attach_progress |= ATTACH_PROGRESS_STATS;
 571 
 572         /*
 573          * Register the driver to the MAC
 574          */
 575         if (igb_register_mac(igb) != IGB_SUCCESS) {
 576                 igb_error(igb, "Failed to register MAC");
 577                 goto attach_fail;
 578         }
 579         igb->attach_progress |= ATTACH_PROGRESS_MAC;
 580 
 581         /*
 582          * Now that mutex locks are initialized, and the chip is also
 583          * initialized, enable interrupts.
 584          */
 585         if (igb_enable_intrs(igb) != IGB_SUCCESS) {
 586                 igb_error(igb, "Failed to enable DDI interrupts");
 587                 goto attach_fail;
 588         }
 589         igb->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
 590 
 591         igb_log(igb, "%s", igb_version);
 592         atomic_or_32(&igb->igb_state, IGB_INITIALIZED);
 593 
 594         /*
 595          * Newer models have Energy Efficient Ethernet, let's disable this by
 596          * default.
 597          */
 598         if (igb->hw.mac.type == e1000_i350)
 599                 (void) e1000_set_eee_i350(&igb->hw);
 600         else if (igb->hw.mac.type == e1000_i354)
 601                 (void) e1000_set_eee_i354(&igb->hw);
 602 
 603         return (DDI_SUCCESS);
 604 
 605 attach_fail:
 606         igb_unconfigure(devinfo, igb);
 607         return (DDI_FAILURE);
 608 }
 609 
 610 /*
 611  * igb_detach - driver detach
 612  *
 613  * The detach() function is the complement of the attach routine.
 614  * If cmd is set to DDI_DETACH, detach() is used to remove  the
 615  * state  associated  with  a  given  instance of a device node
 616  * prior to the removal of that instance from the system.
 617  *
 618  * The detach() function will be called once for each  instance
 619  * of the device for which there has been a successful attach()
 620  * once there are no longer  any  opens  on  the  device.
 621  *
 622  * Interrupts routine are disabled, All memory allocated by this
 623  * driver are freed.
 624  */
 625 static int
 626 igb_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
 627 {
 628         igb_t *igb;
 629 
 630         /*
 631          * Check detach command
 632          */
 633         switch (cmd) {
 634         default:
 635                 return (DDI_FAILURE);
 636 
 637         case DDI_SUSPEND:
 638                 return (igb_suspend(devinfo));
 639 
 640         case DDI_DETACH:
 641                 break;
 642         }
 643 
 644 
 645         /*
 646          * Get the pointer to the driver private data structure
 647          */
 648         igb = (igb_t *)ddi_get_driver_private(devinfo);
 649         if (igb == NULL)
 650                 return (DDI_FAILURE);
 651 
 652         /*
 653          * Unregister MAC. If failed, we have to fail the detach
 654          */
 655         if (mac_unregister(igb->mac_hdl) != 0) {
 656                 igb_error(igb, "Failed to unregister MAC");
 657                 return (DDI_FAILURE);
 658         }
 659         igb->attach_progress &= ~ATTACH_PROGRESS_MAC;
 660 
 661         /*
 662          * If the device is still running, it needs to be stopped first.
 663          * This check is necessary because under some specific circumstances,
 664          * the detach routine can be called without stopping the interface
 665          * first.
 666          */
 667         mutex_enter(&igb->gen_lock);
 668         if (igb->igb_state & IGB_STARTED) {
 669                 atomic_and_32(&igb->igb_state, ~IGB_STARTED);
 670                 igb_stop(igb, B_TRUE);
 671                 mutex_exit(&igb->gen_lock);
 672                 /* Disable and stop the watchdog timer */
 673                 igb_disable_watchdog_timer(igb);
 674         } else
 675                 mutex_exit(&igb->gen_lock);
 676 
 677         /*
 678          * Check if there are still rx buffers held by the upper layer.
 679          * If so, fail the detach.
 680          */
 681         if (!igb_rx_drain(igb))
 682                 return (DDI_FAILURE);
 683 
 684         /*
 685          * Do the remaining unconfigure routines
 686          */
 687         igb_unconfigure(devinfo, igb);
 688 
 689         return (DDI_SUCCESS);
 690 }
 691 
 692 /*
 693  * quiesce(9E) entry point.
 694  *
 695  * This function is called when the system is single-threaded at high
 696  * PIL with preemption disabled. Therefore, this function must not be
 697  * blocked.
 698  *
 699  * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
 700  * DDI_FAILURE indicates an error condition and should almost never happen.
 701  */
 702 static int
 703 igb_quiesce(dev_info_t *devinfo)
 704 {
 705         igb_t *igb;
 706         struct e1000_hw *hw;
 707 
 708         igb = (igb_t *)ddi_get_driver_private(devinfo);
 709 
 710         if (igb == NULL)
 711                 return (DDI_FAILURE);
 712 
 713         hw = &igb->hw;
 714 
 715         /*
 716          * Disable the adapter interrupts
 717          */
 718         igb_disable_adapter_interrupts(igb);
 719 
 720         /* Tell firmware driver is no longer in control */
 721         igb_release_driver_control(hw);
 722 
 723         /*
 724          * Reset the chipset
 725          */
 726         (void) e1000_reset_hw(hw);
 727 
 728         /*
 729          * Reset PHY if possible
 730          */
 731         if (e1000_check_reset_block(hw) == E1000_SUCCESS)
 732                 (void) e1000_phy_hw_reset(hw);
 733 
 734         return (DDI_SUCCESS);
 735 }
 736 
 737 /*
 738  * igb_unconfigure - release all resources held by this instance
 739  */
 740 static void
 741 igb_unconfigure(dev_info_t *devinfo, igb_t *igb)
 742 {
 743         /*
 744          * Disable interrupt
 745          */
 746         if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
 747                 (void) igb_disable_intrs(igb);
 748         }
 749 
 750         /*
 751          * Unregister MAC
 752          */
 753         if (igb->attach_progress & ATTACH_PROGRESS_MAC) {
 754                 (void) mac_unregister(igb->mac_hdl);
 755         }
 756 
 757         /*
 758          * Free statistics
 759          */
 760         if (igb->attach_progress & ATTACH_PROGRESS_STATS) {
 761                 kstat_delete((kstat_t *)igb->igb_ks);
 762         }
 763 
 764         /*
 765          * Remove interrupt handlers
 766          */
 767         if (igb->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
 768                 igb_rem_intr_handlers(igb);
 769         }
 770 
 771         /*
 772          * Remove interrupts
 773          */
 774         if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
 775                 igb_rem_intrs(igb);
 776         }
 777 
 778         /*
 779          * Remove driver properties
 780          */
 781         if (igb->attach_progress & ATTACH_PROGRESS_PROPS) {
 782                 (void) ddi_prop_remove_all(devinfo);
 783         }
 784 
 785         /*
 786          * Stop the adapter
 787          */
 788         if (igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) {
 789                 mutex_enter(&igb->gen_lock);
 790                 igb_stop_adapter(igb);
 791                 mutex_exit(&igb->gen_lock);
 792                 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
 793                         ddi_fm_service_impact(igb->dip, DDI_SERVICE_UNAFFECTED);
 794         }
 795 
 796         /*
 797          * Free multicast table
 798          */
 799         igb_release_multicast(igb);
 800 
 801         /*
 802          * Free register handle
 803          */
 804         if (igb->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
 805                 if (igb->osdep.reg_handle != NULL)
 806                         ddi_regs_map_free(&igb->osdep.reg_handle);
 807         }
 808 
 809         /*
 810          * Free PCI config handle
 811          */
 812         if (igb->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
 813                 if (igb->osdep.cfg_handle != NULL)
 814                         pci_config_teardown(&igb->osdep.cfg_handle);
 815         }
 816 
 817         /*
 818          * Free locks
 819          */
 820         if (igb->attach_progress & ATTACH_PROGRESS_LOCKS) {
 821                 igb_destroy_locks(igb);
 822         }
 823 
 824         /*
 825          * Free the rx/tx rings
 826          */
 827         if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
 828                 igb_free_rings(igb);
 829         }
 830 
 831         /*
 832          * Remove FMA
 833          */
 834         if (igb->attach_progress & ATTACH_PROGRESS_FMINIT) {
 835                 igb_fm_fini(igb);
 836         }
 837 
 838         /*
 839          * Free the driver data structure
 840          */
 841         kmem_free(igb, sizeof (igb_t));
 842 
 843         ddi_set_driver_private(devinfo, NULL);
 844 }
 845 
 846 /*
 847  * igb_register_mac - Register the driver and its function pointers with
 848  * the GLD interface
 849  */
 850 static int
 851 igb_register_mac(igb_t *igb)
 852 {
 853         struct e1000_hw *hw = &igb->hw;
 854         mac_register_t *mac;
 855         int status;
 856 
 857         if ((mac = mac_alloc(MAC_VERSION)) == NULL)
 858                 return (IGB_FAILURE);
 859 
 860         mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
 861         mac->m_driver = igb;
 862         mac->m_dip = igb->dip;
 863         mac->m_src_addr = hw->mac.addr;
 864         mac->m_callbacks = &igb_m_callbacks;
 865         mac->m_min_sdu = 0;
 866         mac->m_max_sdu = igb->max_frame_size -
 867             sizeof (struct ether_vlan_header) - ETHERFCSL;
 868         mac->m_margin = VLAN_TAGSZ;
 869         mac->m_priv_props = igb_priv_props;
 870         mac->m_v12n = MAC_VIRT_LEVEL1;
 871 
 872         status = mac_register(mac, &igb->mac_hdl);
 873 
 874         mac_free(mac);
 875 
 876         return ((status == 0) ? IGB_SUCCESS : IGB_FAILURE);
 877 }
 878 
 879 /*
 880  * igb_identify_hardware - Identify the type of the chipset
 881  */
 882 static int
 883 igb_identify_hardware(igb_t *igb)
 884 {
 885         struct e1000_hw *hw = &igb->hw;
 886         struct igb_osdep *osdep = &igb->osdep;
 887 
 888         /*
 889          * Get the device id
 890          */
 891         hw->vendor_id =
 892             pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
 893         hw->device_id =
 894             pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
 895         hw->revision_id =
 896             pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
 897         hw->subsystem_device_id =
 898             pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
 899         hw->subsystem_vendor_id =
 900             pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
 901 
 902         /*
 903          * Set the mac type of the adapter based on the device id
 904          */
 905         if (e1000_set_mac_type(hw) != E1000_SUCCESS) {
 906                 return (IGB_FAILURE);
 907         }
 908 
 909         /*
 910          * Install adapter capabilities based on mac type
 911          */
 912         switch (hw->mac.type) {
 913         case e1000_82575:
 914                 igb->capab = &igb_82575_cap;
 915                 break;
 916         case e1000_82576:
 917                 igb->capab = &igb_82576_cap;
 918                 break;
 919         case e1000_82580:
 920                 igb->capab = &igb_82580_cap;
 921                 break;
 922         case e1000_i350:
 923                 igb->capab = &igb_i350_cap;
 924                 break;
 925         case e1000_i210:
 926         case e1000_i211:
 927                 igb->capab = &igb_i210_cap;
 928                 break;
 929         case e1000_i354:
 930                 igb->capab = &igb_i354_cap;
 931                 break;
 932         default:
 933                 return (IGB_FAILURE);
 934         }
 935 
 936         return (IGB_SUCCESS);
 937 }
 938 
 939 /*
 940  * igb_regs_map - Map the device registers
 941  */
 942 static int
 943 igb_regs_map(igb_t *igb)
 944 {
 945         dev_info_t *devinfo = igb->dip;
 946         struct e1000_hw *hw = &igb->hw;
 947         struct igb_osdep *osdep = &igb->osdep;
 948         off_t mem_size;
 949 
 950         /*
 951          * First get the size of device registers to be mapped.
 952          */
 953         if (ddi_dev_regsize(devinfo, IGB_ADAPTER_REGSET, &mem_size) !=
 954             DDI_SUCCESS) {
 955                 return (IGB_FAILURE);
 956         }
 957 
 958         /*
 959          * Call ddi_regs_map_setup() to map registers
 960          */
 961         if ((ddi_regs_map_setup(devinfo, IGB_ADAPTER_REGSET,
 962             (caddr_t *)&hw->hw_addr, 0,
 963             mem_size, &igb_regs_acc_attr,
 964             &osdep->reg_handle)) != DDI_SUCCESS) {
 965                 return (IGB_FAILURE);
 966         }
 967 
 968         return (IGB_SUCCESS);
 969 }
 970 
 971 /*
 972  * igb_init_properties - Initialize driver properties
 973  */
 974 static void
 975 igb_init_properties(igb_t *igb)
 976 {
 977         /*
 978          * Get conf file properties, including link settings
 979          * jumbo frames, ring number, descriptor number, etc.
 980          */
 981         igb_get_conf(igb);
 982 }
 983 
 984 /*
 985  * igb_init_driver_settings - Initialize driver settings
 986  *
 987  * The settings include hardware function pointers, bus information,
 988  * rx/tx rings settings, link state, and any other parameters that
 989  * need to be setup during driver initialization.
 990  */
 991 static int
 992 igb_init_driver_settings(igb_t *igb)
 993 {
 994         struct e1000_hw *hw = &igb->hw;
 995         igb_rx_ring_t *rx_ring;
 996         igb_tx_ring_t *tx_ring;
 997         uint32_t rx_size;
 998         uint32_t tx_size;
 999         int i;
1000 
1001         /*
1002          * Initialize chipset specific hardware function pointers
1003          */
1004         if (e1000_setup_init_funcs(hw, B_TRUE) != E1000_SUCCESS) {
1005                 return (IGB_FAILURE);
1006         }
1007 
1008         /*
1009          * Get bus information
1010          */
1011         if (e1000_get_bus_info(hw) != E1000_SUCCESS) {
1012                 return (IGB_FAILURE);
1013         }
1014 
1015         /*
1016          * Get the system page size
1017          */
1018         igb->page_size = ddi_ptob(igb->dip, (ulong_t)1);
1019 
1020         /*
1021          * Set rx buffer size
1022          * The IP header alignment room is counted in the calculation.
1023          * The rx buffer size is in unit of 1K that is required by the
1024          * chipset hardware.
1025          */
1026         rx_size = igb->max_frame_size + IPHDR_ALIGN_ROOM;
1027         igb->rx_buf_size = ((rx_size >> 10) +
1028             ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1029 
1030         /*
1031          * Set tx buffer size
1032          */
1033         tx_size = igb->max_frame_size;
1034         igb->tx_buf_size = ((tx_size >> 10) +
1035             ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1036 
1037         /*
1038          * Initialize rx/tx rings parameters
1039          */
1040         for (i = 0; i < igb->num_rx_rings; i++) {
1041                 rx_ring = &igb->rx_rings[i];
1042                 rx_ring->index = i;
1043                 rx_ring->igb = igb;
1044         }
1045 
1046         for (i = 0; i < igb->num_tx_rings; i++) {
1047                 tx_ring = &igb->tx_rings[i];
1048                 tx_ring->index = i;
1049                 tx_ring->igb = igb;
1050                 if (igb->tx_head_wb_enable)
1051                         tx_ring->tx_recycle = igb_tx_recycle_head_wb;
1052                 else
1053                         tx_ring->tx_recycle = igb_tx_recycle_legacy;
1054 
1055                 tx_ring->ring_size = igb->tx_ring_size;
1056                 tx_ring->free_list_size = igb->tx_ring_size +
1057                     (igb->tx_ring_size >> 1);
1058         }
1059 
1060         /*
1061          * Initialize values of interrupt throttling rates
1062          */
1063         for (i = 1; i < MAX_NUM_EITR; i++)
1064                 igb->intr_throttling[i] = igb->intr_throttling[0];
1065 
1066         /*
1067          * The initial link state should be "unknown"
1068          */
1069         igb->link_state = LINK_STATE_UNKNOWN;
1070 
1071         return (IGB_SUCCESS);
1072 }
1073 
1074 /*
1075  * igb_init_locks - Initialize locks
1076  */
1077 static void
1078 igb_init_locks(igb_t *igb)
1079 {
1080         igb_rx_ring_t *rx_ring;
1081         igb_tx_ring_t *tx_ring;
1082         int i;
1083 
1084         for (i = 0; i < igb->num_rx_rings; i++) {
1085                 rx_ring = &igb->rx_rings[i];
1086                 mutex_init(&rx_ring->rx_lock, NULL,
1087                     MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1088         }
1089 
1090         for (i = 0; i < igb->num_tx_rings; i++) {
1091                 tx_ring = &igb->tx_rings[i];
1092                 mutex_init(&tx_ring->tx_lock, NULL,
1093                     MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1094                 mutex_init(&tx_ring->recycle_lock, NULL,
1095                     MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1096                 mutex_init(&tx_ring->tcb_head_lock, NULL,
1097                     MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1098                 mutex_init(&tx_ring->tcb_tail_lock, NULL,
1099                     MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1100         }
1101 
1102         mutex_init(&igb->gen_lock, NULL,
1103             MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1104 
1105         mutex_init(&igb->watchdog_lock, NULL,
1106             MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1107 
1108         mutex_init(&igb->link_lock, NULL,
1109             MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1110 }
1111 
1112 /*
1113  * igb_destroy_locks - Destroy locks
1114  */
1115 static void
1116 igb_destroy_locks(igb_t *igb)
1117 {
1118         igb_rx_ring_t *rx_ring;
1119         igb_tx_ring_t *tx_ring;
1120         int i;
1121 
1122         for (i = 0; i < igb->num_rx_rings; i++) {
1123                 rx_ring = &igb->rx_rings[i];
1124                 mutex_destroy(&rx_ring->rx_lock);
1125         }
1126 
1127         for (i = 0; i < igb->num_tx_rings; i++) {
1128                 tx_ring = &igb->tx_rings[i];
1129                 mutex_destroy(&tx_ring->tx_lock);
1130                 mutex_destroy(&tx_ring->recycle_lock);
1131                 mutex_destroy(&tx_ring->tcb_head_lock);
1132                 mutex_destroy(&tx_ring->tcb_tail_lock);
1133         }
1134 
1135         mutex_destroy(&igb->gen_lock);
1136         mutex_destroy(&igb->watchdog_lock);
1137         mutex_destroy(&igb->link_lock);
1138 }
1139 
1140 static int
1141 igb_resume(dev_info_t *devinfo)
1142 {
1143         igb_t *igb;
1144 
1145         igb = (igb_t *)ddi_get_driver_private(devinfo);
1146         if (igb == NULL)
1147                 return (DDI_FAILURE);
1148 
1149         mutex_enter(&igb->gen_lock);
1150 
1151         /*
1152          * Enable interrupts
1153          */
1154         if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1155                 if (igb_enable_intrs(igb) != IGB_SUCCESS) {
1156                         igb_error(igb, "Failed to enable DDI interrupts");
1157                         mutex_exit(&igb->gen_lock);
1158                         return (DDI_FAILURE);
1159                 }
1160         }
1161 
1162         if (igb->igb_state & IGB_STARTED) {
1163                 if (igb_start(igb, B_FALSE) != IGB_SUCCESS) {
1164                         mutex_exit(&igb->gen_lock);
1165                         return (DDI_FAILURE);
1166                 }
1167 
1168                 /*
1169                  * Enable and start the watchdog timer
1170                  */
1171                 igb_enable_watchdog_timer(igb);
1172         }
1173 
1174         atomic_and_32(&igb->igb_state, ~IGB_SUSPENDED);
1175 
1176         mutex_exit(&igb->gen_lock);
1177 
1178         return (DDI_SUCCESS);
1179 }
1180 
1181 static int
1182 igb_suspend(dev_info_t *devinfo)
1183 {
1184         igb_t *igb;
1185 
1186         igb = (igb_t *)ddi_get_driver_private(devinfo);
1187         if (igb == NULL)
1188                 return (DDI_FAILURE);
1189 
1190         mutex_enter(&igb->gen_lock);
1191 
1192         atomic_or_32(&igb->igb_state, IGB_SUSPENDED);
1193 
1194         /*
1195          * Disable interrupts
1196          */
1197         if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1198                 (void) igb_disable_intrs(igb);
1199         }
1200 
1201         if (!(igb->igb_state & IGB_STARTED)) {
1202                 mutex_exit(&igb->gen_lock);
1203                 return (DDI_SUCCESS);
1204         }
1205 
1206         igb_stop(igb, B_FALSE);
1207 
1208         mutex_exit(&igb->gen_lock);
1209 
1210         /*
1211          * Disable and stop the watchdog timer
1212          */
1213         igb_disable_watchdog_timer(igb);
1214 
1215         return (DDI_SUCCESS);
1216 }
1217 
1218 static int
1219 igb_init(igb_t *igb)
1220 {
1221         mutex_enter(&igb->gen_lock);
1222 
1223         /*
1224          * Initilize the adapter
1225          */
1226         if (igb_init_adapter(igb) != IGB_SUCCESS) {
1227                 mutex_exit(&igb->gen_lock);
1228                 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1229                 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1230                 return (IGB_FAILURE);
1231         }
1232 
1233         mutex_exit(&igb->gen_lock);
1234 
1235         return (IGB_SUCCESS);
1236 }
1237 
1238 /*
1239  * igb_init_mac_address - Initialize the default MAC address
1240  *
1241  * On success, the MAC address is entered in the igb->hw.mac.addr
1242  * and hw->mac.perm_addr fields and the adapter's RAR(0) receive
1243  * address register.
1244  *
1245  * Important side effects:
1246  * 1. adapter is reset - this is required to put it in a known state.
1247  * 2. all of non-volatile memory (NVM) is read & checksummed - NVM is where
1248  * MAC address and all default settings are stored, so a valid checksum
1249  * is required.
1250  */
1251 static int
1252 igb_init_mac_address(igb_t *igb)
1253 {
1254         struct e1000_hw *hw = &igb->hw;
1255 
1256         ASSERT(mutex_owned(&igb->gen_lock));
1257 
1258         /*
1259          * Reset chipset to put the hardware in a known state
1260          * before we try to get MAC address from NVM.
1261          */
1262         if (e1000_reset_hw(hw) != E1000_SUCCESS) {
1263                 igb_error(igb, "Adapter reset failed.");
1264                 goto init_mac_fail;
1265         }
1266 
1267         /*
1268          * NVM validation
1269          */
1270         if (((igb->hw.mac.type != e1000_i210) &&
1271             (igb->hw.mac.type != e1000_i211)) &&
1272             (e1000_validate_nvm_checksum(hw) < 0)) {
1273                 /*
1274                  * Some PCI-E parts fail the first check due to
1275                  * the link being in sleep state.  Call it again,
1276                  * if it fails a second time its a real issue.
1277                  */
1278                 if (e1000_validate_nvm_checksum(hw) < 0) {
1279                         igb_error(igb,
1280                             "Invalid NVM checksum. Please contact "
1281                             "the vendor to update the NVM.");
1282                         goto init_mac_fail;
1283                 }
1284         }
1285 
1286         /*
1287          * Get the mac address
1288          * This function should handle SPARC case correctly.
1289          */
1290         if (!igb_find_mac_address(igb)) {
1291                 igb_error(igb, "Failed to get the mac address");
1292                 goto init_mac_fail;
1293         }
1294 
1295         /* Validate mac address */
1296         if (!is_valid_mac_addr(hw->mac.addr)) {
1297                 igb_error(igb, "Invalid mac address");
1298                 goto init_mac_fail;
1299         }
1300 
1301         return (IGB_SUCCESS);
1302 
1303 init_mac_fail:
1304         return (IGB_FAILURE);
1305 }
1306 
1307 /*
1308  * igb_init_adapter - Initialize the adapter
1309  */
1310 static int
1311 igb_init_adapter(igb_t *igb)
1312 {
1313         struct e1000_hw *hw = &igb->hw;
1314         uint32_t pba;
1315         int oemid[2];
1316         uint16_t nvmword;
1317         uint32_t hwm;
1318         uint32_t default_mtu;
1319         u8 pbanum[E1000_PBANUM_LENGTH];
1320         char eepromver[5];      /* f.ff */
1321         int i;
1322 
1323         ASSERT(mutex_owned(&igb->gen_lock));
1324 
1325         /*
1326          * In order to obtain the default MAC address, this will reset the
1327          * adapter and validate the NVM that the address and many other
1328          * default settings come from.
1329          */
1330         if (igb_init_mac_address(igb) != IGB_SUCCESS) {
1331                 igb_error(igb, "Failed to initialize MAC address");
1332                 goto init_adapter_fail;
1333         }
1334 
1335         /*
1336          * Packet Buffer Allocation (PBA)
1337          * Writing PBA sets the receive portion of the buffer
1338          * the remainder is used for the transmit buffer.
1339          */
1340         switch (hw->mac.type) {
1341         case e1000_82575:
1342                 pba = E1000_PBA_32K;
1343                 break;
1344         case e1000_82576:
1345                 pba = E1000_READ_REG(hw, E1000_RXPBS);
1346                 pba &= E1000_RXPBS_SIZE_MASK_82576;
1347                 break;
1348         case e1000_82580:
1349         case e1000_i350:
1350         case e1000_i354:
1351                 pba = E1000_READ_REG(hw, E1000_RXPBS);
1352                 pba = e1000_rxpbs_adjust_82580(pba);
1353                 break;
1354         case e1000_i210:
1355         case e1000_i211:
1356                 pba = E1000_PBA_34K;
1357         default:
1358                 break;
1359         }
1360 
1361         /* Special needs in case of Jumbo frames */
1362         default_mtu = igb_get_prop(igb, PROP_DEFAULT_MTU,
1363             MIN_MTU, MAX_MTU, DEFAULT_MTU);
1364         if ((hw->mac.type == e1000_82575) && (default_mtu > ETHERMTU)) {
1365                 u32 tx_space, min_tx, min_rx;
1366                 pba = E1000_READ_REG(hw, E1000_PBA);
1367                 tx_space = pba >> 16;
1368                 pba &= 0xffff;
1369                 min_tx = (igb->max_frame_size +
1370                     sizeof (struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
1371                 min_tx = roundup(min_tx, 1024);
1372                 min_tx >>= 10;
1373                 min_rx = igb->max_frame_size;
1374                 min_rx = roundup(min_rx, 1024);
1375                 min_rx >>= 10;
1376                 if (tx_space < min_tx &&
1377                     ((min_tx - tx_space) < pba)) {
1378                         pba = pba - (min_tx - tx_space);
1379                         /*
1380                          * if short on rx space, rx wins
1381                          * and must trump tx adjustment
1382                          */
1383                         if (pba < min_rx)
1384                                 pba = min_rx;
1385                 }
1386                 E1000_WRITE_REG(hw, E1000_PBA, pba);
1387         }
1388 
1389         DEBUGOUT1("igb_init: pba=%dK", pba);
1390 
1391         /*
1392          * These parameters control the automatic generation (Tx) and
1393          * response (Rx) to Ethernet PAUSE frames.
1394          * - High water mark should allow for at least two frames to be
1395          *   received after sending an XOFF.
1396          * - Low water mark works best when it is very near the high water mark.
1397          *   This allows the receiver to restart by sending XON when it has
1398          *   drained a bit.
1399          */
1400         hwm = min(((pba << 10) * 9 / 10),
1401             ((pba << 10) - 2 * igb->max_frame_size));
1402 
1403         if (hw->mac.type < e1000_82576) {
1404                 hw->fc.high_water = hwm & 0xFFF8;  /* 8-byte granularity */
1405                 hw->fc.low_water = hw->fc.high_water - 8;
1406         } else {
1407                 hw->fc.high_water = hwm & 0xFFF0;  /* 16-byte granularity */
1408                 hw->fc.low_water = hw->fc.high_water - 16;
1409         }
1410 
1411         hw->fc.pause_time = E1000_FC_PAUSE_TIME;
1412         hw->fc.send_xon = B_TRUE;
1413 
1414         (void) e1000_validate_mdi_setting(hw);
1415 
1416         /*
1417          * Reset the chipset hardware the second time to put PBA settings
1418          * into effect.
1419          */
1420         if (e1000_reset_hw(hw) != E1000_SUCCESS) {
1421                 igb_error(igb, "Second reset failed");
1422                 goto init_adapter_fail;
1423         }
1424 
1425         /*
1426          * Don't wait for auto-negotiation to complete
1427          */
1428         hw->phy.autoneg_wait_to_complete = B_FALSE;
1429 
1430         /*
1431          * Copper options
1432          */
1433         if (hw->phy.media_type == e1000_media_type_copper) {
1434                 hw->phy.mdix = 0;    /* AUTO_ALL_MODES */
1435                 hw->phy.disable_polarity_correction = B_FALSE;
1436                 hw->phy.ms_type = e1000_ms_hw_default; /* E1000_MASTER_SLAVE */
1437         }
1438 
1439         /*
1440          * Initialize link settings
1441          */
1442         (void) igb_setup_link(igb, B_FALSE);
1443 
1444         /*
1445          * Configure/Initialize hardware
1446          */
1447         if (e1000_init_hw(hw) != E1000_SUCCESS) {
1448                 igb_error(igb, "Failed to initialize hardware");
1449                 goto init_adapter_fail;
1450         }
1451 
1452         /*
1453          *  Start the link setup timer
1454          */
1455         igb_start_link_timer(igb);
1456 
1457         /*
1458          * Disable wakeup control by default
1459          */
1460         E1000_WRITE_REG(hw, E1000_WUC, 0);
1461 
1462         /*
1463          * Record phy info in hw struct
1464          */
1465         (void) e1000_get_phy_info(hw);
1466 
1467         /*
1468          * Make sure driver has control
1469          */
1470         igb_get_driver_control(hw);
1471 
1472         /*
1473          * Restore LED settings to the default from EEPROM
1474          * to meet the standard for Sun platforms.
1475          */
1476         (void) e1000_cleanup_led(hw);
1477 
1478         /*
1479          * Setup MSI-X interrupts
1480          */
1481         if (igb->intr_type == DDI_INTR_TYPE_MSIX)
1482                 igb->capab->setup_msix(igb);
1483 
1484         /*
1485          * Initialize unicast addresses.
1486          */
1487         igb_init_unicst(igb);
1488 
1489         /*
1490          * Setup and initialize the mctable structures.
1491          */
1492         igb_setup_multicst(igb);
1493 
1494         /*
1495          * Set interrupt throttling rate
1496          */
1497         for (i = 0; i < igb->intr_cnt; i++)
1498                 E1000_WRITE_REG(hw, E1000_EITR(i), igb->intr_throttling[i]);
1499 
1500         /*
1501          * Read identifying information and place in devinfo.
1502          */
1503         nvmword = 0xffff;
1504         (void) e1000_read_nvm(&igb->hw, NVM_OEM_OFFSET_0, 1, &nvmword);
1505         oemid[0] = (int)nvmword;
1506         (void) e1000_read_nvm(&igb->hw, NVM_OEM_OFFSET_1, 1, &nvmword);
1507         oemid[1] = (int)nvmword;
1508         (void) ddi_prop_update_int_array(DDI_DEV_T_NONE, igb->dip,
1509             "oem-identifier", oemid, 2);
1510 
1511         pbanum[0] = '\0';
1512         (void) e1000_read_pba_string(&igb->hw, pbanum, sizeof (pbanum));
1513         if (*pbanum != '\0') {
1514                 (void) ddi_prop_update_string(DDI_DEV_T_NONE, igb->dip,
1515                     "printed-board-assembly", (char *)pbanum);
1516         }
1517 
1518         nvmword = 0xffff;
1519         (void) e1000_read_nvm(&igb->hw, NVM_VERSION, 1, &nvmword);
1520         if ((nvmword & 0xf00) == 0) {
1521                 (void) snprintf(eepromver, sizeof (eepromver), "%x.%x",
1522                     (nvmword & 0xf000) >> 12, (nvmword & 0xff));
1523                 (void) ddi_prop_update_string(DDI_DEV_T_NONE, igb->dip,
1524                     "nvm-version", eepromver);
1525         }
1526 
1527         /*
1528          * Save the state of the phy
1529          */
1530         igb_get_phy_state(igb);
1531 
1532         igb_param_sync(igb);
1533 
1534         return (IGB_SUCCESS);
1535 
1536 init_adapter_fail:
1537         /*
1538          * Reset PHY if possible
1539          */
1540         if (e1000_check_reset_block(hw) == E1000_SUCCESS)
1541                 (void) e1000_phy_hw_reset(hw);
1542 
1543         return (IGB_FAILURE);
1544 }
1545 
1546 /*
1547  * igb_stop_adapter - Stop the adapter
1548  */
1549 static void
1550 igb_stop_adapter(igb_t *igb)
1551 {
1552         struct e1000_hw *hw = &igb->hw;
1553 
1554         ASSERT(mutex_owned(&igb->gen_lock));
1555 
1556         /* Stop the link setup timer */
1557         igb_stop_link_timer(igb);
1558 
1559         /* Tell firmware driver is no longer in control */
1560         igb_release_driver_control(hw);
1561 
1562         /*
1563          * Reset the chipset
1564          */
1565         if (e1000_reset_hw(hw) != E1000_SUCCESS) {
1566                 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1567                 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1568         }
1569 
1570         /*
1571          * e1000_phy_hw_reset is not needed here, MAC reset above is sufficient
1572          */
1573 }
1574 
1575 /*
1576  * igb_reset - Reset the chipset and restart the driver.
1577  *
1578  * It involves stopping and re-starting the chipset,
1579  * and re-configuring the rx/tx rings.
1580  */
1581 static int
1582 igb_reset(igb_t *igb)
1583 {
1584         int i;
1585 
1586         mutex_enter(&igb->gen_lock);
1587 
1588         ASSERT(igb->igb_state & IGB_STARTED);
1589         atomic_and_32(&igb->igb_state, ~IGB_STARTED);
1590 
1591         /*
1592          * Disable the adapter interrupts to stop any rx/tx activities
1593          * before draining pending data and resetting hardware.
1594          */
1595         igb_disable_adapter_interrupts(igb);
1596 
1597         /*
1598          * Drain the pending transmit packets
1599          */
1600         (void) igb_tx_drain(igb);
1601 
1602         for (i = 0; i < igb->num_rx_rings; i++)
1603                 mutex_enter(&igb->rx_rings[i].rx_lock);
1604         for (i = 0; i < igb->num_tx_rings; i++)
1605                 mutex_enter(&igb->tx_rings[i].tx_lock);
1606 
1607         /*
1608          * Stop the adapter
1609          */
1610         igb_stop_adapter(igb);
1611 
1612         /*
1613          * Clean the pending tx data/resources
1614          */
1615         igb_tx_clean(igb);
1616 
1617         /*
1618          * Start the adapter
1619          */
1620         if (igb_init_adapter(igb) != IGB_SUCCESS) {
1621                 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1622                 goto reset_failure;
1623         }
1624 
1625         /*
1626          * Setup the rx/tx rings
1627          */
1628         igb->tx_ring_init = B_FALSE;
1629         igb_setup_rings(igb);
1630 
1631         atomic_and_32(&igb->igb_state, ~(IGB_ERROR | IGB_STALL));
1632 
1633         /*
1634          * Enable adapter interrupts
1635          * The interrupts must be enabled after the driver state is START
1636          */
1637         igb->capab->enable_intr(igb);
1638 
1639         if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK)
1640                 goto reset_failure;
1641 
1642         if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
1643                 goto reset_failure;
1644 
1645         for (i = igb->num_tx_rings - 1; i >= 0; i--)
1646                 mutex_exit(&igb->tx_rings[i].tx_lock);
1647         for (i = igb->num_rx_rings - 1; i >= 0; i--)
1648                 mutex_exit(&igb->rx_rings[i].rx_lock);
1649 
1650         atomic_or_32(&igb->igb_state, IGB_STARTED);
1651 
1652         mutex_exit(&igb->gen_lock);
1653 
1654         return (IGB_SUCCESS);
1655 
1656 reset_failure:
1657         for (i = igb->num_tx_rings - 1; i >= 0; i--)
1658                 mutex_exit(&igb->tx_rings[i].tx_lock);
1659         for (i = igb->num_rx_rings - 1; i >= 0; i--)
1660                 mutex_exit(&igb->rx_rings[i].rx_lock);
1661 
1662         mutex_exit(&igb->gen_lock);
1663 
1664         ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1665 
1666         return (IGB_FAILURE);
1667 }
1668 
1669 /*
1670  * igb_tx_clean - Clean the pending transmit packets and DMA resources
1671  */
1672 static void
1673 igb_tx_clean(igb_t *igb)
1674 {
1675         igb_tx_ring_t *tx_ring;
1676         tx_control_block_t *tcb;
1677         link_list_t pending_list;
1678         uint32_t desc_num;
1679         int i, j;
1680 
1681         LINK_LIST_INIT(&pending_list);
1682 
1683         for (i = 0; i < igb->num_tx_rings; i++) {
1684                 tx_ring = &igb->tx_rings[i];
1685 
1686                 mutex_enter(&tx_ring->recycle_lock);
1687 
1688                 /*
1689                  * Clean the pending tx data - the pending packets in the
1690                  * work_list that have no chances to be transmitted again.
1691                  *
1692                  * We must ensure the chipset is stopped or the link is down
1693                  * before cleaning the transmit packets.
1694                  */
1695                 desc_num = 0;
1696                 for (j = 0; j < tx_ring->ring_size; j++) {
1697                         tcb = tx_ring->work_list[j];
1698                         if (tcb != NULL) {
1699                                 desc_num += tcb->desc_num;
1700 
1701                                 tx_ring->work_list[j] = NULL;
1702 
1703                                 igb_free_tcb(tcb);
1704 
1705                                 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1706                         }
1707                 }
1708 
1709                 if (desc_num > 0) {
1710                         atomic_add_32(&tx_ring->tbd_free, desc_num);
1711                         ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1712 
1713                         /*
1714                          * Reset the head and tail pointers of the tbd ring;
1715                          * Reset the head write-back if it is enabled.
1716                          */
1717                         tx_ring->tbd_head = 0;
1718                         tx_ring->tbd_tail = 0;
1719                         if (igb->tx_head_wb_enable)
1720                                 *tx_ring->tbd_head_wb = 0;
1721 
1722                         E1000_WRITE_REG(&igb->hw, E1000_TDH(tx_ring->index), 0);
1723                         E1000_WRITE_REG(&igb->hw, E1000_TDT(tx_ring->index), 0);
1724                 }
1725 
1726                 mutex_exit(&tx_ring->recycle_lock);
1727 
1728                 /*
1729                  * Add the tx control blocks in the pending list to
1730                  * the free list.
1731                  */
1732                 igb_put_free_list(tx_ring, &pending_list);
1733         }
1734 }
1735 
1736 /*
1737  * igb_tx_drain - Drain the tx rings to allow pending packets to be transmitted
1738  */
1739 static boolean_t
1740 igb_tx_drain(igb_t *igb)
1741 {
1742         igb_tx_ring_t *tx_ring;
1743         boolean_t done;
1744         int i, j;
1745 
1746         /*
1747          * Wait for a specific time to allow pending tx packets
1748          * to be transmitted.
1749          *
1750          * Check the counter tbd_free to see if transmission is done.
1751          * No lock protection is needed here.
1752          *
1753          * Return B_TRUE if all pending packets have been transmitted;
1754          * Otherwise return B_FALSE;
1755          */
1756         for (i = 0; i < TX_DRAIN_TIME; i++) {
1757 
1758                 done = B_TRUE;
1759                 for (j = 0; j < igb->num_tx_rings; j++) {
1760                         tx_ring = &igb->tx_rings[j];
1761                         done = done &&
1762                             (tx_ring->tbd_free == tx_ring->ring_size);
1763                 }
1764 
1765                 if (done)
1766                         break;
1767 
1768                 msec_delay(1);
1769         }
1770 
1771         return (done);
1772 }
1773 
1774 /*
1775  * igb_rx_drain - Wait for all rx buffers to be released by upper layer
1776  */
1777 static boolean_t
1778 igb_rx_drain(igb_t *igb)
1779 {
1780         boolean_t done;
1781         int i;
1782 
1783         /*
1784          * Polling the rx free list to check if those rx buffers held by
1785          * the upper layer are released.
1786          *
1787          * Check the counter rcb_free to see if all pending buffers are
1788          * released. No lock protection is needed here.
1789          *
1790          * Return B_TRUE if all pending buffers have been released;
1791          * Otherwise return B_FALSE;
1792          */
1793         for (i = 0; i < RX_DRAIN_TIME; i++) {
1794                 done = (igb->rcb_pending == 0);
1795 
1796                 if (done)
1797                         break;
1798 
1799                 msec_delay(1);
1800         }
1801 
1802         return (done);
1803 }
1804 
1805 /*
1806  * igb_start - Start the driver/chipset
1807  */
1808 int
1809 igb_start(igb_t *igb, boolean_t alloc_buffer)
1810 {
1811         int i;
1812 
1813         ASSERT(mutex_owned(&igb->gen_lock));
1814 
1815         if (alloc_buffer) {
1816                 if (igb_alloc_rx_data(igb) != IGB_SUCCESS) {
1817                         igb_error(igb,
1818                             "Failed to allocate software receive rings");
1819                         return (IGB_FAILURE);
1820                 }
1821 
1822                 /* Allocate buffers for all the rx/tx rings */
1823                 if (igb_alloc_dma(igb) != IGB_SUCCESS) {
1824                         igb_error(igb, "Failed to allocate DMA resource");
1825                         return (IGB_FAILURE);
1826                 }
1827 
1828                 igb->tx_ring_init = B_TRUE;
1829         } else {
1830                 igb->tx_ring_init = B_FALSE;
1831         }
1832 
1833         for (i = 0; i < igb->num_rx_rings; i++)
1834                 mutex_enter(&igb->rx_rings[i].rx_lock);
1835         for (i = 0; i < igb->num_tx_rings; i++)
1836                 mutex_enter(&igb->tx_rings[i].tx_lock);
1837 
1838         /*
1839          * Start the adapter
1840          */
1841         if ((igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) == 0) {
1842                 if (igb_init_adapter(igb) != IGB_SUCCESS) {
1843                         igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1844                         goto start_failure;
1845                 }
1846                 igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER;
1847         }
1848 
1849         /*
1850          * Setup the rx/tx rings
1851          */
1852         igb_setup_rings(igb);
1853 
1854         /*
1855          * Enable adapter interrupts
1856          * The interrupts must be enabled after the driver state is START
1857          */
1858         igb->capab->enable_intr(igb);
1859 
1860         if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK)
1861                 goto start_failure;
1862 
1863         if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
1864                 goto start_failure;
1865 
1866         if (igb->hw.mac.type == e1000_i350)
1867                 (void) e1000_set_eee_i350(&igb->hw);
1868         else if (igb->hw.mac.type == e1000_i354)
1869                 (void) e1000_set_eee_i354(&igb->hw);
1870 
1871         for (i = igb->num_tx_rings - 1; i >= 0; i--)
1872                 mutex_exit(&igb->tx_rings[i].tx_lock);
1873         for (i = igb->num_rx_rings - 1; i >= 0; i--)
1874                 mutex_exit(&igb->rx_rings[i].rx_lock);
1875 
1876         return (IGB_SUCCESS);
1877 
1878 start_failure:
1879         for (i = igb->num_tx_rings - 1; i >= 0; i--)
1880                 mutex_exit(&igb->tx_rings[i].tx_lock);
1881         for (i = igb->num_rx_rings - 1; i >= 0; i--)
1882                 mutex_exit(&igb->rx_rings[i].rx_lock);
1883 
1884         ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1885 
1886         return (IGB_FAILURE);
1887 }
1888 
1889 /*
1890  * igb_stop - Stop the driver/chipset
1891  */
1892 void
1893 igb_stop(igb_t *igb, boolean_t free_buffer)
1894 {
1895         int i;
1896 
1897         ASSERT(mutex_owned(&igb->gen_lock));
1898 
1899         igb->attach_progress &= ~ATTACH_PROGRESS_INIT_ADAPTER;
1900 
1901         /*
1902          * Disable the adapter interrupts
1903          */
1904         igb_disable_adapter_interrupts(igb);
1905 
1906         /*
1907          * Drain the pending tx packets
1908          */
1909         (void) igb_tx_drain(igb);
1910 
1911         for (i = 0; i < igb->num_rx_rings; i++)
1912                 mutex_enter(&igb->rx_rings[i].rx_lock);
1913         for (i = 0; i < igb->num_tx_rings; i++)
1914                 mutex_enter(&igb->tx_rings[i].tx_lock);
1915 
1916         /*
1917          * Stop the adapter
1918          */
1919         igb_stop_adapter(igb);
1920 
1921         /*
1922          * Clean the pending tx data/resources
1923          */
1924         igb_tx_clean(igb);
1925 
1926         for (i = igb->num_tx_rings - 1; i >= 0; i--)
1927                 mutex_exit(&igb->tx_rings[i].tx_lock);
1928         for (i = igb->num_rx_rings - 1; i >= 0; i--)
1929                 mutex_exit(&igb->rx_rings[i].rx_lock);
1930 
1931         if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
1932                 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1933 
1934         if (igb->link_state == LINK_STATE_UP) {
1935                 igb->link_state = LINK_STATE_UNKNOWN;
1936                 mac_link_update(igb->mac_hdl, igb->link_state);
1937         }
1938 
1939         if (free_buffer) {
1940                 /*
1941                  * Release the DMA/memory resources of rx/tx rings
1942                  */
1943                 igb_free_dma(igb);
1944                 igb_free_rx_data(igb);
1945         }
1946 }
1947 
1948 /*
1949  * igb_alloc_rings - Allocate memory space for rx/tx rings
1950  */
1951 static int
1952 igb_alloc_rings(igb_t *igb)
1953 {
1954         /*
1955          * Allocate memory space for rx rings
1956          */
1957         igb->rx_rings = kmem_zalloc(
1958             sizeof (igb_rx_ring_t) * igb->num_rx_rings,
1959             KM_NOSLEEP);
1960 
1961         if (igb->rx_rings == NULL) {
1962                 return (IGB_FAILURE);
1963         }
1964 
1965         /*
1966          * Allocate memory space for tx rings
1967          */
1968         igb->tx_rings = kmem_zalloc(
1969             sizeof (igb_tx_ring_t) * igb->num_tx_rings,
1970             KM_NOSLEEP);
1971 
1972         if (igb->tx_rings == NULL) {
1973                 kmem_free(igb->rx_rings,
1974                     sizeof (igb_rx_ring_t) * igb->num_rx_rings);
1975                 igb->rx_rings = NULL;
1976                 return (IGB_FAILURE);
1977         }
1978 
1979         /*
1980          * Allocate memory space for rx ring groups
1981          */
1982         igb->rx_groups = kmem_zalloc(
1983             sizeof (igb_rx_group_t) * igb->num_rx_groups,
1984             KM_NOSLEEP);
1985 
1986         if (igb->rx_groups == NULL) {
1987                 kmem_free(igb->rx_rings,
1988                     sizeof (igb_rx_ring_t) * igb->num_rx_rings);
1989                 kmem_free(igb->tx_rings,
1990                     sizeof (igb_tx_ring_t) * igb->num_tx_rings);
1991                 igb->rx_rings = NULL;
1992                 igb->tx_rings = NULL;
1993                 return (IGB_FAILURE);
1994         }
1995 
1996         return (IGB_SUCCESS);
1997 }
1998 
1999 /*
2000  * igb_free_rings - Free the memory space of rx/tx rings.
2001  */
2002 static void
2003 igb_free_rings(igb_t *igb)
2004 {
2005         if (igb->rx_rings != NULL) {
2006                 kmem_free(igb->rx_rings,
2007                     sizeof (igb_rx_ring_t) * igb->num_rx_rings);
2008                 igb->rx_rings = NULL;
2009         }
2010 
2011         if (igb->tx_rings != NULL) {
2012                 kmem_free(igb->tx_rings,
2013                     sizeof (igb_tx_ring_t) * igb->num_tx_rings);
2014                 igb->tx_rings = NULL;
2015         }
2016 
2017         if (igb->rx_groups != NULL) {
2018                 kmem_free(igb->rx_groups,
2019                     sizeof (igb_rx_group_t) * igb->num_rx_groups);
2020                 igb->rx_groups = NULL;
2021         }
2022 }
2023 
2024 static int
2025 igb_alloc_rx_data(igb_t *igb)
2026 {
2027         igb_rx_ring_t *rx_ring;
2028         int i;
2029 
2030         for (i = 0; i < igb->num_rx_rings; i++) {
2031                 rx_ring = &igb->rx_rings[i];
2032                 if (igb_alloc_rx_ring_data(rx_ring) != IGB_SUCCESS)
2033                         goto alloc_rx_rings_failure;
2034         }
2035         return (IGB_SUCCESS);
2036 
2037 alloc_rx_rings_failure:
2038         igb_free_rx_data(igb);
2039         return (IGB_FAILURE);
2040 }
2041 
2042 static void
2043 igb_free_rx_data(igb_t *igb)
2044 {
2045         igb_rx_ring_t *rx_ring;
2046         igb_rx_data_t *rx_data;
2047         int i;
2048 
2049         for (i = 0; i < igb->num_rx_rings; i++) {
2050                 rx_ring = &igb->rx_rings[i];
2051 
2052                 mutex_enter(&igb->rx_pending_lock);
2053                 rx_data = rx_ring->rx_data;
2054 
2055                 if (rx_data != NULL) {
2056                         rx_data->flag |= IGB_RX_STOPPED;
2057 
2058                         if (rx_data->rcb_pending == 0) {
2059                                 igb_free_rx_ring_data(rx_data);
2060                                 rx_ring->rx_data = NULL;
2061                         }
2062                 }
2063 
2064                 mutex_exit(&igb->rx_pending_lock);
2065         }
2066 }
2067 
2068 /*
2069  * igb_setup_rings - Setup rx/tx rings
2070  */
2071 static void
2072 igb_setup_rings(igb_t *igb)
2073 {
2074         /*
2075          * Setup the rx/tx rings, including the following:
2076          *
2077          * 1. Setup the descriptor ring and the control block buffers;
2078          * 2. Initialize necessary registers for receive/transmit;
2079          * 3. Initialize software pointers/parameters for receive/transmit;
2080          */
2081         igb_setup_rx(igb);
2082 
2083         igb_setup_tx(igb);
2084 }
2085 
2086 static void
2087 igb_setup_rx_ring(igb_rx_ring_t *rx_ring)
2088 {
2089         igb_t *igb = rx_ring->igb;
2090         igb_rx_data_t *rx_data = rx_ring->rx_data;
2091         struct e1000_hw *hw = &igb->hw;
2092         rx_control_block_t *rcb;
2093         union e1000_adv_rx_desc *rbd;
2094         uint32_t size;
2095         uint32_t buf_low;
2096         uint32_t buf_high;
2097         uint32_t rxdctl;
2098         int i;
2099 
2100         ASSERT(mutex_owned(&rx_ring->rx_lock));
2101         ASSERT(mutex_owned(&igb->gen_lock));
2102 
2103         /*
2104          * Initialize descriptor ring with buffer addresses
2105          */
2106         for (i = 0; i < igb->rx_ring_size; i++) {
2107                 rcb = rx_data->work_list[i];
2108                 rbd = &rx_data->rbd_ring[i];
2109 
2110                 rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2111                 rbd->read.hdr_addr = NULL;
2112         }
2113 
2114         /*
2115          * Initialize the base address registers
2116          */
2117         buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2118         buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2119         E1000_WRITE_REG(hw, E1000_RDBAH(rx_ring->index), buf_high);
2120         E1000_WRITE_REG(hw, E1000_RDBAL(rx_ring->index), buf_low);
2121 
2122         /*
2123          * Initialize the length register
2124          */
2125         size = rx_data->ring_size * sizeof (union e1000_adv_rx_desc);
2126         E1000_WRITE_REG(hw, E1000_RDLEN(rx_ring->index), size);
2127 
2128         /*
2129          * Initialize buffer size & descriptor type
2130          */
2131         E1000_WRITE_REG(hw, E1000_SRRCTL(rx_ring->index),
2132             ((igb->rx_buf_size >> E1000_SRRCTL_BSIZEPKT_SHIFT) |
2133             E1000_SRRCTL_DESCTYPE_ADV_ONEBUF));
2134 
2135         /*
2136          * Setup the Receive Descriptor Control Register (RXDCTL)
2137          */
2138         rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(rx_ring->index));
2139         rxdctl &= igb->capab->rxdctl_mask;
2140         rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2141         rxdctl |= 16;           /* pthresh */
2142         rxdctl |= 8 << 8; /* hthresh */
2143         rxdctl |= 1 << 16;        /* wthresh */
2144         E1000_WRITE_REG(hw, E1000_RXDCTL(rx_ring->index), rxdctl);
2145 
2146         rx_data->rbd_next = 0;
2147 }
2148 
2149 static void
2150 igb_setup_rx(igb_t *igb)
2151 {
2152         igb_rx_ring_t *rx_ring;
2153         igb_rx_data_t *rx_data;
2154         igb_rx_group_t *rx_group;
2155         struct e1000_hw *hw = &igb->hw;
2156         uint32_t rctl, rxcsum;
2157         uint32_t ring_per_group;
2158         int i;
2159 
2160         /*
2161          * Setup the Receive Control Register (RCTL), and enable the
2162          * receiver. The initial configuration is to: enable the receiver,
2163          * accept broadcasts, discard bad packets, accept long packets,
2164          * disable VLAN filter checking, and set receive buffer size to
2165          * 2k.  For 82575, also set the receive descriptor minimum
2166          * threshold size to 1/2 the ring.
2167          */
2168         rctl = E1000_READ_REG(hw, E1000_RCTL);
2169 
2170         /*
2171          * Clear the field used for wakeup control.  This driver doesn't do
2172          * wakeup but leave this here for completeness.
2173          */
2174         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2175         rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2176 
2177         rctl |= (E1000_RCTL_EN |        /* Enable Receive Unit */
2178             E1000_RCTL_BAM |            /* Accept Broadcast Packets */
2179             E1000_RCTL_LPE |            /* Large Packet Enable */
2180                                         /* Multicast filter offset */
2181             (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT) |
2182             E1000_RCTL_RDMTS_HALF |     /* rx descriptor threshold */
2183             E1000_RCTL_SECRC);          /* Strip Ethernet CRC */
2184 
2185         for (i = 0; i < igb->num_rx_groups; i++) {
2186                 rx_group = &igb->rx_groups[i];
2187                 rx_group->index = i;
2188                 rx_group->igb = igb;
2189         }
2190 
2191         /*
2192          * Set up all rx descriptor rings - must be called before receive unit
2193          * enabled.
2194          */
2195         ring_per_group = igb->num_rx_rings / igb->num_rx_groups;
2196         for (i = 0; i < igb->num_rx_rings; i++) {
2197                 rx_ring = &igb->rx_rings[i];
2198                 igb_setup_rx_ring(rx_ring);
2199 
2200                 /*
2201                  * Map a ring to a group by assigning a group index
2202                  */
2203                 rx_ring->group_index = i / ring_per_group;
2204         }
2205 
2206         /*
2207          * Setup the Rx Long Packet Max Length register
2208          */
2209         E1000_WRITE_REG(hw, E1000_RLPML, igb->max_frame_size);
2210 
2211         /*
2212          * Hardware checksum settings
2213          */
2214         if (igb->rx_hcksum_enable) {
2215                 rxcsum =
2216                     E1000_RXCSUM_TUOFL |        /* TCP/UDP checksum */
2217                     E1000_RXCSUM_IPOFL;         /* IP checksum */
2218 
2219                 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2220         }
2221 
2222         /*
2223          * Setup classify and RSS for multiple receive queues
2224          */
2225         switch (igb->vmdq_mode) {
2226         case E1000_VMDQ_OFF:
2227                 /*
2228                  * One ring group, only RSS is needed when more than
2229                  * one ring enabled.
2230                  */
2231                 if (igb->num_rx_rings > 1)
2232                         igb_setup_rss(igb);
2233                 break;
2234         case E1000_VMDQ_MAC:
2235                 /*
2236                  * Multiple groups, each group has one ring,
2237                  * only the MAC classification is needed.
2238                  */
2239                 igb_setup_mac_classify(igb);
2240                 break;
2241         case E1000_VMDQ_MAC_RSS:
2242                 /*
2243                  * Multiple groups and multiple rings, both
2244                  * MAC classification and RSS are needed.
2245                  */
2246                 igb_setup_mac_rss_classify(igb);
2247                 break;
2248         }
2249 
2250         /*
2251          * Enable the receive unit - must be done after all
2252          * the rx setup above.
2253          */
2254         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2255 
2256         /*
2257          * Initialize all adapter ring head & tail pointers - must
2258          * be done after receive unit is enabled
2259          */
2260         for (i = 0; i < igb->num_rx_rings; i++) {
2261                 rx_ring = &igb->rx_rings[i];
2262                 rx_data = rx_ring->rx_data;
2263                 E1000_WRITE_REG(hw, E1000_RDH(i), 0);
2264                 E1000_WRITE_REG(hw, E1000_RDT(i), rx_data->ring_size - 1);
2265         }
2266 
2267         /*
2268          * 82575 with manageability enabled needs a special flush to make
2269          * sure the fifos start clean.
2270          */
2271         if ((hw->mac.type == e1000_82575) &&
2272             (E1000_READ_REG(hw, E1000_MANC) & E1000_MANC_RCV_TCO_EN)) {
2273                 e1000_rx_fifo_flush_82575(hw);
2274         }
2275 }
2276 
2277 static void
2278 igb_setup_tx_ring(igb_tx_ring_t *tx_ring)
2279 {
2280         igb_t *igb = tx_ring->igb;
2281         struct e1000_hw *hw = &igb->hw;
2282         uint32_t size;
2283         uint32_t buf_low;
2284         uint32_t buf_high;
2285         uint32_t reg_val;
2286 
2287         ASSERT(mutex_owned(&tx_ring->tx_lock));
2288         ASSERT(mutex_owned(&igb->gen_lock));
2289 
2290 
2291         /*
2292          * Initialize the length register
2293          */
2294         size = tx_ring->ring_size * sizeof (union e1000_adv_tx_desc);
2295         E1000_WRITE_REG(hw, E1000_TDLEN(tx_ring->index), size);
2296 
2297         /*
2298          * Initialize the base address registers
2299          */
2300         buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2301         buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2302         E1000_WRITE_REG(hw, E1000_TDBAL(tx_ring->index), buf_low);
2303         E1000_WRITE_REG(hw, E1000_TDBAH(tx_ring->index), buf_high);
2304 
2305         /*
2306          * Setup head & tail pointers
2307          */
2308         E1000_WRITE_REG(hw, E1000_TDH(tx_ring->index), 0);
2309         E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), 0);
2310 
2311         /*
2312          * Setup head write-back
2313          */
2314         if (igb->tx_head_wb_enable) {
2315                 /*
2316                  * The memory of the head write-back is allocated using
2317                  * the extra tbd beyond the tail of the tbd ring.
2318                  */
2319                 tx_ring->tbd_head_wb = (uint32_t *)
2320                     ((uintptr_t)tx_ring->tbd_area.address + size);
2321                 *tx_ring->tbd_head_wb = 0;
2322 
2323                 buf_low = (uint32_t)
2324                     (tx_ring->tbd_area.dma_address + size);
2325                 buf_high = (uint32_t)
2326                     ((tx_ring->tbd_area.dma_address + size) >> 32);
2327 
2328                 /* Set the head write-back enable bit */
2329                 buf_low |= E1000_TX_HEAD_WB_ENABLE;
2330 
2331                 E1000_WRITE_REG(hw, E1000_TDWBAL(tx_ring->index), buf_low);
2332                 E1000_WRITE_REG(hw, E1000_TDWBAH(tx_ring->index), buf_high);
2333 
2334                 /*
2335                  * Turn off relaxed ordering for head write back or it will
2336                  * cause problems with the tx recycling
2337                  */
2338                 reg_val = E1000_READ_REG(hw,
2339                     E1000_DCA_TXCTRL(tx_ring->index));
2340                 reg_val &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN;
2341                 E1000_WRITE_REG(hw,
2342                     E1000_DCA_TXCTRL(tx_ring->index), reg_val);
2343         } else {
2344                 tx_ring->tbd_head_wb = NULL;
2345         }
2346 
2347         tx_ring->tbd_head = 0;
2348         tx_ring->tbd_tail = 0;
2349         tx_ring->tbd_free = tx_ring->ring_size;
2350 
2351         if (igb->tx_ring_init == B_TRUE) {
2352                 tx_ring->tcb_head = 0;
2353                 tx_ring->tcb_tail = 0;
2354                 tx_ring->tcb_free = tx_ring->free_list_size;
2355         }
2356 
2357         /*
2358          * Enable TXDCTL per queue
2359          */
2360         reg_val = E1000_READ_REG(hw, E1000_TXDCTL(tx_ring->index));
2361         reg_val |= E1000_TXDCTL_QUEUE_ENABLE;
2362         E1000_WRITE_REG(hw, E1000_TXDCTL(tx_ring->index), reg_val);
2363 
2364         /*
2365          * Initialize hardware checksum offload settings
2366          */
2367         bzero(&tx_ring->tx_context, sizeof (tx_context_t));
2368 }
2369 
2370 static void
2371 igb_setup_tx(igb_t *igb)
2372 {
2373         igb_tx_ring_t *tx_ring;
2374         struct e1000_hw *hw = &igb->hw;
2375         uint32_t reg_val;
2376         int i;
2377 
2378         for (i = 0; i < igb->num_tx_rings; i++) {
2379                 tx_ring = &igb->tx_rings[i];
2380                 igb_setup_tx_ring(tx_ring);
2381         }
2382 
2383         /*
2384          * Setup the Transmit Control Register (TCTL)
2385          */
2386         reg_val = E1000_READ_REG(hw, E1000_TCTL);
2387         reg_val &= ~E1000_TCTL_CT;
2388         reg_val |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2389             (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2390 
2391         /* Enable transmits */
2392         reg_val |= E1000_TCTL_EN;
2393 
2394         E1000_WRITE_REG(hw, E1000_TCTL, reg_val);
2395 }
2396 
2397 /*
2398  * igb_setup_rss - Setup receive-side scaling feature
2399  */
2400 static void
2401 igb_setup_rss(igb_t *igb)
2402 {
2403         struct e1000_hw *hw = &igb->hw;
2404         uint32_t i, mrqc, rxcsum;
2405         int shift = 0;
2406         uint32_t random;
2407         union e1000_reta {
2408                 uint32_t        dword;
2409                 uint8_t         bytes[4];
2410         } reta;
2411 
2412         /* Setup the Redirection Table */
2413         if (hw->mac.type == e1000_82576) {
2414                 shift = 3;
2415         } else if (hw->mac.type == e1000_82575) {
2416                 shift = 6;
2417         }
2418         for (i = 0; i < (32 * 4); i++) {
2419                 reta.bytes[i & 3] = (i % igb->num_rx_rings) << shift;
2420                 if ((i & 3) == 3) {
2421                         E1000_WRITE_REG(hw,
2422                             (E1000_RETA(0) + (i & ~3)), reta.dword);
2423                 }
2424         }
2425 
2426         /* Fill out hash function seeds */
2427         for (i = 0; i < 10; i++) {
2428                 (void) random_get_pseudo_bytes((uint8_t *)&random,
2429                     sizeof (uint32_t));
2430                 E1000_WRITE_REG(hw, E1000_RSSRK(i), random);
2431         }
2432 
2433         /* Setup the Multiple Receive Queue Control register */
2434         mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2435         mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2436             E1000_MRQC_RSS_FIELD_IPV4_TCP |
2437             E1000_MRQC_RSS_FIELD_IPV6 |
2438             E1000_MRQC_RSS_FIELD_IPV6_TCP |
2439             E1000_MRQC_RSS_FIELD_IPV4_UDP |
2440             E1000_MRQC_RSS_FIELD_IPV6_UDP |
2441             E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2442             E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2443 
2444         E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
2445 
2446         /*
2447          * Disable Packet Checksum to enable RSS for multiple receive queues.
2448          *
2449          * The Packet Checksum is not ethernet CRC. It is another kind of
2450          * checksum offloading provided by the 82575 chipset besides the IP
2451          * header checksum offloading and the TCP/UDP checksum offloading.
2452          * The Packet Checksum is by default computed over the entire packet
2453          * from the first byte of the DA through the last byte of the CRC,
2454          * including the Ethernet and IP headers.
2455          *
2456          * It is a hardware limitation that Packet Checksum is mutually
2457          * exclusive with RSS.
2458          */
2459         rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
2460         rxcsum |= E1000_RXCSUM_PCSD;
2461         E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2462 }
2463 
2464 /*
2465  * igb_setup_mac_rss_classify - Setup MAC classification and rss
2466  */
2467 static void
2468 igb_setup_mac_rss_classify(igb_t *igb)
2469 {
2470         struct e1000_hw *hw = &igb->hw;
2471         uint32_t i, mrqc, vmdctl, rxcsum;
2472         uint32_t ring_per_group;
2473         int shift_group0, shift_group1;
2474         uint32_t random;
2475         union e1000_reta {
2476                 uint32_t        dword;
2477                 uint8_t         bytes[4];
2478         } reta;
2479 
2480         ring_per_group = igb->num_rx_rings / igb->num_rx_groups;
2481 
2482         /* Setup the Redirection Table, it is shared between two groups */
2483         shift_group0 = 2;
2484         shift_group1 = 6;
2485         for (i = 0; i < (32 * 4); i++) {
2486                 reta.bytes[i & 3] = ((i % ring_per_group) << shift_group0) |
2487                     ((ring_per_group + (i % ring_per_group)) << shift_group1);
2488                 if ((i & 3) == 3) {
2489                         E1000_WRITE_REG(hw,
2490                             (E1000_RETA(0) + (i & ~3)), reta.dword);
2491                 }
2492         }
2493 
2494         /* Fill out hash function seeds */
2495         for (i = 0; i < 10; i++) {
2496                 (void) random_get_pseudo_bytes((uint8_t *)&random,
2497                     sizeof (uint32_t));
2498                 E1000_WRITE_REG(hw, E1000_RSSRK(i), random);
2499         }
2500 
2501         /*
2502          * Setup the Multiple Receive Queue Control register,
2503          * enable VMDq based on packet destination MAC address and RSS.
2504          */
2505         mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_RSS_GROUP;
2506         mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2507             E1000_MRQC_RSS_FIELD_IPV4_TCP |
2508             E1000_MRQC_RSS_FIELD_IPV6 |
2509             E1000_MRQC_RSS_FIELD_IPV6_TCP |
2510             E1000_MRQC_RSS_FIELD_IPV4_UDP |
2511             E1000_MRQC_RSS_FIELD_IPV6_UDP |
2512             E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2513             E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2514 
2515         E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
2516 
2517 
2518         /* Define the default group and default queues */
2519         vmdctl = E1000_VMDQ_MAC_GROUP_DEFAULT_QUEUE;
2520         E1000_WRITE_REG(hw, E1000_VT_CTL, vmdctl);
2521 
2522         /*
2523          * Disable Packet Checksum to enable RSS for multiple receive queues.
2524          *
2525          * The Packet Checksum is not ethernet CRC. It is another kind of
2526          * checksum offloading provided by the 82575 chipset besides the IP
2527          * header checksum offloading and the TCP/UDP checksum offloading.
2528          * The Packet Checksum is by default computed over the entire packet
2529          * from the first byte of the DA through the last byte of the CRC,
2530          * including the Ethernet and IP headers.
2531          *
2532          * It is a hardware limitation that Packet Checksum is mutually
2533          * exclusive with RSS.
2534          */
2535         rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
2536         rxcsum |= E1000_RXCSUM_PCSD;
2537         E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2538 }
2539 
2540 /*
2541  * igb_setup_mac_classify - Setup MAC classification feature
2542  */
2543 static void
2544 igb_setup_mac_classify(igb_t *igb)
2545 {
2546         struct e1000_hw *hw = &igb->hw;
2547         uint32_t mrqc, rxcsum;
2548 
2549         /*
2550          * Setup the Multiple Receive Queue Control register,
2551          * enable VMDq based on packet destination MAC address.
2552          */
2553         mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_GROUP;
2554         E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
2555 
2556         /*
2557          * Disable Packet Checksum to enable RSS for multiple receive queues.
2558          *
2559          * The Packet Checksum is not ethernet CRC. It is another kind of
2560          * checksum offloading provided by the 82575 chipset besides the IP
2561          * header checksum offloading and the TCP/UDP checksum offloading.
2562          * The Packet Checksum is by default computed over the entire packet
2563          * from the first byte of the DA through the last byte of the CRC,
2564          * including the Ethernet and IP headers.
2565          *
2566          * It is a hardware limitation that Packet Checksum is mutually
2567          * exclusive with RSS.
2568          */
2569         rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
2570         rxcsum |= E1000_RXCSUM_PCSD;
2571         E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2572 
2573 }
2574 
2575 /*
2576  * igb_init_unicst - Initialize the unicast addresses
2577  */
2578 static void
2579 igb_init_unicst(igb_t *igb)
2580 {
2581         struct e1000_hw *hw = &igb->hw;
2582         int slot;
2583 
2584         /*
2585          * Here we should consider two situations:
2586          *
2587          * 1. Chipset is initialized the first time
2588          *    Initialize the multiple unicast addresses, and
2589          *    save the default MAC address.
2590          *
2591          * 2. Chipset is reset
2592          *    Recover the multiple unicast addresses from the
2593          *    software data structure to the RAR registers.
2594          */
2595 
2596         /*
2597          * Clear the default MAC address in the RAR0 rgister,
2598          * which is loaded from EEPROM when system boot or chipreset,
2599          * this will cause the conficts with add_mac/rem_mac entry
2600          * points when VMDq is enabled. For this reason, the RAR0
2601          * must be cleared for both cases mentioned above.
2602          */
2603         e1000_rar_clear(hw, 0);
2604 
2605         if (!igb->unicst_init) {
2606 
2607                 /* Initialize the multiple unicast addresses */
2608                 igb->unicst_total = MAX_NUM_UNICAST_ADDRESSES;
2609                 igb->unicst_avail = igb->unicst_total;
2610 
2611                 for (slot = 0; slot < igb->unicst_total; slot++)
2612                         igb->unicst_addr[slot].mac.set = 0;
2613 
2614                 igb->unicst_init = B_TRUE;
2615         } else {
2616                 /* Re-configure the RAR registers */
2617                 for (slot = 0; slot < igb->unicst_total; slot++) {
2618                         (void) e1000_rar_set_vmdq(hw,
2619                             igb->unicst_addr[slot].mac.addr,
2620                             slot, igb->vmdq_mode,
2621                             igb->unicst_addr[slot].mac.group_index);
2622                 }
2623         }
2624 }
2625 
2626 /*
2627  * igb_unicst_find - Find the slot for the specified unicast address
2628  */
2629 int
2630 igb_unicst_find(igb_t *igb, const uint8_t *mac_addr)
2631 {
2632         int slot;
2633 
2634         ASSERT(mutex_owned(&igb->gen_lock));
2635 
2636         for (slot = 0; slot < igb->unicst_total; slot++) {
2637                 if (bcmp(igb->unicst_addr[slot].mac.addr,
2638                     mac_addr, ETHERADDRL) == 0)
2639                         return (slot);
2640         }
2641 
2642         return (-1);
2643 }
2644 
2645 /*
2646  * igb_unicst_set - Set the unicast address to the specified slot
2647  */
2648 int
2649 igb_unicst_set(igb_t *igb, const uint8_t *mac_addr,
2650     int slot)
2651 {
2652         struct e1000_hw *hw = &igb->hw;
2653 
2654         ASSERT(mutex_owned(&igb->gen_lock));
2655 
2656         /*
2657          * Save the unicast address in the software data structure
2658          */
2659         bcopy(mac_addr, igb->unicst_addr[slot].mac.addr, ETHERADDRL);
2660 
2661         /*
2662          * Set the unicast address to the RAR register
2663          */
2664         (void) e1000_rar_set(hw, (uint8_t *)mac_addr, slot);
2665 
2666         if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
2667                 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
2668                 return (EIO);
2669         }
2670 
2671         return (0);
2672 }
2673 
2674 /*
2675  * igb_multicst_add - Add a multicst address
2676  */
2677 int
2678 igb_multicst_add(igb_t *igb, const uint8_t *multiaddr)
2679 {
2680         struct ether_addr *new_table;
2681         size_t new_len;
2682         size_t old_len;
2683 
2684         ASSERT(mutex_owned(&igb->gen_lock));
2685 
2686         if ((multiaddr[0] & 01) == 0) {
2687                 igb_error(igb, "Illegal multicast address");
2688                 return (EINVAL);
2689         }
2690 
2691         if (igb->mcast_count >= igb->mcast_max_num) {
2692                 igb_error(igb, "Adapter requested more than %d mcast addresses",
2693                     igb->mcast_max_num);
2694                 return (ENOENT);
2695         }
2696 
2697         if (igb->mcast_count == igb->mcast_alloc_count) {
2698                 old_len = igb->mcast_alloc_count *
2699                     sizeof (struct ether_addr);
2700                 new_len = (igb->mcast_alloc_count + MCAST_ALLOC_COUNT) *
2701                     sizeof (struct ether_addr);
2702 
2703                 new_table = kmem_alloc(new_len, KM_NOSLEEP);
2704                 if (new_table == NULL) {
2705                         igb_error(igb,
2706                             "Not enough memory to alloc mcast table");
2707                         return (ENOMEM);
2708                 }
2709 
2710                 if (igb->mcast_table != NULL) {
2711                         bcopy(igb->mcast_table, new_table, old_len);
2712                         kmem_free(igb->mcast_table, old_len);
2713                 }
2714                 igb->mcast_alloc_count += MCAST_ALLOC_COUNT;
2715                 igb->mcast_table = new_table;
2716         }
2717 
2718         bcopy(multiaddr,
2719             &igb->mcast_table[igb->mcast_count], ETHERADDRL);
2720         igb->mcast_count++;
2721 
2722         /*
2723          * Update the multicast table in the hardware
2724          */
2725         igb_setup_multicst(igb);
2726 
2727         if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
2728                 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
2729                 return (EIO);
2730         }
2731 
2732         return (0);
2733 }
2734 
2735 /*
2736  * igb_multicst_remove - Remove a multicst address
2737  */
2738 int
2739 igb_multicst_remove(igb_t *igb, const uint8_t *multiaddr)
2740 {
2741         struct ether_addr *new_table;
2742         size_t new_len;
2743         size_t old_len;
2744         int i;
2745 
2746         ASSERT(mutex_owned(&igb->gen_lock));
2747 
2748         for (i = 0; i < igb->mcast_count; i++) {
2749                 if (bcmp(multiaddr, &igb->mcast_table[i],
2750                     ETHERADDRL) == 0) {
2751                         for (i++; i < igb->mcast_count; i++) {
2752                                 igb->mcast_table[i - 1] =
2753                                     igb->mcast_table[i];
2754                         }
2755                         igb->mcast_count--;
2756                         break;
2757                 }
2758         }
2759 
2760         if ((igb->mcast_alloc_count - igb->mcast_count) >
2761             MCAST_ALLOC_COUNT) {
2762                 old_len = igb->mcast_alloc_count *
2763                     sizeof (struct ether_addr);
2764                 new_len = (igb->mcast_alloc_count - MCAST_ALLOC_COUNT) *
2765                     sizeof (struct ether_addr);
2766 
2767                 new_table = kmem_alloc(new_len, KM_NOSLEEP);
2768                 if (new_table != NULL) {
2769                         bcopy(igb->mcast_table, new_table, new_len);
2770                         kmem_free(igb->mcast_table, old_len);
2771                         igb->mcast_alloc_count -= MCAST_ALLOC_COUNT;
2772                         igb->mcast_table = new_table;
2773                 }
2774         }
2775 
2776         /*
2777          * Update the multicast table in the hardware
2778          */
2779         igb_setup_multicst(igb);
2780 
2781         if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
2782                 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
2783                 return (EIO);
2784         }
2785 
2786         return (0);
2787 }
2788 
2789 static void
2790 igb_release_multicast(igb_t *igb)
2791 {
2792         if (igb->mcast_table != NULL) {
2793                 kmem_free(igb->mcast_table,
2794                     igb->mcast_alloc_count * sizeof (struct ether_addr));
2795                 igb->mcast_table = NULL;
2796         }
2797 }
2798 
2799 /*
2800  * igb_setup_multicast - setup multicast data structures
2801  *
2802  * This routine initializes all of the multicast related structures
2803  * and save them in the hardware registers.
2804  */
2805 static void
2806 igb_setup_multicst(igb_t *igb)
2807 {
2808         uint8_t *mc_addr_list;
2809         uint32_t mc_addr_count;
2810         struct e1000_hw *hw = &igb->hw;
2811 
2812         ASSERT(mutex_owned(&igb->gen_lock));
2813         ASSERT(igb->mcast_count <= igb->mcast_max_num);
2814 
2815         mc_addr_list = (uint8_t *)igb->mcast_table;
2816         mc_addr_count = igb->mcast_count;
2817 
2818         /*
2819          * Update the multicase addresses to the MTA registers
2820          */
2821         e1000_update_mc_addr_list(hw, mc_addr_list, mc_addr_count);
2822 }
2823 
2824 /*
2825  * igb_get_conf - Get driver configurations set in driver.conf
2826  *
2827  * This routine gets user-configured values out of the configuration
2828  * file igb.conf.
2829  *
2830  * For each configurable value, there is a minimum, a maximum, and a
2831  * default.
2832  * If user does not configure a value, use the default.
2833  * If user configures below the minimum, use the minumum.
2834  * If user configures above the maximum, use the maxumum.
2835  */
2836 static void
2837 igb_get_conf(igb_t *igb)
2838 {
2839         struct e1000_hw *hw = &igb->hw;
2840         uint32_t default_mtu;
2841         uint32_t flow_control;
2842         uint32_t ring_per_group;
2843         int i;
2844 
2845         /*
2846          * igb driver supports the following user configurations:
2847          *
2848          * Link configurations:
2849          *    adv_autoneg_cap
2850          *    adv_1000fdx_cap
2851          *    adv_100fdx_cap
2852          *    adv_100hdx_cap
2853          *    adv_10fdx_cap
2854          *    adv_10hdx_cap
2855          * Note: 1000hdx is not supported.
2856          *
2857          * Jumbo frame configuration:
2858          *    default_mtu
2859          *
2860          * Ethernet flow control configuration:
2861          *    flow_control
2862          *
2863          * Multiple rings configurations:
2864          *    tx_queue_number
2865          *    tx_ring_size
2866          *    rx_queue_number
2867          *    rx_ring_size
2868          *
2869          * Call igb_get_prop() to get the value for a specific
2870          * configuration parameter.
2871          */
2872 
2873         /*
2874          * Link configurations
2875          */
2876         igb->param_adv_autoneg_cap = igb_get_prop(igb,
2877             PROP_ADV_AUTONEG_CAP, 0, 1, 1);
2878         igb->param_adv_1000fdx_cap = igb_get_prop(igb,
2879             PROP_ADV_1000FDX_CAP, 0, 1, 1);
2880         igb->param_adv_100fdx_cap = igb_get_prop(igb,
2881             PROP_ADV_100FDX_CAP, 0, 1, 1);
2882         igb->param_adv_100hdx_cap = igb_get_prop(igb,
2883             PROP_ADV_100HDX_CAP, 0, 1, 1);
2884         igb->param_adv_10fdx_cap = igb_get_prop(igb,
2885             PROP_ADV_10FDX_CAP, 0, 1, 1);
2886         igb->param_adv_10hdx_cap = igb_get_prop(igb,
2887             PROP_ADV_10HDX_CAP, 0, 1, 1);
2888 
2889         /*
2890          * Jumbo frame configurations
2891          */
2892         default_mtu = igb_get_prop(igb, PROP_DEFAULT_MTU,
2893             MIN_MTU, MAX_MTU, DEFAULT_MTU);
2894 
2895         igb->max_frame_size = default_mtu +
2896             sizeof (struct ether_vlan_header) + ETHERFCSL;
2897 
2898         /*
2899          * Ethernet flow control configuration
2900          */
2901         flow_control = igb_get_prop(igb, PROP_FLOW_CONTROL,
2902             e1000_fc_none, 4, e1000_fc_full);
2903         if (flow_control == 4)
2904                 flow_control = e1000_fc_default;
2905 
2906         hw->fc.requested_mode = flow_control;
2907 
2908         /*
2909          * Multiple rings configurations
2910          */
2911         igb->tx_ring_size = igb_get_prop(igb, PROP_TX_RING_SIZE,
2912             MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
2913         igb->rx_ring_size = igb_get_prop(igb, PROP_RX_RING_SIZE,
2914             MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
2915 
2916         igb->mr_enable = igb_get_prop(igb, PROP_MR_ENABLE, 0, 1, 0);
2917         igb->num_rx_groups = igb_get_prop(igb, PROP_RX_GROUP_NUM,
2918             MIN_RX_GROUP_NUM, MAX_RX_GROUP_NUM, DEFAULT_RX_GROUP_NUM);
2919         /*
2920          * Currently we do not support VMDq for 82576 and 82580.
2921          * If it is e1000_82576, set num_rx_groups to 1.
2922          */
2923         if (hw->mac.type >= e1000_82576)
2924                 igb->num_rx_groups = 1;
2925 
2926         if (igb->mr_enable) {
2927                 igb->num_tx_rings = igb->capab->def_tx_que_num;
2928                 igb->num_rx_rings = igb->capab->def_rx_que_num;
2929         } else {
2930                 igb->num_tx_rings = 1;
2931                 igb->num_rx_rings = 1;
2932 
2933                 if (igb->num_rx_groups > 1) {
2934                         igb_error(igb,
2935                             "Invalid rx groups number. Please enable multiple "
2936                             "rings first");
2937                         igb->num_rx_groups = 1;
2938                 }
2939         }
2940 
2941         /*
2942          * Check the divisibility between rx rings and rx groups.
2943          */
2944         for (i = igb->num_rx_groups; i > 0; i--) {
2945                 if ((igb->num_rx_rings % i) == 0)
2946                         break;
2947         }
2948         if (i != igb->num_rx_groups) {
2949                 igb_error(igb,
2950                     "Invalid rx groups number. Downgrade the rx group "
2951                     "number to %d.", i);
2952                 igb->num_rx_groups = i;
2953         }
2954 
2955         /*
2956          * Get the ring number per group.
2957          */
2958         ring_per_group = igb->num_rx_rings / igb->num_rx_groups;
2959 
2960         if (igb->num_rx_groups == 1) {
2961                 /*
2962                  * One rx ring group, the rx ring number is num_rx_rings.
2963                  */
2964                 igb->vmdq_mode = E1000_VMDQ_OFF;
2965         } else if (ring_per_group == 1) {
2966                 /*
2967                  * Multiple rx groups, each group has one rx ring.
2968                  */
2969                 igb->vmdq_mode = E1000_VMDQ_MAC;
2970         } else {
2971                 /*
2972                  * Multiple groups and multiple rings.
2973                  */
2974                 igb->vmdq_mode = E1000_VMDQ_MAC_RSS;
2975         }
2976 
2977         /*
2978          * Tunable used to force an interrupt type. The only use is
2979          * for testing of the lesser interrupt types.
2980          * 0 = don't force interrupt type
2981          * 1 = force interrupt type MSIX
2982          * 2 = force interrupt type MSI
2983          * 3 = force interrupt type Legacy
2984          */
2985         igb->intr_force = igb_get_prop(igb, PROP_INTR_FORCE,
2986             IGB_INTR_NONE, IGB_INTR_LEGACY, IGB_INTR_NONE);
2987 
2988         igb->tx_hcksum_enable = igb_get_prop(igb, PROP_TX_HCKSUM_ENABLE,
2989             0, 1, 1);
2990         igb->rx_hcksum_enable = igb_get_prop(igb, PROP_RX_HCKSUM_ENABLE,
2991             0, 1, 1);
2992         igb->lso_enable = igb_get_prop(igb, PROP_LSO_ENABLE,
2993             0, 1, 1);
2994         igb->tx_head_wb_enable = igb_get_prop(igb, PROP_TX_HEAD_WB_ENABLE,
2995             0, 1, 1);
2996 
2997         /*
2998          * igb LSO needs the tx h/w checksum support.
2999          * Here LSO will be disabled if tx h/w checksum has been disabled.
3000          */
3001         if (igb->tx_hcksum_enable == B_FALSE)
3002                 igb->lso_enable = B_FALSE;
3003 
3004         igb->tx_copy_thresh = igb_get_prop(igb, PROP_TX_COPY_THRESHOLD,
3005             MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3006             DEFAULT_TX_COPY_THRESHOLD);
3007         igb->tx_recycle_thresh = igb_get_prop(igb, PROP_TX_RECYCLE_THRESHOLD,
3008             MIN_TX_RECYCLE_THRESHOLD, MAX_TX_RECYCLE_THRESHOLD,
3009             DEFAULT_TX_RECYCLE_THRESHOLD);
3010         igb->tx_overload_thresh = igb_get_prop(igb, PROP_TX_OVERLOAD_THRESHOLD,
3011             MIN_TX_OVERLOAD_THRESHOLD, MAX_TX_OVERLOAD_THRESHOLD,
3012             DEFAULT_TX_OVERLOAD_THRESHOLD);
3013         igb->tx_resched_thresh = igb_get_prop(igb, PROP_TX_RESCHED_THRESHOLD,
3014             MIN_TX_RESCHED_THRESHOLD,
3015             MIN(igb->tx_ring_size, MAX_TX_RESCHED_THRESHOLD),
3016             igb->tx_ring_size > DEFAULT_TX_RESCHED_THRESHOLD ?
3017             DEFAULT_TX_RESCHED_THRESHOLD : DEFAULT_TX_RESCHED_THRESHOLD_LOW);
3018 
3019         igb->rx_copy_thresh = igb_get_prop(igb, PROP_RX_COPY_THRESHOLD,
3020             MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3021             DEFAULT_RX_COPY_THRESHOLD);
3022         igb->rx_limit_per_intr = igb_get_prop(igb, PROP_RX_LIMIT_PER_INTR,
3023             MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3024             DEFAULT_RX_LIMIT_PER_INTR);
3025 
3026         igb->intr_throttling[0] = igb_get_prop(igb, PROP_INTR_THROTTLING,
3027             igb->capab->min_intr_throttle,
3028             igb->capab->max_intr_throttle,
3029             igb->capab->def_intr_throttle);
3030 
3031         /*
3032          * Max number of multicast addresses
3033          */
3034         igb->mcast_max_num =
3035             igb_get_prop(igb, PROP_MCAST_MAX_NUM,
3036             MIN_MCAST_NUM, MAX_MCAST_NUM, DEFAULT_MCAST_NUM);
3037 }
3038 
3039 /*
3040  * igb_get_prop - Get a property value out of the configuration file igb.conf
3041  *
3042  * Caller provides the name of the property, a default value, a minimum
3043  * value, and a maximum value.
3044  *
3045  * Return configured value of the property, with default, minimum and
3046  * maximum properly applied.
3047  */
3048 static int
3049 igb_get_prop(igb_t *igb,
3050     char *propname,     /* name of the property */
3051     int minval,         /* minimum acceptable value */
3052     int maxval,         /* maximim acceptable value */
3053     int defval)         /* default value */
3054 {
3055         int value;
3056 
3057         /*
3058          * Call ddi_prop_get_int() to read the conf settings
3059          */
3060         value = ddi_prop_get_int(DDI_DEV_T_ANY, igb->dip,
3061             DDI_PROP_DONTPASS, propname, defval);
3062 
3063         if (value > maxval)
3064                 value = maxval;
3065 
3066         if (value < minval)
3067                 value = minval;
3068 
3069         return (value);
3070 }
3071 
3072 /*
3073  * igb_setup_link - Using the link properties to setup the link
3074  */
3075 int
3076 igb_setup_link(igb_t *igb, boolean_t setup_hw)
3077 {
3078         struct e1000_mac_info *mac;
3079         struct e1000_phy_info *phy;
3080         boolean_t invalid;
3081 
3082         mac = &igb->hw.mac;
3083         phy = &igb->hw.phy;
3084         invalid = B_FALSE;
3085 
3086         if (igb->param_adv_autoneg_cap == 1) {
3087                 mac->autoneg = B_TRUE;
3088                 phy->autoneg_advertised = 0;
3089 
3090                 /*
3091                  * 1000hdx is not supported for autonegotiation
3092                  */
3093                 if (igb->param_adv_1000fdx_cap == 1)
3094                         phy->autoneg_advertised |= ADVERTISE_1000_FULL;
3095 
3096                 if (igb->param_adv_100fdx_cap == 1)
3097                         phy->autoneg_advertised |= ADVERTISE_100_FULL;
3098 
3099                 if (igb->param_adv_100hdx_cap == 1)
3100                         phy->autoneg_advertised |= ADVERTISE_100_HALF;
3101 
3102                 if (igb->param_adv_10fdx_cap == 1)
3103                         phy->autoneg_advertised |= ADVERTISE_10_FULL;
3104 
3105                 if (igb->param_adv_10hdx_cap == 1)
3106                         phy->autoneg_advertised |= ADVERTISE_10_HALF;
3107 
3108                 if (phy->autoneg_advertised == 0)
3109                         invalid = B_TRUE;
3110         } else {
3111                 mac->autoneg = B_FALSE;
3112 
3113                 /*
3114                  * 1000fdx and 1000hdx are not supported for forced link
3115                  */
3116                 if (igb->param_adv_100fdx_cap == 1)
3117                         mac->forced_speed_duplex = ADVERTISE_100_FULL;
3118                 else if (igb->param_adv_100hdx_cap == 1)
3119                         mac->forced_speed_duplex = ADVERTISE_100_HALF;
3120                 else if (igb->param_adv_10fdx_cap == 1)
3121                         mac->forced_speed_duplex = ADVERTISE_10_FULL;
3122                 else if (igb->param_adv_10hdx_cap == 1)
3123                         mac->forced_speed_duplex = ADVERTISE_10_HALF;
3124                 else
3125                         invalid = B_TRUE;
3126         }
3127 
3128         if (invalid) {
3129                 igb_notice(igb, "Invalid link settings. Setup link to "
3130                     "autonegotiation with full link capabilities.");
3131                 mac->autoneg = B_TRUE;
3132                 phy->autoneg_advertised = ADVERTISE_1000_FULL |
3133                     ADVERTISE_100_FULL | ADVERTISE_100_HALF |
3134                     ADVERTISE_10_FULL | ADVERTISE_10_HALF;
3135         }
3136 
3137         if (setup_hw) {
3138                 if (e1000_setup_link(&igb->hw) != E1000_SUCCESS)
3139                         return (IGB_FAILURE);
3140         }
3141 
3142         return (IGB_SUCCESS);
3143 }
3144 
3145 
3146 /*
3147  * igb_is_link_up - Check if the link is up
3148  */
3149 static boolean_t
3150 igb_is_link_up(igb_t *igb)
3151 {
3152         struct e1000_hw *hw = &igb->hw;
3153         boolean_t link_up = B_FALSE;
3154 
3155         ASSERT(mutex_owned(&igb->gen_lock));
3156 
3157         /*
3158          * get_link_status is set in the interrupt handler on link-status-change
3159          * or rx sequence error interrupt.  get_link_status will stay
3160          * false until the e1000_check_for_link establishes link only
3161          * for copper adapters.
3162          */
3163         switch (hw->phy.media_type) {
3164         case e1000_media_type_copper:
3165                 if (hw->mac.get_link_status) {
3166                         (void) e1000_check_for_link(hw);
3167                         link_up = !hw->mac.get_link_status;
3168                 } else {
3169                         link_up = B_TRUE;
3170                 }
3171                 break;
3172         case e1000_media_type_fiber:
3173                 (void) e1000_check_for_link(hw);
3174                 link_up = (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU);
3175                 break;
3176         case e1000_media_type_internal_serdes:
3177                 (void) e1000_check_for_link(hw);
3178                 link_up = hw->mac.serdes_has_link;
3179                 break;
3180         }
3181 
3182         return (link_up);
3183 }
3184 
3185 /*
3186  * igb_link_check - Link status processing
3187  */
3188 static boolean_t
3189 igb_link_check(igb_t *igb)
3190 {
3191         struct e1000_hw *hw = &igb->hw;
3192         uint16_t speed = 0, duplex = 0;
3193         boolean_t link_changed = B_FALSE;
3194 
3195         ASSERT(mutex_owned(&igb->gen_lock));
3196 
3197         if (igb_is_link_up(igb)) {
3198                 /*
3199                  * The Link is up, check whether it was marked as down earlier
3200                  */
3201                 if (igb->link_state != LINK_STATE_UP) {
3202                         (void) e1000_get_speed_and_duplex(hw, &speed, &duplex);
3203                         igb->link_speed = speed;
3204                         igb->link_duplex = duplex;
3205                         igb->link_state = LINK_STATE_UP;
3206                         link_changed = B_TRUE;
3207                         if (!igb->link_complete)
3208                                 igb_stop_link_timer(igb);
3209                 }
3210         } else if (igb->link_complete) {
3211                 if (igb->link_state != LINK_STATE_DOWN) {
3212                         igb->link_speed = 0;
3213                         igb->link_duplex = 0;
3214                         igb->link_state = LINK_STATE_DOWN;
3215                         link_changed = B_TRUE;
3216                 }
3217         }
3218 
3219         if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
3220                 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
3221                 return (B_FALSE);
3222         }
3223 
3224         return (link_changed);
3225 }
3226 
3227 /*
3228  * igb_local_timer - driver watchdog function
3229  *
3230  * This function will handle the hardware stall check, link status
3231  * check and other routines.
3232  */
3233 static void
3234 igb_local_timer(void *arg)
3235 {
3236         igb_t *igb = (igb_t *)arg;
3237         boolean_t link_changed = B_FALSE;
3238 
3239         if (igb->igb_state & IGB_ERROR) {
3240                 igb->reset_count++;
3241                 if (igb_reset(igb) == IGB_SUCCESS)
3242                         ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED);
3243 
3244                 igb_restart_watchdog_timer(igb);
3245                 return;
3246         }
3247 
3248         if (igb_stall_check(igb) || (igb->igb_state & IGB_STALL)) {
3249                 igb_fm_ereport(igb, DDI_FM_DEVICE_STALL);
3250                 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
3251                 igb->reset_count++;
3252                 if (igb_reset(igb) == IGB_SUCCESS)
3253                         ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED);
3254 
3255                 igb_restart_watchdog_timer(igb);
3256                 return;
3257         }
3258 
3259         mutex_enter(&igb->gen_lock);
3260         if (!(igb->igb_state & IGB_SUSPENDED) && (igb->igb_state & IGB_STARTED))
3261                 link_changed = igb_link_check(igb);
3262         mutex_exit(&igb->gen_lock);
3263 
3264         if (link_changed)
3265                 mac_link_update(igb->mac_hdl, igb->link_state);
3266 
3267         igb_restart_watchdog_timer(igb);
3268 }
3269 
3270 /*
3271  * igb_link_timer - link setup timer function
3272  *
3273  * It is called when the timer for link setup is expired, which indicates
3274  * the completion of the link setup. The link state will not be updated
3275  * until the link setup is completed. And the link state will not be sent
3276  * to the upper layer through mac_link_update() in this function. It will
3277  * be updated in the local timer routine or the interrupts service routine
3278  * after the interface is started (plumbed).
3279  */
3280 static void
3281 igb_link_timer(void *arg)
3282 {
3283         igb_t *igb = (igb_t *)arg;
3284 
3285         mutex_enter(&igb->link_lock);
3286         igb->link_complete = B_TRUE;
3287         igb->link_tid = 0;
3288         mutex_exit(&igb->link_lock);
3289 }
3290 /*
3291  * igb_stall_check - check for transmit stall
3292  *
3293  * This function checks if the adapter is stalled (in transmit).
3294  *
3295  * It is called each time the watchdog timeout is invoked.
3296  * If the transmit descriptor reclaim continuously fails,
3297  * the watchdog value will increment by 1. If the watchdog
3298  * value exceeds the threshold, the igb is assumed to
3299  * have stalled and need to be reset.
3300  */
3301 static boolean_t
3302 igb_stall_check(igb_t *igb)
3303 {
3304         igb_tx_ring_t *tx_ring;
3305         struct e1000_hw *hw = &igb->hw;
3306         boolean_t result;
3307         int i;
3308 
3309         if (igb->link_state != LINK_STATE_UP)
3310                 return (B_FALSE);
3311 
3312         /*
3313          * If any tx ring is stalled, we'll reset the chipset
3314          */
3315         result = B_FALSE;
3316         for (i = 0; i < igb->num_tx_rings; i++) {
3317                 tx_ring = &igb->tx_rings[i];
3318 
3319                 if (tx_ring->recycle_fail > 0)
3320                         tx_ring->stall_watchdog++;
3321                 else
3322                         tx_ring->stall_watchdog = 0;
3323 
3324                 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
3325                         result = B_TRUE;
3326                         if (hw->mac.type == e1000_82580) {
3327                                 hw->dev_spec._82575.global_device_reset
3328                                     = B_TRUE;
3329                         }
3330                         break;
3331                 }
3332         }
3333 
3334         if (result) {
3335                 tx_ring->stall_watchdog = 0;
3336                 tx_ring->recycle_fail = 0;
3337         }
3338 
3339         return (result);
3340 }
3341 
3342 
3343 /*
3344  * is_valid_mac_addr - Check if the mac address is valid
3345  */
3346 static boolean_t
3347 is_valid_mac_addr(uint8_t *mac_addr)
3348 {
3349         const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
3350         const uint8_t addr_test2[6] =
3351             { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
3352 
3353         if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
3354             !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
3355                 return (B_FALSE);
3356 
3357         return (B_TRUE);
3358 }
3359 
3360 static boolean_t
3361 igb_find_mac_address(igb_t *igb)
3362 {
3363         struct e1000_hw *hw = &igb->hw;
3364 #ifdef __sparc
3365         uchar_t *bytes;
3366         struct ether_addr sysaddr;
3367         uint_t nelts;
3368         int err;
3369         boolean_t found = B_FALSE;
3370 
3371         /*
3372          * The "vendor's factory-set address" may already have
3373          * been extracted from the chip, but if the property
3374          * "local-mac-address" is set we use that instead.
3375          *
3376          * We check whether it looks like an array of 6
3377          * bytes (which it should, if OBP set it).  If we can't
3378          * make sense of it this way, we'll ignore it.
3379          */
3380         err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip,
3381             DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
3382         if (err == DDI_PROP_SUCCESS) {
3383                 if (nelts == ETHERADDRL) {
3384                         while (nelts--)
3385                                 hw->mac.addr[nelts] = bytes[nelts];
3386                         found = B_TRUE;
3387                 }
3388                 ddi_prop_free(bytes);
3389         }
3390 
3391         /*
3392          * Look up the OBP property "local-mac-address?". If the user has set
3393          * 'local-mac-address? = false', use "the system address" instead.
3394          */
3395         if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 0,
3396             "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
3397                 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
3398                         if (localetheraddr(NULL, &sysaddr) != 0) {
3399                                 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
3400                                 found = B_TRUE;
3401                         }
3402                 }
3403                 ddi_prop_free(bytes);
3404         }
3405 
3406         /*
3407          * Finally(!), if there's a valid "mac-address" property (created
3408          * if we netbooted from this interface), we must use this instead
3409          * of any of the above to ensure that the NFS/install server doesn't
3410          * get confused by the address changing as Solaris takes over!
3411          */
3412         err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip,
3413             DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
3414         if (err == DDI_PROP_SUCCESS) {
3415                 if (nelts == ETHERADDRL) {
3416                         while (nelts--)
3417                                 hw->mac.addr[nelts] = bytes[nelts];
3418                         found = B_TRUE;
3419                 }
3420                 ddi_prop_free(bytes);
3421         }
3422 
3423         if (found) {
3424                 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
3425                 return (B_TRUE);
3426         }
3427 #endif
3428 
3429         /*
3430          * Read the device MAC address from the EEPROM
3431          */
3432         if (e1000_read_mac_addr(hw) != E1000_SUCCESS)
3433                 return (B_FALSE);
3434 
3435         return (B_TRUE);
3436 }
3437 
3438 #pragma inline(igb_arm_watchdog_timer)
3439 
3440 static void
3441 igb_arm_watchdog_timer(igb_t *igb)
3442 {
3443         /*
3444          * Fire a watchdog timer
3445          */
3446         igb->watchdog_tid =
3447             timeout(igb_local_timer,
3448             (void *)igb, 1 * drv_usectohz(1000000));
3449 
3450 }
3451 
3452 /*
3453  * igb_enable_watchdog_timer - Enable and start the driver watchdog timer
3454  */
3455 void
3456 igb_enable_watchdog_timer(igb_t *igb)
3457 {
3458         mutex_enter(&igb->watchdog_lock);
3459 
3460         if (!igb->watchdog_enable) {
3461                 igb->watchdog_enable = B_TRUE;
3462                 igb->watchdog_start = B_TRUE;
3463                 igb_arm_watchdog_timer(igb);
3464         }
3465 
3466         mutex_exit(&igb->watchdog_lock);
3467 
3468 }
3469 
3470 /*
3471  * igb_disable_watchdog_timer - Disable and stop the driver watchdog timer
3472  */
3473 void
3474 igb_disable_watchdog_timer(igb_t *igb)
3475 {
3476         timeout_id_t tid;
3477 
3478         mutex_enter(&igb->watchdog_lock);
3479 
3480         igb->watchdog_enable = B_FALSE;
3481         igb->watchdog_start = B_FALSE;
3482         tid = igb->watchdog_tid;
3483         igb->watchdog_tid = 0;
3484 
3485         mutex_exit(&igb->watchdog_lock);
3486 
3487         if (tid != 0)
3488                 (void) untimeout(tid);
3489 
3490 }
3491 
3492 /*
3493  * igb_start_watchdog_timer - Start the driver watchdog timer
3494  */
3495 static void
3496 igb_start_watchdog_timer(igb_t *igb)
3497 {
3498         mutex_enter(&igb->watchdog_lock);
3499 
3500         if (igb->watchdog_enable) {
3501                 if (!igb->watchdog_start) {
3502                         igb->watchdog_start = B_TRUE;
3503                         igb_arm_watchdog_timer(igb);
3504                 }
3505         }
3506 
3507         mutex_exit(&igb->watchdog_lock);
3508 }
3509 
3510 /*
3511  * igb_restart_watchdog_timer - Restart the driver watchdog timer
3512  */
3513 static void
3514 igb_restart_watchdog_timer(igb_t *igb)
3515 {
3516         mutex_enter(&igb->watchdog_lock);
3517 
3518         if (igb->watchdog_start)
3519                 igb_arm_watchdog_timer(igb);
3520 
3521         mutex_exit(&igb->watchdog_lock);
3522 }
3523 
3524 /*
3525  * igb_stop_watchdog_timer - Stop the driver watchdog timer
3526  */
3527 static void
3528 igb_stop_watchdog_timer(igb_t *igb)
3529 {
3530         timeout_id_t tid;
3531 
3532         mutex_enter(&igb->watchdog_lock);
3533 
3534         igb->watchdog_start = B_FALSE;
3535         tid = igb->watchdog_tid;
3536         igb->watchdog_tid = 0;
3537 
3538         mutex_exit(&igb->watchdog_lock);
3539 
3540         if (tid != 0)
3541                 (void) untimeout(tid);
3542 }
3543 
3544 /*
3545  * igb_start_link_timer - Start the link setup timer
3546  */
3547 static void
3548 igb_start_link_timer(struct igb *igb)
3549 {
3550         struct e1000_hw *hw = &igb->hw;
3551         clock_t link_timeout;
3552 
3553         if (hw->mac.autoneg)
3554                 link_timeout = PHY_AUTO_NEG_LIMIT *
3555                     drv_usectohz(100000);
3556         else
3557                 link_timeout = PHY_FORCE_LIMIT * drv_usectohz(100000);
3558 
3559         mutex_enter(&igb->link_lock);
3560         if (hw->phy.autoneg_wait_to_complete) {
3561                 igb->link_complete = B_TRUE;
3562         } else {
3563                 igb->link_complete = B_FALSE;
3564                 igb->link_tid = timeout(igb_link_timer, (void *)igb,
3565                     link_timeout);
3566         }
3567         mutex_exit(&igb->link_lock);
3568 }
3569 
3570 /*
3571  * igb_stop_link_timer - Stop the link setup timer
3572  */
3573 static void
3574 igb_stop_link_timer(struct igb *igb)
3575 {
3576         timeout_id_t tid;
3577 
3578         mutex_enter(&igb->link_lock);
3579         igb->link_complete = B_TRUE;
3580         tid = igb->link_tid;
3581         igb->link_tid = 0;
3582         mutex_exit(&igb->link_lock);
3583 
3584         if (tid != 0)
3585                 (void) untimeout(tid);
3586 }
3587 
3588 /*
3589  * igb_disable_adapter_interrupts - Clear/disable all hardware interrupts
3590  */
3591 static void
3592 igb_disable_adapter_interrupts(igb_t *igb)
3593 {
3594         struct e1000_hw *hw = &igb->hw;
3595 
3596         /*
3597          * Set the IMC register to mask all the interrupts,
3598          * including the tx interrupts.
3599          */
3600         E1000_WRITE_REG(hw, E1000_IMC, ~0);
3601         E1000_WRITE_REG(hw, E1000_IAM, 0);
3602 
3603         /*
3604          * Additional disabling for MSI-X
3605          */
3606         if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3607                 E1000_WRITE_REG(hw, E1000_EIMC, ~0);
3608                 E1000_WRITE_REG(hw, E1000_EIAC, 0);
3609                 E1000_WRITE_REG(hw, E1000_EIAM, 0);
3610         }
3611 
3612         E1000_WRITE_FLUSH(hw);
3613 }
3614 
3615 /*
3616  * igb_enable_adapter_interrupts_82580 - Enable NIC interrupts for 82580
3617  */
3618 static void
3619 igb_enable_adapter_interrupts_82580(igb_t *igb)
3620 {
3621         struct e1000_hw *hw = &igb->hw;
3622 
3623         /* Clear any pending interrupts */
3624         (void) E1000_READ_REG(hw, E1000_ICR);
3625         igb->ims_mask |= E1000_IMS_DRSTA;
3626 
3627         if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3628 
3629                 /* Interrupt enabling for MSI-X */
3630                 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask);
3631                 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask);
3632                 igb->ims_mask = (E1000_IMS_LSC | E1000_IMS_DRSTA);
3633                 E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask);
3634         } else { /* Interrupt enabling for MSI and legacy */
3635                 E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID);
3636                 igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE;
3637                 igb->ims_mask |= E1000_IMS_DRSTA;
3638                 E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask);
3639         }
3640 
3641         /* Disable auto-mask for ICR interrupt bits */
3642         E1000_WRITE_REG(hw, E1000_IAM, 0);
3643 
3644         E1000_WRITE_FLUSH(hw);
3645 }
3646 
3647 /*
3648  * igb_enable_adapter_interrupts_82576 - Enable NIC interrupts for 82576
3649  */
3650 static void
3651 igb_enable_adapter_interrupts_82576(igb_t *igb)
3652 {
3653         struct e1000_hw *hw = &igb->hw;
3654 
3655         /* Clear any pending interrupts */
3656         (void) E1000_READ_REG(hw, E1000_ICR);
3657 
3658         if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3659 
3660                 /* Interrupt enabling for MSI-X */
3661                 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask);
3662                 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask);
3663                 igb->ims_mask = E1000_IMS_LSC;
3664                 E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC);
3665         } else {
3666                 /* Interrupt enabling for MSI and legacy */
3667                 E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID);
3668                 igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE;
3669                 E1000_WRITE_REG(hw, E1000_IMS,
3670                     (IMS_ENABLE_MASK | E1000_IMS_TXQE));
3671         }
3672 
3673         /* Disable auto-mask for ICR interrupt bits */
3674         E1000_WRITE_REG(hw, E1000_IAM, 0);
3675 
3676         E1000_WRITE_FLUSH(hw);
3677 }
3678 
3679 /*
3680  * igb_enable_adapter_interrupts_82575 - Enable NIC interrupts for 82575
3681  */
3682 static void
3683 igb_enable_adapter_interrupts_82575(igb_t *igb)
3684 {
3685         struct e1000_hw *hw = &igb->hw;
3686         uint32_t reg;
3687 
3688         /* Clear any pending interrupts */
3689         (void) E1000_READ_REG(hw, E1000_ICR);
3690 
3691         if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3692                 /* Interrupt enabling for MSI-X */
3693                 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask);
3694                 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask);
3695                 igb->ims_mask = E1000_IMS_LSC;
3696                 E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC);
3697 
3698                 /* Enable MSI-X PBA support */
3699                 reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
3700                 reg |= E1000_CTRL_EXT_PBA_CLR;
3701 
3702                 /* Non-selective interrupt clear-on-read */
3703                 reg |= E1000_CTRL_EXT_IRCA;     /* Called NSICR in the EAS */
3704 
3705                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
3706         } else {
3707                 /* Interrupt enabling for MSI and legacy */
3708                 igb->ims_mask = IMS_ENABLE_MASK;
3709                 E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK);
3710         }
3711 
3712         E1000_WRITE_FLUSH(hw);
3713 }
3714 
3715 /*
3716  * Loopback Support
3717  */
3718 static lb_property_t lb_normal =
3719         { normal,       "normal",       IGB_LB_NONE             };
3720 static lb_property_t lb_external =
3721         { external,     "External",     IGB_LB_EXTERNAL         };
3722 static lb_property_t lb_phy =
3723         { internal,     "PHY",          IGB_LB_INTERNAL_PHY     };
3724 static lb_property_t lb_serdes =
3725         { internal,     "SerDes",       IGB_LB_INTERNAL_SERDES  };
3726 
3727 enum ioc_reply
3728 igb_loopback_ioctl(igb_t *igb, struct iocblk *iocp, mblk_t *mp)
3729 {
3730         lb_info_sz_t *lbsp;
3731         lb_property_t *lbpp;
3732         struct e1000_hw *hw;
3733         uint32_t *lbmp;
3734         uint32_t size;
3735         uint32_t value;
3736 
3737         hw = &igb->hw;
3738 
3739         if (mp->b_cont == NULL)
3740                 return (IOC_INVAL);
3741 
3742         switch (iocp->ioc_cmd) {
3743         default:
3744                 return (IOC_INVAL);
3745 
3746         case LB_GET_INFO_SIZE:
3747                 size = sizeof (lb_info_sz_t);
3748                 if (iocp->ioc_count != size)
3749                         return (IOC_INVAL);
3750 
3751                 value = sizeof (lb_normal);
3752                 if (hw->phy.media_type == e1000_media_type_copper)
3753                         value += sizeof (lb_phy);
3754                 else
3755                         value += sizeof (lb_serdes);
3756                 value += sizeof (lb_external);
3757 
3758                 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
3759                 *lbsp = value;
3760                 break;
3761 
3762         case LB_GET_INFO:
3763                 value = sizeof (lb_normal);
3764                 if (hw->phy.media_type == e1000_media_type_copper)
3765                         value += sizeof (lb_phy);
3766                 else
3767                         value += sizeof (lb_serdes);
3768                 value += sizeof (lb_external);
3769 
3770                 size = value;
3771                 if (iocp->ioc_count != size)
3772                         return (IOC_INVAL);
3773 
3774                 value = 0;
3775                 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
3776 
3777                 lbpp[value++] = lb_normal;
3778                 if (hw->phy.media_type == e1000_media_type_copper)
3779                         lbpp[value++] = lb_phy;
3780                 else
3781                         lbpp[value++] = lb_serdes;
3782                 lbpp[value++] = lb_external;
3783                 break;
3784 
3785         case LB_GET_MODE:
3786                 size = sizeof (uint32_t);
3787                 if (iocp->ioc_count != size)
3788                         return (IOC_INVAL);
3789 
3790                 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3791                 *lbmp = igb->loopback_mode;
3792                 break;
3793 
3794         case LB_SET_MODE:
3795                 size = 0;
3796                 if (iocp->ioc_count != sizeof (uint32_t))
3797                         return (IOC_INVAL);
3798 
3799                 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3800                 if (!igb_set_loopback_mode(igb, *lbmp))
3801                         return (IOC_INVAL);
3802                 break;
3803         }
3804 
3805         iocp->ioc_count = size;
3806         iocp->ioc_error = 0;
3807 
3808         if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
3809                 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
3810                 return (IOC_INVAL);
3811         }
3812 
3813         return (IOC_REPLY);
3814 }
3815 
3816 /*
3817  * igb_set_loopback_mode - Setup loopback based on the loopback mode
3818  */
3819 static boolean_t
3820 igb_set_loopback_mode(igb_t *igb, uint32_t mode)
3821 {
3822         struct e1000_hw *hw;
3823         int i;
3824 
3825         if (mode == igb->loopback_mode)
3826                 return (B_TRUE);
3827 
3828         hw = &igb->hw;
3829 
3830         igb->loopback_mode = mode;
3831 
3832         if (mode == IGB_LB_NONE) {
3833                 /* Reset the chip */
3834                 hw->phy.autoneg_wait_to_complete = B_TRUE;
3835                 (void) igb_reset(igb);
3836                 hw->phy.autoneg_wait_to_complete = B_FALSE;
3837                 return (B_TRUE);
3838         }
3839 
3840         mutex_enter(&igb->gen_lock);
3841 
3842         switch (mode) {
3843         default:
3844                 mutex_exit(&igb->gen_lock);
3845                 return (B_FALSE);
3846 
3847         case IGB_LB_EXTERNAL:
3848                 igb_set_external_loopback(igb);
3849                 break;
3850 
3851         case IGB_LB_INTERNAL_PHY:
3852                 igb_set_internal_phy_loopback(igb);
3853                 break;
3854 
3855         case IGB_LB_INTERNAL_SERDES:
3856                 igb_set_internal_serdes_loopback(igb);
3857                 break;
3858         }
3859 
3860         mutex_exit(&igb->gen_lock);
3861 
3862         /*
3863          * When external loopback is set, wait up to 1000ms to get the link up.
3864          * According to test, 1000ms can work and it's an experimental value.
3865          */
3866         if (mode == IGB_LB_EXTERNAL) {
3867                 for (i = 0; i <= 10; i++) {
3868                         mutex_enter(&igb->gen_lock);
3869                         (void) igb_link_check(igb);
3870                         mutex_exit(&igb->gen_lock);
3871 
3872                         if (igb->link_state == LINK_STATE_UP)
3873                                 break;
3874 
3875                         msec_delay(100);
3876                 }
3877 
3878                 if (igb->link_state != LINK_STATE_UP) {
3879                         /*
3880                          * Does not support external loopback.
3881                          * Reset driver to loopback none.
3882                          */
3883                         igb->loopback_mode = IGB_LB_NONE;
3884 
3885                         /* Reset the chip */
3886                         hw->phy.autoneg_wait_to_complete = B_TRUE;
3887                         (void) igb_reset(igb);
3888                         hw->phy.autoneg_wait_to_complete = B_FALSE;
3889 
3890                         IGB_DEBUGLOG_0(igb, "Set external loopback failed, "
3891                             "reset to loopback none.");
3892 
3893                         return (B_FALSE);
3894                 }
3895         }
3896 
3897         return (B_TRUE);
3898 }
3899 
3900 /*
3901  * igb_set_external_loopback - Set the external loopback mode
3902  */
3903 static void
3904 igb_set_external_loopback(igb_t *igb)
3905 {
3906         struct e1000_hw *hw;
3907         uint32_t ctrl_ext;
3908 
3909         hw = &igb->hw;
3910 
3911         /* Set link mode to PHY (00b) in the Extended Control register */
3912         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
3913         ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK;
3914         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
3915 
3916         (void) e1000_write_phy_reg(hw, 0x0, 0x0140);
3917         (void) e1000_write_phy_reg(hw, 0x9, 0x1a00);
3918         (void) e1000_write_phy_reg(hw, 0x12, 0x1610);
3919         (void) e1000_write_phy_reg(hw, 0x1f37, 0x3f1c);
3920 }
3921 
3922 /*
3923  * igb_set_internal_phy_loopback - Set the internal PHY loopback mode
3924  */
3925 static void
3926 igb_set_internal_phy_loopback(igb_t *igb)
3927 {
3928         struct e1000_hw *hw;
3929         uint32_t ctrl_ext;
3930         uint16_t phy_ctrl;
3931         uint16_t phy_pconf;
3932 
3933         hw = &igb->hw;
3934 
3935         /* Set link mode to PHY (00b) in the Extended Control register */
3936         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
3937         ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK;
3938         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
3939 
3940         /*
3941          * Set PHY control register (0x4140):
3942          *    Set full duplex mode
3943          *    Set loopback bit
3944          *    Clear auto-neg enable bit
3945          *    Set PHY speed
3946          */
3947         phy_ctrl = MII_CR_FULL_DUPLEX | MII_CR_SPEED_1000 | MII_CR_LOOPBACK;
3948         (void) e1000_write_phy_reg(hw, PHY_CONTROL, phy_ctrl);
3949 
3950         /* Set the link disable bit in the Port Configuration register */
3951         (void) e1000_read_phy_reg(hw, 0x10, &phy_pconf);
3952         phy_pconf |= (uint16_t)1 << 14;
3953         (void) e1000_write_phy_reg(hw, 0x10, phy_pconf);
3954 }
3955 
3956 /*
3957  * igb_set_internal_serdes_loopback - Set the internal SerDes loopback mode
3958  */
3959 static void
3960 igb_set_internal_serdes_loopback(igb_t *igb)
3961 {
3962         struct e1000_hw *hw;
3963         uint32_t ctrl_ext;
3964         uint32_t ctrl;
3965         uint32_t pcs_lctl;
3966         uint32_t connsw;
3967 
3968         hw = &igb->hw;
3969 
3970         /* Set link mode to SerDes (11b) in the Extended Control register */
3971         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
3972         ctrl_ext |= E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
3973         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
3974 
3975         /* Configure the SerDes to loopback */
3976         E1000_WRITE_REG(hw, E1000_SCTL, 0x410);
3977 
3978         /* Set Device Control register */
3979         ctrl = E1000_READ_REG(hw, E1000_CTRL);
3980         ctrl |= (E1000_CTRL_FD |        /* Force full duplex */
3981             E1000_CTRL_SLU);            /* Force link up */
3982         ctrl &= ~(E1000_CTRL_RFCE | /* Disable receive flow control */
3983             E1000_CTRL_TFCE |           /* Disable transmit flow control */
3984             E1000_CTRL_LRST);           /* Clear link reset */
3985         E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
3986 
3987         /* Set PCS Link Control register */
3988         pcs_lctl = E1000_READ_REG(hw, E1000_PCS_LCTL);
3989         pcs_lctl |= (E1000_PCS_LCTL_FORCE_LINK |
3990             E1000_PCS_LCTL_FSD |
3991             E1000_PCS_LCTL_FDV_FULL |
3992             E1000_PCS_LCTL_FLV_LINK_UP);
3993         pcs_lctl &= ~E1000_PCS_LCTL_AN_ENABLE;
3994         E1000_WRITE_REG(hw, E1000_PCS_LCTL, pcs_lctl);
3995 
3996         /* Set the Copper/Fiber Switch Control - CONNSW register */
3997         connsw = E1000_READ_REG(hw, E1000_CONNSW);
3998         connsw &= ~E1000_CONNSW_ENRGSRC;
3999         E1000_WRITE_REG(hw, E1000_CONNSW, connsw);
4000 }
4001 
4002 #pragma inline(igb_intr_rx_work)
4003 /*
4004  * igb_intr_rx_work - rx processing of ISR
4005  */
4006 static void
4007 igb_intr_rx_work(igb_rx_ring_t *rx_ring)
4008 {
4009         mblk_t *mp;
4010 
4011         mutex_enter(&rx_ring->rx_lock);
4012         mp = igb_rx(rx_ring, IGB_NO_POLL);
4013         mutex_exit(&rx_ring->rx_lock);
4014 
4015         if (mp != NULL)
4016                 mac_rx_ring(rx_ring->igb->mac_hdl, rx_ring->ring_handle, mp,
4017                     rx_ring->ring_gen_num);
4018 }
4019 
4020 #pragma inline(igb_intr_tx_work)
4021 /*
4022  * igb_intr_tx_work - tx processing of ISR
4023  */
4024 static void
4025 igb_intr_tx_work(igb_tx_ring_t *tx_ring)
4026 {
4027         igb_t *igb = tx_ring->igb;
4028 
4029         /* Recycle the tx descriptors */
4030         tx_ring->tx_recycle(tx_ring);
4031 
4032         /* Schedule the re-transmit */
4033         if (tx_ring->reschedule &&
4034             (tx_ring->tbd_free >= igb->tx_resched_thresh)) {
4035                 tx_ring->reschedule = B_FALSE;
4036                 mac_tx_ring_update(tx_ring->igb->mac_hdl, tx_ring->ring_handle);
4037                 IGB_DEBUG_STAT(tx_ring->stat_reschedule);
4038         }
4039 }
4040 
4041 #pragma inline(igb_intr_link_work)
4042 /*
4043  * igb_intr_link_work - link-status-change processing of ISR
4044  */
4045 static void
4046 igb_intr_link_work(igb_t *igb)
4047 {
4048         boolean_t link_changed;
4049 
4050         igb_stop_watchdog_timer(igb);
4051 
4052         mutex_enter(&igb->gen_lock);
4053 
4054         /*
4055          * Because we got a link-status-change interrupt, force
4056          * e1000_check_for_link() to look at phy
4057          */
4058         igb->hw.mac.get_link_status = B_TRUE;
4059 
4060         /* igb_link_check takes care of link status change */
4061         link_changed = igb_link_check(igb);
4062 
4063         /* Get new phy state */
4064         igb_get_phy_state(igb);
4065 
4066         mutex_exit(&igb->gen_lock);
4067 
4068         if (link_changed)
4069                 mac_link_update(igb->mac_hdl, igb->link_state);
4070 
4071         igb_start_watchdog_timer(igb);
4072 }
4073 
4074 /*
4075  * igb_intr_legacy - Interrupt handler for legacy interrupts
4076  */
4077 static uint_t
4078 igb_intr_legacy(void *arg1, void *arg2)
4079 {
4080         igb_t *igb = (igb_t *)arg1;
4081         igb_tx_ring_t *tx_ring;
4082         uint32_t icr;
4083         mblk_t *mp;
4084         boolean_t tx_reschedule;
4085         boolean_t link_changed;
4086         uint_t result;
4087 
4088         _NOTE(ARGUNUSED(arg2));
4089 
4090         mutex_enter(&igb->gen_lock);
4091 
4092         if (igb->igb_state & IGB_SUSPENDED) {
4093                 mutex_exit(&igb->gen_lock);
4094                 return (DDI_INTR_UNCLAIMED);
4095         }
4096 
4097         mp = NULL;
4098         tx_reschedule = B_FALSE;
4099         link_changed = B_FALSE;
4100         icr = E1000_READ_REG(&igb->hw, E1000_ICR);
4101 
4102         if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
4103                 mutex_exit(&igb->gen_lock);
4104                 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
4105                 atomic_or_32(&igb->igb_state, IGB_ERROR);
4106                 return (DDI_INTR_UNCLAIMED);
4107         }
4108 
4109         if (icr & E1000_ICR_INT_ASSERTED) {
4110                 /*
4111                  * E1000_ICR_INT_ASSERTED bit was set:
4112                  * Read(Clear) the ICR, claim this interrupt,
4113                  * look for work to do.
4114                  */
4115                 ASSERT(igb->num_rx_rings == 1);
4116                 ASSERT(igb->num_tx_rings == 1);
4117 
4118                 /* Make sure all interrupt causes cleared */
4119                 (void) E1000_READ_REG(&igb->hw, E1000_EICR);
4120 
4121                 if (icr & E1000_ICR_RXT0) {
4122                         mp = igb_rx(&igb->rx_rings[0], IGB_NO_POLL);
4123                 }
4124 
4125                 if (icr & E1000_ICR_TXDW) {
4126                         tx_ring = &igb->tx_rings[0];
4127 
4128                         /* Recycle the tx descriptors */
4129                         tx_ring->tx_recycle(tx_ring);
4130 
4131                         /* Schedule the re-transmit */
4132                         tx_reschedule = (tx_ring->reschedule &&
4133                             (tx_ring->tbd_free >= igb->tx_resched_thresh));
4134                 }
4135 
4136                 if (icr & E1000_ICR_LSC) {
4137                         /*
4138                          * Because we got a link-status-change interrupt, force
4139                          * e1000_check_for_link() to look at phy
4140                          */
4141                         igb->hw.mac.get_link_status = B_TRUE;
4142 
4143                         /* igb_link_check takes care of link status change */
4144                         link_changed = igb_link_check(igb);
4145 
4146                         /* Get new phy state */
4147                         igb_get_phy_state(igb);
4148                 }
4149 
4150                 if (icr & E1000_ICR_DRSTA) {
4151                         /* 82580 Full Device Reset needed */
4152                         atomic_or_32(&igb->igb_state, IGB_STALL);
4153                 }
4154 
4155                 result = DDI_INTR_CLAIMED;
4156         } else {
4157                 /*
4158                  * E1000_ICR_INT_ASSERTED bit was not set:
4159                  * Don't claim this interrupt.
4160                  */
4161                 result = DDI_INTR_UNCLAIMED;
4162         }
4163 
4164         mutex_exit(&igb->gen_lock);
4165 
4166         /*
4167          * Do the following work outside of the gen_lock
4168          */
4169         if (mp != NULL)
4170                 mac_rx(igb->mac_hdl, NULL, mp);
4171 
4172         if (tx_reschedule)  {
4173                 tx_ring->reschedule = B_FALSE;
4174                 mac_tx_ring_update(igb->mac_hdl, tx_ring->ring_handle);
4175                 IGB_DEBUG_STAT(tx_ring->stat_reschedule);
4176         }
4177 
4178         if (link_changed)
4179                 mac_link_update(igb->mac_hdl, igb->link_state);
4180 
4181         return (result);
4182 }
4183 
4184 /*
4185  * igb_intr_msi - Interrupt handler for MSI
4186  */
4187 static uint_t
4188 igb_intr_msi(void *arg1, void *arg2)
4189 {
4190         igb_t *igb = (igb_t *)arg1;
4191         uint32_t icr;
4192 
4193         _NOTE(ARGUNUSED(arg2));
4194 
4195         icr = E1000_READ_REG(&igb->hw, E1000_ICR);
4196 
4197         if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
4198                 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
4199                 atomic_or_32(&igb->igb_state, IGB_ERROR);
4200                 return (DDI_INTR_CLAIMED);
4201         }
4202 
4203         /* Make sure all interrupt causes cleared */
4204         (void) E1000_READ_REG(&igb->hw, E1000_EICR);
4205 
4206         /*
4207          * For MSI interrupt, we have only one vector,
4208          * so we have only one rx ring and one tx ring enabled.
4209          */
4210         ASSERT(igb->num_rx_rings == 1);
4211         ASSERT(igb->num_tx_rings == 1);
4212 
4213         if (icr & E1000_ICR_RXT0) {
4214                 igb_intr_rx_work(&igb->rx_rings[0]);
4215         }
4216 
4217         if (icr & E1000_ICR_TXDW) {
4218                 igb_intr_tx_work(&igb->tx_rings[0]);
4219         }
4220 
4221         if (icr & E1000_ICR_LSC) {
4222                 igb_intr_link_work(igb);
4223         }
4224 
4225         if (icr & E1000_ICR_DRSTA) {
4226                 /* 82580 Full Device Reset needed */
4227                 atomic_or_32(&igb->igb_state, IGB_STALL);
4228         }
4229 
4230         return (DDI_INTR_CLAIMED);
4231 }
4232 
4233 /*
4234  * igb_intr_rx - Interrupt handler for rx
4235  */
4236 static uint_t
4237 igb_intr_rx(void *arg1, void *arg2)
4238 {
4239         igb_rx_ring_t *rx_ring = (igb_rx_ring_t *)arg1;
4240 
4241         _NOTE(ARGUNUSED(arg2));
4242 
4243         /*
4244          * Only used via MSI-X vector so don't check cause bits
4245          * and only clean the given ring.
4246          */
4247         igb_intr_rx_work(rx_ring);
4248 
4249         return (DDI_INTR_CLAIMED);
4250 }
4251 
4252 /*
4253  * igb_intr_tx - Interrupt handler for tx
4254  */
4255 static uint_t
4256 igb_intr_tx(void *arg1, void *arg2)
4257 {
4258         igb_tx_ring_t *tx_ring = (igb_tx_ring_t *)arg1;
4259 
4260         _NOTE(ARGUNUSED(arg2));
4261 
4262         /*
4263          * Only used via MSI-X vector so don't check cause bits
4264          * and only clean the given ring.
4265          */
4266         igb_intr_tx_work(tx_ring);
4267 
4268         return (DDI_INTR_CLAIMED);
4269 }
4270 
4271 /*
4272  * igb_intr_tx_other - Interrupt handler for both tx and other
4273  *
4274  */
4275 static uint_t
4276 igb_intr_tx_other(void *arg1, void *arg2)
4277 {
4278         igb_t *igb = (igb_t *)arg1;
4279         uint32_t icr;
4280 
4281         _NOTE(ARGUNUSED(arg2));
4282 
4283         icr = E1000_READ_REG(&igb->hw, E1000_ICR);
4284 
4285         if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
4286                 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
4287                 atomic_or_32(&igb->igb_state, IGB_ERROR);
4288                 return (DDI_INTR_CLAIMED);
4289         }
4290 
4291         /*
4292          * Look for tx reclaiming work first. Remember, in the
4293          * case of only interrupt sharing, only one tx ring is
4294          * used
4295          */
4296         igb_intr_tx_work(&igb->tx_rings[0]);
4297 
4298         /*
4299          * Check for "other" causes.
4300          */
4301         if (icr & E1000_ICR_LSC) {
4302                 igb_intr_link_work(igb);
4303         }
4304 
4305         /*
4306          * The DOUTSYNC bit indicates a tx packet dropped because
4307          * DMA engine gets "out of sync". There isn't a real fix
4308          * for this. The Intel recommendation is to count the number
4309          * of occurrences so user can detect when it is happening.
4310          * The issue is non-fatal and there's no recovery action
4311          * available.
4312          */
4313         if (icr & E1000_ICR_DOUTSYNC) {
4314                 IGB_STAT(igb->dout_sync);
4315         }
4316 
4317         if (icr & E1000_ICR_DRSTA) {
4318                 /* 82580 Full Device Reset needed */
4319                 atomic_or_32(&igb->igb_state, IGB_STALL);
4320         }
4321 
4322         return (DDI_INTR_CLAIMED);
4323 }
4324 
4325 /*
4326  * igb_alloc_intrs - Allocate interrupts for the driver
4327  *
4328  * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4329  * if not successful, try Legacy.
4330  * igb->intr_force can be used to force sequence to start with
4331  * any of the 3 types.
4332  * If MSI-X is not used, number of tx/rx rings is forced to 1.
4333  */
4334 static int
4335 igb_alloc_intrs(igb_t *igb)
4336 {
4337         dev_info_t *devinfo;
4338         int intr_types;
4339         int rc;
4340 
4341         devinfo = igb->dip;
4342 
4343         /* Get supported interrupt types */
4344         rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4345 
4346         if (rc != DDI_SUCCESS) {
4347                 igb_log(igb,
4348                     "Get supported interrupt types failed: %d", rc);
4349                 return (IGB_FAILURE);
4350         }
4351         IGB_DEBUGLOG_1(igb, "Supported interrupt types: %x", intr_types);
4352 
4353         igb->intr_type = 0;
4354 
4355         /* Install MSI-X interrupts */
4356         if ((intr_types & DDI_INTR_TYPE_MSIX) &&
4357             (igb->intr_force <= IGB_INTR_MSIX)) {
4358                 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSIX);
4359 
4360                 if (rc == IGB_SUCCESS)
4361                         return (IGB_SUCCESS);
4362 
4363                 igb_log(igb,
4364                     "Allocate MSI-X failed, trying MSI interrupts...");
4365         }
4366 
4367         /* MSI-X not used, force rings to 1 */
4368         igb->num_rx_rings = 1;
4369         igb->num_tx_rings = 1;
4370         igb_log(igb,
4371             "MSI-X not used, force rx and tx queue number to 1");
4372 
4373         /* Install MSI interrupts */
4374         if ((intr_types & DDI_INTR_TYPE_MSI) &&
4375             (igb->intr_force <= IGB_INTR_MSI)) {
4376                 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSI);
4377 
4378                 if (rc == IGB_SUCCESS)
4379                         return (IGB_SUCCESS);
4380 
4381                 igb_log(igb,
4382                     "Allocate MSI failed, trying Legacy interrupts...");
4383         }
4384 
4385         /* Install legacy interrupts */
4386         if (intr_types & DDI_INTR_TYPE_FIXED) {
4387                 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_FIXED);
4388 
4389                 if (rc == IGB_SUCCESS)
4390                         return (IGB_SUCCESS);
4391 
4392                 igb_log(igb,
4393                     "Allocate Legacy interrupts failed");
4394         }
4395 
4396         /* If none of the 3 types succeeded, return failure */
4397         return (IGB_FAILURE);
4398 }
4399 
4400 /*
4401  * igb_alloc_intr_handles - Allocate interrupt handles.
4402  *
4403  * For legacy and MSI, only 1 handle is needed.  For MSI-X,
4404  * if fewer than 2 handles are available, return failure.
4405  * Upon success, this sets the number of Rx rings to a number that
4406  * matches the handles available for Rx interrupts.
4407  */
4408 static int
4409 igb_alloc_intr_handles(igb_t *igb, int intr_type)
4410 {
4411         dev_info_t *devinfo;
4412         int orig, request, count, avail, actual;
4413         int diff, minimum;
4414         int rc;
4415 
4416         devinfo = igb->dip;
4417 
4418         switch (intr_type) {
4419         case DDI_INTR_TYPE_FIXED:
4420                 request = 1;    /* Request 1 legacy interrupt handle */
4421                 minimum = 1;
4422                 IGB_DEBUGLOG_0(igb, "interrupt type: legacy");
4423                 break;
4424 
4425         case DDI_INTR_TYPE_MSI:
4426                 request = 1;    /* Request 1 MSI interrupt handle */
4427                 minimum = 1;
4428                 IGB_DEBUGLOG_0(igb, "interrupt type: MSI");
4429                 break;
4430 
4431         case DDI_INTR_TYPE_MSIX:
4432                 /*
4433                  * Number of vectors for the adapter is
4434                  * # rx rings + # tx rings
4435                  * One of tx vectors is for tx & other
4436                  */
4437                 request = igb->num_rx_rings + igb->num_tx_rings;
4438                 orig = request;
4439                 minimum = 2;
4440                 IGB_DEBUGLOG_0(igb, "interrupt type: MSI-X");
4441                 break;
4442 
4443         default:
4444                 igb_log(igb,
4445                     "invalid call to igb_alloc_intr_handles(): %d\n",
4446                     intr_type);
4447                 return (IGB_FAILURE);
4448         }
4449         IGB_DEBUGLOG_2(igb, "interrupt handles requested: %d  minimum: %d",
4450             request, minimum);
4451 
4452         /*
4453          * Get number of supported interrupts
4454          */
4455         rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
4456         if ((rc != DDI_SUCCESS) || (count < minimum)) {
4457                 igb_log(igb,
4458                     "Get supported interrupt number failed. "
4459                     "Return: %d, count: %d", rc, count);
4460                 return (IGB_FAILURE);
4461         }
4462         IGB_DEBUGLOG_1(igb, "interrupts supported: %d", count);
4463 
4464         /*
4465          * Get number of available interrupts
4466          */
4467         rc = ddi_intr_get_navail(devinfo, intr_type, &avail);
4468         if ((rc != DDI_SUCCESS) || (avail < minimum)) {
4469                 igb_log(igb,
4470                     "Get available interrupt number failed. "
4471                     "Return: %d, available: %d", rc, avail);
4472                 return (IGB_FAILURE);
4473         }
4474         IGB_DEBUGLOG_1(igb, "interrupts available: %d", avail);
4475 
4476         if (avail < request) {
4477                 igb_log(igb, "Request %d handles, %d available",
4478                     request, avail);
4479                 request = avail;
4480         }
4481 
4482         actual = 0;
4483         igb->intr_cnt = 0;
4484 
4485         /*
4486          * Allocate an array of interrupt handles
4487          */
4488         igb->intr_size = request * sizeof (ddi_intr_handle_t);
4489         igb->htable = kmem_alloc(igb->intr_size, KM_SLEEP);
4490 
4491         rc = ddi_intr_alloc(devinfo, igb->htable, intr_type, 0,
4492             request, &actual, DDI_INTR_ALLOC_NORMAL);
4493         if (rc != DDI_SUCCESS) {
4494                 igb_log(igb, "Allocate interrupts failed. "
4495                     "return: %d, request: %d, actual: %d",
4496                     rc, request, actual);
4497                 goto alloc_handle_fail;
4498         }
4499         IGB_DEBUGLOG_1(igb, "interrupts actually allocated: %d", actual);
4500 
4501         igb->intr_cnt = actual;
4502 
4503         if (actual < minimum) {
4504                 igb_log(igb, "Insufficient interrupt handles allocated: %d",
4505                     actual);
4506                 goto alloc_handle_fail;
4507         }
4508 
4509         /*
4510          * For MSI-X, actual might force us to reduce number of tx & rx rings
4511          */
4512         if ((intr_type == DDI_INTR_TYPE_MSIX) && (orig > actual)) {
4513                 diff = orig - actual;
4514                 if (diff < igb->num_tx_rings) {
4515                         igb_log(igb,
4516                             "MSI-X vectors force Tx queue number to %d",
4517                             igb->num_tx_rings - diff);
4518                         igb->num_tx_rings -= diff;
4519                 } else {
4520                         igb_log(igb,
4521                             "MSI-X vectors force Tx queue number to 1");
4522                         igb->num_tx_rings = 1;
4523 
4524                         igb_log(igb,
4525                             "MSI-X vectors force Rx queue number to %d",
4526                             actual - 1);
4527                         igb->num_rx_rings = actual - 1;
4528                 }
4529         }
4530 
4531         /*
4532          * Get priority for first vector, assume remaining are all the same
4533          */
4534         rc = ddi_intr_get_pri(igb->htable[0], &igb->intr_pri);
4535         if (rc != DDI_SUCCESS) {
4536                 igb_log(igb,
4537                     "Get interrupt priority failed: %d", rc);
4538                 goto alloc_handle_fail;
4539         }
4540 
4541         rc = ddi_intr_get_cap(igb->htable[0], &igb->intr_cap);
4542         if (rc != DDI_SUCCESS) {
4543                 igb_log(igb,
4544                     "Get interrupt cap failed: %d", rc);
4545                 goto alloc_handle_fail;
4546         }
4547 
4548         igb->intr_type = intr_type;
4549 
4550         return (IGB_SUCCESS);
4551 
4552 alloc_handle_fail:
4553         igb_rem_intrs(igb);
4554 
4555         return (IGB_FAILURE);
4556 }
4557 
4558 /*
4559  * igb_add_intr_handlers - Add interrupt handlers based on the interrupt type
4560  *
4561  * Before adding the interrupt handlers, the interrupt vectors have
4562  * been allocated, and the rx/tx rings have also been allocated.
4563  */
4564 static int
4565 igb_add_intr_handlers(igb_t *igb)
4566 {
4567         igb_rx_ring_t *rx_ring;
4568         igb_tx_ring_t *tx_ring;
4569         int vector;
4570         int rc;
4571         int i;
4572 
4573         vector = 0;
4574 
4575         switch (igb->intr_type) {
4576         case DDI_INTR_TYPE_MSIX:
4577                 /* Add interrupt handler for tx + other */
4578                 tx_ring = &igb->tx_rings[0];
4579                 rc = ddi_intr_add_handler(igb->htable[vector],
4580                     (ddi_intr_handler_t *)igb_intr_tx_other,
4581                     (void *)igb, NULL);
4582 
4583                 if (rc != DDI_SUCCESS) {
4584                         igb_log(igb,
4585                             "Add tx/other interrupt handler failed: %d", rc);
4586                         return (IGB_FAILURE);
4587                 }
4588                 tx_ring->intr_vector = vector;
4589                 vector++;
4590 
4591                 /* Add interrupt handler for each rx ring */
4592                 for (i = 0; i < igb->num_rx_rings; i++) {
4593                         rx_ring = &igb->rx_rings[i];
4594 
4595                         rc = ddi_intr_add_handler(igb->htable[vector],
4596                             (ddi_intr_handler_t *)igb_intr_rx,
4597                             (void *)rx_ring, NULL);
4598 
4599                         if (rc != DDI_SUCCESS) {
4600                                 igb_log(igb,
4601                                     "Add rx interrupt handler failed. "
4602                                     "return: %d, rx ring: %d", rc, i);
4603                                 for (vector--; vector >= 0; vector--) {
4604                                         (void) ddi_intr_remove_handler(
4605                                             igb->htable[vector]);
4606                                 }
4607                                 return (IGB_FAILURE);
4608                         }
4609 
4610                         rx_ring->intr_vector = vector;
4611 
4612                         vector++;
4613                 }
4614 
4615                 /* Add interrupt handler for each tx ring from 2nd ring */
4616                 for (i = 1; i < igb->num_tx_rings; i++) {
4617                         tx_ring = &igb->tx_rings[i];
4618 
4619                         rc = ddi_intr_add_handler(igb->htable[vector],
4620                             (ddi_intr_handler_t *)igb_intr_tx,
4621                             (void *)tx_ring, NULL);
4622 
4623                         if (rc != DDI_SUCCESS) {
4624                                 igb_log(igb,
4625                                     "Add tx interrupt handler failed. "
4626                                     "return: %d, tx ring: %d", rc, i);
4627                                 for (vector--; vector >= 0; vector--) {
4628                                         (void) ddi_intr_remove_handler(
4629                                             igb->htable[vector]);
4630                                 }
4631                                 return (IGB_FAILURE);
4632                         }
4633 
4634                         tx_ring->intr_vector = vector;
4635 
4636                         vector++;
4637                 }
4638 
4639                 break;
4640 
4641         case DDI_INTR_TYPE_MSI:
4642                 /* Add interrupt handlers for the only vector */
4643                 rc = ddi_intr_add_handler(igb->htable[vector],
4644                     (ddi_intr_handler_t *)igb_intr_msi,
4645                     (void *)igb, NULL);
4646 
4647                 if (rc != DDI_SUCCESS) {
4648                         igb_log(igb,
4649                             "Add MSI interrupt handler failed: %d", rc);
4650                         return (IGB_FAILURE);
4651                 }
4652 
4653                 rx_ring = &igb->rx_rings[0];
4654                 rx_ring->intr_vector = vector;
4655 
4656                 vector++;
4657                 break;
4658 
4659         case DDI_INTR_TYPE_FIXED:
4660                 /* Add interrupt handlers for the only vector */
4661                 rc = ddi_intr_add_handler(igb->htable[vector],
4662                     (ddi_intr_handler_t *)igb_intr_legacy,
4663                     (void *)igb, NULL);
4664 
4665                 if (rc != DDI_SUCCESS) {
4666                         igb_log(igb,
4667                             "Add legacy interrupt handler failed: %d", rc);
4668                         return (IGB_FAILURE);
4669                 }
4670 
4671                 rx_ring = &igb->rx_rings[0];
4672                 rx_ring->intr_vector = vector;
4673 
4674                 vector++;
4675                 break;
4676 
4677         default:
4678                 return (IGB_FAILURE);
4679         }
4680 
4681         ASSERT(vector == igb->intr_cnt);
4682 
4683         return (IGB_SUCCESS);
4684 }
4685 
4686 /*
4687  * igb_setup_msix_82575 - setup 82575 adapter to use MSI-X interrupts
4688  *
4689  * For each vector enabled on the adapter, Set the MSIXBM register accordingly
4690  */
4691 static void
4692 igb_setup_msix_82575(igb_t *igb)
4693 {
4694         uint32_t eims = 0;
4695         int i, vector;
4696         struct e1000_hw *hw = &igb->hw;
4697 
4698         /*
4699          * Set vector for tx ring 0 and other causes.
4700          * NOTE assumption that it is vector 0.
4701          */
4702         vector = 0;
4703 
4704         igb->eims_mask = E1000_EICR_TX_QUEUE0 | E1000_EICR_OTHER;
4705         E1000_WRITE_REG(hw, E1000_MSIXBM(vector), igb->eims_mask);
4706         vector++;
4707 
4708         for (i = 0; i < igb->num_rx_rings; i++) {
4709                 /*
4710                  * Set vector for each rx ring
4711                  */
4712                 eims = (E1000_EICR_RX_QUEUE0 << i);
4713                 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims);
4714 
4715                 /*
4716                  * Accumulate bits to enable in
4717                  * igb_enable_adapter_interrupts_82575()
4718                  */
4719                 igb->eims_mask |= eims;
4720 
4721                 vector++;
4722         }
4723 
4724         for (i = 1; i < igb->num_tx_rings; i++) {
4725                 /*
4726                  * Set vector for each tx ring from 2nd tx ring
4727                  */
4728                 eims = (E1000_EICR_TX_QUEUE0 << i);
4729                 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims);
4730 
4731                 /*
4732                  * Accumulate bits to enable in
4733                  * igb_enable_adapter_interrupts_82575()
4734                  */
4735                 igb->eims_mask |= eims;
4736 
4737                 vector++;
4738         }
4739 
4740         ASSERT(vector == igb->intr_cnt);
4741 
4742         /*
4743          * Disable IAM for ICR interrupt bits
4744          */
4745         E1000_WRITE_REG(hw, E1000_IAM, 0);
4746         E1000_WRITE_FLUSH(hw);
4747 }
4748 
4749 /*
4750  * igb_setup_msix_82576 - setup 82576 adapter to use MSI-X interrupts
4751  *
4752  * 82576 uses a table based method for assigning vectors.  Each queue has a
4753  * single entry in the table to which we write a vector number along with a
4754  * "valid" bit.  The entry is a single byte in a 4-byte register.  Vectors
4755  * take a different position in the 4-byte register depending on whether
4756  * they are numbered above or below 8.
4757  */
4758 static void
4759 igb_setup_msix_82576(igb_t *igb)
4760 {
4761         struct e1000_hw *hw = &igb->hw;
4762         uint32_t ivar, index, vector;
4763         int i;
4764 
4765         /* must enable msi-x capability before IVAR settings */
4766         E1000_WRITE_REG(hw, E1000_GPIE,
4767             (E1000_GPIE_MSIX_MODE | E1000_GPIE_PBA | E1000_GPIE_NSICR));
4768 
4769         /*
4770          * Set vector for tx ring 0 and other causes.
4771          * NOTE assumption that it is vector 0.
4772          * This is also interdependent with installation of interrupt service
4773          * routines in igb_add_intr_handlers().
4774          */
4775 
4776         /* assign "other" causes to vector 0 */
4777         vector = 0;
4778         ivar = ((vector | E1000_IVAR_VALID) << 8);
4779         E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
4780 
4781         /* assign tx ring 0 to vector 0 */
4782         ivar = ((vector | E1000_IVAR_VALID) << 8);
4783         E1000_WRITE_REG(hw, E1000_IVAR0, ivar);
4784 
4785         /* prepare to enable tx & other interrupt causes */
4786         igb->eims_mask = (1 << vector);
4787 
4788         vector ++;
4789         for (i = 0; i < igb->num_rx_rings; i++) {
4790                 /*
4791                  * Set vector for each rx ring
4792                  */
4793                 index = (i & 0x7);
4794                 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4795 
4796                 if (i < 8) {
4797                         /* vector goes into low byte of register */
4798                         ivar = ivar & 0xFFFFFF00;
4799                         ivar |= (vector | E1000_IVAR_VALID);
4800                 } else {
4801                         /* vector goes into third byte of register */
4802                         ivar = ivar & 0xFF00FFFF;
4803                         ivar |= ((vector | E1000_IVAR_VALID) << 16);
4804                 }
4805                 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4806 
4807                 /* Accumulate interrupt-cause bits to enable */
4808                 igb->eims_mask |= (1 << vector);
4809 
4810                 vector ++;
4811         }
4812 
4813         for (i = 1; i < igb->num_tx_rings; i++) {
4814                 /*
4815                  * Set vector for each tx ring from 2nd tx ring.
4816                  * Note assumption that tx vectors numericall follow rx vectors.
4817                  */
4818                 index = (i & 0x7);
4819                 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4820 
4821                 if (i < 8) {
4822                         /* vector goes into second byte of register */
4823                         ivar = ivar & 0xFFFF00FF;
4824                         ivar |= ((vector | E1000_IVAR_VALID) << 8);
4825                 } else {
4826                         /* vector goes into fourth byte of register */
4827                         ivar = ivar & 0x00FFFFFF;
4828                         ivar |= (vector | E1000_IVAR_VALID) << 24;
4829                 }
4830                 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4831 
4832                 /* Accumulate interrupt-cause bits to enable */
4833                 igb->eims_mask |= (1 << vector);
4834 
4835                 vector ++;
4836         }
4837 
4838         ASSERT(vector == igb->intr_cnt);
4839 }
4840 
4841 /*
4842  * igb_setup_msix_82580 - setup 82580 adapter to use MSI-X interrupts
4843  *
4844  * 82580 uses same table approach at 82576 but has fewer entries.  Each
4845  * queue has a single entry in the table to which we write a vector number
4846  * along with a "valid" bit.  Vectors take a different position in the
4847  * register depending on * whether * they are numbered above or below 4.
4848  */
4849 static void
4850 igb_setup_msix_82580(igb_t *igb)
4851 {
4852         struct e1000_hw *hw = &igb->hw;
4853         uint32_t ivar, index, vector;
4854         int i;
4855 
4856         /* must enable msi-x capability before IVAR settings */
4857         E1000_WRITE_REG(hw, E1000_GPIE, (E1000_GPIE_MSIX_MODE |
4858             E1000_GPIE_PBA | E1000_GPIE_NSICR | E1000_GPIE_EIAME));
4859         /*
4860          * Set vector for tx ring 0 and other causes.
4861          * NOTE assumption that it is vector 0.
4862          * This is also interdependent with installation of interrupt service
4863          * routines in igb_add_intr_handlers().
4864          */
4865 
4866         /* assign "other" causes to vector 0 */
4867         vector = 0;
4868         ivar = ((vector | E1000_IVAR_VALID) << 8);
4869         E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
4870 
4871         /* assign tx ring 0 to vector 0 */
4872         ivar = ((vector | E1000_IVAR_VALID) << 8);
4873         E1000_WRITE_REG(hw, E1000_IVAR0, ivar);
4874 
4875         /* prepare to enable tx & other interrupt causes */
4876         igb->eims_mask = (1 << vector);
4877 
4878         vector ++;
4879 
4880         for (i = 0; i < igb->num_rx_rings; i++) {
4881                 /*
4882                  * Set vector for each rx ring
4883                  */
4884                 index = (i >> 1);
4885                 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4886 
4887                 if (i & 1) {
4888                         /* vector goes into third byte of register */
4889                         ivar = ivar & 0xFF00FFFF;
4890                         ivar |= ((vector | E1000_IVAR_VALID) << 16);
4891                 } else {
4892                         /* vector goes into low byte of register */
4893                         ivar = ivar & 0xFFFFFF00;
4894                         ivar |= (vector | E1000_IVAR_VALID);
4895                 }
4896                 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4897 
4898                 /* Accumulate interrupt-cause bits to enable */
4899                 igb->eims_mask |= (1 << vector);
4900 
4901                 vector ++;
4902         }
4903 
4904         for (i = 1; i < igb->num_tx_rings; i++) {
4905                 /*
4906                  * Set vector for each tx ring from 2nd tx ring.
4907                  * Note assumption that tx vectors numericall follow rx vectors.
4908                  */
4909                 index = (i >> 1);
4910                 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4911 
4912                 if (i & 1) {
4913                         /* vector goes into high byte of register */
4914                         ivar = ivar & 0x00FFFFFF;
4915                         ivar |= ((vector | E1000_IVAR_VALID) << 24);
4916                 } else {
4917                         /* vector goes into second byte of register */
4918                         ivar = ivar & 0xFFFF00FF;
4919                         ivar |= (vector | E1000_IVAR_VALID) << 8;
4920                 }
4921                 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4922 
4923                 /* Accumulate interrupt-cause bits to enable */
4924                 igb->eims_mask |= (1 << vector);
4925 
4926                 vector ++;
4927         }
4928         ASSERT(vector == igb->intr_cnt);
4929 }
4930 
4931 /*
4932  * igb_rem_intr_handlers - remove the interrupt handlers
4933  */
4934 static void
4935 igb_rem_intr_handlers(igb_t *igb)
4936 {
4937         int i;
4938         int rc;
4939 
4940         for (i = 0; i < igb->intr_cnt; i++) {
4941                 rc = ddi_intr_remove_handler(igb->htable[i]);
4942                 if (rc != DDI_SUCCESS) {
4943                         IGB_DEBUGLOG_1(igb,
4944                             "Remove intr handler failed: %d", rc);
4945                 }
4946         }
4947 }
4948 
4949 /*
4950  * igb_rem_intrs - remove the allocated interrupts
4951  */
4952 static void
4953 igb_rem_intrs(igb_t *igb)
4954 {
4955         int i;
4956         int rc;
4957 
4958         for (i = 0; i < igb->intr_cnt; i++) {
4959                 rc = ddi_intr_free(igb->htable[i]);
4960                 if (rc != DDI_SUCCESS) {
4961                         IGB_DEBUGLOG_1(igb,
4962                             "Free intr failed: %d", rc);
4963                 }
4964         }
4965 
4966         kmem_free(igb->htable, igb->intr_size);
4967         igb->htable = NULL;
4968 }
4969 
4970 /*
4971  * igb_enable_intrs - enable all the ddi interrupts
4972  */
4973 static int
4974 igb_enable_intrs(igb_t *igb)
4975 {
4976         int i;
4977         int rc;
4978 
4979         /* Enable interrupts */
4980         if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) {
4981                 /* Call ddi_intr_block_enable() for MSI */
4982                 rc = ddi_intr_block_enable(igb->htable, igb->intr_cnt);
4983                 if (rc != DDI_SUCCESS) {
4984                         igb_log(igb,
4985                             "Enable block intr failed: %d", rc);
4986                         return (IGB_FAILURE);
4987                 }
4988         } else {
4989                 /* Call ddi_intr_enable() for Legacy/MSI non block enable */
4990                 for (i = 0; i < igb->intr_cnt; i++) {
4991                         rc = ddi_intr_enable(igb->htable[i]);
4992                         if (rc != DDI_SUCCESS) {
4993                                 igb_log(igb,
4994                                     "Enable intr failed: %d", rc);
4995                                 return (IGB_FAILURE);
4996                         }
4997                 }
4998         }
4999 
5000         return (IGB_SUCCESS);
5001 }
5002 
5003 /*
5004  * igb_disable_intrs - disable all the ddi interrupts
5005  */
5006 static int
5007 igb_disable_intrs(igb_t *igb)
5008 {
5009         int i;
5010         int rc;
5011 
5012         /* Disable all interrupts */
5013         if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) {
5014                 rc = ddi_intr_block_disable(igb->htable, igb->intr_cnt);
5015                 if (rc != DDI_SUCCESS) {
5016                         igb_log(igb,
5017                             "Disable block intr failed: %d", rc);
5018                         return (IGB_FAILURE);
5019                 }
5020         } else {
5021                 for (i = 0; i < igb->intr_cnt; i++) {
5022                         rc = ddi_intr_disable(igb->htable[i]);
5023                         if (rc != DDI_SUCCESS) {
5024                                 igb_log(igb,
5025                                     "Disable intr failed: %d", rc);
5026                                 return (IGB_FAILURE);
5027                         }
5028                 }
5029         }
5030 
5031         return (IGB_SUCCESS);
5032 }
5033 
5034 /*
5035  * igb_get_phy_state - Get and save the parameters read from PHY registers
5036  */
5037 static void
5038 igb_get_phy_state(igb_t *igb)
5039 {
5040         struct e1000_hw *hw = &igb->hw;
5041         uint16_t phy_ctrl;
5042         uint16_t phy_status;
5043         uint16_t phy_an_adv;
5044         uint16_t phy_an_exp;
5045         uint16_t phy_ext_status;
5046         uint16_t phy_1000t_ctrl;
5047         uint16_t phy_1000t_status;
5048         uint16_t phy_lp_able;
5049 
5050         ASSERT(mutex_owned(&igb->gen_lock));
5051 
5052         if (hw->phy.media_type == e1000_media_type_copper) {
5053                 (void) e1000_read_phy_reg(hw, PHY_CONTROL, &phy_ctrl);
5054                 (void) e1000_read_phy_reg(hw, PHY_STATUS, &phy_status);
5055                 (void) e1000_read_phy_reg(hw, PHY_AUTONEG_ADV, &phy_an_adv);
5056                 (void) e1000_read_phy_reg(hw, PHY_AUTONEG_EXP, &phy_an_exp);
5057                 (void) e1000_read_phy_reg(hw, PHY_EXT_STATUS, &phy_ext_status);
5058                 (void) e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_1000t_ctrl);
5059                 (void) e1000_read_phy_reg(hw,
5060                     PHY_1000T_STATUS, &phy_1000t_status);
5061                 (void) e1000_read_phy_reg(hw, PHY_LP_ABILITY, &phy_lp_able);
5062 
5063                 igb->param_autoneg_cap =
5064                     (phy_status & MII_SR_AUTONEG_CAPS) ? 1 : 0;
5065                 igb->param_pause_cap =
5066                     (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0;
5067                 igb->param_asym_pause_cap =
5068                     (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0;
5069                 igb->param_1000fdx_cap =
5070                     ((phy_ext_status & IEEE_ESR_1000T_FD_CAPS) ||
5071                     (phy_ext_status & IEEE_ESR_1000X_FD_CAPS)) ? 1 : 0;
5072                 igb->param_1000hdx_cap =
5073                     ((phy_ext_status & IEEE_ESR_1000T_HD_CAPS) ||
5074                     (phy_ext_status & IEEE_ESR_1000X_HD_CAPS)) ? 1 : 0;
5075                 igb->param_100t4_cap =
5076                     (phy_status & MII_SR_100T4_CAPS) ? 1 : 0;
5077                 igb->param_100fdx_cap = ((phy_status & MII_SR_100X_FD_CAPS) ||
5078                     (phy_status & MII_SR_100T2_FD_CAPS)) ? 1 : 0;
5079                 igb->param_100hdx_cap = ((phy_status & MII_SR_100X_HD_CAPS) ||
5080                     (phy_status & MII_SR_100T2_HD_CAPS)) ? 1 : 0;
5081                 igb->param_10fdx_cap =
5082                     (phy_status & MII_SR_10T_FD_CAPS) ? 1 : 0;
5083                 igb->param_10hdx_cap =
5084                     (phy_status & MII_SR_10T_HD_CAPS) ? 1 : 0;
5085                 igb->param_rem_fault =
5086                     (phy_status & MII_SR_REMOTE_FAULT) ? 1 : 0;
5087 
5088                 igb->param_adv_autoneg_cap = hw->mac.autoneg;
5089                 igb->param_adv_pause_cap =
5090                     (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0;
5091                 igb->param_adv_asym_pause_cap =
5092                     (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0;
5093                 igb->param_adv_1000hdx_cap =
5094                     (phy_1000t_ctrl & CR_1000T_HD_CAPS) ? 1 : 0;
5095                 igb->param_adv_100t4_cap =
5096                     (phy_an_adv & NWAY_AR_100T4_CAPS) ? 1 : 0;
5097                 igb->param_adv_rem_fault =
5098                     (phy_an_adv & NWAY_AR_REMOTE_FAULT) ? 1 : 0;
5099                 if (igb->param_adv_autoneg_cap == 1) {
5100                         igb->param_adv_1000fdx_cap =
5101                             (phy_1000t_ctrl & CR_1000T_FD_CAPS) ? 1 : 0;
5102                         igb->param_adv_100fdx_cap =
5103                             (phy_an_adv & NWAY_AR_100TX_FD_CAPS) ? 1 : 0;
5104                         igb->param_adv_100hdx_cap =
5105                             (phy_an_adv & NWAY_AR_100TX_HD_CAPS) ? 1 : 0;
5106                         igb->param_adv_10fdx_cap =
5107                             (phy_an_adv & NWAY_AR_10T_FD_CAPS) ? 1 : 0;
5108                         igb->param_adv_10hdx_cap =
5109                             (phy_an_adv & NWAY_AR_10T_HD_CAPS) ? 1 : 0;
5110                 }
5111 
5112                 igb->param_lp_autoneg_cap =
5113                     (phy_an_exp & NWAY_ER_LP_NWAY_CAPS) ? 1 : 0;
5114                 igb->param_lp_pause_cap =
5115                     (phy_lp_able & NWAY_LPAR_PAUSE) ? 1 : 0;
5116                 igb->param_lp_asym_pause_cap =
5117                     (phy_lp_able & NWAY_LPAR_ASM_DIR) ? 1 : 0;
5118                 igb->param_lp_1000fdx_cap =
5119                     (phy_1000t_status & SR_1000T_LP_FD_CAPS) ? 1 : 0;
5120                 igb->param_lp_1000hdx_cap =
5121                     (phy_1000t_status & SR_1000T_LP_HD_CAPS) ? 1 : 0;
5122                 igb->param_lp_100t4_cap =
5123                     (phy_lp_able & NWAY_LPAR_100T4_CAPS) ? 1 : 0;
5124                 igb->param_lp_100fdx_cap =
5125                     (phy_lp_able & NWAY_LPAR_100TX_FD_CAPS) ? 1 : 0;
5126                 igb->param_lp_100hdx_cap =
5127                     (phy_lp_able & NWAY_LPAR_100TX_HD_CAPS) ? 1 : 0;
5128                 igb->param_lp_10fdx_cap =
5129                     (phy_lp_able & NWAY_LPAR_10T_FD_CAPS) ? 1 : 0;
5130                 igb->param_lp_10hdx_cap =
5131                     (phy_lp_able & NWAY_LPAR_10T_HD_CAPS) ? 1 : 0;
5132                 igb->param_lp_rem_fault =
5133                     (phy_lp_able & NWAY_LPAR_REMOTE_FAULT) ? 1 : 0;
5134         } else {
5135                 /*
5136                  * 1Gig Fiber adapter only offers 1Gig Full Duplex.
5137                  */
5138                 igb->param_autoneg_cap = 0;
5139                 igb->param_pause_cap = 1;
5140                 igb->param_asym_pause_cap = 1;
5141                 igb->param_1000fdx_cap = 1;
5142                 igb->param_1000hdx_cap = 0;
5143                 igb->param_100t4_cap = 0;
5144                 igb->param_100fdx_cap = 0;
5145                 igb->param_100hdx_cap = 0;
5146                 igb->param_10fdx_cap = 0;
5147                 igb->param_10hdx_cap = 0;
5148 
5149                 igb->param_adv_autoneg_cap = 0;
5150                 igb->param_adv_pause_cap = 1;
5151                 igb->param_adv_asym_pause_cap = 1;
5152                 igb->param_adv_1000fdx_cap = 1;
5153                 igb->param_adv_1000hdx_cap = 0;
5154                 igb->param_adv_100t4_cap = 0;
5155                 igb->param_adv_100fdx_cap = 0;
5156                 igb->param_adv_100hdx_cap = 0;
5157                 igb->param_adv_10fdx_cap = 0;
5158                 igb->param_adv_10hdx_cap = 0;
5159 
5160                 igb->param_lp_autoneg_cap = 0;
5161                 igb->param_lp_pause_cap = 0;
5162                 igb->param_lp_asym_pause_cap = 0;
5163                 igb->param_lp_1000fdx_cap = 0;
5164                 igb->param_lp_1000hdx_cap = 0;
5165                 igb->param_lp_100t4_cap = 0;
5166                 igb->param_lp_100fdx_cap = 0;
5167                 igb->param_lp_100hdx_cap = 0;
5168                 igb->param_lp_10fdx_cap = 0;
5169                 igb->param_lp_10hdx_cap = 0;
5170                 igb->param_lp_rem_fault = 0;
5171         }
5172 }
5173 
5174 /*
5175  * synchronize the adv* and en* parameters.
5176  *
5177  * See comments in <sys/dld.h> for details of the *_en_*
5178  * parameters. The usage of ndd for setting adv parameters will
5179  * synchronize all the en parameters with the e1000g parameters,
5180  * implicitly disabling any settings made via dladm.
5181  */
5182 static void
5183 igb_param_sync(igb_t *igb)
5184 {
5185         igb->param_en_1000fdx_cap = igb->param_adv_1000fdx_cap;
5186         igb->param_en_1000hdx_cap = igb->param_adv_1000hdx_cap;
5187         igb->param_en_100t4_cap = igb->param_adv_100t4_cap;
5188         igb->param_en_100fdx_cap = igb->param_adv_100fdx_cap;
5189         igb->param_en_100hdx_cap = igb->param_adv_100hdx_cap;
5190         igb->param_en_10fdx_cap = igb->param_adv_10fdx_cap;
5191         igb->param_en_10hdx_cap = igb->param_adv_10hdx_cap;
5192 }
5193 
5194 /*
5195  * igb_get_driver_control
5196  */
5197 static void
5198 igb_get_driver_control(struct e1000_hw *hw)
5199 {
5200         uint32_t ctrl_ext;
5201 
5202         /* Notify firmware that driver is in control of device */
5203         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
5204         ctrl_ext |= E1000_CTRL_EXT_DRV_LOAD;
5205         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
5206 }
5207 
5208 /*
5209  * igb_release_driver_control
5210  */
5211 static void
5212 igb_release_driver_control(struct e1000_hw *hw)
5213 {
5214         uint32_t ctrl_ext;
5215 
5216         /* Notify firmware that driver is no longer in control of device */
5217         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
5218         ctrl_ext &= ~E1000_CTRL_EXT_DRV_LOAD;
5219         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
5220 }
5221 
5222 /*
5223  * igb_atomic_reserve - Atomic decrease operation
5224  */
5225 int
5226 igb_atomic_reserve(uint32_t *count_p, uint32_t n)
5227 {
5228         uint32_t oldval;
5229         uint32_t newval;
5230 
5231         /* ATOMICALLY */
5232         do {
5233                 oldval = *count_p;
5234                 if (oldval < n)
5235                         return (-1);
5236                 newval = oldval - n;
5237         } while (atomic_cas_32(count_p, oldval, newval) != oldval);
5238 
5239         return (newval);
5240 }
5241 
5242 /*
5243  * FMA support
5244  */
5245 
5246 int
5247 igb_check_acc_handle(ddi_acc_handle_t handle)
5248 {
5249         ddi_fm_error_t de;
5250 
5251         ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5252         ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5253         return (de.fme_status);
5254 }
5255 
5256 int
5257 igb_check_dma_handle(ddi_dma_handle_t handle)
5258 {
5259         ddi_fm_error_t de;
5260 
5261         ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5262         return (de.fme_status);
5263 }
5264 
5265 /*
5266  * The IO fault service error handling callback function
5267  */
5268 /*ARGSUSED*/
5269 static int
5270 igb_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5271 {
5272         /*
5273          * as the driver can always deal with an error in any dma or
5274          * access handle, we can just return the fme_status value.
5275          */
5276         pci_ereport_post(dip, err, NULL);
5277         return (err->fme_status);
5278 }
5279 
5280 static void
5281 igb_fm_init(igb_t *igb)
5282 {
5283         ddi_iblock_cookie_t iblk;
5284         int fma_dma_flag;
5285 
5286         /* Only register with IO Fault Services if we have some capability */
5287         if (igb->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
5288                 igb_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
5289         } else {
5290                 igb_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
5291         }
5292 
5293         if (igb->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
5294                 fma_dma_flag = 1;
5295         } else {
5296                 fma_dma_flag = 0;
5297         }
5298 
5299         (void) igb_set_fma_flags(fma_dma_flag);
5300 
5301         if (igb->fm_capabilities) {
5302 
5303                 /* Register capabilities with IO Fault Services */
5304                 ddi_fm_init(igb->dip, &igb->fm_capabilities, &iblk);
5305 
5306                 /*
5307                  * Initialize pci ereport capabilities if ereport capable
5308                  */
5309                 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) ||
5310                     DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5311                         pci_ereport_setup(igb->dip);
5312 
5313                 /*
5314                  * Register error callback if error callback capable
5315                  */
5316                 if (DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5317                         ddi_fm_handler_register(igb->dip,
5318                             igb_fm_error_cb, (void*) igb);
5319         }
5320 }
5321 
5322 static void
5323 igb_fm_fini(igb_t *igb)
5324 {
5325         /* Only unregister FMA capabilities if we registered some */
5326         if (igb->fm_capabilities) {
5327 
5328                 /*
5329                  * Release any resources allocated by pci_ereport_setup()
5330                  */
5331                 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) ||
5332                     DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5333                         pci_ereport_teardown(igb->dip);
5334 
5335                 /*
5336                  * Un-register error callback if error callback capable
5337                  */
5338                 if (DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5339                         ddi_fm_handler_unregister(igb->dip);
5340 
5341                 /* Unregister from IO Fault Services */
5342                 ddi_fm_fini(igb->dip);
5343         }
5344 }
5345 
5346 void
5347 igb_fm_ereport(igb_t *igb, char *detail)
5348 {
5349         uint64_t ena;
5350         char buf[FM_MAX_CLASS];
5351 
5352         (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
5353         ena = fm_ena_generate(0, FM_ENA_FMT1);
5354         if (DDI_FM_EREPORT_CAP(igb->fm_capabilities)) {
5355                 ddi_fm_ereport_post(igb->dip, buf, ena, DDI_NOSLEEP,
5356                     FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
5357         }
5358 }