drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c

   1 /*
   2  * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/module.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/init.h>
  35 #include <linux/pci.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/netdevice.h>
  38 #include <linux/etherdevice.h>
  39 #include <linux/if_vlan.h>
  40 #include <linux/mdio.h>
  41 #include <linux/sockios.h>
  42 #include <linux/workqueue.h>
  43 #include <linux/proc_fs.h>
  44 #include <linux/rtnetlink.h>
  45 #include <linux/firmware.h>
  46 #include <linux/log2.h>
  47 #include <linux/stringify.h>
  48 #include <linux/sched.h>
  49 #include <linux/slab.h>
  50 #include <asm/uaccess.h>
  51
  52 #include "common.h"
  53 #include "cxgb3_ioctl.h"
  54 #include "regs.h"
  55 #include "cxgb3_offload.h"
  56 #include "version.h"
  57
  58 #include "cxgb3_ctl_defs.h"
  59 #include "t3_cpl.h"
  60 #include "firmware_exports.h"
  61
  62 enum {
  63         MAX_TXQ_ENTRIES = 16384,
  64         MAX_CTRL_TXQ_ENTRIES = 1024,
  65         MAX_RSPQ_ENTRIES = 16384,
  66         MAX_RX_BUFFERS = 16384,
  67         MAX_RX_JUMBO_BUFFERS = 16384,
  68         MIN_TXQ_ENTRIES = 4,
  69         MIN_CTRL_TXQ_ENTRIES = 4,
  70         MIN_RSPQ_ENTRIES = 32,
  71         MIN_FL_ENTRIES = 32
  72 };
  73
  74 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  75
  76 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  77                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
  78                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
  79
  80 #define EEPROM_MAGIC 0x38E2F10C
  81
  82 #define CH_DEVICE(devid, idx) \
  83         { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
  84
  85 static DEFINE_PCI_DEVICE_TABLE(cxgb3_pci_tbl) = {
  86         CH_DEVICE(0x20, 0),     /* PE9000 */
  87         CH_DEVICE(0x21, 1),     /* T302E */
  88         CH_DEVICE(0x22, 2),     /* T310E */
  89         CH_DEVICE(0x23, 3),     /* T320X */
  90         CH_DEVICE(0x24, 1),     /* T302X */
  91         CH_DEVICE(0x25, 3),     /* T320E */
  92         CH_DEVICE(0x26, 2),     /* T310X */
  93         CH_DEVICE(0x30, 2),     /* T3B10 */
  94         CH_DEVICE(0x31, 3),     /* T3B20 */
  95         CH_DEVICE(0x32, 1),     /* T3B02 */
  96         CH_DEVICE(0x35, 6),     /* T3C20-derived T3C10 */
  97         CH_DEVICE(0x36, 3),     /* S320E-CR */
  98         CH_DEVICE(0x37, 7),     /* N320E-G2 */
  99         {0,}
 100 };
 101
 102 MODULE_DESCRIPTION(DRV_DESC);
 103 MODULE_AUTHOR("Chelsio Communications");
 104 MODULE_LICENSE("Dual BSD/GPL");
 105 MODULE_VERSION(DRV_VERSION);
 106 MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
 107
 108 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 109
 110 module_param(dflt_msg_enable, int, 0644);
 111 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
 112
 113 /*
 114  * The driver uses the best interrupt scheme available on a platform in the
 115  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 116  * of these schemes the driver may consider as follows:
 117  *
 118  * msi = 2: choose from among all three options
 119  * msi = 1: only consider MSI and pin interrupts
 120  * msi = 0: force pin interrupts
 121  */
 122 static int msi = 2;
 123
 124 module_param(msi, int, 0644);
 125 MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
 126
 127 /*
 128  * The driver enables offload as a default.
 129  * To disable it, use ofld_disable = 1.
 130  */
 131
 132 static int ofld_disable = 0;
 133
 134 module_param(ofld_disable, int, 0644);
 135 MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
 136
 137 /*
 138  * We have work elements that we need to cancel when an interface is taken
 139  * down.  Normally the work elements would be executed by keventd but that
 140  * can deadlock because of linkwatch.  If our close method takes the rtnl
 141  * lock and linkwatch is ahead of our work elements in keventd, linkwatch
 142  * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
 143  * for our work to complete.  Get our own work queue to solve this.
 144  */
 145 struct workqueue_struct *cxgb3_wq;
 146
 147 /**
 148  *      link_report - show link status and link speed/duplex
 149  *      @p: the port whose settings are to be reported
 150  *
 151  *      Shows the link status, speed, and duplex of a port.
 152  */
 153 static void link_report(struct net_device *dev)
 154 {
 155         if (!netif_carrier_ok(dev))
 156                 printk(KERN_INFO "%s: link down\n", dev->name);
 157         else {
 158                 const char *s = "10Mbps";
 159                 const struct port_info *p = netdev_priv(dev);
 160
 161                 switch (p->link_config.speed) {
 162                 case SPEED_10000:
 163                         s = "10Gbps";
 164                         break;
 165                 case SPEED_1000:
 166                         s = "1000Mbps";
 167                         break;
 168                 case SPEED_100:
 169                         s = "100Mbps";
 170                         break;
 171                 }
 172
 173                 printk(KERN_INFO "%s: link up, %s, %s-duplex\n", dev->name, s,
 174                        p->link_config.duplex == DUPLEX_FULL ? "full" : "half");
 175         }
 176 }
 177
 178 static void enable_tx_fifo_drain(struct adapter *adapter,
 179                                  struct port_info *pi)
 180 {
 181         t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, 0,
 182                          F_ENDROPPKT);
 183         t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, 0);
 184         t3_write_reg(adapter, A_XGM_TX_CTRL + pi->mac.offset, F_TXEN);
 185         t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, F_RXEN);
 186 }
 187
 188 static void disable_tx_fifo_drain(struct adapter *adapter,
 189                                   struct port_info *pi)
 190 {
 191         t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset,
 192                          F_ENDROPPKT, 0);
 193 }
 194
 195 void t3_os_link_fault(struct adapter *adap, int port_id, int state)
 196 {
 197         struct net_device *dev = adap->port[port_id];
 198         struct port_info *pi = netdev_priv(dev);
 199
 200         if (state == netif_carrier_ok(dev))
 201                 return;
 202
 203         if (state) {
 204                 struct cmac *mac = &pi->mac;
 205
 206                 netif_carrier_on(dev);
 207
 208                 disable_tx_fifo_drain(adap, pi);
 209
 210                 /* Clear local faults */
 211                 t3_xgm_intr_disable(adap, pi->port_id);
 212                 t3_read_reg(adap, A_XGM_INT_STATUS +
 213                                     pi->mac.offset);
 214                 t3_write_reg(adap,
 215                              A_XGM_INT_CAUSE + pi->mac.offset,
 216                              F_XGM_INT);
 217
 218                 t3_set_reg_field(adap,
 219                                  A_XGM_INT_ENABLE +
 220                                  pi->mac.offset,
 221                                  F_XGM_INT, F_XGM_INT);
 222                 t3_xgm_intr_enable(adap, pi->port_id);
 223
 224                 t3_mac_enable(mac, MAC_DIRECTION_TX);
 225         } else {
 226                 netif_carrier_off(dev);
 227
 228                 /* Flush TX FIFO */
 229                 enable_tx_fifo_drain(adap, pi);
 230         }
 231         link_report(dev);
 232 }
 233
 234 /**
 235  *      t3_os_link_changed - handle link status changes
 236  *      @adapter: the adapter associated with the link change
 237  *      @port_id: the port index whose limk status has changed
 238  *      @link_stat: the new status of the link
 239  *      @speed: the new speed setting
 240  *      @duplex: the new duplex setting
 241  *      @pause: the new flow-control setting
 242  *
 243  *      This is the OS-dependent handler for link status changes.  The OS
 244  *      neutral handler takes care of most of the processing for these events,
 245  *      then calls this handler for any OS-specific processing.
 246  */
 247 void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
 248                         int speed, int duplex, int pause)
 249 {
 250         struct net_device *dev = adapter->port[port_id];
 251         struct port_info *pi = netdev_priv(dev);
 252         struct cmac *mac = &pi->mac;
 253
 254         /* Skip changes from disabled ports. */
 255         if (!netif_running(dev))
 256                 return;
 257
 258         if (link_stat != netif_carrier_ok(dev)) {
 259                 if (link_stat) {
 260                         disable_tx_fifo_drain(adapter, pi);
 261
 262                         t3_mac_enable(mac, MAC_DIRECTION_RX);
 263
 264                         /* Clear local faults */
 265                         t3_xgm_intr_disable(adapter, pi->port_id);
 266                         t3_read_reg(adapter, A_XGM_INT_STATUS +
 267                                     pi->mac.offset);
 268                         t3_write_reg(adapter,
 269                                      A_XGM_INT_CAUSE + pi->mac.offset,
 270                                      F_XGM_INT);
 271
 272                         t3_set_reg_field(adapter,
 273                                          A_XGM_INT_ENABLE + pi->mac.offset,
 274                                          F_XGM_INT, F_XGM_INT);
 275                         t3_xgm_intr_enable(adapter, pi->port_id);
 276
 277                         netif_carrier_on(dev);
 278                 } else {
 279                         netif_carrier_off(dev);
 280
 281                         t3_xgm_intr_disable(adapter, pi->port_id);
 282                         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
 283                         t3_set_reg_field(adapter,
 284                                          A_XGM_INT_ENABLE + pi->mac.offset,
 285                                          F_XGM_INT, 0);
 286
 287                         if (is_10G(adapter))
 288                                 pi->phy.ops->power_down(&pi->phy, 1);
 289
 290                         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
 291                         t3_mac_disable(mac, MAC_DIRECTION_RX);
 292                         t3_link_start(&pi->phy, mac, &pi->link_config);
 293
 294                         /* Flush TX FIFO */
 295                         enable_tx_fifo_drain(adapter, pi);
 296                 }
 297
 298                 link_report(dev);
 299         }
 300 }
 301
 302 /**
 303  *      t3_os_phymod_changed - handle PHY module changes
 304  *      @phy: the PHY reporting the module change
 305  *      @mod_type: new module type
 306  *
 307  *      This is the OS-dependent handler for PHY module changes.  It is
 308  *      invoked when a PHY module is removed or inserted for any OS-specific
 309  *      processing.
 310  */
 311 void t3_os_phymod_changed(struct adapter *adap, int port_id)
 312 {
 313         static const char *mod_str[] = {
 314                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
 315         };
 316
 317         const struct net_device *dev = adap->port[port_id];
 318         const struct port_info *pi = netdev_priv(dev);
 319
 320         if (pi->phy.modtype == phy_modtype_none)
 321                 printk(KERN_INFO "%s: PHY module unplugged\n", dev->name);
 322         else
 323                 printk(KERN_INFO "%s: %s PHY module inserted\n", dev->name,
 324                        mod_str[pi->phy.modtype]);
 325 }
 326
 327 static void cxgb_set_rxmode(struct net_device *dev)
 328 {
 329         struct port_info *pi = netdev_priv(dev);
 330
 331         t3_mac_set_rx_mode(&pi->mac, dev);
 332 }
 333
 334 /**
 335  *      link_start - enable a port
 336  *      @dev: the device to enable
 337  *
 338  *      Performs the MAC and PHY actions needed to enable a port.
 339  */
 340 static void link_start(struct net_device *dev)
 341 {
 342         struct port_info *pi = netdev_priv(dev);
 343         struct cmac *mac = &pi->mac;
 344
 345         t3_mac_reset(mac);
 346         t3_mac_set_num_ucast(mac, MAX_MAC_IDX);
 347         t3_mac_set_mtu(mac, dev->mtu);
 348         t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
 349         t3_mac_set_address(mac, SAN_MAC_IDX, pi->iscsic.mac_addr);
 350         t3_mac_set_rx_mode(mac, dev);
 351         t3_link_start(&pi->phy, mac, &pi->link_config);
 352         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 353 }
 354
 355 static inline void cxgb_disable_msi(struct adapter *adapter)
 356 {
 357         if (adapter->flags & USING_MSIX) {
 358                 pci_disable_msix(adapter->pdev);
 359                 adapter->flags &= ~USING_MSIX;
 360         } else if (adapter->flags & USING_MSI) {
 361                 pci_disable_msi(adapter->pdev);
 362                 adapter->flags &= ~USING_MSI;
 363         }
 364 }
 365
 366 /*
 367  * Interrupt handler for asynchronous events used with MSI-X.
 368  */
 369 static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
 370 {
 371         t3_slow_intr_handler(cookie);
 372         return IRQ_HANDLED;
 373 }
 374
 375 /*
 376  * Name the MSI-X interrupts.
 377  */
 378 static void name_msix_vecs(struct adapter *adap)
 379 {
 380         int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
 381
 382         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
 383         adap->msix_info[0].desc[n] = 0;
 384
 385         for_each_port(adap, j) {
 386                 struct net_device *d = adap->port[j];
 387                 const struct port_info *pi = netdev_priv(d);
 388
 389                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 390                         snprintf(adap->msix_info[msi_idx].desc, n,
 391                                  "%s-%d", d->name, pi->first_qset + i);
 392                         adap->msix_info[msi_idx].desc[n] = 0;
 393                 }
 394         }
 395 }
 396
 397 static int request_msix_data_irqs(struct adapter *adap)
 398 {
 399         int i, j, err, qidx = 0;
 400
 401         for_each_port(adap, i) {
 402                 int nqsets = adap2pinfo(adap, i)->nqsets;
 403
 404                 for (j = 0; j < nqsets; ++j) {
 405                         err = request_irq(adap->msix_info[qidx + 1].vec,
 406                                           t3_intr_handler(adap,
 407                                                           adap->sge.qs[qidx].
 408                                                           rspq.polling), 0,
 409                                           adap->msix_info[qidx + 1].desc,
 410                                           &adap->sge.qs[qidx]);
 411                         if (err) {
 412                                 while (--qidx >= 0)
 413                                         free_irq(adap->msix_info[qidx + 1].vec,
 414                                                  &adap->sge.qs[qidx]);
 415                                 return err;
 416                         }
 417                         qidx++;
 418                 }
 419         }
 420         return 0;
 421 }
 422
 423 static void free_irq_resources(struct adapter *adapter)
 424 {
 425         if (adapter->flags & USING_MSIX) {
 426                 int i, n = 0;
 427
 428                 free_irq(adapter->msix_info[0].vec, adapter);
 429                 for_each_port(adapter, i)
 430                         n += adap2pinfo(adapter, i)->nqsets;
 431
 432                 for (i = 0; i < n; ++i)
 433                         free_irq(adapter->msix_info[i + 1].vec,
 434                                  &adapter->sge.qs[i]);
 435         } else
 436                 free_irq(adapter->pdev->irq, adapter);
 437 }
 438
 439 static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
 440                               unsigned long n)
 441 {
 442         int attempts = 10;
 443
 444         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
 445                 if (!--attempts)
 446                         return -ETIMEDOUT;
 447                 msleep(10);
 448         }
 449         return 0;
 450 }
 451
 452 static int init_tp_parity(struct adapter *adap)
 453 {
 454         int i;
 455         struct sk_buff *skb;
 456         struct cpl_set_tcb_field *greq;
 457         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
 458
 459         t3_tp_set_offload_mode(adap, 1);
 460
 461         for (i = 0; i < 16; i++) {
 462                 struct cpl_smt_write_req *req;
 463
 464                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 465                 if (!skb)
 466                         skb = adap->nofail_skb;
 467                 if (!skb)
 468                         goto alloc_skb_fail;
 469
 470                 req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 471                 memset(req, 0, sizeof(*req));
 472                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 473                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
 474                 req->mtu_idx = NMTUS - 1;
 475                 req->iff = i;
 476                 t3_mgmt_tx(adap, skb);
 477                 if (skb == adap->nofail_skb) {
 478                         await_mgmt_replies(adap, cnt, i + 1);
 479                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 480                         if (!adap->nofail_skb)
 481                                 goto alloc_skb_fail;
 482                 }
 483         }
 484
 485         for (i = 0; i < 2048; i++) {
 486                 struct cpl_l2t_write_req *req;
 487
 488                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 489                 if (!skb)
 490                         skb = adap->nofail_skb;
 491                 if (!skb)
 492                         goto alloc_skb_fail;
 493
 494                 req = (struct cpl_l2t_write_req *)__skb_put(skb, sizeof(*req));
 495                 memset(req, 0, sizeof(*req));
 496                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 497                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
 498                 req->params = htonl(V_L2T_W_IDX(i));
 499                 t3_mgmt_tx(adap, skb);
 500                 if (skb == adap->nofail_skb) {
 501                         await_mgmt_replies(adap, cnt, 16 + i + 1);
 502                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 503                         if (!adap->nofail_skb)
 504                                 goto alloc_skb_fail;
 505                 }
 506         }
 507
 508         for (i = 0; i < 2048; i++) {
 509                 struct cpl_rte_write_req *req;
 510
 511                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 512                 if (!skb)
 513                         skb = adap->nofail_skb;
 514                 if (!skb)
 515                         goto alloc_skb_fail;
 516
 517                 req = (struct cpl_rte_write_req *)__skb_put(skb, sizeof(*req));
 518                 memset(req, 0, sizeof(*req));
 519                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 520                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
 521                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
 522                 t3_mgmt_tx(adap, skb);
 523                 if (skb == adap->nofail_skb) {
 524                         await_mgmt_replies(adap, cnt, 16 + 2048 + i + 1);
 525                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 526                         if (!adap->nofail_skb)
 527                                 goto alloc_skb_fail;
 528                 }
 529         }
 530
 531         skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 532         if (!skb)
 533                 skb = adap->nofail_skb;
 534         if (!skb)
 535                 goto alloc_skb_fail;
 536
 537         greq = (struct cpl_set_tcb_field *)__skb_put(skb, sizeof(*greq));
 538         memset(greq, 0, sizeof(*greq));
 539         greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 540         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
 541         greq->mask = cpu_to_be64(1);
 542         t3_mgmt_tx(adap, skb);
 543
 544         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 545         if (skb == adap->nofail_skb) {
 546                 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 547                 adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 548         }
 549
 550         t3_tp_set_offload_mode(adap, 0);
 551         return i;
 552
 553 alloc_skb_fail:
 554         t3_tp_set_offload_mode(adap, 0);
 555         return -ENOMEM;
 556 }
 557
 558 /**
 559  *      setup_rss - configure RSS
 560  *      @adap: the adapter
 561  *
 562  *      Sets up RSS to distribute packets to multiple receive queues.  We
 563  *      configure the RSS CPU lookup table to distribute to the number of HW
 564  *      receive queues, and the response queue lookup table to narrow that
 565  *      down to the response queues actually configured for each port.
 566  *      We always configure the RSS mapping for two ports since the mapping
 567  *      table has plenty of entries.
 568  */
 569 static void setup_rss(struct adapter *adap)
 570 {
 571         int i;
 572         unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
 573         unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
 574         u8 cpus[SGE_QSETS + 1];
 575         u16 rspq_map[RSS_TABLE_SIZE];
 576
 577         for (i = 0; i < SGE_QSETS; ++i)
 578                 cpus[i] = i;
 579         cpus[SGE_QSETS] = 0xff; /* terminator */
 580
 581         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 582                 rspq_map[i] = i % nq0;
 583                 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
 584         }
 585
 586         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 587                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
 588                       V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, cpus, rspq_map);
 589 }
 590
 591 static void ring_dbs(struct adapter *adap)
 592 {
 593         int i, j;
 594
 595         for (i = 0; i < SGE_QSETS; i++) {
 596                 struct sge_qset *qs = &adap->sge.qs[i];
 597
 598                 if (qs->adap)
 599                         for (j = 0; j < SGE_TXQ_PER_SET; j++)
 600                                 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(qs->txq[j].cntxt_id));
 601         }
 602 }
 603
 604 static void init_napi(struct adapter *adap)
 605 {
 606         int i;
 607
 608         for (i = 0; i < SGE_QSETS; i++) {
 609                 struct sge_qset *qs = &adap->sge.qs[i];
 610
 611                 if (qs->adap)
 612                         netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
 613                                        64);
 614         }
 615
 616         /*
 617          * netif_napi_add() can be called only once per napi_struct because it
 618          * adds each new napi_struct to a list.  Be careful not to call it a
 619          * second time, e.g., during EEH recovery, by making a note of it.
 620          */
 621         adap->flags |= NAPI_INIT;
 622 }
 623
 624 /*
 625  * Wait until all NAPI handlers are descheduled.  This includes the handlers of
 626  * both netdevices representing interfaces and the dummy ones for the extra
 627  * queues.
 628  */
 629 static void quiesce_rx(struct adapter *adap)
 630 {
 631         int i;
 632
 633         for (i = 0; i < SGE_QSETS; i++)
 634                 if (adap->sge.qs[i].adap)
 635                         napi_disable(&adap->sge.qs[i].napi);
 636 }
 637
 638 static void enable_all_napi(struct adapter *adap)
 639 {
 640         int i;
 641         for (i = 0; i < SGE_QSETS; i++)
 642                 if (adap->sge.qs[i].adap)
 643                         napi_enable(&adap->sge.qs[i].napi);
 644 }
 645
 646 /**
 647  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 648  *      @adap: the adapter
 649  *
 650  *      Determines how many sets of SGE queues to use and initializes them.
 651  *      We support multiple queue sets per port if we have MSI-X, otherwise
 652  *      just one queue set per port.
 653  */
 654 static int setup_sge_qsets(struct adapter *adap)
 655 {
 656         int i, j, err, irq_idx = 0, qset_idx = 0;
 657         unsigned int ntxq = SGE_TXQ_PER_SET;
 658
 659         if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
 660                 irq_idx = -1;
 661
 662         for_each_port(adap, i) {
 663                 struct net_device *dev = adap->port[i];
 664                 struct port_info *pi = netdev_priv(dev);
 665
 666                 pi->qs = &adap->sge.qs[pi->first_qset];
 667                 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
 668                         err = t3_sge_alloc_qset(adap, qset_idx, 1,
 669                                 (adap->flags & USING_MSIX) ? qset_idx + 1 :
 670                                                              irq_idx,
 671                                 &adap->params.sge.qset[qset_idx], ntxq, dev,
 672                                 netdev_get_tx_queue(dev, j));
 673                         if (err) {
 674                                 t3_free_sge_resources(adap);
 675                                 return err;
 676                         }
 677                 }
 678         }
 679
 680         return 0;
 681 }
 682
 683 static ssize_t attr_show(struct device *d, char *buf,
 684                          ssize_t(*format) (struct net_device *, char *))
 685 {
 686         ssize_t len;
 687
 688         /* Synchronize with ioctls that may shut down the device */
 689         rtnl_lock();
 690         len = (*format) (to_net_dev(d), buf);
 691         rtnl_unlock();
 692         return len;
 693 }
 694
 695 static ssize_t attr_store(struct device *d,
 696                           const char *buf, size_t len,
 697                           ssize_t(*set) (struct net_device *, unsigned int),
 698                           unsigned int min_val, unsigned int max_val)
 699 {
 700         char *endp;
 701         ssize_t ret;
 702         unsigned int val;
 703
 704         if (!capable(CAP_NET_ADMIN))
 705                 return -EPERM;
 706
 707         val = simple_strtoul(buf, &endp, 0);
 708         if (endp == buf || val < min_val || val > max_val)
 709                 return -EINVAL;
 710
 711         rtnl_lock();
 712         ret = (*set) (to_net_dev(d), val);
 713         if (!ret)
 714                 ret = len;
 715         rtnl_unlock();
 716         return ret;
 717 }
 718
 719 #define CXGB3_SHOW(name, val_expr) \
 720 static ssize_t format_##name(struct net_device *dev, char *buf) \
 721 { \
 722         struct port_info *pi = netdev_priv(dev); \
 723         struct adapter *adap = pi->adapter; \
 724         return sprintf(buf, "%u\n", val_expr); \
 725 } \
 726 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 727                            char *buf) \
 728 { \
 729         return attr_show(d, buf, format_##name); \
 730 }
 731
 732 static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
 733 {
 734         struct port_info *pi = netdev_priv(dev);
 735         struct adapter *adap = pi->adapter;
 736         int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
 737
 738         if (adap->flags & FULL_INIT_DONE)
 739                 return -EBUSY;
 740         if (val && adap->params.rev == 0)
 741                 return -EINVAL;
 742         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
 743             min_tids)
 744                 return -EINVAL;
 745         adap->params.mc5.nfilters = val;
 746         return 0;
 747 }
 748
 749 static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
 750                               const char *buf, size_t len)
 751 {
 752         return attr_store(d, buf, len, set_nfilters, 0, ~0);
 753 }
 754
 755 static ssize_t set_nservers(struct net_device *dev, unsigned int val)
 756 {
 757         struct port_info *pi = netdev_priv(dev);
 758         struct adapter *adap = pi->adapter;
 759
 760         if (adap->flags & FULL_INIT_DONE)
 761                 return -EBUSY;
 762         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
 763             MC5_MIN_TIDS)
 764                 return -EINVAL;
 765         adap->params.mc5.nservers = val;
 766         return 0;
 767 }
 768
 769 static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
 770                               const char *buf, size_t len)
 771 {
 772         return attr_store(d, buf, len, set_nservers, 0, ~0);
 773 }
 774
 775 #define CXGB3_ATTR_R(name, val_expr) \
 776 CXGB3_SHOW(name, val_expr) \
 777 static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 778
 779 #define CXGB3_ATTR_RW(name, val_expr, store_method) \
 780 CXGB3_SHOW(name, val_expr) \
 781 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_method)
 782
 783 CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
 784 CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
 785 CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
 786
 787 static struct attribute *cxgb3_attrs[] = {
 788         &dev_attr_cam_size.attr,
 789         &dev_attr_nfilters.attr,
 790         &dev_attr_nservers.attr,
 791         NULL
 792 };
 793
 794 static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
 795
 796 static ssize_t tm_attr_show(struct device *d,
 797                             char *buf, int sched)
 798 {
 799         struct port_info *pi = netdev_priv(to_net_dev(d));
 800         struct adapter *adap = pi->adapter;
 801         unsigned int v, addr, bpt, cpt;
 802         ssize_t len;
 803
 804         addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
 805         rtnl_lock();
 806         t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 807         v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
 808         if (sched & 1)
 809                 v >>= 16;
 810         bpt = (v >> 8) & 0xff;
 811         cpt = v & 0xff;
 812         if (!cpt)
 813                 len = sprintf(buf, "disabled\n");
 814         else {
 815                 v = (adap->params.vpd.cclk * 1000) / cpt;
 816                 len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
 817         }
 818         rtnl_unlock();
 819         return len;
 820 }
 821
 822 static ssize_t tm_attr_store(struct device *d,
 823                              const char *buf, size_t len, int sched)
 824 {
 825         struct port_info *pi = netdev_priv(to_net_dev(d));
 826         struct adapter *adap = pi->adapter;
 827         unsigned int val;
 828         char *endp;
 829         ssize_t ret;
 830
 831         if (!capable(CAP_NET_ADMIN))
 832                 return -EPERM;
 833
 834         val = simple_strtoul(buf, &endp, 0);
 835         if (endp == buf || val > 10000000)
 836                 return -EINVAL;
 837
 838         rtnl_lock();
 839         ret = t3_config_sched(adap, val, sched);
 840         if (!ret)
 841                 ret = len;
 842         rtnl_unlock();
 843         return ret;
 844 }
 845
 846 #define TM_ATTR(name, sched) \
 847 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 848                            char *buf) \
 849 { \
 850         return tm_attr_show(d, buf, sched); \
 851 } \
 852 static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
 853                             const char *buf, size_t len) \
 854 { \
 855         return tm_attr_store(d, buf, len, sched); \
 856 } \
 857 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_##name)
 858
 859 TM_ATTR(sched0, 0);
 860 TM_ATTR(sched1, 1);
 861 TM_ATTR(sched2, 2);
 862 TM_ATTR(sched3, 3);
 863 TM_ATTR(sched4, 4);
 864 TM_ATTR(sched5, 5);
 865 TM_ATTR(sched6, 6);
 866 TM_ATTR(sched7, 7);
 867
 868 static struct attribute *offload_attrs[] = {
 869         &dev_attr_sched0.attr,
 870         &dev_attr_sched1.attr,
 871         &dev_attr_sched2.attr,
 872         &dev_attr_sched3.attr,
 873         &dev_attr_sched4.attr,
 874         &dev_attr_sched5.attr,
 875         &dev_attr_sched6.attr,
 876         &dev_attr_sched7.attr,
 877         NULL
 878 };
 879
 880 static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
 881
 882 /*
 883  * Sends an sk_buff to an offload queue driver
 884  * after dealing with any active network taps.
 885  */
 886 static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
 887 {
 888         int ret;
 889
 890         local_bh_disable();
 891         ret = t3_offload_tx(tdev, skb);
 892         local_bh_enable();
 893         return ret;
 894 }
 895
 896 static int write_smt_entry(struct adapter *adapter, int idx)
 897 {
 898         struct cpl_smt_write_req *req;
 899         struct port_info *pi = netdev_priv(adapter->port[idx]);
 900         struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 901
 902         if (!skb)
 903                 return -ENOMEM;
 904
 905         req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 906         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 907         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
 908         req->mtu_idx = NMTUS - 1;       /* should be 0 but there's a T3 bug */
 909         req->iff = idx;
 910         memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
 911         memcpy(req->src_mac1, pi->iscsic.mac_addr, ETH_ALEN);
 912         skb->priority = 1;
 913         offload_tx(&adapter->tdev, skb);
 914         return 0;
 915 }
 916
 917 static int init_smt(struct adapter *adapter)
 918 {
 919         int i;
 920
 921         for_each_port(adapter, i)
 922             write_smt_entry(adapter, i);
 923         return 0;
 924 }
 925
 926 static void init_port_mtus(struct adapter *adapter)
 927 {
 928         unsigned int mtus = adapter->port[0]->mtu;
 929
 930         if (adapter->port[1])
 931                 mtus |= adapter->port[1]->mtu << 16;
 932         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
 933 }
 934
 935 static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 936                               int hi, int port)
 937 {
 938         struct sk_buff *skb;
 939         struct mngt_pktsched_wr *req;
 940         int ret;
 941
 942         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 943         if (!skb)
 944                 skb = adap->nofail_skb;
 945         if (!skb)
 946                 return -ENOMEM;
 947
 948         req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
 949         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 950         req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 951         req->sched = sched;
 952         req->idx = qidx;
 953         req->min = lo;
 954         req->max = hi;
 955         req->binding = port;
 956         ret = t3_mgmt_tx(adap, skb);
 957         if (skb == adap->nofail_skb) {
 958                 adap->nofail_skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
 959                                              GFP_KERNEL);
 960                 if (!adap->nofail_skb)
 961                         ret = -ENOMEM;
 962         }
 963
 964         return ret;
 965 }
 966
 967 static int bind_qsets(struct adapter *adap)
 968 {
 969         int i, j, err = 0;
 970
 971         for_each_port(adap, i) {
 972                 const struct port_info *pi = adap2pinfo(adap, i);
 973
 974                 for (j = 0; j < pi->nqsets; ++j) {
 975                         int ret = send_pktsched_cmd(adap, 1,
 976                                                     pi->first_qset + j, -1,
 977                                                     -1, i);
 978                         if (ret)
 979                                 err = ret;
 980                 }
 981         }
 982
 983         return err;
 984 }
 985
 986 #define FW_VERSION __stringify(FW_VERSION_MAJOR) "."                    \
 987         __stringify(FW_VERSION_MINOR) "." __stringify(FW_VERSION_MICRO)
 988 #define FW_FNAME "cxgb3/t3fw-" FW_VERSION ".bin"
 989 #define TPSRAM_VERSION __stringify(TP_VERSION_MAJOR) "."                \
 990         __stringify(TP_VERSION_MINOR) "." __stringify(TP_VERSION_MICRO)
 991 #define TPSRAM_NAME "cxgb3/t3%c_psram-" TPSRAM_VERSION ".bin"
 992 #define AEL2005_OPT_EDC_NAME "cxgb3/ael2005_opt_edc.bin"
 993 #define AEL2005_TWX_EDC_NAME "cxgb3/ael2005_twx_edc.bin"
 994 #define AEL2020_TWX_EDC_NAME "cxgb3/ael2020_twx_edc.bin"
 995 MODULE_FIRMWARE(FW_FNAME);
 996 MODULE_FIRMWARE("cxgb3/t3b_psram-" TPSRAM_VERSION ".bin");
 997 MODULE_FIRMWARE("cxgb3/t3c_psram-" TPSRAM_VERSION ".bin");
 998 MODULE_FIRMWARE(AEL2005_OPT_EDC_NAME);
 999 MODULE_FIRMWARE(AEL2005_TWX_EDC_NAME);
1000 MODULE_FIRMWARE(AEL2020_TWX_EDC_NAME);
1001
1002 static inline const char *get_edc_fw_name(int edc_idx)
1003 {
1004         const char *fw_name = NULL;
1005
1006         switch (edc_idx) {
1007         case EDC_OPT_AEL2005:
1008                 fw_name = AEL2005_OPT_EDC_NAME;
1009                 break;
1010         case EDC_TWX_AEL2005:
1011                 fw_name = AEL2005_TWX_EDC_NAME;
1012                 break;
1013         case EDC_TWX_AEL2020:
1014                 fw_name = AEL2020_TWX_EDC_NAME;
1015                 break;
1016         }
1017         return fw_name;
1018 }
1019
1020 int t3_get_edc_fw(struct cphy *phy, int edc_idx, int size)
1021 {
1022         struct adapter *adapter = phy->adapter;
1023         const struct firmware *fw;
1024         char buf[64];
1025         u32 csum;
1026         const __be32 *p;
1027         u16 *cache = phy->phy_cache;
1028         int i, ret;
1029
1030         snprintf(buf, sizeof(buf), get_edc_fw_name(edc_idx));
1031
1032         ret = request_firmware(&fw, buf, &adapter->pdev->dev);
1033         if (ret < 0) {
1034                 dev_err(&adapter->pdev->dev,
1035                         "could not upgrade firmware: unable to load %s\n",
1036                         buf);
1037                 return ret;
1038         }
1039
1040         /* check size, take checksum in account */
1041         if (fw->size > size + 4) {
1042                 CH_ERR(adapter, "firmware image too large %u, expected %d\n",
1043                        (unsigned int)fw->size, size + 4);
1044                 ret = -EINVAL;
1045         }
1046
1047         /* compute checksum */
1048         p = (const __be32 *)fw->data;
1049         for (csum = 0, i = 0; i < fw->size / sizeof(csum); i++)
1050                 csum += ntohl(p[i]);
1051
1052         if (csum != 0xffffffff) {
1053                 CH_ERR(adapter, "corrupted firmware image, checksum %u\n",
1054                        csum);
1055                 ret = -EINVAL;
1056         }
1057
1058         for (i = 0; i < size / 4 ; i++) {
1059                 *cache++ = (be32_to_cpu(p[i]) & 0xffff0000) >> 16;
1060                 *cache++ = be32_to_cpu(p[i]) & 0xffff;
1061         }
1062
1063         release_firmware(fw);
1064
1065         return ret;
1066 }
1067
1068 static int upgrade_fw(struct adapter *adap)
1069 {
1070         int ret;
1071         const struct firmware *fw;
1072         struct device *dev = &adap->pdev->dev;
1073
1074         ret = request_firmware(&fw, FW_FNAME, dev);
1075         if (ret < 0) {
1076                 dev_err(dev, "could not upgrade firmware: unable to load %s\n",
1077                         FW_FNAME);
1078                 return ret;
1079         }
1080         ret = t3_load_fw(adap, fw->data, fw->size);
1081         release_firmware(fw);
1082
1083         if (ret == 0)
1084                 dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
1085                          FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1086         else
1087                 dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
1088                         FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1089
1090         return ret;
1091 }
1092
1093 static inline char t3rev2char(struct adapter *adapter)
1094 {
1095         char rev = 0;
1096
1097         switch(adapter->params.rev) {
1098         case T3_REV_B:
1099         case T3_REV_B2:
1100                 rev = 'b';
1101                 break;
1102         case T3_REV_C:
1103                 rev = 'c';
1104                 break;
1105         }
1106         return rev;
1107 }
1108
1109 static int update_tpsram(struct adapter *adap)
1110 {
1111         const struct firmware *tpsram;
1112         char buf[64];
1113         struct device *dev = &adap->pdev->dev;
1114         int ret;
1115         char rev;
1116
1117         rev = t3rev2char(adap);
1118         if (!rev)
1119                 return 0;
1120
1121         snprintf(buf, sizeof(buf), TPSRAM_NAME, rev);
1122
1123         ret = request_firmware(&tpsram, buf, dev);
1124         if (ret < 0) {
1125                 dev_err(dev, "could not load TP SRAM: unable to load %s\n",
1126                         buf);
1127                 return ret;
1128         }
1129
1130         ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
1131         if (ret)
1132                 goto release_tpsram;
1133
1134         ret = t3_set_proto_sram(adap, tpsram->data);
1135         if (ret == 0)
1136                 dev_info(dev,
1137                          "successful update of protocol engine "
1138                          "to %d.%d.%d\n",
1139                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1140         else
1141                 dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
1142                         TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1143         if (ret)
1144                 dev_err(dev, "loading protocol SRAM failed\n");
1145
1146 release_tpsram:
1147         release_firmware(tpsram);
1148
1149         return ret;
1150 }
1151
1152 /**
1153  *      cxgb_up - enable the adapter
1154  *      @adapter: adapter being enabled
1155  *
1156  *      Called when the first port is enabled, this function performs the
1157  *      actions necessary to make an adapter operational, such as completing
1158  *      the initialization of HW modules, and enabling interrupts.
1159  *
1160  *      Must be called with the rtnl lock held.
1161  */
1162 static int cxgb_up(struct adapter *adap)
1163 {
1164         int err;
1165
1166         if (!(adap->flags & FULL_INIT_DONE)) {
1167                 err = t3_check_fw_version(adap);
1168                 if (err == -EINVAL) {
1169                         err = upgrade_fw(adap);
1170                         CH_WARN(adap, "FW upgrade to %d.%d.%d %s\n",
1171                                 FW_VERSION_MAJOR, FW_VERSION_MINOR,
1172                                 FW_VERSION_MICRO, err ? "failed" : "succeeded");
1173                 }
1174
1175                 err = t3_check_tpsram_version(adap);
1176                 if (err == -EINVAL) {
1177                         err = update_tpsram(adap);
1178                         CH_WARN(adap, "TP upgrade to %d.%d.%d %s\n",
1179                                 TP_VERSION_MAJOR, TP_VERSION_MINOR,
1180                                 TP_VERSION_MICRO, err ? "failed" : "succeeded");
1181                 }
1182
1183                 /*
1184                  * Clear interrupts now to catch errors if t3_init_hw fails.
1185                  * We clear them again later as initialization may trigger
1186                  * conditions that can interrupt.
1187                  */
1188                 t3_intr_clear(adap);
1189
1190                 err = t3_init_hw(adap, 0);
1191                 if (err)
1192                         goto out;
1193
1194                 t3_set_reg_field(adap, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1195                 t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1196
1197                 err = setup_sge_qsets(adap);
1198                 if (err)
1199                         goto out;
1200
1201                 setup_rss(adap);
1202                 if (!(adap->flags & NAPI_INIT))
1203                         init_napi(adap);
1204
1205                 t3_start_sge_timers(adap);
1206                 adap->flags |= FULL_INIT_DONE;
1207         }
1208
1209         t3_intr_clear(adap);
1210
1211         if (adap->flags & USING_MSIX) {
1212                 name_msix_vecs(adap);
1213                 err = request_irq(adap->msix_info[0].vec,
1214                                   t3_async_intr_handler, 0,
1215                                   adap->msix_info[0].desc, adap);
1216                 if (err)
1217                         goto irq_err;
1218
1219                 err = request_msix_data_irqs(adap);
1220                 if (err) {
1221                         free_irq(adap->msix_info[0].vec, adap);
1222                         goto irq_err;
1223                 }
1224         } else if ((err = request_irq(adap->pdev->irq,
1225                                       t3_intr_handler(adap,
1226                                                       adap->sge.qs[0].rspq.
1227                                                       polling),
1228                                       (adap->flags & USING_MSI) ?
1229                                        0 : IRQF_SHARED,
1230                                       adap->name, adap)))
1231                 goto irq_err;
1232
1233         enable_all_napi(adap);
1234         t3_sge_start(adap);
1235         t3_intr_enable(adap);
1236
1237         if (adap->params.rev >= T3_REV_C && !(adap->flags & TP_PARITY_INIT) &&
1238             is_offload(adap) && init_tp_parity(adap) == 0)
1239                 adap->flags |= TP_PARITY_INIT;
1240
1241         if (adap->flags & TP_PARITY_INIT) {
1242                 t3_write_reg(adap, A_TP_INT_CAUSE,
1243                              F_CMCACHEPERR | F_ARPLUTPERR);
1244                 t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff);
1245         }
1246
1247         if (!(adap->flags & QUEUES_BOUND)) {
1248                 int ret = bind_qsets(adap);
1249
1250                 if (ret < 0) {
1251                         CH_ERR(adap, "failed to bind qsets, err %d\n", ret);
1252                         t3_intr_disable(adap);
1253                         free_irq_resources(adap);
1254                         err = ret;
1255                         goto out;
1256                 }
1257                 adap->flags |= QUEUES_BOUND;
1258         }
1259
1260 out:
1261         return err;
1262 irq_err:
1263         CH_ERR(adap, "request_irq failed, err %d\n", err);
1264         goto out;
1265 }
1266
1267 /*
1268  * Release resources when all the ports and offloading have been stopped.
1269  */
1270 static void cxgb_down(struct adapter *adapter, int on_wq)
1271 {
1272         t3_sge_stop(adapter);
1273         spin_lock_irq(&adapter->work_lock);     /* sync with PHY intr task */
1274         t3_intr_disable(adapter);
1275         spin_unlock_irq(&adapter->work_lock);
1276
1277         free_irq_resources(adapter);
1278         quiesce_rx(adapter);
1279         t3_sge_stop(adapter);
1280         if (!on_wq)
1281                 flush_workqueue(cxgb3_wq);/* wait for external IRQ handler */
1282 }
1283
1284 static void schedule_chk_task(struct adapter *adap)
1285 {
1286         unsigned int timeo;
1287
1288         timeo = adap->params.linkpoll_period ?
1289             (HZ * adap->params.linkpoll_period) / 10 :
1290             adap->params.stats_update_period * HZ;
1291         if (timeo)
1292                 queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
1293 }
1294
1295 static int offload_open(struct net_device *dev)
1296 {
1297         struct port_info *pi = netdev_priv(dev);
1298         struct adapter *adapter = pi->adapter;
1299         struct t3cdev *tdev = dev2t3cdev(dev);
1300         int adap_up = adapter->open_device_map & PORT_MASK;
1301         int err;
1302
1303         if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1304                 return 0;
1305
1306         if (!adap_up && (err = cxgb_up(adapter)) < 0)
1307                 goto out;
1308
1309         t3_tp_set_offload_mode(adapter, 1);
1310         tdev->lldev = adapter->port[0];
1311         err = cxgb3_offload_activate(adapter);
1312         if (err)
1313                 goto out;
1314
1315         init_port_mtus(adapter);
1316         t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1317                      adapter->params.b_wnd,
1318                      adapter->params.rev == 0 ?
1319                      adapter->port[0]->mtu : 0xffff);
1320         init_smt(adapter);
1321
1322         if (sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group))
1323                 dev_dbg(&dev->dev, "cannot create sysfs group\n");
1324
1325         /* Call back all registered clients */
1326         cxgb3_add_clients(tdev);
1327
1328 out:
1329         /* restore them in case the offload module has changed them */
1330         if (err) {
1331                 t3_tp_set_offload_mode(adapter, 0);
1332                 clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1333                 cxgb3_set_dummy_ops(tdev);
1334         }
1335         return err;
1336 }
1337
1338 static int offload_close(struct t3cdev *tdev)
1339 {
1340         struct adapter *adapter = tdev2adap(tdev);
1341         struct t3c_data *td = T3C_DATA(tdev);
1342
1343         if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1344                 return 0;
1345
1346         /* Call back all registered clients */
1347         cxgb3_remove_clients(tdev);
1348
1349         sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
1350
1351         /* Flush work scheduled while releasing TIDs */
1352         flush_work_sync(&td->tid_release_task);
1353
1354         tdev->lldev = NULL;
1355         cxgb3_set_dummy_ops(tdev);
1356         t3_tp_set_offload_mode(adapter, 0);
1357         clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1358
1359         if (!adapter->open_device_map)
1360                 cxgb_down(adapter, 0);
1361
1362         cxgb3_offload_deactivate(adapter);
1363         return 0;
1364 }
1365
1366 static int cxgb_open(struct net_device *dev)
1367 {
1368         struct port_info *pi = netdev_priv(dev);
1369         struct adapter *adapter = pi->adapter;
1370         int other_ports = adapter->open_device_map & PORT_MASK;
1371         int err;
1372
1373         if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0)
1374                 return err;
1375
1376         set_bit(pi->port_id, &adapter->open_device_map);
1377         if (is_offload(adapter) && !ofld_disable) {
1378                 err = offload_open(dev);
1379                 if (err)
1380                         printk(KERN_WARNING
1381                                "Could not initialize offload capabilities\n");
1382         }
1383
1384         netif_set_real_num_tx_queues(dev, pi->nqsets);
1385         err = netif_set_real_num_rx_queues(dev, pi->nqsets);
1386         if (err)
1387                 return err;
1388         link_start(dev);
1389         t3_port_intr_enable(adapter, pi->port_id);
1390         netif_tx_start_all_queues(dev);
1391         if (!other_ports)
1392                 schedule_chk_task(adapter);
1393
1394         cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_UP, pi->port_id);
1395         return 0;
1396 }
1397
1398 static int __cxgb_close(struct net_device *dev, int on_wq)
1399 {
1400         struct port_info *pi = netdev_priv(dev);
1401         struct adapter *adapter = pi->adapter;
1402
1403
1404         if (!adapter->open_device_map)
1405                 return 0;
1406
1407         /* Stop link fault interrupts */
1408         t3_xgm_intr_disable(adapter, pi->port_id);
1409         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
1410
1411         t3_port_intr_disable(adapter, pi->port_id);
1412         netif_tx_stop_all_queues(dev);
1413         pi->phy.ops->power_down(&pi->phy, 1);
1414         netif_carrier_off(dev);
1415         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1416
1417         spin_lock_irq(&adapter->work_lock);     /* sync with update task */
1418         clear_bit(pi->port_id, &adapter->open_device_map);
1419         spin_unlock_irq(&adapter->work_lock);
1420
1421         if (!(adapter->open_device_map & PORT_MASK))
1422                 cancel_delayed_work_sync(&adapter->adap_check_task);
1423
1424         if (!adapter->open_device_map)
1425                 cxgb_down(adapter, on_wq);
1426
1427         cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_DOWN, pi->port_id);
1428         return 0;
1429 }
1430
1431 static int cxgb_close(struct net_device *dev)
1432 {
1433         return __cxgb_close(dev, 0);
1434 }
1435
1436 static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1437 {
1438         struct port_info *pi = netdev_priv(dev);
1439         struct adapter *adapter = pi->adapter;
1440         struct net_device_stats *ns = &pi->netstats;
1441         const struct mac_stats *pstats;
1442
1443         spin_lock(&adapter->stats_lock);
1444         pstats = t3_mac_update_stats(&pi->mac);
1445         spin_unlock(&adapter->stats_lock);
1446
1447         ns->tx_bytes = pstats->tx_octets;
1448         ns->tx_packets = pstats->tx_frames;
1449         ns->rx_bytes = pstats->rx_octets;
1450         ns->rx_packets = pstats->rx_frames;
1451         ns->multicast = pstats->rx_mcast_frames;
1452
1453         ns->tx_errors = pstats->tx_underrun;
1454         ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1455             pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1456             pstats->rx_fifo_ovfl;
1457
1458         /* detailed rx_errors */
1459         ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1460         ns->rx_over_errors = 0;
1461         ns->rx_crc_errors = pstats->rx_fcs_errs;
1462         ns->rx_frame_errors = pstats->rx_symbol_errs;
1463         ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1464         ns->rx_missed_errors = pstats->rx_cong_drops;
1465
1466         /* detailed tx_errors */
1467         ns->tx_aborted_errors = 0;
1468         ns->tx_carrier_errors = 0;
1469         ns->tx_fifo_errors = pstats->tx_underrun;
1470         ns->tx_heartbeat_errors = 0;
1471         ns->tx_window_errors = 0;
1472         return ns;
1473 }
1474
1475 static u32 get_msglevel(struct net_device *dev)
1476 {
1477         struct port_info *pi = netdev_priv(dev);
1478         struct adapter *adapter = pi->adapter;
1479
1480         return adapter->msg_enable;
1481 }
1482
1483 static void set_msglevel(struct net_device *dev, u32 val)
1484 {
1485         struct port_info *pi = netdev_priv(dev);
1486         struct adapter *adapter = pi->adapter;
1487
1488         adapter->msg_enable = val;
1489 }
1490
1491 static char stats_strings[][ETH_GSTRING_LEN] = {
1492         "TxOctetsOK         ",
1493         "TxFramesOK         ",
1494         "TxMulticastFramesOK",
1495         "TxBroadcastFramesOK",
1496         "TxPauseFrames      ",
1497         "TxUnderrun         ",
1498         "TxExtUnderrun      ",
1499
1500         "TxFrames64         ",
1501         "TxFrames65To127    ",
1502         "TxFrames128To255   ",
1503         "TxFrames256To511   ",
1504         "TxFrames512To1023  ",
1505         "TxFrames1024To1518 ",
1506         "TxFrames1519ToMax  ",
1507
1508         "RxOctetsOK         ",
1509         "RxFramesOK         ",
1510         "RxMulticastFramesOK",
1511         "RxBroadcastFramesOK",
1512         "RxPauseFrames      ",
1513         "RxFCSErrors        ",
1514         "RxSymbolErrors     ",
1515         "RxShortErrors      ",
1516         "RxJabberErrors     ",
1517         "RxLengthErrors     ",
1518         "RxFIFOoverflow     ",
1519
1520         "RxFrames64         ",
1521         "RxFrames65To127    ",
1522         "RxFrames128To255   ",
1523         "RxFrames256To511   ",
1524         "RxFrames512To1023  ",
1525         "RxFrames1024To1518 ",
1526         "RxFrames1519ToMax  ",
1527
1528         "PhyFIFOErrors      ",
1529         "TSO                ",
1530         "VLANextractions    ",
1531         "VLANinsertions     ",
1532         "TxCsumOffload      ",
1533         "RxCsumGood         ",
1534         "LroAggregated      ",
1535         "LroFlushed         ",
1536         "LroNoDesc          ",
1537         "RxDrops            ",
1538
1539         "CheckTXEnToggled   ",
1540         "CheckResets        ",
1541
1542         "LinkFaults         ",
1543 };
1544
1545 static int get_sset_count(struct net_device *dev, int sset)
1546 {
1547         switch (sset) {
1548         case ETH_SS_STATS:
1549                 return ARRAY_SIZE(stats_strings);
1550         default:
1551                 return -EOPNOTSUPP;
1552         }
1553 }
1554
1555 #define T3_REGMAP_SIZE (3 * 1024)
1556
1557 static int get_regs_len(struct net_device *dev)
1558 {
1559         return T3_REGMAP_SIZE;
1560 }
1561
1562 static int get_eeprom_len(struct net_device *dev)
1563 {
1564         return EEPROMSIZE;
1565 }
1566
1567 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1568 {
1569         struct port_info *pi = netdev_priv(dev);
1570         struct adapter *adapter = pi->adapter;
1571         u32 fw_vers = 0;
1572         u32 tp_vers = 0;
1573
1574         spin_lock(&adapter->stats_lock);
1575         t3_get_fw_version(adapter, &fw_vers);
1576         t3_get_tp_version(adapter, &tp_vers);
1577         spin_unlock(&adapter->stats_lock);
1578
1579         strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
1580         strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1581         strlcpy(info->bus_info, pci_name(adapter->pdev),
1582                 sizeof(info->bus_info));
1583         if (fw_vers)
1584                 snprintf(info->fw_version, sizeof(info->fw_version),
1585                          "%s %u.%u.%u TP %u.%u.%u",
1586                          G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1587                          G_FW_VERSION_MAJOR(fw_vers),
1588                          G_FW_VERSION_MINOR(fw_vers),
1589                          G_FW_VERSION_MICRO(fw_vers),
1590                          G_TP_VERSION_MAJOR(tp_vers),
1591                          G_TP_VERSION_MINOR(tp_vers),
1592                          G_TP_VERSION_MICRO(tp_vers));
1593 }
1594
1595 static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1596 {
1597         if (stringset == ETH_SS_STATS)
1598                 memcpy(data, stats_strings, sizeof(stats_strings));
1599 }
1600
1601 static unsigned long collect_sge_port_stats(struct adapter *adapter,
1602                                             struct port_info *p, int idx)
1603 {
1604         int i;
1605         unsigned long tot = 0;
1606
1607         for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i)
1608                 tot += adapter->sge.qs[i].port_stats[idx];
1609         return tot;
1610 }
1611
1612 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1613                       u64 *data)
1614 {
1615         struct port_info *pi = netdev_priv(dev);
1616         struct adapter *adapter = pi->adapter;
1617         const struct mac_stats *s;
1618
1619         spin_lock(&adapter->stats_lock);
1620         s = t3_mac_update_stats(&pi->mac);
1621         spin_unlock(&adapter->stats_lock);
1622
1623         *data++ = s->tx_octets;
1624         *data++ = s->tx_frames;
1625         *data++ = s->tx_mcast_frames;
1626         *data++ = s->tx_bcast_frames;
1627         *data++ = s->tx_pause;
1628         *data++ = s->tx_underrun;
1629         *data++ = s->tx_fifo_urun;
1630
1631         *data++ = s->tx_frames_64;
1632         *data++ = s->tx_frames_65_127;
1633         *data++ = s->tx_frames_128_255;
1634         *data++ = s->tx_frames_256_511;
1635         *data++ = s->tx_frames_512_1023;
1636         *data++ = s->tx_frames_1024_1518;
1637         *data++ = s->tx_frames_1519_max;
1638
1639         *data++ = s->rx_octets;
1640         *data++ = s->rx_frames;
1641         *data++ = s->rx_mcast_frames;
1642         *data++ = s->rx_bcast_frames;
1643         *data++ = s->rx_pause;
1644         *data++ = s->rx_fcs_errs;
1645         *data++ = s->rx_symbol_errs;
1646         *data++ = s->rx_short;
1647         *data++ = s->rx_jabber;
1648         *data++ = s->rx_too_long;
1649         *data++ = s->rx_fifo_ovfl;
1650
1651         *data++ = s->rx_frames_64;
1652         *data++ = s->rx_frames_65_127;
1653         *data++ = s->rx_frames_128_255;
1654         *data++ = s->rx_frames_256_511;
1655         *data++ = s->rx_frames_512_1023;
1656         *data++ = s->rx_frames_1024_1518;
1657         *data++ = s->rx_frames_1519_max;
1658
1659         *data++ = pi->phy.fifo_errors;
1660
1661         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1662         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1663         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1664         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1665         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
1666         *data++ = 0;
1667         *data++ = 0;
1668         *data++ = 0;
1669         *data++ = s->rx_cong_drops;
1670
1671         *data++ = s->num_toggled;
1672         *data++ = s->num_resets;
1673
1674         *data++ = s->link_faults;
1675 }
1676
1677 static inline void reg_block_dump(struct adapter *ap, void *buf,
1678                                   unsigned int start, unsigned int end)
1679 {
1680         u32 *p = buf + start;
1681
1682         for (; start <= end; start += sizeof(u32))
1683                 *p++ = t3_read_reg(ap, start);
1684 }
1685
1686 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1687                      void *buf)
1688 {
1689         struct port_info *pi = netdev_priv(dev);
1690         struct adapter *ap = pi->adapter;
1691
1692         /*
1693          * Version scheme:
1694          * bits 0..9: chip version
1695          * bits 10..15: chip revision
1696          * bit 31: set for PCIe cards
1697          */
1698         regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1699
1700         /*
1701          * We skip the MAC statistics registers because they are clear-on-read.
1702          * Also reading multi-register stats would need to synchronize with the
1703          * periodic mac stats accumulation.  Hard to justify the complexity.
1704          */
1705         memset(buf, 0, T3_REGMAP_SIZE);
1706         reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1707         reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1708         reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1709         reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1710         reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1711         reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1712                        XGM_REG(A_XGM_SERDES_STAT3, 1));
1713         reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1714                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1715 }
1716
1717 static int restart_autoneg(struct net_device *dev)
1718 {
1719         struct port_info *p = netdev_priv(dev);
1720
1721         if (!netif_running(dev))
1722                 return -EAGAIN;
1723         if (p->link_config.autoneg != AUTONEG_ENABLE)
1724                 return -EINVAL;
1725         p->phy.ops->autoneg_restart(&p->phy);
1726         return 0;
1727 }
1728
1729 static int set_phys_id(struct net_device *dev,
1730                        enum ethtool_phys_id_state state)
1731 {
1732         struct port_info *pi = netdev_priv(dev);
1733         struct adapter *adapter = pi->adapter;
1734
1735         switch (state) {
1736         case ETHTOOL_ID_ACTIVE:
1737                 return 1;       /* cycle on/off once per second */
1738
1739         case ETHTOOL_ID_OFF:
1740                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL, 0);
1741                 break;
1742
1743         case ETHTOOL_ID_ON:
1744         case ETHTOOL_ID_INACTIVE:
1745                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1746                          F_GPIO0_OUT_VAL);
1747         }
1748
1749         return 0;
1750 }
1751
1752 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1753 {
1754         struct port_info *p = netdev_priv(dev);
1755
1756         cmd->supported = p->link_config.supported;
1757         cmd->advertising = p->link_config.advertising;
1758
1759         if (netif_carrier_ok(dev)) {
1760                 ethtool_cmd_speed_set(cmd, p->link_config.speed);
1761                 cmd->duplex = p->link_config.duplex;
1762         } else {
1763                 ethtool_cmd_speed_set(cmd, -1);
1764                 cmd->duplex = -1;
1765         }
1766
1767         cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1768         cmd->phy_address = p->phy.mdio.prtad;
1769         cmd->transceiver = XCVR_EXTERNAL;
1770         cmd->autoneg = p->link_config.autoneg;
1771         cmd->maxtxpkt = 0;
1772         cmd->maxrxpkt = 0;
1773         return 0;
1774 }
1775
1776 static int speed_duplex_to_caps(int speed, int duplex)
1777 {
1778         int cap = 0;
1779
1780         switch (speed) {
1781         case SPEED_10:
1782                 if (duplex == DUPLEX_FULL)
1783                         cap = SUPPORTED_10baseT_Full;
1784                 else
1785                         cap = SUPPORTED_10baseT_Half;
1786                 break;
1787         case SPEED_100:
1788                 if (duplex == DUPLEX_FULL)
1789                         cap = SUPPORTED_100baseT_Full;
1790                 else
1791                         cap = SUPPORTED_100baseT_Half;
1792                 break;
1793         case SPEED_1000:
1794                 if (duplex == DUPLEX_FULL)
1795                         cap = SUPPORTED_1000baseT_Full;
1796                 else
1797                         cap = SUPPORTED_1000baseT_Half;
1798                 break;
1799         case SPEED_10000:
1800                 if (duplex == DUPLEX_FULL)
1801                         cap = SUPPORTED_10000baseT_Full;
1802         }
1803         return cap;
1804 }
1805
1806 #define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1807                       ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1808                       ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1809                       ADVERTISED_10000baseT_Full)
1810
1811 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1812 {
1813         struct port_info *p = netdev_priv(dev);
1814         struct link_config *lc = &p->link_config;
1815
1816         if (!(lc->supported & SUPPORTED_Autoneg)) {
1817                 /*
1818                  * PHY offers a single speed/duplex.  See if that's what's
1819                  * being requested.
1820                  */
1821                 if (cmd->autoneg == AUTONEG_DISABLE) {
1822                         u32 speed = ethtool_cmd_speed(cmd);
1823                         int cap = speed_duplex_to_caps(speed, cmd->duplex);
1824                         if (lc->supported & cap)
1825                                 return 0;
1826                 }
1827                 return -EINVAL;
1828         }
1829
1830         if (cmd->autoneg == AUTONEG_DISABLE) {
1831                 u32 speed = ethtool_cmd_speed(cmd);
1832                 int cap = speed_duplex_to_caps(speed, cmd->duplex);
1833
1834                 if (!(lc->supported & cap) || (speed == SPEED_1000))
1835                         return -EINVAL;
1836                 lc->requested_speed = speed;
1837                 lc->requested_duplex = cmd->duplex;
1838                 lc->advertising = 0;
1839         } else {
1840                 cmd->advertising &= ADVERTISED_MASK;
1841                 cmd->advertising &= lc->supported;
1842                 if (!cmd->advertising)
1843                         return -EINVAL;
1844                 lc->requested_speed = SPEED_INVALID;
1845                 lc->requested_duplex = DUPLEX_INVALID;
1846                 lc->advertising = cmd->advertising | ADVERTISED_Autoneg;
1847         }
1848         lc->autoneg = cmd->autoneg;
1849         if (netif_running(dev))
1850                 t3_link_start(&p->phy, &p->mac, lc);
1851         return 0;
1852 }
1853
1854 static void get_pauseparam(struct net_device *dev,
1855                            struct ethtool_pauseparam *epause)
1856 {
1857         struct port_info *p = netdev_priv(dev);
1858
1859         epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1860         epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1861         epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1862 }
1863
1864 static int set_pauseparam(struct net_device *dev,
1865                           struct ethtool_pauseparam *epause)
1866 {
1867         struct port_info *p = netdev_priv(dev);
1868         struct link_config *lc = &p->link_config;
1869
1870         if (epause->autoneg == AUTONEG_DISABLE)
1871                 lc->requested_fc = 0;
1872         else if (lc->supported & SUPPORTED_Autoneg)
1873                 lc->requested_fc = PAUSE_AUTONEG;
1874         else
1875                 return -EINVAL;
1876
1877         if (epause->rx_pause)
1878                 lc->requested_fc |= PAUSE_RX;
1879         if (epause->tx_pause)
1880                 lc->requested_fc |= PAUSE_TX;
1881         if (lc->autoneg == AUTONEG_ENABLE) {
1882                 if (netif_running(dev))
1883                         t3_link_start(&p->phy, &p->mac, lc);
1884         } else {
1885                 lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1886                 if (netif_running(dev))
1887                         t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1888         }
1889         return 0;
1890 }
1891
1892 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1893 {
1894         struct port_info *pi = netdev_priv(dev);
1895         struct adapter *adapter = pi->adapter;
1896         const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
1897
1898         e->rx_max_pending = MAX_RX_BUFFERS;
1899         e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1900         e->tx_max_pending = MAX_TXQ_ENTRIES;
1901
1902         e->rx_pending = q->fl_size;
1903         e->rx_mini_pending = q->rspq_size;
1904         e->rx_jumbo_pending = q->jumbo_size;
1905         e->tx_pending = q->txq_size[0];
1906 }
1907
1908 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1909 {
1910         struct port_info *pi = netdev_priv(dev);
1911         struct adapter *adapter = pi->adapter;
1912         struct qset_params *q;
1913         int i;
1914
1915         if (e->rx_pending > MAX_RX_BUFFERS ||
1916             e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1917             e->tx_pending > MAX_TXQ_ENTRIES ||
1918             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1919             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1920             e->rx_pending < MIN_FL_ENTRIES ||
1921             e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1922             e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1923                 return -EINVAL;
1924
1925         if (adapter->flags & FULL_INIT_DONE)
1926                 return -EBUSY;
1927
1928         q = &adapter->params.sge.qset[pi->first_qset];
1929         for (i = 0; i < pi->nqsets; ++i, ++q) {
1930                 q->rspq_size = e->rx_mini_pending;
1931                 q->fl_size = e->rx_pending;
1932                 q->jumbo_size = e->rx_jumbo_pending;
1933                 q->txq_size[0] = e->tx_pending;
1934                 q->txq_size[1] = e->tx_pending;
1935                 q->txq_size[2] = e->tx_pending;
1936         }
1937         return 0;
1938 }
1939
1940 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1941 {
1942         struct port_info *pi = netdev_priv(dev);
1943         struct adapter *adapter = pi->adapter;
1944         struct qset_params *qsp;
1945         struct sge_qset *qs;
1946         int i;
1947
1948         if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
1949                 return -EINVAL;
1950
1951         for (i = 0; i < pi->nqsets; i++) {
1952                 qsp = &adapter->params.sge.qset[i];
1953                 qs = &adapter->sge.qs[i];
1954                 qsp->coalesce_usecs = c->rx_coalesce_usecs;
1955                 t3_update_qset_coalesce(qs, qsp);
1956         }
1957
1958         return 0;
1959 }
1960
1961 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1962 {
1963         struct port_info *pi = netdev_priv(dev);
1964         struct adapter *adapter = pi->adapter;
1965         struct qset_params *q = adapter->params.sge.qset;
1966
1967         c->rx_coalesce_usecs = q->coalesce_usecs;
1968         return 0;
1969 }
1970
1971 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1972                       u8 * data)
1973 {
1974         struct port_info *pi = netdev_priv(dev);
1975         struct adapter *adapter = pi->adapter;
1976         int i, err = 0;
1977
1978         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1979         if (!buf)
1980                 return -ENOMEM;
1981
1982         e->magic = EEPROM_MAGIC;
1983         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1984                 err = t3_seeprom_read(adapter, i, (__le32 *) & buf[i]);
1985
1986         if (!err)
1987                 memcpy(data, buf + e->offset, e->len);
1988         kfree(buf);
1989         return err;
1990 }
1991
1992 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1993                       u8 * data)
1994 {
1995         struct port_info *pi = netdev_priv(dev);
1996         struct adapter *adapter = pi->adapter;
1997         u32 aligned_offset, aligned_len;
1998         __le32 *p;
1999         u8 *buf;
2000         int err;
2001
2002         if (eeprom->magic != EEPROM_MAGIC)
2003                 return -EINVAL;
2004
2005         aligned_offset = eeprom->offset & ~3;
2006         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2007
2008         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2009                 buf = kmalloc(aligned_len, GFP_KERNEL);
2010                 if (!buf)
2011                         return -ENOMEM;
2012                 err = t3_seeprom_read(adapter, aligned_offset, (__le32 *) buf);
2013                 if (!err && aligned_len > 4)
2014                         err = t3_seeprom_read(adapter,
2015                                               aligned_offset + aligned_len - 4,
2016                                               (__le32 *) & buf[aligned_len - 4]);
2017                 if (err)
2018                         goto out;
2019                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2020         } else
2021                 buf = data;
2022
2023         err = t3_seeprom_wp(adapter, 0);
2024         if (err)
2025                 goto out;
2026
2027         for (p = (__le32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
2028                 err = t3_seeprom_write(adapter, aligned_offset, *p);
2029                 aligned_offset += 4;
2030         }
2031
2032         if (!err)
2033                 err = t3_seeprom_wp(adapter, 1);
2034 out:
2035         if (buf != data)
2036                 kfree(buf);
2037         return err;
2038 }
2039
2040 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2041 {
2042         wol->supported = 0;
2043         wol->wolopts = 0;
2044         memset(&wol->sopass, 0, sizeof(wol->sopass));
2045 }
2046
2047 static const struct ethtool_ops cxgb_ethtool_ops = {
2048         .get_settings = get_settings,
2049         .set_settings = set_settings,
2050         .get_drvinfo = get_drvinfo,
2051         .get_msglevel = get_msglevel,
2052         .set_msglevel = set_msglevel,
2053         .get_ringparam = get_sge_param,
2054         .set_ringparam = set_sge_param,
2055         .get_coalesce = get_coalesce,
2056         .set_coalesce = set_coalesce,
2057         .get_eeprom_len = get_eeprom_len,
2058         .get_eeprom = get_eeprom,
2059         .set_eeprom = set_eeprom,
2060         .get_pauseparam = get_pauseparam,
2061         .set_pauseparam = set_pauseparam,
2062         .get_link = ethtool_op_get_link,
2063         .get_strings = get_strings,
2064         .set_phys_id = set_phys_id,
2065         .nway_reset = restart_autoneg,
2066         .get_sset_count = get_sset_count,
2067         .get_ethtool_stats = get_stats,
2068         .get_regs_len = get_regs_len,
2069         .get_regs = get_regs,
2070         .get_wol = get_wol,
2071 };
2072
2073 static int in_range(int val, int lo, int hi)
2074 {
2075         return val < 0 || (val <= hi && val >= lo);
2076 }
2077
2078 static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
2079 {
2080         struct port_info *pi = netdev_priv(dev);
2081         struct adapter *adapter = pi->adapter;
2082         u32 cmd;
2083         int ret;
2084
2085         if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
2086                 return -EFAULT;
2087
2088         switch (cmd) {
2089         case CHELSIO_SET_QSET_PARAMS:{
2090                 int i;
2091                 struct qset_params *q;
2092                 struct ch_qset_params t;
2093                 int q1 = pi->first_qset;
2094                 int nqsets = pi->nqsets;
2095
2096                 if (!capable(CAP_NET_ADMIN))
2097                         return -EPERM;
2098                 if (copy_from_user(&t, useraddr, sizeof(t)))
2099                         return -EFAULT;
2100                 if (t.qset_idx >= SGE_QSETS)
2101                         return -EINVAL;
2102                 if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
2103                     !in_range(t.cong_thres, 0, 255) ||
2104                     !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
2105                               MAX_TXQ_ENTRIES) ||
2106                     !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
2107                               MAX_TXQ_ENTRIES) ||
2108                     !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
2109                               MAX_CTRL_TXQ_ENTRIES) ||
2110                     !in_range(t.fl_size[0], MIN_FL_ENTRIES,
2111                               MAX_RX_BUFFERS) ||
2112                     !in_range(t.fl_size[1], MIN_FL_ENTRIES,
2113                               MAX_RX_JUMBO_BUFFERS) ||
2114                     !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
2115                               MAX_RSPQ_ENTRIES))
2116                         return -EINVAL;
2117
2118                 if ((adapter->flags & FULL_INIT_DONE) &&
2119                         (t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
2120                         t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
2121                         t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
2122                         t.polling >= 0 || t.cong_thres >= 0))
2123                         return -EBUSY;
2124
2125                 /* Allow setting of any available qset when offload enabled */
2126                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2127                         q1 = 0;
2128                         for_each_port(adapter, i) {
2129                                 pi = adap2pinfo(adapter, i);
2130                                 nqsets += pi->first_qset + pi->nqsets;
2131                         }
2132                 }
2133
2134                 if (t.qset_idx < q1)
2135                         return -EINVAL;
2136                 if (t.qset_idx > q1 + nqsets - 1)
2137                         return -EINVAL;
2138
2139                 q = &adapter->params.sge.qset[t.qset_idx];
2140
2141                 if (t.rspq_size >= 0)
2142                         q->rspq_size = t.rspq_size;
2143                 if (t.fl_size[0] >= 0)
2144                         q->fl_size = t.fl_size[0];
2145                 if (t.fl_size[1] >= 0)
2146                         q->jumbo_size = t.fl_size[1];
2147                 if (t.txq_size[0] >= 0)
2148                         q->txq_size[0] = t.txq_size[0];
2149                 if (t.txq_size[1] >= 0)
2150                         q->txq_size[1] = t.txq_size[1];
2151                 if (t.txq_size[2] >= 0)
2152                         q->txq_size[2] = t.txq_size[2];
2153                 if (t.cong_thres >= 0)
2154                         q->cong_thres = t.cong_thres;
2155                 if (t.intr_lat >= 0) {
2156                         struct sge_qset *qs =
2157                                 &adapter->sge.qs[t.qset_idx];
2158
2159                         q->coalesce_usecs = t.intr_lat;
2160                         t3_update_qset_coalesce(qs, q);
2161                 }
2162                 if (t.polling >= 0) {
2163                         if (adapter->flags & USING_MSIX)
2164                                 q->polling = t.polling;
2165                         else {
2166                                 /* No polling with INTx for T3A */
2167                                 if (adapter->params.rev == 0 &&
2168                                         !(adapter->flags & USING_MSI))
2169                                         t.polling = 0;
2170
2171                                 for (i = 0; i < SGE_QSETS; i++) {
2172                                         q = &adapter->params.sge.
2173                                                 qset[i];
2174                                         q->polling = t.polling;
2175                                 }
2176                         }
2177                 }
2178
2179                 if (t.lro >= 0) {
2180                         if (t.lro)
2181                                 dev->wanted_features |= NETIF_F_GRO;
2182                         else
2183                                 dev->wanted_features &= ~NETIF_F_GRO;
2184                         netdev_update_features(dev);
2185                 }
2186
2187                 break;
2188         }
2189         case CHELSIO_GET_QSET_PARAMS:{
2190                 struct qset_params *q;
2191                 struct ch_qset_params t;
2192                 int q1 = pi->first_qset;
2193                 int nqsets = pi->nqsets;
2194                 int i;
2195
2196                 if (copy_from_user(&t, useraddr, sizeof(t)))
2197                         return -EFAULT;
2198
2199                 /* Display qsets for all ports when offload enabled */
2200                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2201                         q1 = 0;
2202                         for_each_port(adapter, i) {
2203                                 pi = adap2pinfo(adapter, i);
2204                                 nqsets = pi->first_qset + pi->nqsets;
2205                         }
2206                 }
2207
2208                 if (t.qset_idx >= nqsets)
2209                         return -EINVAL;
2210
2211                 q = &adapter->params.sge.qset[q1 + t.qset_idx];
2212                 t.rspq_size = q->rspq_size;
2213                 t.txq_size[0] = q->txq_size[0];
2214                 t.txq_size[1] = q->txq_size[1];
2215                 t.txq_size[2] = q->txq_size[2];
2216                 t.fl_size[0] = q->fl_size;
2217                 t.fl_size[1] = q->jumbo_size;
2218                 t.polling = q->polling;
2219                 t.lro = !!(dev->features & NETIF_F_GRO);
2220                 t.intr_lat = q->coalesce_usecs;
2221                 t.cong_thres = q->cong_thres;
2222                 t.qnum = q1;
2223
2224                 if (adapter->flags & USING_MSIX)
2225                         t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec;
2226                 else
2227                         t.vector = adapter->pdev->irq;
2228
2229                 if (copy_to_user(useraddr, &t, sizeof(t)))
2230                         return -EFAULT;
2231                 break;
2232         }
2233         case CHELSIO_SET_QSET_NUM:{
2234                 struct ch_reg edata;
2235                 unsigned int i, first_qset = 0, other_qsets = 0;
2236
2237                 if (!capable(CAP_NET_ADMIN))
2238                         return -EPERM;
2239                 if (adapter->flags & FULL_INIT_DONE)
2240                         return -EBUSY;
2241                 if (copy_from_user(&edata, useraddr, sizeof(edata)))
2242                         return -EFAULT;
2243                 if (edata.val < 1 ||
2244                         (edata.val > 1 && !(adapter->flags & USING_MSIX)))
2245                         return -EINVAL;
2246
2247                 for_each_port(adapter, i)
2248                         if (adapter->port[i] && adapter->port[i] != dev)
2249                                 other_qsets += adap2pinfo(adapter, i)->nqsets;
2250
2251                 if (edata.val + other_qsets > SGE_QSETS)
2252                         return -EINVAL;
2253
2254                 pi->nqsets = edata.val;
2255
2256                 for_each_port(adapter, i)
2257                         if (adapter->port[i]) {
2258                                 pi = adap2pinfo(adapter, i);
2259                                 pi->first_qset = first_qset;
2260                                 first_qset += pi->nqsets;
2261                         }
2262                 break;
2263         }
2264         case CHELSIO_GET_QSET_NUM:{
2265                 struct ch_reg edata;
2266
2267                 memset(&edata, 0, sizeof(struct ch_reg));
2268
2269                 edata.cmd = CHELSIO_GET_QSET_NUM;
2270                 edata.val = pi->nqsets;
2271                 if (copy_to_user(useraddr, &edata, sizeof(edata)))
2272                         return -EFAULT;
2273                 break;
2274         }
2275         case CHELSIO_LOAD_FW:{
2276                 u8 *fw_data;
2277                 struct ch_mem_range t;
2278
2279                 if (!capable(CAP_SYS_RAWIO))
2280                         return -EPERM;
2281                 if (copy_from_user(&t, useraddr, sizeof(t)))
2282                         return -EFAULT;
2283                 /* Check t.len sanity ? */
2284                 fw_data = memdup_user(useraddr + sizeof(t), t.len);
2285                 if (IS_ERR(fw_data))
2286                         return PTR_ERR(fw_data);
2287
2288                 ret = t3_load_fw(adapter, fw_data, t.len);
2289                 kfree(fw_data);
2290                 if (ret)
2291                         return ret;
2292                 break;
2293         }
2294         case CHELSIO_SETMTUTAB:{
2295                 struct ch_mtus m;
2296                 int i;
2297
2298                 if (!is_offload(adapter))
2299                         return -EOPNOTSUPP;
2300                 if (!capable(CAP_NET_ADMIN))
2301                         return -EPERM;
2302                 if (offload_running(adapter))
2303                         return -EBUSY;
2304                 if (copy_from_user(&m, useraddr, sizeof(m)))
2305                         return -EFAULT;
2306                 if (m.nmtus != NMTUS)
2307                         return -EINVAL;
2308                 if (m.mtus[0] < 81)     /* accommodate SACK */
2309                         return -EINVAL;
2310
2311                 /* MTUs must be in ascending order */
2312                 for (i = 1; i < NMTUS; ++i)
2313                         if (m.mtus[i] < m.mtus[i - 1])
2314                                 return -EINVAL;
2315
2316                 memcpy(adapter->params.mtus, m.mtus,
2317                         sizeof(adapter->params.mtus));
2318                 break;
2319         }
2320         case CHELSIO_GET_PM:{
2321                 struct tp_params *p = &adapter->params.tp;
2322                 struct ch_pm m = {.cmd = CHELSIO_GET_PM };
2323
2324                 if (!is_offload(adapter))
2325                         return -EOPNOTSUPP;
2326                 m.tx_pg_sz = p->tx_pg_size;
2327                 m.tx_num_pg = p->tx_num_pgs;
2328                 m.rx_pg_sz = p->rx_pg_size;
2329                 m.rx_num_pg = p->rx_num_pgs;
2330                 m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2331                 if (copy_to_user(useraddr, &m, sizeof(m)))
2332                         return -EFAULT;
2333                 break;
2334         }
2335         case CHELSIO_SET_PM:{
2336                 struct ch_pm m;
2337                 struct tp_params *p = &adapter->params.tp;
2338
2339                 if (!is_offload(adapter))
2340                         return -EOPNOTSUPP;
2341                 if (!capable(CAP_NET_ADMIN))
2342                         return -EPERM;
2343                 if (adapter->flags & FULL_INIT_DONE)
2344                         return -EBUSY;
2345                 if (copy_from_user(&m, useraddr, sizeof(m)))
2346                         return -EFAULT;
2347                 if (!is_power_of_2(m.rx_pg_sz) ||
2348                         !is_power_of_2(m.tx_pg_sz))
2349                         return -EINVAL; /* not power of 2 */
2350                 if (!(m.rx_pg_sz & 0x14000))
2351                         return -EINVAL; /* not 16KB or 64KB */
2352                 if (!(m.tx_pg_sz & 0x1554000))
2353                         return -EINVAL;
2354                 if (m.tx_num_pg == -1)
2355                         m.tx_num_pg = p->tx_num_pgs;
2356                 if (m.rx_num_pg == -1)
2357                         m.rx_num_pg = p->rx_num_pgs;
2358                 if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
2359                         return -EINVAL;
2360                 if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
2361                         m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
2362                         return -EINVAL;
2363                 p->rx_pg_size = m.rx_pg_sz;
2364                 p->tx_pg_size = m.tx_pg_sz;
2365                 p->rx_num_pgs = m.rx_num_pg;
2366                 p->tx_num_pgs = m.tx_num_pg;
2367                 break;
2368         }
2369         case CHELSIO_GET_MEM:{
2370                 struct ch_mem_range t;
2371                 struct mc7 *mem;
2372                 u64 buf[32];
2373
2374                 if (!is_offload(adapter))
2375                         return -EOPNOTSUPP;
2376                 if (!(adapter->flags & FULL_INIT_DONE))
2377                         return -EIO;    /* need the memory controllers */
2378                 if (copy_from_user(&t, useraddr, sizeof(t)))
2379                         return -EFAULT;
2380                 if ((t.addr & 7) || (t.len & 7))
2381                         return -EINVAL;
2382                 if (t.mem_id == MEM_CM)
2383                         mem = &adapter->cm;
2384                 else if (t.mem_id == MEM_PMRX)
2385                         mem = &adapter->pmrx;
2386                 else if (t.mem_id == MEM_PMTX)
2387                         mem = &adapter->pmtx;
2388                 else
2389                         return -EINVAL;
2390
2391                 /*
2392                  * Version scheme:
2393                  * bits 0..9: chip version
2394                  * bits 10..15: chip revision
2395                  */
2396                 t.version = 3 | (adapter->params.rev << 10);
2397                 if (copy_to_user(useraddr, &t, sizeof(t)))
2398                         return -EFAULT;
2399
2400                 /*
2401                  * Read 256 bytes at a time as len can be large and we don't
2402                  * want to use huge intermediate buffers.
2403                  */
2404                 useraddr += sizeof(t);  /* advance to start of buffer */
2405                 while (t.len) {
2406                         unsigned int chunk =
2407                                 min_t(unsigned int, t.len, sizeof(buf));
2408
2409                         ret =
2410                                 t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
2411                                                 buf);
2412                         if (ret)
2413                                 return ret;
2414                         if (copy_to_user(useraddr, buf, chunk))
2415                                 return -EFAULT;
2416                         useraddr += chunk;
2417                         t.addr += chunk;
2418                         t.len -= chunk;
2419                 }
2420                 break;
2421         }
2422         case CHELSIO_SET_TRACE_FILTER:{
2423                 struct ch_trace t;
2424                 const struct trace_params *tp;
2425
2426                 if (!capable(CAP_NET_ADMIN))
2427                         return -EPERM;
2428                 if (!offload_running(adapter))
2429                         return -EAGAIN;
2430                 if (copy_from_user(&t, useraddr, sizeof(t)))
2431                         return -EFAULT;
2432
2433                 tp = (const struct trace_params *)&t.sip;
2434                 if (t.config_tx)
2435                         t3_config_trace_filter(adapter, tp, 0,
2436                                                 t.invert_match,
2437                                                 t.trace_tx);
2438                 if (t.config_rx)
2439                         t3_config_trace_filter(adapter, tp, 1,
2440                                                 t.invert_match,
2441                                                 t.trace_rx);
2442                 break;
2443         }
2444         default:
2445                 return -EOPNOTSUPP;
2446         }
2447         return 0;
2448 }
2449
2450 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2451 {
2452         struct mii_ioctl_data *data = if_mii(req);
2453         struct port_info *pi = netdev_priv(dev);
2454         struct adapter *adapter = pi->adapter;
2455
2456         switch (cmd) {
2457         case SIOCGMIIREG:
2458         case SIOCSMIIREG:
2459                 /* Convert phy_id from older PRTAD/DEVAD format */
2460                 if (is_10G(adapter) &&
2461                     !mdio_phy_id_is_c45(data->phy_id) &&
2462                     (data->phy_id & 0x1f00) &&
2463                     !(data->phy_id & 0xe0e0))
2464                         data->phy_id = mdio_phy_id_c45(data->phy_id >> 8,
2465                                                        data->phy_id & 0x1f);
2466                 /* FALLTHRU */
2467         case SIOCGMIIPHY:
2468                 return mdio_mii_ioctl(&pi->phy.mdio, data, cmd);
2469         case SIOCCHIOCTL:
2470                 return cxgb_extension_ioctl(dev, req->ifr_data);
2471         default:
2472                 return -EOPNOTSUPP;
2473         }
2474 }
2475
2476 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2477 {
2478         struct port_info *pi = netdev_priv(dev);
2479         struct adapter *adapter = pi->adapter;
2480         int ret;
2481
2482         if (new_mtu < 81)       /* accommodate SACK */
2483                 return -EINVAL;
2484         if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2485                 return ret;
2486         dev->mtu = new_mtu;
2487         init_port_mtus(adapter);
2488         if (adapter->params.rev == 0 && offload_running(adapter))
2489                 t3_load_mtus(adapter, adapter->params.mtus,
2490                              adapter->params.a_wnd, adapter->params.b_wnd,
2491                              adapter->port[0]->mtu);
2492         return 0;
2493 }
2494
2495 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2496 {
2497         struct port_info *pi = netdev_priv(dev);
2498         struct adapter *adapter = pi->adapter;
2499         struct sockaddr *addr = p;
2500
2501         if (!is_valid_ether_addr(addr->sa_data))
2502                 return -EINVAL;
2503
2504         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2505         t3_mac_set_address(&pi->mac, LAN_MAC_IDX, dev->dev_addr);
2506         if (offload_running(adapter))
2507                 write_smt_entry(adapter, pi->port_id);
2508         return 0;
2509 }
2510
2511 /**
2512  * t3_synchronize_rx - wait for current Rx processing on a port to complete
2513  * @adap: the adapter
2514  * @p: the port
2515  *
2516  * Ensures that current Rx processing on any of the queues associated with
2517  * the given port completes before returning.  We do this by acquiring and
2518  * releasing the locks of the response queues associated with the port.
2519  */
2520 static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
2521 {
2522         int i;
2523
2524         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
2525                 struct sge_rspq *q = &adap->sge.qs[i].rspq;
2526
2527                 spin_lock_irq(&q->lock);
2528                 spin_unlock_irq(&q->lock);
2529         }
2530 }
2531
2532 static void cxgb_vlan_mode(struct net_device *dev, netdev_features_t features)
2533 {
2534         struct port_info *pi = netdev_priv(dev);
2535         struct adapter *adapter = pi->adapter;
2536
2537         if (adapter->params.rev > 0) {
2538                 t3_set_vlan_accel(adapter, 1 << pi->port_id,
2539                                   features & NETIF_F_HW_VLAN_RX);
2540         } else {
2541                 /* single control for all ports */
2542                 unsigned int i, have_vlans = features & NETIF_F_HW_VLAN_RX;
2543
2544                 for_each_port(adapter, i)
2545                         have_vlans |=
2546                                 adapter->port[i]->features & NETIF_F_HW_VLAN_RX;
2547
2548                 t3_set_vlan_accel(adapter, 1, have_vlans);
2549         }
2550         t3_synchronize_rx(adapter, pi);
2551 }
2552
2553 static netdev_features_t cxgb_fix_features(struct net_device *dev,
2554         netdev_features_t features)
2555 {
2556         /*
2557          * Since there is no support for separate rx/tx vlan accel
2558          * enable/disable make sure tx flag is always in same state as rx.
2559          */
2560         if (features & NETIF_F_HW_VLAN_RX)
2561                 features |= NETIF_F_HW_VLAN_TX;
2562         else
2563                 features &= ~NETIF_F_HW_VLAN_TX;
2564
2565         return features;
2566 }
2567
2568 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
2569 {
2570         netdev_features_t changed = dev->features ^ features;
2571
2572         if (changed & NETIF_F_HW_VLAN_RX)
2573                 cxgb_vlan_mode(dev, features);
2574
2575         return 0;
2576 }
2577
2578 #ifdef CONFIG_NET_POLL_CONTROLLER
2579 static void cxgb_netpoll(struct net_device *dev)
2580 {
2581         struct port_info *pi = netdev_priv(dev);
2582         struct adapter *adapter = pi->adapter;
2583         int qidx;
2584
2585         for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2586                 struct sge_qset *qs = &adapter->sge.qs[qidx];
2587                 void *source;
2588
2589                 if (adapter->flags & USING_MSIX)
2590                         source = qs;
2591                 else
2592                         source = adapter;
2593
2594                 t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2595         }
2596 }
2597 #endif
2598
2599 /*
2600  * Periodic accumulation of MAC statistics.
2601  */
2602 static void mac_stats_update(struct adapter *adapter)
2603 {
2604         int i;
2605
2606         for_each_port(adapter, i) {
2607                 struct net_device *dev = adapter->port[i];
2608                 struct port_info *p = netdev_priv(dev);
2609
2610                 if (netif_running(dev)) {
2611                         spin_lock(&adapter->stats_lock);
2612                         t3_mac_update_stats(&p->mac);
2613                         spin_unlock(&adapter->stats_lock);
2614                 }
2615         }
2616 }
2617
2618 static void check_link_status(struct adapter *adapter)
2619 {
2620         int i;
2621
2622         for_each_port(adapter, i) {
2623                 struct net_device *dev = adapter->port[i];
2624                 struct port_info *p = netdev_priv(dev);
2625                 int link_fault;
2626
2627                 spin_lock_irq(&adapter->work_lock);
2628                 link_fault = p->link_fault;
2629                 spin_unlock_irq(&adapter->work_lock);
2630
2631                 if (link_fault) {
2632                         t3_link_fault(adapter, i);
2633                         continue;
2634                 }
2635
2636                 if (!(p->phy.caps & SUPPORTED_IRQ) && netif_running(dev)) {
2637                         t3_xgm_intr_disable(adapter, i);
2638                         t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2639
2640                         t3_link_changed(adapter, i);
2641                         t3_xgm_intr_enable(adapter, i);
2642                 }
2643         }
2644 }
2645
2646 static void check_t3b2_mac(struct adapter *adapter)
2647 {
2648         int i;
2649
2650         if (!rtnl_trylock())    /* synchronize with ifdown */
2651                 return;
2652
2653         for_each_port(adapter, i) {
2654                 struct net_device *dev = adapter->port[i];
2655                 struct port_info *p = netdev_priv(dev);
2656                 int status;
2657
2658                 if (!netif_running(dev))
2659                         continue;
2660
2661                 status = 0;
2662                 if (netif_running(dev) && netif_carrier_ok(dev))
2663                         status = t3b2_mac_watchdog_task(&p->mac);
2664                 if (status == 1)
2665                         p->mac.stats.num_toggled++;
2666                 else if (status == 2) {
2667                         struct cmac *mac = &p->mac;
2668
2669                         t3_mac_set_mtu(mac, dev->mtu);
2670                         t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
2671                         cxgb_set_rxmode(dev);
2672                         t3_link_start(&p->phy, mac, &p->link_config);
2673                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2674                         t3_port_intr_enable(adapter, p->port_id);
2675                         p->mac.stats.num_resets++;
2676                 }
2677         }
2678         rtnl_unlock();
2679 }
2680
2681
2682 static void t3_adap_check_task(struct work_struct *work)
2683 {
2684         struct adapter *adapter = container_of(work, struct adapter,
2685                                                adap_check_task.work);
2686         const struct adapter_params *p = &adapter->params;
2687         int port;
2688         unsigned int v, status, reset;
2689
2690         adapter->check_task_cnt++;
2691
2692         check_link_status(adapter);
2693
2694         /* Accumulate MAC stats if needed */
2695         if (!p->linkpoll_period ||
2696             (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2697             p->stats_update_period) {
2698                 mac_stats_update(adapter);
2699                 adapter->check_task_cnt = 0;
2700         }
2701
2702         if (p->rev == T3_REV_B2)
2703                 check_t3b2_mac(adapter);
2704
2705         /*
2706          * Scan the XGMAC's to check for various conditions which we want to
2707          * monitor in a periodic polling manner rather than via an interrupt
2708          * condition.  This is used for conditions which would otherwise flood
2709          * the system with interrupts and we only really need to know that the
2710          * conditions are "happening" ...  For each condition we count the
2711          * detection of the condition and reset it for the next polling loop.
2712          */
2713         for_each_port(adapter, port) {
2714                 struct cmac *mac =  &adap2pinfo(adapter, port)->mac;
2715                 u32 cause;
2716
2717                 cause = t3_read_reg(adapter, A_XGM_INT_CAUSE + mac->offset);
2718                 reset = 0;
2719                 if (cause & F_RXFIFO_OVERFLOW) {
2720                         mac->stats.rx_fifo_ovfl++;
2721                         reset |= F_RXFIFO_OVERFLOW;
2722                 }
2723
2724                 t3_write_reg(adapter, A_XGM_INT_CAUSE + mac->offset, reset);
2725         }
2726
2727         /*
2728          * We do the same as above for FL_EMPTY interrupts.
2729          */
2730         status = t3_read_reg(adapter, A_SG_INT_CAUSE);
2731         reset = 0;
2732
2733         if (status & F_FLEMPTY) {
2734                 struct sge_qset *qs = &adapter->sge.qs[0];
2735                 int i = 0;
2736
2737                 reset |= F_FLEMPTY;
2738
2739                 v = (t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS) >> S_FL0EMPTY) &
2740                     0xffff;
2741
2742                 while (v) {
2743                         qs->fl[i].empty += (v & 1);
2744                         if (i)
2745                                 qs++;
2746                         i ^= 1;
2747                         v >>= 1;
2748                 }
2749         }
2750
2751         t3_write_reg(adapter, A_SG_INT_CAUSE, reset);
2752
2753         /* Schedule the next check update if any port is active. */
2754         spin_lock_irq(&adapter->work_lock);
2755         if (adapter->open_device_map & PORT_MASK)
2756                 schedule_chk_task(adapter);
2757         spin_unlock_irq(&adapter->work_lock);
2758 }
2759
2760 static void db_full_task(struct work_struct *work)
2761 {
2762         struct adapter *adapter = container_of(work, struct adapter,
2763                                                db_full_task);
2764
2765         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_FULL, 0);
2766 }
2767
2768 static void db_empty_task(struct work_struct *work)
2769 {
2770         struct adapter *adapter = container_of(work, struct adapter,
2771                                                db_empty_task);
2772
2773         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_EMPTY, 0);
2774 }
2775
2776 static void db_drop_task(struct work_struct *work)
2777 {
2778         struct adapter *adapter = container_of(work, struct adapter,
2779                                                db_drop_task);
2780         unsigned long delay = 1000;
2781         unsigned short r;
2782
2783         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_DROP, 0);
2784
2785         /*
2786          * Sleep a while before ringing the driver qset dbs.
2787          * The delay is between 1000-2023 usecs.
2788          */
2789         get_random_bytes(&r, 2);
2790         delay += r & 1023;
2791         set_current_state(TASK_UNINTERRUPTIBLE);
2792         schedule_timeout(usecs_to_jiffies(delay));
2793         ring_dbs(adapter);
2794 }
2795
2796 /*
2797  * Processes external (PHY) interrupts in process context.
2798  */
2799 static void ext_intr_task(struct work_struct *work)
2800 {
2801         struct adapter *adapter = container_of(work, struct adapter,
2802                                                ext_intr_handler_task);
2803         int i;
2804
2805         /* Disable link fault interrupts */
2806         for_each_port(adapter, i) {
2807                 struct net_device *dev = adapter->port[i];
2808                 struct port_info *p = netdev_priv(dev);
2809
2810                 t3_xgm_intr_disable(adapter, i);
2811                 t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2812         }
2813
2814         /* Re-enable link fault interrupts */
2815         t3_phy_intr_handler(adapter);
2816
2817         for_each_port(adapter, i)
2818                 t3_xgm_intr_enable(adapter, i);
2819
2820         /* Now reenable external interrupts */
2821         spin_lock_irq(&adapter->work_lock);
2822         if (adapter->slow_intr_mask) {
2823                 adapter->slow_intr_mask |= F_T3DBG;
2824                 t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2825                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2826                              adapter->slow_intr_mask);
2827         }
2828         spin_unlock_irq(&adapter->work_lock);
2829 }
2830
2831 /*
2832  * Interrupt-context handler for external (PHY) interrupts.
2833  */
2834 void t3_os_ext_intr_handler(struct adapter *adapter)
2835 {
2836         /*
2837          * Schedule a task to handle external interrupts as they may be slow
2838          * and we use a mutex to protect MDIO registers.  We disable PHY
2839          * interrupts in the meantime and let the task reenable them when
2840          * it's done.
2841          */
2842         spin_lock(&adapter->work_lock);
2843         if (adapter->slow_intr_mask) {
2844                 adapter->slow_intr_mask &= ~F_T3DBG;
2845                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2846                              adapter->slow_intr_mask);
2847                 queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2848         }
2849         spin_unlock(&adapter->work_lock);
2850 }
2851
2852 void t3_os_link_fault_handler(struct adapter *adapter, int port_id)
2853 {
2854         struct net_device *netdev = adapter->port[port_id];
2855         struct port_info *pi = netdev_priv(netdev);
2856
2857         spin_lock(&adapter->work_lock);
2858         pi->link_fault = 1;
2859         spin_unlock(&adapter->work_lock);
2860 }
2861
2862 static int t3_adapter_error(struct adapter *adapter, int reset, int on_wq)
2863 {
2864         int i, ret = 0;
2865
2866         if (is_offload(adapter) &&
2867             test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2868                 cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0);
2869                 offload_close(&adapter->tdev);
2870         }
2871
2872         /* Stop all ports */
2873         for_each_port(adapter, i) {
2874                 struct net_device *netdev = adapter->port[i];
2875
2876                 if (netif_running(netdev))
2877                         __cxgb_close(netdev, on_wq);
2878         }
2879
2880         /* Stop SGE timers */
2881         t3_stop_sge_timers(adapter);
2882
2883         adapter->flags &= ~FULL_INIT_DONE;
2884
2885         if (reset)
2886                 ret = t3_reset_adapter(adapter);
2887
2888         pci_disable_device(adapter->pdev);
2889
2890         return ret;
2891 }
2892
2893 static int t3_reenable_adapter(struct adapter *adapter)
2894 {
2895         if (pci_enable_device(adapter->pdev)) {
2896                 dev_err(&adapter->pdev->dev,
2897                         "Cannot re-enable PCI device after reset.\n");
2898                 goto err;
2899         }
2900         pci_set_master(adapter->pdev);
2901         pci_restore_state(adapter->pdev);
2902         pci_save_state(adapter->pdev);
2903
2904         /* Free sge resources */
2905         t3_free_sge_resources(adapter);
2906
2907         if (t3_replay_prep_adapter(adapter))
2908                 goto err;
2909
2910         return 0;
2911 err:
2912         return -1;
2913 }
2914
2915 static void t3_resume_ports(struct adapter *adapter)
2916 {
2917         int i;
2918
2919         /* Restart the ports */
2920         for_each_port(adapter, i) {
2921                 struct net_device *netdev = adapter->port[i];
2922
2923                 if (netif_running(netdev)) {
2924                         if (cxgb_open(netdev)) {
2925                                 dev_err(&adapter->pdev->dev,
2926                                         "can't bring device back up"
2927                                         " after reset\n");
2928                                 continue;
2929                         }
2930                 }
2931         }
2932
2933         if (is_offload(adapter) && !ofld_disable)
2934                 cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0);
2935 }
2936
2937 /*
2938  * processes a fatal error.
2939  * Bring the ports down, reset the chip, bring the ports back up.
2940  */
2941 static void fatal_error_task(struct work_struct *work)
2942 {
2943         struct adapter *adapter = container_of(work, struct adapter,
2944                                                fatal_error_handler_task);
2945         int err = 0;
2946
2947         rtnl_lock();
2948         err = t3_adapter_error(adapter, 1, 1);
2949         if (!err)
2950                 err = t3_reenable_adapter(adapter);
2951         if (!err)
2952                 t3_resume_ports(adapter);
2953
2954         CH_ALERT(adapter, "adapter reset %s\n", err ? "failed" : "succeeded");
2955         rtnl_unlock();
2956 }
2957
2958 void t3_fatal_err(struct adapter *adapter)
2959 {
2960         unsigned int fw_status[4];
2961
2962         if (adapter->flags & FULL_INIT_DONE) {
2963                 t3_sge_stop(adapter);
2964                 t3_write_reg(adapter, A_XGM_TX_CTRL, 0);
2965                 t3_write_reg(adapter, A_XGM_RX_CTRL, 0);
2966                 t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0);
2967                 t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0);
2968
2969                 spin_lock(&adapter->work_lock);
2970                 t3_intr_disable(adapter);
2971                 queue_work(cxgb3_wq, &adapter->fatal_error_handler_task);
2972                 spin_unlock(&adapter->work_lock);
2973         }
2974         CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
2975         if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
2976                 CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
2977                          fw_status[0], fw_status[1],
2978                          fw_status[2], fw_status[3]);
2979 }
2980
2981 /**
2982  * t3_io_error_detected - called when PCI error is detected
2983  * @pdev: Pointer to PCI device
2984  * @state: The current pci connection state
2985  *
2986  * This function is called after a PCI bus error affecting
2987  * this device has been detected.
2988  */
2989 static pci_ers_result_t t3_io_error_detected(struct pci_dev *pdev,
2990                                              pci_channel_state_t state)
2991 {
2992         struct adapter *adapter = pci_get_drvdata(pdev);
2993
2994         if (state == pci_channel_io_perm_failure)
2995                 return PCI_ERS_RESULT_DISCONNECT;
2996
2997         t3_adapter_error(adapter, 0, 0);
2998
2999         /* Request a slot reset. */
3000         return PCI_ERS_RESULT_NEED_RESET;
3001 }
3002
3003 /**
3004  * t3_io_slot_reset - called after the pci bus has been reset.
3005  * @pdev: Pointer to PCI device
3006  *
3007  * Restart the card from scratch, as if from a cold-boot.
3008  */
3009 static pci_ers_result_t t3_io_slot_reset(struct pci_dev *pdev)
3010 {
3011         struct adapter *adapter = pci_get_drvdata(pdev);
3012
3013         if (!t3_reenable_adapter(adapter))
3014                 return PCI_ERS_RESULT_RECOVERED;
3015
3016         return PCI_ERS_RESULT_DISCONNECT;
3017 }
3018
3019 /**
3020  * t3_io_resume - called when traffic can start flowing again.
3021  * @pdev: Pointer to PCI device
3022  *
3023  * This callback is called when the error recovery driver tells us that
3024  * its OK to resume normal operation.
3025  */
3026 static void t3_io_resume(struct pci_dev *pdev)
3027 {
3028         struct adapter *adapter = pci_get_drvdata(pdev);
3029
3030         CH_ALERT(adapter, "adapter recovering, PEX ERR 0x%x\n",
3031                  t3_read_reg(adapter, A_PCIE_PEX_ERR));
3032
3033         t3_resume_ports(adapter);
3034 }
3035
3036 static struct pci_error_handlers t3_err_handler = {
3037         .error_detected = t3_io_error_detected,
3038         .slot_reset = t3_io_slot_reset,
3039         .resume = t3_io_resume,
3040 };
3041
3042 /*
3043  * Set the number of qsets based on the number of CPUs and the number of ports,
3044  * not to exceed the number of available qsets, assuming there are enough qsets
3045  * per port in HW.
3046  */
3047 static void set_nqsets(struct adapter *adap)
3048 {
3049         int i, j = 0;
3050         int num_cpus = num_online_cpus();
3051         int hwports = adap->params.nports;
3052         int nqsets = adap->msix_nvectors - 1;
3053
3054         if (adap->params.rev > 0 && adap->flags & USING_MSIX) {
3055                 if (hwports == 2 &&
3056                     (hwports * nqsets > SGE_QSETS ||
3057                      num_cpus >= nqsets / hwports))
3058                         nqsets /= hwports;
3059                 if (nqsets > num_cpus)
3060                         nqsets = num_cpus;
3061                 if (nqsets < 1 || hwports == 4)
3062                         nqsets = 1;
3063         } else
3064                 nqsets = 1;
3065
3066         for_each_port(adap, i) {
3067                 struct port_info *pi = adap2pinfo(adap, i);
3068
3069                 pi->first_qset = j;
3070                 pi->nqsets = nqsets;
3071                 j = pi->first_qset + nqsets;
3072
3073                 dev_info(&adap->pdev->dev,
3074                          "Port %d using %d queue sets.\n", i, nqsets);
3075         }
3076 }
3077
3078 static int __devinit cxgb_enable_msix(struct adapter *adap)
3079 {
3080         struct msix_entry entries[SGE_QSETS + 1];
3081         int vectors;
3082         int i, err;
3083
3084         vectors = ARRAY_SIZE(entries);
3085         for (i = 0; i < vectors; ++i)
3086                 entries[i].entry = i;
3087
3088         while ((err = pci_enable_msix(adap->pdev, entries, vectors)) > 0)
3089                 vectors = err;
3090
3091         if (err < 0)
3092                 pci_disable_msix(adap->pdev);
3093
3094         if (!err && vectors < (adap->params.nports + 1)) {
3095                 pci_disable_msix(adap->pdev);
3096                 err = -1;
3097         }
3098
3099         if (!err) {
3100                 for (i = 0; i < vectors; ++i)
3101                         adap->msix_info[i].vec = entries[i].vector;
3102                 adap->msix_nvectors = vectors;
3103         }
3104
3105         return err;
3106 }
3107
3108 static void __devinit print_port_info(struct adapter *adap,
3109                                       const struct adapter_info *ai)
3110 {
3111         static const char *pci_variant[] = {
3112                 "PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
3113         };
3114
3115         int i;
3116         char buf[80];
3117
3118         if (is_pcie(adap))
3119                 snprintf(buf, sizeof(buf), "%s x%d",
3120                          pci_variant[adap->params.pci.variant],
3121                          adap->params.pci.width);
3122         else
3123                 snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
3124                          pci_variant[adap->params.pci.variant],
3125                          adap->params.pci.speed, adap->params.pci.width);
3126
3127         for_each_port(adap, i) {
3128                 struct net_device *dev = adap->port[i];
3129                 const struct port_info *pi = netdev_priv(dev);
3130
3131                 if (!test_bit(i, &adap->registered_device_map))
3132                         continue;
3133                 printk(KERN_INFO "%s: %s %s %sNIC (rev %d) %s%s\n",
3134                        dev->name, ai->desc, pi->phy.desc,
3135                        is_offload(adap) ? "R" : "", adap->params.rev, buf,
3136                        (adap->flags & USING_MSIX) ? " MSI-X" :
3137                        (adap->flags & USING_MSI) ? " MSI" : "");
3138                 if (adap->name == dev->name && adap->params.vpd.mclk)
3139                         printk(KERN_INFO
3140                                "%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n",
3141                                adap->name, t3_mc7_size(&adap->cm) >> 20,
3142                                t3_mc7_size(&adap->pmtx) >> 20,
3143                                t3_mc7_size(&adap->pmrx) >> 20,
3144                                adap->params.vpd.sn);
3145         }
3146 }
3147
3148 static const struct net_device_ops cxgb_netdev_ops = {
3149         .ndo_open               = cxgb_open,
3150         .ndo_stop               = cxgb_close,
3151         .ndo_start_xmit         = t3_eth_xmit,
3152         .ndo_get_stats          = cxgb_get_stats,
3153         .ndo_validate_addr      = eth_validate_addr,
3154         .ndo_set_rx_mode        = cxgb_set_rxmode,
3155         .ndo_do_ioctl           = cxgb_ioctl,
3156         .ndo_change_mtu         = cxgb_change_mtu,
3157         .ndo_set_mac_address    = cxgb_set_mac_addr,
3158         .ndo_fix_features       = cxgb_fix_features,
3159         .ndo_set_features       = cxgb_set_features,
3160 #ifdef CONFIG_NET_POLL_CONTROLLER
3161         .ndo_poll_controller    = cxgb_netpoll,
3162 #endif
3163 };
3164
3165 static void __devinit cxgb3_init_iscsi_mac(struct net_device *dev)
3166 {
3167         struct port_info *pi = netdev_priv(dev);
3168
3169         memcpy(pi->iscsic.mac_addr, dev->dev_addr, ETH_ALEN);
3170         pi->iscsic.mac_addr[3] |= 0x80;
3171 }
3172
3173 static int __devinit init_one(struct pci_dev *pdev,
3174                               const struct pci_device_id *ent)
3175 {
3176         static int version_printed;
3177
3178         int i, err, pci_using_dac = 0;
3179         resource_size_t mmio_start, mmio_len;
3180         const struct adapter_info *ai;
3181         struct adapter *adapter = NULL;
3182         struct port_info *pi;
3183
3184         if (!version_printed) {
3185                 printk(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3186                 ++version_printed;
3187         }
3188
3189         if (!cxgb3_wq) {
3190                 cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
3191                 if (!cxgb3_wq) {
3192                         printk(KERN_ERR DRV_NAME
3193                                ": cannot initialize work queue\n");
3194                         return -ENOMEM;
3195                 }
3196         }
3197
3198         err = pci_enable_device(pdev);
3199         if (err) {
3200                 dev_err(&pdev->dev, "cannot enable PCI device\n");
3201                 goto out;
3202         }
3203
3204         err = pci_request_regions(pdev, DRV_NAME);
3205         if (err) {
3206                 /* Just info, some other driver may have claimed the device. */
3207                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3208                 goto out_disable_device;
3209         }
3210
3211         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3212                 pci_using_dac = 1;
3213                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3214                 if (err) {
3215                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3216                                "coherent allocations\n");
3217                         goto out_release_regions;
3218                 }
3219         } else if ((err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) != 0) {
3220                 dev_err(&pdev->dev, "no usable DMA configuration\n");
3221                 goto out_release_regions;
3222         }
3223
3224         pci_set_master(pdev);
3225         pci_save_state(pdev);
3226
3227         mmio_start = pci_resource_start(pdev, 0);
3228         mmio_len = pci_resource_len(pdev, 0);
3229         ai = t3_get_adapter_info(ent->driver_data);
3230
3231         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3232         if (!adapter) {
3233                 err = -ENOMEM;
3234                 goto out_release_regions;
3235         }
3236
3237         adapter->nofail_skb =
3238                 alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_KERNEL);
3239         if (!adapter->nofail_skb) {
3240                 dev_err(&pdev->dev, "cannot allocate nofail buffer\n");
3241                 err = -ENOMEM;
3242                 goto out_free_adapter;
3243         }
3244
3245         adapter->regs = ioremap_nocache(mmio_start, mmio_len);
3246         if (!adapter->regs) {
3247                 dev_err(&pdev->dev, "cannot map device registers\n");
3248                 err = -ENOMEM;
3249                 goto out_free_adapter;
3250         }
3251
3252         adapter->pdev = pdev;
3253         adapter->name = pci_name(pdev);
3254         adapter->msg_enable = dflt_msg_enable;
3255         adapter->mmio_len = mmio_len;
3256
3257         mutex_init(&adapter->mdio_lock);
3258         spin_lock_init(&adapter->work_lock);
3259         spin_lock_init(&adapter->stats_lock);
3260
3261         INIT_LIST_HEAD(&adapter->adapter_list);
3262         INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
3263         INIT_WORK(&adapter->fatal_error_handler_task, fatal_error_task);
3264
3265         INIT_WORK(&adapter->db_full_task, db_full_task);
3266         INIT_WORK(&adapter->db_empty_task, db_empty_task);
3267         INIT_WORK(&adapter->db_drop_task, db_drop_task);
3268
3269         INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
3270
3271         for (i = 0; i < ai->nports0 + ai->nports1; ++i) {
3272                 struct net_device *netdev;
3273
3274                 netdev = alloc_etherdev_mq(sizeof(struct port_info), SGE_QSETS);
3275                 if (!netdev) {
3276                         err = -ENOMEM;
3277                         goto out_free_dev;
3278                 }
3279
3280                 SET_NETDEV_DEV(netdev, &pdev->dev);
3281
3282                 adapter->port[i] = netdev;
3283                 pi = netdev_priv(netdev);
3284                 pi->adapter = adapter;
3285                 pi->port_id = i;
3286                 netif_carrier_off(netdev);
3287                 netdev->irq = pdev->irq;
3288                 netdev->mem_start = mmio_start;
3289                 netdev->mem_end = mmio_start + mmio_len - 1;
3290                 netdev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM |
3291                         NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_RX;
3292                 netdev->features |= netdev->hw_features | NETIF_F_HW_VLAN_TX;
3293                 if (pci_using_dac)
3294                         netdev->features |= NETIF_F_HIGHDMA;
3295
3296                 netdev->netdev_ops = &cxgb_netdev_ops;
3297                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3298         }
3299
3300         pci_set_drvdata(pdev, adapter);
3301         if (t3_prep_adapter(adapter, ai, 1) < 0) {
3302                 err = -ENODEV;
3303                 goto out_free_dev;
3304         }
3305
3306         /*
3307          * The card is now ready to go.  If any errors occur during device
3308          * registration we do not fail the whole card but rather proceed only
3309          * with the ports we manage to register successfully.  However we must
3310          * register at least one net device.
3311          */
3312         for_each_port(adapter, i) {
3313 #ifndef CONFIG_XEN
3314                 err = register_netdev(adapter->port[i]);
3315 #else
3316                 rtnl_lock();
3317                 err = register_netdevice(adapter->port[i]);
3318                 if (!err) {
3319                         adapter->port[i]->wanted_features &= ~NETIF_F_GRO;
3320                         netdev_update_features(adapter->port[i]);
3321                 }
3322                 rtnl_unlock();
3323 #endif
3324                 if (err)
3325                         dev_warn(&pdev->dev,
3326                                  "cannot register net device %s, skipping\n",
3327                                  adapter->port[i]->name);
3328                 else {
3329                         /*
3330                          * Change the name we use for messages to the name of
3331                          * the first successfully registered interface.
3332                          */
3333                         if (!adapter->registered_device_map)
3334                                 adapter->name = adapter->port[i]->name;
3335
3336                         __set_bit(i, &adapter->registered_device_map);
3337                 }
3338         }
3339         if (!adapter->registered_device_map) {
3340                 dev_err(&pdev->dev, "could not register any net devices\n");
3341                 goto out_free_dev;
3342         }
3343
3344         for_each_port(adapter, i)
3345                 cxgb3_init_iscsi_mac(adapter->port[i]);
3346
3347         /* Driver's ready. Reflect it on LEDs */
3348         t3_led_ready(adapter);
3349
3350         if (is_offload(adapter)) {
3351                 __set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
3352                 cxgb3_adapter_ofld(adapter);
3353         }
3354
3355         /* See what interrupts we'll be using */
3356         if (msi > 1 && cxgb_enable_msix(adapter) == 0)
3357                 adapter->flags |= USING_MSIX;
3358         else if (msi > 0 && pci_enable_msi(pdev) == 0)
3359                 adapter->flags |= USING_MSI;
3360
3361         set_nqsets(adapter);
3362
3363         err = sysfs_create_group(&adapter->port[0]->dev.kobj,
3364                                  &cxgb3_attr_group);
3365
3366         for_each_port(adapter, i)
3367                 cxgb_vlan_mode(adapter->port[i], adapter->port[i]->features);
3368
3369         print_port_info(adapter, ai);
3370         return 0;
3371
3372 out_free_dev:
3373         iounmap(adapter->regs);
3374         for (i = ai->nports0 + ai->nports1 - 1; i >= 0; --i)
3375                 if (adapter->port[i])
3376                         free_netdev(adapter->port[i]);
3377
3378 out_free_adapter:
3379         kfree(adapter);
3380
3381 out_release_regions:
3382         pci_release_regions(pdev);
3383 out_disable_device:
3384         pci_disable_device(pdev);
3385         pci_set_drvdata(pdev, NULL);
3386 out:
3387         return err;
3388 }
3389
3390 static void __devexit remove_one(struct pci_dev *pdev)
3391 {
3392         struct adapter *adapter = pci_get_drvdata(pdev);
3393
3394         if (adapter) {
3395                 int i;
3396
3397                 t3_sge_stop(adapter);
3398                 sysfs_remove_group(&adapter->port[0]->dev.kobj,
3399                                    &cxgb3_attr_group);
3400
3401                 if (is_offload(adapter)) {
3402                         cxgb3_adapter_unofld(adapter);
3403                         if (test_bit(OFFLOAD_DEVMAP_BIT,
3404                                      &adapter->open_device_map))
3405                                 offload_close(&adapter->tdev);
3406                 }
3407
3408                 for_each_port(adapter, i)
3409                     if (test_bit(i, &adapter->registered_device_map))
3410                         unregister_netdev(adapter->port[i]);
3411
3412                 t3_stop_sge_timers(adapter);
3413                 t3_free_sge_resources(adapter);
3414                 cxgb_disable_msi(adapter);
3415
3416                 for_each_port(adapter, i)
3417                         if (adapter->port[i])
3418                                 free_netdev(adapter->port[i]);
3419
3420                 iounmap(adapter->regs);
3421                 if (adapter->nofail_skb)
3422                         kfree_skb(adapter->nofail_skb);
3423                 kfree(adapter);
3424                 pci_release_regions(pdev);
3425                 pci_disable_device(pdev);
3426                 pci_set_drvdata(pdev, NULL);
3427         }
3428 }
3429
3430 static struct pci_driver driver = {
3431         .name = DRV_NAME,
3432         .id_table = cxgb3_pci_tbl,
3433         .probe = init_one,
3434         .remove = __devexit_p(remove_one),
3435         .err_handler = &t3_err_handler,
3436 };
3437
3438 static int __init cxgb3_init_module(void)
3439 {
3440         int ret;
3441
3442         cxgb3_offload_init();
3443
3444         ret = pci_register_driver(&driver);
3445         return ret;
3446 }
3447
3448 static void __exit cxgb3_cleanup_module(void)
3449 {
3450         pci_unregister_driver(&driver);
3451         if (cxgb3_wq)
3452                 destroy_workqueue(cxgb3_wq);
3453 }
3454
3455 module_init(cxgb3_init_module);
3456 module_exit(cxgb3_cleanup_module);