cxgb4: don't offload Rx checksums for IPv6 fragments
[linux-2.6.git] / drivers / net / cxgb4 / cxgb4_main.c
1 /*
2  * This file is part of the Chelsio T4 Ethernet driver for Linux.
3  *
4  * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  */
34
35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37 #include <linux/bitmap.h>
38 #include <linux/crc32.h>
39 #include <linux/ctype.h>
40 #include <linux/debugfs.h>
41 #include <linux/err.h>
42 #include <linux/etherdevice.h>
43 #include <linux/firmware.h>
44 #include <linux/if_vlan.h>
45 #include <linux/init.h>
46 #include <linux/log2.h>
47 #include <linux/mdio.h>
48 #include <linux/module.h>
49 #include <linux/moduleparam.h>
50 #include <linux/mutex.h>
51 #include <linux/netdevice.h>
52 #include <linux/pci.h>
53 #include <linux/aer.h>
54 #include <linux/rtnetlink.h>
55 #include <linux/sched.h>
56 #include <linux/seq_file.h>
57 #include <linux/sockios.h>
58 #include <linux/vmalloc.h>
59 #include <linux/workqueue.h>
60 #include <net/neighbour.h>
61 #include <net/netevent.h>
62 #include <asm/uaccess.h>
63
64 #include "cxgb4.h"
65 #include "t4_regs.h"
66 #include "t4_msg.h"
67 #include "t4fw_api.h"
68 #include "l2t.h"
69
70 #define DRV_VERSION "1.0.0-ko"
71 #define DRV_DESC "Chelsio T4 Network Driver"
72
73 /*
74  * Max interrupt hold-off timer value in us.  Queues fall back to this value
75  * under extreme memory pressure so it's largish to give the system time to
76  * recover.
77  */
78 #define MAX_SGE_TIMERVAL 200U
79
80 #ifdef CONFIG_PCI_IOV
81 /*
82  * Virtual Function provisioning constants.  We need two extra Ingress Queues
83  * with Interrupt capability to serve as the VF's Firmware Event Queue and
84  * Forwarded Interrupt Queue (when using MSI mode) -- neither will have Free
85  * Lists associated with them).  For each Ethernet/Control Egress Queue and
86  * for each Free List, we need an Egress Context.
87  */
88 enum {
89         VFRES_NPORTS = 1,               /* # of "ports" per VF */
90         VFRES_NQSETS = 2,               /* # of "Queue Sets" per VF */
91
92         VFRES_NVI = VFRES_NPORTS,       /* # of Virtual Interfaces */
93         VFRES_NETHCTRL = VFRES_NQSETS,  /* # of EQs used for ETH or CTRL Qs */
94         VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
95         VFRES_NIQ = 0,                  /* # of non-fl/int ingress queues */
96         VFRES_NEQ = VFRES_NQSETS*2,     /* # of egress queues */
97         VFRES_TC = 0,                   /* PCI-E traffic class */
98         VFRES_NEXACTF = 16,             /* # of exact MPS filters */
99
100         VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
101         VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
102 };
103
104 /*
105  * Provide a Port Access Rights Mask for the specified PF/VF.  This is very
106  * static and likely not to be useful in the long run.  We really need to
107  * implement some form of persistent configuration which the firmware
108  * controls.
109  */
110 static unsigned int pfvfres_pmask(struct adapter *adapter,
111                                   unsigned int pf, unsigned int vf)
112 {
113         unsigned int portn, portvec;
114
115         /*
116          * Give PF's access to all of the ports.
117          */
118         if (vf == 0)
119                 return FW_PFVF_CMD_PMASK_MASK;
120
121         /*
122          * For VFs, we'll assign them access to the ports based purely on the
123          * PF.  We assign active ports in order, wrapping around if there are
124          * fewer active ports than PFs: e.g. active port[pf % nports].
125          * Unfortunately the adapter's port_info structs haven't been
126          * initialized yet so we have to compute this.
127          */
128         if (adapter->params.nports == 0)
129                 return 0;
130
131         portn = pf % adapter->params.nports;
132         portvec = adapter->params.portvec;
133         for (;;) {
134                 /*
135                  * Isolate the lowest set bit in the port vector.  If we're at
136                  * the port number that we want, return that as the pmask.
137                  * otherwise mask that bit out of the port vector and
138                  * decrement our port number ...
139                  */
140                 unsigned int pmask = portvec ^ (portvec & (portvec-1));
141                 if (portn == 0)
142                         return pmask;
143                 portn--;
144                 portvec &= ~pmask;
145         }
146         /*NOTREACHED*/
147 }
148 #endif
149
150 enum {
151         MEMWIN0_APERTURE = 65536,
152         MEMWIN0_BASE     = 0x30000,
153         MEMWIN1_APERTURE = 32768,
154         MEMWIN1_BASE     = 0x28000,
155         MEMWIN2_APERTURE = 2048,
156         MEMWIN2_BASE     = 0x1b800,
157 };
158
159 enum {
160         MAX_TXQ_ENTRIES      = 16384,
161         MAX_CTRL_TXQ_ENTRIES = 1024,
162         MAX_RSPQ_ENTRIES     = 16384,
163         MAX_RX_BUFFERS       = 16384,
164         MIN_TXQ_ENTRIES      = 32,
165         MIN_CTRL_TXQ_ENTRIES = 32,
166         MIN_RSPQ_ENTRIES     = 128,
167         MIN_FL_ENTRIES       = 16
168 };
169
170 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
171                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
172                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
173
174 #define CH_DEVICE(devid) { PCI_VDEVICE(CHELSIO, devid), 0 }
175
176 static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
177         CH_DEVICE(0xa000),  /* PE10K */
178         { 0, }
179 };
180
181 #define FW_FNAME "cxgb4/t4fw.bin"
182
183 MODULE_DESCRIPTION(DRV_DESC);
184 MODULE_AUTHOR("Chelsio Communications");
185 MODULE_LICENSE("Dual BSD/GPL");
186 MODULE_VERSION(DRV_VERSION);
187 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
188 MODULE_FIRMWARE(FW_FNAME);
189
190 static int dflt_msg_enable = DFLT_MSG_ENABLE;
191
192 module_param(dflt_msg_enable, int, 0644);
193 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
194
195 /*
196  * The driver uses the best interrupt scheme available on a platform in the
197  * order MSI-X, MSI, legacy INTx interrupts.  This parameter determines which
198  * of these schemes the driver may consider as follows:
199  *
200  * msi = 2: choose from among all three options
201  * msi = 1: only consider MSI and INTx interrupts
202  * msi = 0: force INTx interrupts
203  */
204 static int msi = 2;
205
206 module_param(msi, int, 0644);
207 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
208
209 /*
210  * Queue interrupt hold-off timer values.  Queues default to the first of these
211  * upon creation.
212  */
213 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
214
215 module_param_array(intr_holdoff, uint, NULL, 0644);
216 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
217                  "0..4 in microseconds");
218
219 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
220
221 module_param_array(intr_cnt, uint, NULL, 0644);
222 MODULE_PARM_DESC(intr_cnt,
223                  "thresholds 1..3 for queue interrupt packet counters");
224
225 static int vf_acls;
226
227 #ifdef CONFIG_PCI_IOV
228 module_param(vf_acls, bool, 0644);
229 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
230
231 static unsigned int num_vf[4];
232
233 module_param_array(num_vf, uint, NULL, 0644);
234 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
235 #endif
236
237 static struct dentry *cxgb4_debugfs_root;
238
239 static LIST_HEAD(adapter_list);
240 static DEFINE_MUTEX(uld_mutex);
241 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
242 static const char *uld_str[] = { "RDMA", "iSCSI" };
243
244 static void link_report(struct net_device *dev)
245 {
246         if (!netif_carrier_ok(dev))
247                 netdev_info(dev, "link down\n");
248         else {
249                 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
250
251                 const char *s = "10Mbps";
252                 const struct port_info *p = netdev_priv(dev);
253
254                 switch (p->link_cfg.speed) {
255                 case SPEED_10000:
256                         s = "10Gbps";
257                         break;
258                 case SPEED_1000:
259                         s = "1000Mbps";
260                         break;
261                 case SPEED_100:
262                         s = "100Mbps";
263                         break;
264                 }
265
266                 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
267                             fc[p->link_cfg.fc]);
268         }
269 }
270
271 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
272 {
273         struct net_device *dev = adapter->port[port_id];
274
275         /* Skip changes from disabled ports. */
276         if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
277                 if (link_stat)
278                         netif_carrier_on(dev);
279                 else
280                         netif_carrier_off(dev);
281
282                 link_report(dev);
283         }
284 }
285
286 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
287 {
288         static const char *mod_str[] = {
289                 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
290         };
291
292         const struct net_device *dev = adap->port[port_id];
293         const struct port_info *pi = netdev_priv(dev);
294
295         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
296                 netdev_info(dev, "port module unplugged\n");
297         else if (pi->mod_type < ARRAY_SIZE(mod_str))
298                 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
299 }
300
301 /*
302  * Configure the exact and hash address filters to handle a port's multicast
303  * and secondary unicast MAC addresses.
304  */
305 static int set_addr_filters(const struct net_device *dev, bool sleep)
306 {
307         u64 mhash = 0;
308         u64 uhash = 0;
309         bool free = true;
310         u16 filt_idx[7];
311         const u8 *addr[7];
312         int ret, naddr = 0;
313         const struct netdev_hw_addr *ha;
314         int uc_cnt = netdev_uc_count(dev);
315         int mc_cnt = netdev_mc_count(dev);
316         const struct port_info *pi = netdev_priv(dev);
317
318         /* first do the secondary unicast addresses */
319         netdev_for_each_uc_addr(ha, dev) {
320                 addr[naddr++] = ha->addr;
321                 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
322                         ret = t4_alloc_mac_filt(pi->adapter, 0, pi->viid, free,
323                                         naddr, addr, filt_idx, &uhash, sleep);
324                         if (ret < 0)
325                                 return ret;
326
327                         free = false;
328                         naddr = 0;
329                 }
330         }
331
332         /* next set up the multicast addresses */
333         netdev_for_each_mc_addr(ha, dev) {
334                 addr[naddr++] = ha->addr;
335                 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
336                         ret = t4_alloc_mac_filt(pi->adapter, 0, pi->viid, free,
337                                         naddr, addr, filt_idx, &mhash, sleep);
338                         if (ret < 0)
339                                 return ret;
340
341                         free = false;
342                         naddr = 0;
343                 }
344         }
345
346         return t4_set_addr_hash(pi->adapter, 0, pi->viid, uhash != 0,
347                                 uhash | mhash, sleep);
348 }
349
350 /*
351  * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
352  * If @mtu is -1 it is left unchanged.
353  */
354 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
355 {
356         int ret;
357         struct port_info *pi = netdev_priv(dev);
358
359         ret = set_addr_filters(dev, sleep_ok);
360         if (ret == 0)
361                 ret = t4_set_rxmode(pi->adapter, 0, pi->viid, mtu,
362                                     (dev->flags & IFF_PROMISC) ? 1 : 0,
363                                     (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
364                                     sleep_ok);
365         return ret;
366 }
367
368 /**
369  *      link_start - enable a port
370  *      @dev: the port to enable
371  *
372  *      Performs the MAC and PHY actions needed to enable a port.
373  */
374 static int link_start(struct net_device *dev)
375 {
376         int ret;
377         struct port_info *pi = netdev_priv(dev);
378
379         /*
380          * We do not set address filters and promiscuity here, the stack does
381          * that step explicitly.
382          */
383         ret = t4_set_rxmode(pi->adapter, 0, pi->viid, dev->mtu, -1, -1, -1,
384                             pi->vlan_grp != NULL, true);
385         if (ret == 0) {
386                 ret = t4_change_mac(pi->adapter, 0, pi->viid,
387                                     pi->xact_addr_filt, dev->dev_addr, true,
388                                     true);
389                 if (ret >= 0) {
390                         pi->xact_addr_filt = ret;
391                         ret = 0;
392                 }
393         }
394         if (ret == 0)
395                 ret = t4_link_start(pi->adapter, 0, pi->tx_chan, &pi->link_cfg);
396         if (ret == 0)
397                 ret = t4_enable_vi(pi->adapter, 0, pi->viid, true, true);
398         return ret;
399 }
400
401 /*
402  * Response queue handler for the FW event queue.
403  */
404 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
405                           const struct pkt_gl *gl)
406 {
407         u8 opcode = ((const struct rss_header *)rsp)->opcode;
408
409         rsp++;                                          /* skip RSS header */
410         if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
411                 const struct cpl_sge_egr_update *p = (void *)rsp;
412                 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
413                 struct sge_txq *txq = q->adap->sge.egr_map[qid];
414
415                 txq->restarts++;
416                 if ((u8 *)txq < (u8 *)q->adap->sge.ethrxq) {
417                         struct sge_eth_txq *eq;
418
419                         eq = container_of(txq, struct sge_eth_txq, q);
420                         netif_tx_wake_queue(eq->txq);
421                 } else {
422                         struct sge_ofld_txq *oq;
423
424                         oq = container_of(txq, struct sge_ofld_txq, q);
425                         tasklet_schedule(&oq->qresume_tsk);
426                 }
427         } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
428                 const struct cpl_fw6_msg *p = (void *)rsp;
429
430                 if (p->type == 0)
431                         t4_handle_fw_rpl(q->adap, p->data);
432         } else if (opcode == CPL_L2T_WRITE_RPL) {
433                 const struct cpl_l2t_write_rpl *p = (void *)rsp;
434
435                 do_l2t_write_rpl(q->adap, p);
436         } else
437                 dev_err(q->adap->pdev_dev,
438                         "unexpected CPL %#x on FW event queue\n", opcode);
439         return 0;
440 }
441
442 /**
443  *      uldrx_handler - response queue handler for ULD queues
444  *      @q: the response queue that received the packet
445  *      @rsp: the response queue descriptor holding the offload message
446  *      @gl: the gather list of packet fragments
447  *
448  *      Deliver an ingress offload packet to a ULD.  All processing is done by
449  *      the ULD, we just maintain statistics.
450  */
451 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
452                          const struct pkt_gl *gl)
453 {
454         struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
455
456         if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
457                 rxq->stats.nomem++;
458                 return -1;
459         }
460         if (gl == NULL)
461                 rxq->stats.imm++;
462         else if (gl == CXGB4_MSG_AN)
463                 rxq->stats.an++;
464         else
465                 rxq->stats.pkts++;
466         return 0;
467 }
468
469 static void disable_msi(struct adapter *adapter)
470 {
471         if (adapter->flags & USING_MSIX) {
472                 pci_disable_msix(adapter->pdev);
473                 adapter->flags &= ~USING_MSIX;
474         } else if (adapter->flags & USING_MSI) {
475                 pci_disable_msi(adapter->pdev);
476                 adapter->flags &= ~USING_MSI;
477         }
478 }
479
480 /*
481  * Interrupt handler for non-data events used with MSI-X.
482  */
483 static irqreturn_t t4_nondata_intr(int irq, void *cookie)
484 {
485         struct adapter *adap = cookie;
486
487         u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
488         if (v & PFSW) {
489                 adap->swintr = 1;
490                 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
491         }
492         t4_slow_intr_handler(adap);
493         return IRQ_HANDLED;
494 }
495
496 /*
497  * Name the MSI-X interrupts.
498  */
499 static void name_msix_vecs(struct adapter *adap)
500 {
501         int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc) - 1;
502
503         /* non-data interrupts */
504         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
505         adap->msix_info[0].desc[n] = 0;
506
507         /* FW events */
508         snprintf(adap->msix_info[1].desc, n, "%s-FWeventq", adap->name);
509         adap->msix_info[1].desc[n] = 0;
510
511         /* Ethernet queues */
512         for_each_port(adap, j) {
513                 struct net_device *d = adap->port[j];
514                 const struct port_info *pi = netdev_priv(d);
515
516                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
517                         snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
518                                  d->name, i);
519                         adap->msix_info[msi_idx].desc[n] = 0;
520                 }
521         }
522
523         /* offload queues */
524         for_each_ofldrxq(&adap->sge, i) {
525                 snprintf(adap->msix_info[msi_idx].desc, n, "%s-ofld%d",
526                          adap->name, i);
527                 adap->msix_info[msi_idx++].desc[n] = 0;
528         }
529         for_each_rdmarxq(&adap->sge, i) {
530                 snprintf(adap->msix_info[msi_idx].desc, n, "%s-rdma%d",
531                          adap->name, i);
532                 adap->msix_info[msi_idx++].desc[n] = 0;
533         }
534 }
535
536 static int request_msix_queue_irqs(struct adapter *adap)
537 {
538         struct sge *s = &adap->sge;
539         int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi = 2;
540
541         err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
542                           adap->msix_info[1].desc, &s->fw_evtq);
543         if (err)
544                 return err;
545
546         for_each_ethrxq(s, ethqidx) {
547                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
548                                   adap->msix_info[msi].desc,
549                                   &s->ethrxq[ethqidx].rspq);
550                 if (err)
551                         goto unwind;
552                 msi++;
553         }
554         for_each_ofldrxq(s, ofldqidx) {
555                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
556                                   adap->msix_info[msi].desc,
557                                   &s->ofldrxq[ofldqidx].rspq);
558                 if (err)
559                         goto unwind;
560                 msi++;
561         }
562         for_each_rdmarxq(s, rdmaqidx) {
563                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
564                                   adap->msix_info[msi].desc,
565                                   &s->rdmarxq[rdmaqidx].rspq);
566                 if (err)
567                         goto unwind;
568                 msi++;
569         }
570         return 0;
571
572 unwind:
573         while (--rdmaqidx >= 0)
574                 free_irq(adap->msix_info[--msi].vec,
575                          &s->rdmarxq[rdmaqidx].rspq);
576         while (--ofldqidx >= 0)
577                 free_irq(adap->msix_info[--msi].vec,
578                          &s->ofldrxq[ofldqidx].rspq);
579         while (--ethqidx >= 0)
580                 free_irq(adap->msix_info[--msi].vec, &s->ethrxq[ethqidx].rspq);
581         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
582         return err;
583 }
584
585 static void free_msix_queue_irqs(struct adapter *adap)
586 {
587         int i, msi = 2;
588         struct sge *s = &adap->sge;
589
590         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
591         for_each_ethrxq(s, i)
592                 free_irq(adap->msix_info[msi++].vec, &s->ethrxq[i].rspq);
593         for_each_ofldrxq(s, i)
594                 free_irq(adap->msix_info[msi++].vec, &s->ofldrxq[i].rspq);
595         for_each_rdmarxq(s, i)
596                 free_irq(adap->msix_info[msi++].vec, &s->rdmarxq[i].rspq);
597 }
598
599 /**
600  *      write_rss - write the RSS table for a given port
601  *      @pi: the port
602  *      @queues: array of queue indices for RSS
603  *
604  *      Sets up the portion of the HW RSS table for the port's VI to distribute
605  *      packets to the Rx queues in @queues.
606  */
607 static int write_rss(const struct port_info *pi, const u16 *queues)
608 {
609         u16 *rss;
610         int i, err;
611         const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
612
613         rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
614         if (!rss)
615                 return -ENOMEM;
616
617         /* map the queue indices to queue ids */
618         for (i = 0; i < pi->rss_size; i++, queues++)
619                 rss[i] = q[*queues].rspq.abs_id;
620
621         err = t4_config_rss_range(pi->adapter, 0, pi->viid, 0, pi->rss_size,
622                                   rss, pi->rss_size);
623         kfree(rss);
624         return err;
625 }
626
627 /**
628  *      setup_rss - configure RSS
629  *      @adap: the adapter
630  *
631  *      Sets up RSS for each port.
632  */
633 static int setup_rss(struct adapter *adap)
634 {
635         int i, err;
636
637         for_each_port(adap, i) {
638                 const struct port_info *pi = adap2pinfo(adap, i);
639
640                 err = write_rss(pi, pi->rss);
641                 if (err)
642                         return err;
643         }
644         return 0;
645 }
646
647 /*
648  * Wait until all NAPI handlers are descheduled.
649  */
650 static void quiesce_rx(struct adapter *adap)
651 {
652         int i;
653
654         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
655                 struct sge_rspq *q = adap->sge.ingr_map[i];
656
657                 if (q && q->handler)
658                         napi_disable(&q->napi);
659         }
660 }
661
662 /*
663  * Enable NAPI scheduling and interrupt generation for all Rx queues.
664  */
665 static void enable_rx(struct adapter *adap)
666 {
667         int i;
668
669         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
670                 struct sge_rspq *q = adap->sge.ingr_map[i];
671
672                 if (!q)
673                         continue;
674                 if (q->handler)
675                         napi_enable(&q->napi);
676                 /* 0-increment GTS to start the timer and enable interrupts */
677                 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
678                              SEINTARM(q->intr_params) |
679                              INGRESSQID(q->cntxt_id));
680         }
681 }
682
683 /**
684  *      setup_sge_queues - configure SGE Tx/Rx/response queues
685  *      @adap: the adapter
686  *
687  *      Determines how many sets of SGE queues to use and initializes them.
688  *      We support multiple queue sets per port if we have MSI-X, otherwise
689  *      just one queue set per port.
690  */
691 static int setup_sge_queues(struct adapter *adap)
692 {
693         int err, msi_idx, i, j;
694         struct sge *s = &adap->sge;
695
696         bitmap_zero(s->starving_fl, MAX_EGRQ);
697         bitmap_zero(s->txq_maperr, MAX_EGRQ);
698
699         if (adap->flags & USING_MSIX)
700                 msi_idx = 1;         /* vector 0 is for non-queue interrupts */
701         else {
702                 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
703                                        NULL, NULL);
704                 if (err)
705                         return err;
706                 msi_idx = -((int)s->intrq.abs_id + 1);
707         }
708
709         err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
710                                msi_idx, NULL, fwevtq_handler);
711         if (err) {
712 freeout:        t4_free_sge_resources(adap);
713                 return err;
714         }
715
716         for_each_port(adap, i) {
717                 struct net_device *dev = adap->port[i];
718                 struct port_info *pi = netdev_priv(dev);
719                 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
720                 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
721
722                 for (j = 0; j < pi->nqsets; j++, q++) {
723                         if (msi_idx > 0)
724                                 msi_idx++;
725                         err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
726                                                msi_idx, &q->fl,
727                                                t4_ethrx_handler);
728                         if (err)
729                                 goto freeout;
730                         q->rspq.idx = j;
731                         memset(&q->stats, 0, sizeof(q->stats));
732                 }
733                 for (j = 0; j < pi->nqsets; j++, t++) {
734                         err = t4_sge_alloc_eth_txq(adap, t, dev,
735                                         netdev_get_tx_queue(dev, j),
736                                         s->fw_evtq.cntxt_id);
737                         if (err)
738                                 goto freeout;
739                 }
740         }
741
742         j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
743         for_each_ofldrxq(s, i) {
744                 struct sge_ofld_rxq *q = &s->ofldrxq[i];
745                 struct net_device *dev = adap->port[i / j];
746
747                 if (msi_idx > 0)
748                         msi_idx++;
749                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
750                                        &q->fl, uldrx_handler);
751                 if (err)
752                         goto freeout;
753                 memset(&q->stats, 0, sizeof(q->stats));
754                 s->ofld_rxq[i] = q->rspq.abs_id;
755                 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
756                                             s->fw_evtq.cntxt_id);
757                 if (err)
758                         goto freeout;
759         }
760
761         for_each_rdmarxq(s, i) {
762                 struct sge_ofld_rxq *q = &s->rdmarxq[i];
763
764                 if (msi_idx > 0)
765                         msi_idx++;
766                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
767                                        msi_idx, &q->fl, uldrx_handler);
768                 if (err)
769                         goto freeout;
770                 memset(&q->stats, 0, sizeof(q->stats));
771                 s->rdma_rxq[i] = q->rspq.abs_id;
772         }
773
774         for_each_port(adap, i) {
775                 /*
776                  * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
777                  * have RDMA queues, and that's the right value.
778                  */
779                 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
780                                             s->fw_evtq.cntxt_id,
781                                             s->rdmarxq[i].rspq.cntxt_id);
782                 if (err)
783                         goto freeout;
784         }
785
786         t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
787                      RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
788                      QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
789         return 0;
790 }
791
792 /*
793  * Returns 0 if new FW was successfully loaded, a positive errno if a load was
794  * started but failed, and a negative errno if flash load couldn't start.
795  */
796 static int upgrade_fw(struct adapter *adap)
797 {
798         int ret;
799         u32 vers;
800         const struct fw_hdr *hdr;
801         const struct firmware *fw;
802         struct device *dev = adap->pdev_dev;
803
804         ret = request_firmware(&fw, FW_FNAME, dev);
805         if (ret < 0) {
806                 dev_err(dev, "unable to load firmware image " FW_FNAME
807                         ", error %d\n", ret);
808                 return ret;
809         }
810
811         hdr = (const struct fw_hdr *)fw->data;
812         vers = ntohl(hdr->fw_ver);
813         if (FW_HDR_FW_VER_MAJOR_GET(vers) != FW_VERSION_MAJOR) {
814                 ret = -EINVAL;              /* wrong major version, won't do */
815                 goto out;
816         }
817
818         /*
819          * If the flash FW is unusable or we found something newer, load it.
820          */
821         if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != FW_VERSION_MAJOR ||
822             vers > adap->params.fw_vers) {
823                 ret = -t4_load_fw(adap, fw->data, fw->size);
824                 if (!ret)
825                         dev_info(dev, "firmware upgraded to version %pI4 from "
826                                  FW_FNAME "\n", &hdr->fw_ver);
827         }
828 out:    release_firmware(fw);
829         return ret;
830 }
831
832 /*
833  * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
834  * The allocated memory is cleared.
835  */
836 void *t4_alloc_mem(size_t size)
837 {
838         void *p = kmalloc(size, GFP_KERNEL);
839
840         if (!p)
841                 p = vmalloc(size);
842         if (p)
843                 memset(p, 0, size);
844         return p;
845 }
846
847 /*
848  * Free memory allocated through alloc_mem().
849  */
850 void t4_free_mem(void *addr)
851 {
852         if (is_vmalloc_addr(addr))
853                 vfree(addr);
854         else
855                 kfree(addr);
856 }
857
858 static inline int is_offload(const struct adapter *adap)
859 {
860         return adap->params.offload;
861 }
862
863 /*
864  * Implementation of ethtool operations.
865  */
866
867 static u32 get_msglevel(struct net_device *dev)
868 {
869         return netdev2adap(dev)->msg_enable;
870 }
871
872 static void set_msglevel(struct net_device *dev, u32 val)
873 {
874         netdev2adap(dev)->msg_enable = val;
875 }
876
877 static char stats_strings[][ETH_GSTRING_LEN] = {
878         "TxOctetsOK         ",
879         "TxFramesOK         ",
880         "TxBroadcastFrames  ",
881         "TxMulticastFrames  ",
882         "TxUnicastFrames    ",
883         "TxErrorFrames      ",
884
885         "TxFrames64         ",
886         "TxFrames65To127    ",
887         "TxFrames128To255   ",
888         "TxFrames256To511   ",
889         "TxFrames512To1023  ",
890         "TxFrames1024To1518 ",
891         "TxFrames1519ToMax  ",
892
893         "TxFramesDropped    ",
894         "TxPauseFrames      ",
895         "TxPPP0Frames       ",
896         "TxPPP1Frames       ",
897         "TxPPP2Frames       ",
898         "TxPPP3Frames       ",
899         "TxPPP4Frames       ",
900         "TxPPP5Frames       ",
901         "TxPPP6Frames       ",
902         "TxPPP7Frames       ",
903
904         "RxOctetsOK         ",
905         "RxFramesOK         ",
906         "RxBroadcastFrames  ",
907         "RxMulticastFrames  ",
908         "RxUnicastFrames    ",
909
910         "RxFramesTooLong    ",
911         "RxJabberErrors     ",
912         "RxFCSErrors        ",
913         "RxLengthErrors     ",
914         "RxSymbolErrors     ",
915         "RxRuntFrames       ",
916
917         "RxFrames64         ",
918         "RxFrames65To127    ",
919         "RxFrames128To255   ",
920         "RxFrames256To511   ",
921         "RxFrames512To1023  ",
922         "RxFrames1024To1518 ",
923         "RxFrames1519ToMax  ",
924
925         "RxPauseFrames      ",
926         "RxPPP0Frames       ",
927         "RxPPP1Frames       ",
928         "RxPPP2Frames       ",
929         "RxPPP3Frames       ",
930         "RxPPP4Frames       ",
931         "RxPPP5Frames       ",
932         "RxPPP6Frames       ",
933         "RxPPP7Frames       ",
934
935         "RxBG0FramesDropped ",
936         "RxBG1FramesDropped ",
937         "RxBG2FramesDropped ",
938         "RxBG3FramesDropped ",
939         "RxBG0FramesTrunc   ",
940         "RxBG1FramesTrunc   ",
941         "RxBG2FramesTrunc   ",
942         "RxBG3FramesTrunc   ",
943
944         "TSO                ",
945         "TxCsumOffload      ",
946         "RxCsumGood         ",
947         "VLANextractions    ",
948         "VLANinsertions     ",
949         "GROpackets         ",
950         "GROmerged          ",
951 };
952
953 static int get_sset_count(struct net_device *dev, int sset)
954 {
955         switch (sset) {
956         case ETH_SS_STATS:
957                 return ARRAY_SIZE(stats_strings);
958         default:
959                 return -EOPNOTSUPP;
960         }
961 }
962
963 #define T4_REGMAP_SIZE (160 * 1024)
964
965 static int get_regs_len(struct net_device *dev)
966 {
967         return T4_REGMAP_SIZE;
968 }
969
970 static int get_eeprom_len(struct net_device *dev)
971 {
972         return EEPROMSIZE;
973 }
974
975 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
976 {
977         struct adapter *adapter = netdev2adap(dev);
978
979         strcpy(info->driver, KBUILD_MODNAME);
980         strcpy(info->version, DRV_VERSION);
981         strcpy(info->bus_info, pci_name(adapter->pdev));
982
983         if (!adapter->params.fw_vers)
984                 strcpy(info->fw_version, "N/A");
985         else
986                 snprintf(info->fw_version, sizeof(info->fw_version),
987                         "%u.%u.%u.%u, TP %u.%u.%u.%u",
988                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
989                         FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
990                         FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
991                         FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
992                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
993                         FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
994                         FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
995                         FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
996 }
997
998 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
999 {
1000         if (stringset == ETH_SS_STATS)
1001                 memcpy(data, stats_strings, sizeof(stats_strings));
1002 }
1003
1004 /*
1005  * port stats maintained per queue of the port.  They should be in the same
1006  * order as in stats_strings above.
1007  */
1008 struct queue_port_stats {
1009         u64 tso;
1010         u64 tx_csum;
1011         u64 rx_csum;
1012         u64 vlan_ex;
1013         u64 vlan_ins;
1014         u64 gro_pkts;
1015         u64 gro_merged;
1016 };
1017
1018 static void collect_sge_port_stats(const struct adapter *adap,
1019                 const struct port_info *p, struct queue_port_stats *s)
1020 {
1021         int i;
1022         const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1023         const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1024
1025         memset(s, 0, sizeof(*s));
1026         for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1027                 s->tso += tx->tso;
1028                 s->tx_csum += tx->tx_cso;
1029                 s->rx_csum += rx->stats.rx_cso;
1030                 s->vlan_ex += rx->stats.vlan_ex;
1031                 s->vlan_ins += tx->vlan_ins;
1032                 s->gro_pkts += rx->stats.lro_pkts;
1033                 s->gro_merged += rx->stats.lro_merged;
1034         }
1035 }
1036
1037 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1038                       u64 *data)
1039 {
1040         struct port_info *pi = netdev_priv(dev);
1041         struct adapter *adapter = pi->adapter;
1042
1043         t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1044
1045         data += sizeof(struct port_stats) / sizeof(u64);
1046         collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1047 }
1048
1049 /*
1050  * Return a version number to identify the type of adapter.  The scheme is:
1051  * - bits 0..9: chip version
1052  * - bits 10..15: chip revision
1053  * - bits 16..23: register dump version
1054  */
1055 static inline unsigned int mk_adap_vers(const struct adapter *ap)
1056 {
1057         return 4 | (ap->params.rev << 10) | (1 << 16);
1058 }
1059
1060 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1061                            unsigned int end)
1062 {
1063         u32 *p = buf + start;
1064
1065         for ( ; start <= end; start += sizeof(u32))
1066                 *p++ = t4_read_reg(ap, start);
1067 }
1068
1069 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1070                      void *buf)
1071 {
1072         static const unsigned int reg_ranges[] = {
1073                 0x1008, 0x1108,
1074                 0x1180, 0x11b4,
1075                 0x11fc, 0x123c,
1076                 0x1300, 0x173c,
1077                 0x1800, 0x18fc,
1078                 0x3000, 0x30d8,
1079                 0x30e0, 0x5924,
1080                 0x5960, 0x59d4,
1081                 0x5a00, 0x5af8,
1082                 0x6000, 0x6098,
1083                 0x6100, 0x6150,
1084                 0x6200, 0x6208,
1085                 0x6240, 0x6248,
1086                 0x6280, 0x6338,
1087                 0x6370, 0x638c,
1088                 0x6400, 0x643c,
1089                 0x6500, 0x6524,
1090                 0x6a00, 0x6a38,
1091                 0x6a60, 0x6a78,
1092                 0x6b00, 0x6b84,
1093                 0x6bf0, 0x6c84,
1094                 0x6cf0, 0x6d84,
1095                 0x6df0, 0x6e84,
1096                 0x6ef0, 0x6f84,
1097                 0x6ff0, 0x7084,
1098                 0x70f0, 0x7184,
1099                 0x71f0, 0x7284,
1100                 0x72f0, 0x7384,
1101                 0x73f0, 0x7450,
1102                 0x7500, 0x7530,
1103                 0x7600, 0x761c,
1104                 0x7680, 0x76cc,
1105                 0x7700, 0x7798,
1106                 0x77c0, 0x77fc,
1107                 0x7900, 0x79fc,
1108                 0x7b00, 0x7c38,
1109                 0x7d00, 0x7efc,
1110                 0x8dc0, 0x8e1c,
1111                 0x8e30, 0x8e78,
1112                 0x8ea0, 0x8f6c,
1113                 0x8fc0, 0x9074,
1114                 0x90fc, 0x90fc,
1115                 0x9400, 0x9458,
1116                 0x9600, 0x96bc,
1117                 0x9800, 0x9808,
1118                 0x9820, 0x983c,
1119                 0x9850, 0x9864,
1120                 0x9c00, 0x9c6c,
1121                 0x9c80, 0x9cec,
1122                 0x9d00, 0x9d6c,
1123                 0x9d80, 0x9dec,
1124                 0x9e00, 0x9e6c,
1125                 0x9e80, 0x9eec,
1126                 0x9f00, 0x9f6c,
1127                 0x9f80, 0x9fec,
1128                 0xd004, 0xd03c,
1129                 0xdfc0, 0xdfe0,
1130                 0xe000, 0xea7c,
1131                 0xf000, 0x11190,
1132                 0x19040, 0x1906c,
1133                 0x19078, 0x19080,
1134                 0x1908c, 0x19124,
1135                 0x19150, 0x191b0,
1136                 0x191d0, 0x191e8,
1137                 0x19238, 0x1924c,
1138                 0x193f8, 0x19474,
1139                 0x19490, 0x194f8,
1140                 0x19800, 0x19f30,
1141                 0x1a000, 0x1a06c,
1142                 0x1a0b0, 0x1a120,
1143                 0x1a128, 0x1a138,
1144                 0x1a190, 0x1a1c4,
1145                 0x1a1fc, 0x1a1fc,
1146                 0x1e040, 0x1e04c,
1147                 0x1e284, 0x1e28c,
1148                 0x1e2c0, 0x1e2c0,
1149                 0x1e2e0, 0x1e2e0,
1150                 0x1e300, 0x1e384,
1151                 0x1e3c0, 0x1e3c8,
1152                 0x1e440, 0x1e44c,
1153                 0x1e684, 0x1e68c,
1154                 0x1e6c0, 0x1e6c0,
1155                 0x1e6e0, 0x1e6e0,
1156                 0x1e700, 0x1e784,
1157                 0x1e7c0, 0x1e7c8,
1158                 0x1e840, 0x1e84c,
1159                 0x1ea84, 0x1ea8c,
1160                 0x1eac0, 0x1eac0,
1161                 0x1eae0, 0x1eae0,
1162                 0x1eb00, 0x1eb84,
1163                 0x1ebc0, 0x1ebc8,
1164                 0x1ec40, 0x1ec4c,
1165                 0x1ee84, 0x1ee8c,
1166                 0x1eec0, 0x1eec0,
1167                 0x1eee0, 0x1eee0,
1168                 0x1ef00, 0x1ef84,
1169                 0x1efc0, 0x1efc8,
1170                 0x1f040, 0x1f04c,
1171                 0x1f284, 0x1f28c,
1172                 0x1f2c0, 0x1f2c0,
1173                 0x1f2e0, 0x1f2e0,
1174                 0x1f300, 0x1f384,
1175                 0x1f3c0, 0x1f3c8,
1176                 0x1f440, 0x1f44c,
1177                 0x1f684, 0x1f68c,
1178                 0x1f6c0, 0x1f6c0,
1179                 0x1f6e0, 0x1f6e0,
1180                 0x1f700, 0x1f784,
1181                 0x1f7c0, 0x1f7c8,
1182                 0x1f840, 0x1f84c,
1183                 0x1fa84, 0x1fa8c,
1184                 0x1fac0, 0x1fac0,
1185                 0x1fae0, 0x1fae0,
1186                 0x1fb00, 0x1fb84,
1187                 0x1fbc0, 0x1fbc8,
1188                 0x1fc40, 0x1fc4c,
1189                 0x1fe84, 0x1fe8c,
1190                 0x1fec0, 0x1fec0,
1191                 0x1fee0, 0x1fee0,
1192                 0x1ff00, 0x1ff84,
1193                 0x1ffc0, 0x1ffc8,
1194                 0x20000, 0x2002c,
1195                 0x20100, 0x2013c,
1196                 0x20190, 0x201c8,
1197                 0x20200, 0x20318,
1198                 0x20400, 0x20528,
1199                 0x20540, 0x20614,
1200                 0x21000, 0x21040,
1201                 0x2104c, 0x21060,
1202                 0x210c0, 0x210ec,
1203                 0x21200, 0x21268,
1204                 0x21270, 0x21284,
1205                 0x212fc, 0x21388,
1206                 0x21400, 0x21404,
1207                 0x21500, 0x21518,
1208                 0x2152c, 0x2153c,
1209                 0x21550, 0x21554,
1210                 0x21600, 0x21600,
1211                 0x21608, 0x21628,
1212                 0x21630, 0x2163c,
1213                 0x21700, 0x2171c,
1214                 0x21780, 0x2178c,
1215                 0x21800, 0x21c38,
1216                 0x21c80, 0x21d7c,
1217                 0x21e00, 0x21e04,
1218                 0x22000, 0x2202c,
1219                 0x22100, 0x2213c,
1220                 0x22190, 0x221c8,
1221                 0x22200, 0x22318,
1222                 0x22400, 0x22528,
1223                 0x22540, 0x22614,
1224                 0x23000, 0x23040,
1225                 0x2304c, 0x23060,
1226                 0x230c0, 0x230ec,
1227                 0x23200, 0x23268,
1228                 0x23270, 0x23284,
1229                 0x232fc, 0x23388,
1230                 0x23400, 0x23404,
1231                 0x23500, 0x23518,
1232                 0x2352c, 0x2353c,
1233                 0x23550, 0x23554,
1234                 0x23600, 0x23600,
1235                 0x23608, 0x23628,
1236                 0x23630, 0x2363c,
1237                 0x23700, 0x2371c,
1238                 0x23780, 0x2378c,
1239                 0x23800, 0x23c38,
1240                 0x23c80, 0x23d7c,
1241                 0x23e00, 0x23e04,
1242                 0x24000, 0x2402c,
1243                 0x24100, 0x2413c,
1244                 0x24190, 0x241c8,
1245                 0x24200, 0x24318,
1246                 0x24400, 0x24528,
1247                 0x24540, 0x24614,
1248                 0x25000, 0x25040,
1249                 0x2504c, 0x25060,
1250                 0x250c0, 0x250ec,
1251                 0x25200, 0x25268,
1252                 0x25270, 0x25284,
1253                 0x252fc, 0x25388,
1254                 0x25400, 0x25404,
1255                 0x25500, 0x25518,
1256                 0x2552c, 0x2553c,
1257                 0x25550, 0x25554,
1258                 0x25600, 0x25600,
1259                 0x25608, 0x25628,
1260                 0x25630, 0x2563c,
1261                 0x25700, 0x2571c,
1262                 0x25780, 0x2578c,
1263                 0x25800, 0x25c38,
1264                 0x25c80, 0x25d7c,
1265                 0x25e00, 0x25e04,
1266                 0x26000, 0x2602c,
1267                 0x26100, 0x2613c,
1268                 0x26190, 0x261c8,
1269                 0x26200, 0x26318,
1270                 0x26400, 0x26528,
1271                 0x26540, 0x26614,
1272                 0x27000, 0x27040,
1273                 0x2704c, 0x27060,
1274                 0x270c0, 0x270ec,
1275                 0x27200, 0x27268,
1276                 0x27270, 0x27284,
1277                 0x272fc, 0x27388,
1278                 0x27400, 0x27404,
1279                 0x27500, 0x27518,
1280                 0x2752c, 0x2753c,
1281                 0x27550, 0x27554,
1282                 0x27600, 0x27600,
1283                 0x27608, 0x27628,
1284                 0x27630, 0x2763c,
1285                 0x27700, 0x2771c,
1286                 0x27780, 0x2778c,
1287                 0x27800, 0x27c38,
1288                 0x27c80, 0x27d7c,
1289                 0x27e00, 0x27e04
1290         };
1291
1292         int i;
1293         struct adapter *ap = netdev2adap(dev);
1294
1295         regs->version = mk_adap_vers(ap);
1296
1297         memset(buf, 0, T4_REGMAP_SIZE);
1298         for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
1299                 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
1300 }
1301
1302 static int restart_autoneg(struct net_device *dev)
1303 {
1304         struct port_info *p = netdev_priv(dev);
1305
1306         if (!netif_running(dev))
1307                 return -EAGAIN;
1308         if (p->link_cfg.autoneg != AUTONEG_ENABLE)
1309                 return -EINVAL;
1310         t4_restart_aneg(p->adapter, 0, p->tx_chan);
1311         return 0;
1312 }
1313
1314 static int identify_port(struct net_device *dev, u32 data)
1315 {
1316         if (data == 0)
1317                 data = 2;     /* default to 2 seconds */
1318
1319         return t4_identify_port(netdev2adap(dev), 0, netdev2pinfo(dev)->viid,
1320                                 data * 5);
1321 }
1322
1323 static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
1324 {
1325         unsigned int v = 0;
1326
1327         if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
1328             type == FW_PORT_TYPE_BT_XAUI) {
1329                 v |= SUPPORTED_TP;
1330                 if (caps & FW_PORT_CAP_SPEED_100M)
1331                         v |= SUPPORTED_100baseT_Full;
1332                 if (caps & FW_PORT_CAP_SPEED_1G)
1333                         v |= SUPPORTED_1000baseT_Full;
1334                 if (caps & FW_PORT_CAP_SPEED_10G)
1335                         v |= SUPPORTED_10000baseT_Full;
1336         } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1337                 v |= SUPPORTED_Backplane;
1338                 if (caps & FW_PORT_CAP_SPEED_1G)
1339                         v |= SUPPORTED_1000baseKX_Full;
1340                 if (caps & FW_PORT_CAP_SPEED_10G)
1341                         v |= SUPPORTED_10000baseKX4_Full;
1342         } else if (type == FW_PORT_TYPE_KR)
1343                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
1344         else if (type == FW_PORT_TYPE_BP_AP)
1345                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC;
1346         else if (type == FW_PORT_TYPE_FIBER_XFI ||
1347                  type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
1348                 v |= SUPPORTED_FIBRE;
1349
1350         if (caps & FW_PORT_CAP_ANEG)
1351                 v |= SUPPORTED_Autoneg;
1352         return v;
1353 }
1354
1355 static unsigned int to_fw_linkcaps(unsigned int caps)
1356 {
1357         unsigned int v = 0;
1358
1359         if (caps & ADVERTISED_100baseT_Full)
1360                 v |= FW_PORT_CAP_SPEED_100M;
1361         if (caps & ADVERTISED_1000baseT_Full)
1362                 v |= FW_PORT_CAP_SPEED_1G;
1363         if (caps & ADVERTISED_10000baseT_Full)
1364                 v |= FW_PORT_CAP_SPEED_10G;
1365         return v;
1366 }
1367
1368 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1369 {
1370         const struct port_info *p = netdev_priv(dev);
1371
1372         if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
1373             p->port_type == FW_PORT_TYPE_BT_XFI ||
1374             p->port_type == FW_PORT_TYPE_BT_XAUI)
1375                 cmd->port = PORT_TP;
1376         else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
1377                  p->port_type == FW_PORT_TYPE_FIBER_XAUI)
1378                 cmd->port = PORT_FIBRE;
1379         else if (p->port_type == FW_PORT_TYPE_SFP) {
1380                 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1381                     p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1382                         cmd->port = PORT_DA;
1383                 else
1384                         cmd->port = PORT_FIBRE;
1385         } else
1386                 cmd->port = PORT_OTHER;
1387
1388         if (p->mdio_addr >= 0) {
1389                 cmd->phy_address = p->mdio_addr;
1390                 cmd->transceiver = XCVR_EXTERNAL;
1391                 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1392                         MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1393         } else {
1394                 cmd->phy_address = 0;  /* not really, but no better option */
1395                 cmd->transceiver = XCVR_INTERNAL;
1396                 cmd->mdio_support = 0;
1397         }
1398
1399         cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
1400         cmd->advertising = from_fw_linkcaps(p->port_type,
1401                                             p->link_cfg.advertising);
1402         cmd->speed = netif_carrier_ok(dev) ? p->link_cfg.speed : 0;
1403         cmd->duplex = DUPLEX_FULL;
1404         cmd->autoneg = p->link_cfg.autoneg;
1405         cmd->maxtxpkt = 0;
1406         cmd->maxrxpkt = 0;
1407         return 0;
1408 }
1409
1410 static unsigned int speed_to_caps(int speed)
1411 {
1412         if (speed == SPEED_100)
1413                 return FW_PORT_CAP_SPEED_100M;
1414         if (speed == SPEED_1000)
1415                 return FW_PORT_CAP_SPEED_1G;
1416         if (speed == SPEED_10000)
1417                 return FW_PORT_CAP_SPEED_10G;
1418         return 0;
1419 }
1420
1421 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1422 {
1423         unsigned int cap;
1424         struct port_info *p = netdev_priv(dev);
1425         struct link_config *lc = &p->link_cfg;
1426
1427         if (cmd->duplex != DUPLEX_FULL)     /* only full-duplex supported */
1428                 return -EINVAL;
1429
1430         if (!(lc->supported & FW_PORT_CAP_ANEG)) {
1431                 /*
1432                  * PHY offers a single speed.  See if that's what's
1433                  * being requested.
1434                  */
1435                 if (cmd->autoneg == AUTONEG_DISABLE &&
1436                     (lc->supported & speed_to_caps(cmd->speed)))
1437                                 return 0;
1438                 return -EINVAL;
1439         }
1440
1441         if (cmd->autoneg == AUTONEG_DISABLE) {
1442                 cap = speed_to_caps(cmd->speed);
1443
1444                 if (!(lc->supported & cap) || cmd->speed == SPEED_1000 ||
1445                     cmd->speed == SPEED_10000)
1446                         return -EINVAL;
1447                 lc->requested_speed = cap;
1448                 lc->advertising = 0;
1449         } else {
1450                 cap = to_fw_linkcaps(cmd->advertising);
1451                 if (!(lc->supported & cap))
1452                         return -EINVAL;
1453                 lc->requested_speed = 0;
1454                 lc->advertising = cap | FW_PORT_CAP_ANEG;
1455         }
1456         lc->autoneg = cmd->autoneg;
1457
1458         if (netif_running(dev))
1459                 return t4_link_start(p->adapter, 0, p->tx_chan, lc);
1460         return 0;
1461 }
1462
1463 static void get_pauseparam(struct net_device *dev,
1464                            struct ethtool_pauseparam *epause)
1465 {
1466         struct port_info *p = netdev_priv(dev);
1467
1468         epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1469         epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
1470         epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
1471 }
1472
1473 static int set_pauseparam(struct net_device *dev,
1474                           struct ethtool_pauseparam *epause)
1475 {
1476         struct port_info *p = netdev_priv(dev);
1477         struct link_config *lc = &p->link_cfg;
1478
1479         if (epause->autoneg == AUTONEG_DISABLE)
1480                 lc->requested_fc = 0;
1481         else if (lc->supported & FW_PORT_CAP_ANEG)
1482                 lc->requested_fc = PAUSE_AUTONEG;
1483         else
1484                 return -EINVAL;
1485
1486         if (epause->rx_pause)
1487                 lc->requested_fc |= PAUSE_RX;
1488         if (epause->tx_pause)
1489                 lc->requested_fc |= PAUSE_TX;
1490         if (netif_running(dev))
1491                 return t4_link_start(p->adapter, 0, p->tx_chan, lc);
1492         return 0;
1493 }
1494
1495 static u32 get_rx_csum(struct net_device *dev)
1496 {
1497         struct port_info *p = netdev_priv(dev);
1498
1499         return p->rx_offload & RX_CSO;
1500 }
1501
1502 static int set_rx_csum(struct net_device *dev, u32 data)
1503 {
1504         struct port_info *p = netdev_priv(dev);
1505
1506         if (data)
1507                 p->rx_offload |= RX_CSO;
1508         else
1509                 p->rx_offload &= ~RX_CSO;
1510         return 0;
1511 }
1512
1513 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1514 {
1515         const struct port_info *pi = netdev_priv(dev);
1516         const struct sge *s = &pi->adapter->sge;
1517
1518         e->rx_max_pending = MAX_RX_BUFFERS;
1519         e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1520         e->rx_jumbo_max_pending = 0;
1521         e->tx_max_pending = MAX_TXQ_ENTRIES;
1522
1523         e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
1524         e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1525         e->rx_jumbo_pending = 0;
1526         e->tx_pending = s->ethtxq[pi->first_qset].q.size;
1527 }
1528
1529 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1530 {
1531         int i;
1532         const struct port_info *pi = netdev_priv(dev);
1533         struct adapter *adapter = pi->adapter;
1534         struct sge *s = &adapter->sge;
1535
1536         if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
1537             e->tx_pending > MAX_TXQ_ENTRIES ||
1538             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1539             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1540             e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
1541                 return -EINVAL;
1542
1543         if (adapter->flags & FULL_INIT_DONE)
1544                 return -EBUSY;
1545
1546         for (i = 0; i < pi->nqsets; ++i) {
1547                 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
1548                 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
1549                 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
1550         }
1551         return 0;
1552 }
1553
1554 static int closest_timer(const struct sge *s, int time)
1555 {
1556         int i, delta, match = 0, min_delta = INT_MAX;
1557
1558         for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1559                 delta = time - s->timer_val[i];
1560                 if (delta < 0)
1561                         delta = -delta;
1562                 if (delta < min_delta) {
1563                         min_delta = delta;
1564                         match = i;
1565                 }
1566         }
1567         return match;
1568 }
1569
1570 static int closest_thres(const struct sge *s, int thres)
1571 {
1572         int i, delta, match = 0, min_delta = INT_MAX;
1573
1574         for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1575                 delta = thres - s->counter_val[i];
1576                 if (delta < 0)
1577                         delta = -delta;
1578                 if (delta < min_delta) {
1579                         min_delta = delta;
1580                         match = i;
1581                 }
1582         }
1583         return match;
1584 }
1585
1586 /*
1587  * Return a queue's interrupt hold-off time in us.  0 means no timer.
1588  */
1589 static unsigned int qtimer_val(const struct adapter *adap,
1590                                const struct sge_rspq *q)
1591 {
1592         unsigned int idx = q->intr_params >> 1;
1593
1594         return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
1595 }
1596
1597 /**
1598  *      set_rxq_intr_params - set a queue's interrupt holdoff parameters
1599  *      @adap: the adapter
1600  *      @q: the Rx queue
1601  *      @us: the hold-off time in us, or 0 to disable timer
1602  *      @cnt: the hold-off packet count, or 0 to disable counter
1603  *
1604  *      Sets an Rx queue's interrupt hold-off time and packet count.  At least
1605  *      one of the two needs to be enabled for the queue to generate interrupts.
1606  */
1607 static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
1608                                unsigned int us, unsigned int cnt)
1609 {
1610         if ((us | cnt) == 0)
1611                 cnt = 1;
1612
1613         if (cnt) {
1614                 int err;
1615                 u32 v, new_idx;
1616
1617                 new_idx = closest_thres(&adap->sge, cnt);
1618                 if (q->desc && q->pktcnt_idx != new_idx) {
1619                         /* the queue has already been created, update it */
1620                         v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1621                             FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1622                             FW_PARAMS_PARAM_YZ(q->cntxt_id);
1623                         err = t4_set_params(adap, 0, 0, 0, 1, &v, &new_idx);
1624                         if (err)
1625                                 return err;
1626                 }
1627                 q->pktcnt_idx = new_idx;
1628         }
1629
1630         us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1631         q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
1632         return 0;
1633 }
1634
1635 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1636 {
1637         const struct port_info *pi = netdev_priv(dev);
1638         struct adapter *adap = pi->adapter;
1639
1640         return set_rxq_intr_params(adap, &adap->sge.ethrxq[pi->first_qset].rspq,
1641                         c->rx_coalesce_usecs, c->rx_max_coalesced_frames);
1642 }
1643
1644 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1645 {
1646         const struct port_info *pi = netdev_priv(dev);
1647         const struct adapter *adap = pi->adapter;
1648         const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
1649
1650         c->rx_coalesce_usecs = qtimer_val(adap, rq);
1651         c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
1652                 adap->sge.counter_val[rq->pktcnt_idx] : 0;
1653         return 0;
1654 }
1655
1656 /*
1657  * Translate a physical EEPROM address to virtual.  The first 1K is accessed
1658  * through virtual addresses starting at 31K, the rest is accessed through
1659  * virtual addresses starting at 0.  This mapping is correct only for PF0.
1660  */
1661 static int eeprom_ptov(unsigned int phys_addr)
1662 {
1663         if (phys_addr < 1024)
1664                 return phys_addr + (31 << 10);
1665         if (phys_addr < EEPROMSIZE)
1666                 return phys_addr - 1024;
1667         return -EINVAL;
1668 }
1669
1670 /*
1671  * The next two routines implement eeprom read/write from physical addresses.
1672  * The physical->virtual translation is correct only for PF0.
1673  */
1674 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
1675 {
1676         int vaddr = eeprom_ptov(phys_addr);
1677
1678         if (vaddr >= 0)
1679                 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
1680         return vaddr < 0 ? vaddr : 0;
1681 }
1682
1683 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
1684 {
1685         int vaddr = eeprom_ptov(phys_addr);
1686
1687         if (vaddr >= 0)
1688                 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
1689         return vaddr < 0 ? vaddr : 0;
1690 }
1691
1692 #define EEPROM_MAGIC 0x38E2F10C
1693
1694 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1695                       u8 *data)
1696 {
1697         int i, err = 0;
1698         struct adapter *adapter = netdev2adap(dev);
1699
1700         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1701         if (!buf)
1702                 return -ENOMEM;
1703
1704         e->magic = EEPROM_MAGIC;
1705         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1706                 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
1707
1708         if (!err)
1709                 memcpy(data, buf + e->offset, e->len);
1710         kfree(buf);
1711         return err;
1712 }
1713
1714 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1715                       u8 *data)
1716 {
1717         u8 *buf;
1718         int err = 0;
1719         u32 aligned_offset, aligned_len, *p;
1720         struct adapter *adapter = netdev2adap(dev);
1721
1722         if (eeprom->magic != EEPROM_MAGIC)
1723                 return -EINVAL;
1724
1725         aligned_offset = eeprom->offset & ~3;
1726         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1727
1728         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1729                 /*
1730                  * RMW possibly needed for first or last words.
1731                  */
1732                 buf = kmalloc(aligned_len, GFP_KERNEL);
1733                 if (!buf)
1734                         return -ENOMEM;
1735                 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
1736                 if (!err && aligned_len > 4)
1737                         err = eeprom_rd_phys(adapter,
1738                                              aligned_offset + aligned_len - 4,
1739                                              (u32 *)&buf[aligned_len - 4]);
1740                 if (err)
1741                         goto out;
1742                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1743         } else
1744                 buf = data;
1745
1746         err = t4_seeprom_wp(adapter, false);
1747         if (err)
1748                 goto out;
1749
1750         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
1751                 err = eeprom_wr_phys(adapter, aligned_offset, *p);
1752                 aligned_offset += 4;
1753         }
1754
1755         if (!err)
1756                 err = t4_seeprom_wp(adapter, true);
1757 out:
1758         if (buf != data)
1759                 kfree(buf);
1760         return err;
1761 }
1762
1763 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
1764 {
1765         int ret;
1766         const struct firmware *fw;
1767         struct adapter *adap = netdev2adap(netdev);
1768
1769         ef->data[sizeof(ef->data) - 1] = '\0';
1770         ret = request_firmware(&fw, ef->data, adap->pdev_dev);
1771         if (ret < 0)
1772                 return ret;
1773
1774         ret = t4_load_fw(adap, fw->data, fw->size);
1775         release_firmware(fw);
1776         if (!ret)
1777                 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
1778         return ret;
1779 }
1780
1781 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
1782 #define BCAST_CRC 0xa0ccc1a6
1783
1784 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1785 {
1786         wol->supported = WAKE_BCAST | WAKE_MAGIC;
1787         wol->wolopts = netdev2adap(dev)->wol;
1788         memset(&wol->sopass, 0, sizeof(wol->sopass));
1789 }
1790
1791 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1792 {
1793         int err = 0;
1794         struct port_info *pi = netdev_priv(dev);
1795
1796         if (wol->wolopts & ~WOL_SUPPORTED)
1797                 return -EINVAL;
1798         t4_wol_magic_enable(pi->adapter, pi->tx_chan,
1799                             (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
1800         if (wol->wolopts & WAKE_BCAST) {
1801                 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
1802                                         ~0ULL, 0, false);
1803                 if (!err)
1804                         err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
1805                                                 ~6ULL, ~0ULL, BCAST_CRC, true);
1806         } else
1807                 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
1808         return err;
1809 }
1810
1811 static int set_tso(struct net_device *dev, u32 value)
1812 {
1813         if (value)
1814                 dev->features |= NETIF_F_TSO | NETIF_F_TSO6;
1815         else
1816                 dev->features &= ~(NETIF_F_TSO | NETIF_F_TSO6);
1817         return 0;
1818 }
1819
1820 static int set_flags(struct net_device *dev, u32 flags)
1821 {
1822         return ethtool_op_set_flags(dev, flags, ETH_FLAG_RXHASH);
1823 }
1824
1825 static int get_rss_table(struct net_device *dev, struct ethtool_rxfh_indir *p)
1826 {
1827         const struct port_info *pi = netdev_priv(dev);
1828         unsigned int n = min_t(unsigned int, p->size, pi->rss_size);
1829
1830         p->size = pi->rss_size;
1831         while (n--)
1832                 p->ring_index[n] = pi->rss[n];
1833         return 0;
1834 }
1835
1836 static int set_rss_table(struct net_device *dev,
1837                          const struct ethtool_rxfh_indir *p)
1838 {
1839         unsigned int i;
1840         struct port_info *pi = netdev_priv(dev);
1841
1842         if (p->size != pi->rss_size)
1843                 return -EINVAL;
1844         for (i = 0; i < p->size; i++)
1845                 if (p->ring_index[i] >= pi->nqsets)
1846                         return -EINVAL;
1847         for (i = 0; i < p->size; i++)
1848                 pi->rss[i] = p->ring_index[i];
1849         if (pi->adapter->flags & FULL_INIT_DONE)
1850                 return write_rss(pi, pi->rss);
1851         return 0;
1852 }
1853
1854 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1855                      void *rules)
1856 {
1857         const struct port_info *pi = netdev_priv(dev);
1858
1859         switch (info->cmd) {
1860         case ETHTOOL_GRXFH: {
1861                 unsigned int v = pi->rss_mode;
1862
1863                 info->data = 0;
1864                 switch (info->flow_type) {
1865                 case TCP_V4_FLOW:
1866                         if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
1867                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1868                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1869                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1870                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1871                         break;
1872                 case UDP_V4_FLOW:
1873                         if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
1874                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1875                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1876                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1877                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1878                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1879                         break;
1880                 case SCTP_V4_FLOW:
1881                 case AH_ESP_V4_FLOW:
1882                 case IPV4_FLOW:
1883                         if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1884                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1885                         break;
1886                 case TCP_V6_FLOW:
1887                         if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
1888                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1889                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1890                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1891                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1892                         break;
1893                 case UDP_V6_FLOW:
1894                         if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
1895                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1896                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1897                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1898                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1899                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1900                         break;
1901                 case SCTP_V6_FLOW:
1902                 case AH_ESP_V6_FLOW:
1903                 case IPV6_FLOW:
1904                         if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1905                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1906                         break;
1907                 }
1908                 return 0;
1909         }
1910         case ETHTOOL_GRXRINGS:
1911                 info->data = pi->nqsets;
1912                 return 0;
1913         }
1914         return -EOPNOTSUPP;
1915 }
1916
1917 static struct ethtool_ops cxgb_ethtool_ops = {
1918         .get_settings      = get_settings,
1919         .set_settings      = set_settings,
1920         .get_drvinfo       = get_drvinfo,
1921         .get_msglevel      = get_msglevel,
1922         .set_msglevel      = set_msglevel,
1923         .get_ringparam     = get_sge_param,
1924         .set_ringparam     = set_sge_param,
1925         .get_coalesce      = get_coalesce,
1926         .set_coalesce      = set_coalesce,
1927         .get_eeprom_len    = get_eeprom_len,
1928         .get_eeprom        = get_eeprom,
1929         .set_eeprom        = set_eeprom,
1930         .get_pauseparam    = get_pauseparam,
1931         .set_pauseparam    = set_pauseparam,
1932         .get_rx_csum       = get_rx_csum,
1933         .set_rx_csum       = set_rx_csum,
1934         .set_tx_csum       = ethtool_op_set_tx_ipv6_csum,
1935         .set_sg            = ethtool_op_set_sg,
1936         .get_link          = ethtool_op_get_link,
1937         .get_strings       = get_strings,
1938         .phys_id           = identify_port,
1939         .nway_reset        = restart_autoneg,
1940         .get_sset_count    = get_sset_count,
1941         .get_ethtool_stats = get_stats,
1942         .get_regs_len      = get_regs_len,
1943         .get_regs          = get_regs,
1944         .get_wol           = get_wol,
1945         .set_wol           = set_wol,
1946         .set_tso           = set_tso,
1947         .set_flags         = set_flags,
1948         .get_rxnfc         = get_rxnfc,
1949         .get_rxfh_indir    = get_rss_table,
1950         .set_rxfh_indir    = set_rss_table,
1951         .flash_device      = set_flash,
1952 };
1953
1954 /*
1955  * debugfs support
1956  */
1957
1958 static int mem_open(struct inode *inode, struct file *file)
1959 {
1960         file->private_data = inode->i_private;
1961         return 0;
1962 }
1963
1964 static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
1965                         loff_t *ppos)
1966 {
1967         loff_t pos = *ppos;
1968         loff_t avail = file->f_path.dentry->d_inode->i_size;
1969         unsigned int mem = (uintptr_t)file->private_data & 3;
1970         struct adapter *adap = file->private_data - mem;
1971
1972         if (pos < 0)
1973                 return -EINVAL;
1974         if (pos >= avail)
1975                 return 0;
1976         if (count > avail - pos)
1977                 count = avail - pos;
1978
1979         while (count) {
1980                 size_t len;
1981                 int ret, ofst;
1982                 __be32 data[16];
1983
1984                 if (mem == MEM_MC)
1985                         ret = t4_mc_read(adap, pos, data, NULL);
1986                 else
1987                         ret = t4_edc_read(adap, mem, pos, data, NULL);
1988                 if (ret)
1989                         return ret;
1990
1991                 ofst = pos % sizeof(data);
1992                 len = min(count, sizeof(data) - ofst);
1993                 if (copy_to_user(buf, (u8 *)data + ofst, len))
1994                         return -EFAULT;
1995
1996                 buf += len;
1997                 pos += len;
1998                 count -= len;
1999         }
2000         count = pos - *ppos;
2001         *ppos = pos;
2002         return count;
2003 }
2004
2005 static const struct file_operations mem_debugfs_fops = {
2006         .owner   = THIS_MODULE,
2007         .open    = mem_open,
2008         .read    = mem_read,
2009 };
2010
2011 static void __devinit add_debugfs_mem(struct adapter *adap, const char *name,
2012                                       unsigned int idx, unsigned int size_mb)
2013 {
2014         struct dentry *de;
2015
2016         de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2017                                  (void *)adap + idx, &mem_debugfs_fops);
2018         if (de && de->d_inode)
2019                 de->d_inode->i_size = size_mb << 20;
2020 }
2021
2022 static int __devinit setup_debugfs(struct adapter *adap)
2023 {
2024         int i;
2025
2026         if (IS_ERR_OR_NULL(adap->debugfs_root))
2027                 return -1;
2028
2029         i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2030         if (i & EDRAM0_ENABLE)
2031                 add_debugfs_mem(adap, "edc0", MEM_EDC0, 5);
2032         if (i & EDRAM1_ENABLE)
2033                 add_debugfs_mem(adap, "edc1", MEM_EDC1, 5);
2034         if (i & EXT_MEM_ENABLE)
2035                 add_debugfs_mem(adap, "mc", MEM_MC,
2036                         EXT_MEM_SIZE_GET(t4_read_reg(adap, MA_EXT_MEMORY_BAR)));
2037         if (adap->l2t)
2038                 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2039                                     &t4_l2t_fops);
2040         return 0;
2041 }
2042
2043 /*
2044  * upper-layer driver support
2045  */
2046
2047 /*
2048  * Allocate an active-open TID and set it to the supplied value.
2049  */
2050 int cxgb4_alloc_atid(struct tid_info *t, void *data)
2051 {
2052         int atid = -1;
2053
2054         spin_lock_bh(&t->atid_lock);
2055         if (t->afree) {
2056                 union aopen_entry *p = t->afree;
2057
2058                 atid = p - t->atid_tab;
2059                 t->afree = p->next;
2060                 p->data = data;
2061                 t->atids_in_use++;
2062         }
2063         spin_unlock_bh(&t->atid_lock);
2064         return atid;
2065 }
2066 EXPORT_SYMBOL(cxgb4_alloc_atid);
2067
2068 /*
2069  * Release an active-open TID.
2070  */
2071 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2072 {
2073         union aopen_entry *p = &t->atid_tab[atid];
2074
2075         spin_lock_bh(&t->atid_lock);
2076         p->next = t->afree;
2077         t->afree = p;
2078         t->atids_in_use--;
2079         spin_unlock_bh(&t->atid_lock);
2080 }
2081 EXPORT_SYMBOL(cxgb4_free_atid);
2082
2083 /*
2084  * Allocate a server TID and set it to the supplied value.
2085  */
2086 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2087 {
2088         int stid;
2089
2090         spin_lock_bh(&t->stid_lock);
2091         if (family == PF_INET) {
2092                 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2093                 if (stid < t->nstids)
2094                         __set_bit(stid, t->stid_bmap);
2095                 else
2096                         stid = -1;
2097         } else {
2098                 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2099                 if (stid < 0)
2100                         stid = -1;
2101         }
2102         if (stid >= 0) {
2103                 t->stid_tab[stid].data = data;
2104                 stid += t->stid_base;
2105                 t->stids_in_use++;
2106         }
2107         spin_unlock_bh(&t->stid_lock);
2108         return stid;
2109 }
2110 EXPORT_SYMBOL(cxgb4_alloc_stid);
2111
2112 /*
2113  * Release a server TID.
2114  */
2115 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
2116 {
2117         stid -= t->stid_base;
2118         spin_lock_bh(&t->stid_lock);
2119         if (family == PF_INET)
2120                 __clear_bit(stid, t->stid_bmap);
2121         else
2122                 bitmap_release_region(t->stid_bmap, stid, 2);
2123         t->stid_tab[stid].data = NULL;
2124         t->stids_in_use--;
2125         spin_unlock_bh(&t->stid_lock);
2126 }
2127 EXPORT_SYMBOL(cxgb4_free_stid);
2128
2129 /*
2130  * Populate a TID_RELEASE WR.  Caller must properly size the skb.
2131  */
2132 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
2133                            unsigned int tid)
2134 {
2135         struct cpl_tid_release *req;
2136
2137         set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
2138         req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
2139         INIT_TP_WR(req, tid);
2140         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
2141 }
2142
2143 /*
2144  * Queue a TID release request and if necessary schedule a work queue to
2145  * process it.
2146  */
2147 void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
2148                              unsigned int tid)
2149 {
2150         void **p = &t->tid_tab[tid];
2151         struct adapter *adap = container_of(t, struct adapter, tids);
2152
2153         spin_lock_bh(&adap->tid_release_lock);
2154         *p = adap->tid_release_head;
2155         /* Low 2 bits encode the Tx channel number */
2156         adap->tid_release_head = (void **)((uintptr_t)p | chan);
2157         if (!adap->tid_release_task_busy) {
2158                 adap->tid_release_task_busy = true;
2159                 schedule_work(&adap->tid_release_task);
2160         }
2161         spin_unlock_bh(&adap->tid_release_lock);
2162 }
2163 EXPORT_SYMBOL(cxgb4_queue_tid_release);
2164
2165 /*
2166  * Process the list of pending TID release requests.
2167  */
2168 static void process_tid_release_list(struct work_struct *work)
2169 {
2170         struct sk_buff *skb;
2171         struct adapter *adap;
2172
2173         adap = container_of(work, struct adapter, tid_release_task);
2174
2175         spin_lock_bh(&adap->tid_release_lock);
2176         while (adap->tid_release_head) {
2177                 void **p = adap->tid_release_head;
2178                 unsigned int chan = (uintptr_t)p & 3;
2179                 p = (void *)p - chan;
2180
2181                 adap->tid_release_head = *p;
2182                 *p = NULL;
2183                 spin_unlock_bh(&adap->tid_release_lock);
2184
2185                 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
2186                                          GFP_KERNEL)))
2187                         schedule_timeout_uninterruptible(1);
2188
2189                 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
2190                 t4_ofld_send(adap, skb);
2191                 spin_lock_bh(&adap->tid_release_lock);
2192         }
2193         adap->tid_release_task_busy = false;
2194         spin_unlock_bh(&adap->tid_release_lock);
2195 }
2196
2197 /*
2198  * Release a TID and inform HW.  If we are unable to allocate the release
2199  * message we defer to a work queue.
2200  */
2201 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
2202 {
2203         void *old;
2204         struct sk_buff *skb;
2205         struct adapter *adap = container_of(t, struct adapter, tids);
2206
2207         old = t->tid_tab[tid];
2208         skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
2209         if (likely(skb)) {
2210                 t->tid_tab[tid] = NULL;
2211                 mk_tid_release(skb, chan, tid);
2212                 t4_ofld_send(adap, skb);
2213         } else
2214                 cxgb4_queue_tid_release(t, chan, tid);
2215         if (old)
2216                 atomic_dec(&t->tids_in_use);
2217 }
2218 EXPORT_SYMBOL(cxgb4_remove_tid);
2219
2220 /*
2221  * Allocate and initialize the TID tables.  Returns 0 on success.
2222  */
2223 static int tid_init(struct tid_info *t)
2224 {
2225         size_t size;
2226         unsigned int natids = t->natids;
2227
2228         size = t->ntids * sizeof(*t->tid_tab) + natids * sizeof(*t->atid_tab) +
2229                t->nstids * sizeof(*t->stid_tab) +
2230                BITS_TO_LONGS(t->nstids) * sizeof(long);
2231         t->tid_tab = t4_alloc_mem(size);
2232         if (!t->tid_tab)
2233                 return -ENOMEM;
2234
2235         t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
2236         t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
2237         t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids];
2238         spin_lock_init(&t->stid_lock);
2239         spin_lock_init(&t->atid_lock);
2240
2241         t->stids_in_use = 0;
2242         t->afree = NULL;
2243         t->atids_in_use = 0;
2244         atomic_set(&t->tids_in_use, 0);
2245
2246         /* Setup the free list for atid_tab and clear the stid bitmap. */
2247         if (natids) {
2248                 while (--natids)
2249                         t->atid_tab[natids - 1].next = &t->atid_tab[natids];
2250                 t->afree = t->atid_tab;
2251         }
2252         bitmap_zero(t->stid_bmap, t->nstids);
2253         return 0;
2254 }
2255
2256 /**
2257  *      cxgb4_create_server - create an IP server
2258  *      @dev: the device
2259  *      @stid: the server TID
2260  *      @sip: local IP address to bind server to
2261  *      @sport: the server's TCP port
2262  *      @queue: queue to direct messages from this server to
2263  *
2264  *      Create an IP server for the given port and address.
2265  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
2266  */
2267 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
2268                         __be32 sip, __be16 sport, unsigned int queue)
2269 {
2270         unsigned int chan;
2271         struct sk_buff *skb;
2272         struct adapter *adap;
2273         struct cpl_pass_open_req *req;
2274
2275         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2276         if (!skb)
2277                 return -ENOMEM;
2278
2279         adap = netdev2adap(dev);
2280         req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
2281         INIT_TP_WR(req, 0);
2282         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
2283         req->local_port = sport;
2284         req->peer_port = htons(0);
2285         req->local_ip = sip;
2286         req->peer_ip = htonl(0);
2287         chan = netdev2pinfo(adap->sge.ingr_map[queue]->netdev)->tx_chan;
2288         req->opt0 = cpu_to_be64(TX_CHAN(chan));
2289         req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2290                                 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2291         return t4_mgmt_tx(adap, skb);
2292 }
2293 EXPORT_SYMBOL(cxgb4_create_server);
2294
2295 /**
2296  *      cxgb4_create_server6 - create an IPv6 server
2297  *      @dev: the device
2298  *      @stid: the server TID
2299  *      @sip: local IPv6 address to bind server to
2300  *      @sport: the server's TCP port
2301  *      @queue: queue to direct messages from this server to
2302  *
2303  *      Create an IPv6 server for the given port and address.
2304  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
2305  */
2306 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
2307                          const struct in6_addr *sip, __be16 sport,
2308                          unsigned int queue)
2309 {
2310         unsigned int chan;
2311         struct sk_buff *skb;
2312         struct adapter *adap;
2313         struct cpl_pass_open_req6 *req;
2314
2315         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2316         if (!skb)
2317                 return -ENOMEM;
2318
2319         adap = netdev2adap(dev);
2320         req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
2321         INIT_TP_WR(req, 0);
2322         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
2323         req->local_port = sport;
2324         req->peer_port = htons(0);
2325         req->local_ip_hi = *(__be64 *)(sip->s6_addr);
2326         req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
2327         req->peer_ip_hi = cpu_to_be64(0);
2328         req->peer_ip_lo = cpu_to_be64(0);
2329         chan = netdev2pinfo(adap->sge.ingr_map[queue]->netdev)->tx_chan;
2330         req->opt0 = cpu_to_be64(TX_CHAN(chan));
2331         req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2332                                 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2333         return t4_mgmt_tx(adap, skb);
2334 }
2335 EXPORT_SYMBOL(cxgb4_create_server6);
2336
2337 /**
2338  *      cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2339  *      @mtus: the HW MTU table
2340  *      @mtu: the target MTU
2341  *      @idx: index of selected entry in the MTU table
2342  *
2343  *      Returns the index and the value in the HW MTU table that is closest to
2344  *      but does not exceed @mtu, unless @mtu is smaller than any value in the
2345  *      table, in which case that smallest available value is selected.
2346  */
2347 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
2348                             unsigned int *idx)
2349 {
2350         unsigned int i = 0;
2351
2352         while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
2353                 ++i;
2354         if (idx)
2355                 *idx = i;
2356         return mtus[i];
2357 }
2358 EXPORT_SYMBOL(cxgb4_best_mtu);
2359
2360 /**
2361  *      cxgb4_port_chan - get the HW channel of a port
2362  *      @dev: the net device for the port
2363  *
2364  *      Return the HW Tx channel of the given port.
2365  */
2366 unsigned int cxgb4_port_chan(const struct net_device *dev)
2367 {
2368         return netdev2pinfo(dev)->tx_chan;
2369 }
2370 EXPORT_SYMBOL(cxgb4_port_chan);
2371
2372 /**
2373  *      cxgb4_port_viid - get the VI id of a port
2374  *      @dev: the net device for the port
2375  *
2376  *      Return the VI id of the given port.
2377  */
2378 unsigned int cxgb4_port_viid(const struct net_device *dev)
2379 {
2380         return netdev2pinfo(dev)->viid;
2381 }
2382 EXPORT_SYMBOL(cxgb4_port_viid);
2383
2384 /**
2385  *      cxgb4_port_idx - get the index of a port
2386  *      @dev: the net device for the port
2387  *
2388  *      Return the index of the given port.
2389  */
2390 unsigned int cxgb4_port_idx(const struct net_device *dev)
2391 {
2392         return netdev2pinfo(dev)->port_id;
2393 }
2394 EXPORT_SYMBOL(cxgb4_port_idx);
2395
2396 /**
2397  *      cxgb4_netdev_by_hwid - return the net device of a HW port
2398  *      @pdev: identifies the adapter
2399  *      @id: the HW port id
2400  *
2401  *      Return the net device associated with the interface with the given HW
2402  *      id.
2403  */
2404 struct net_device *cxgb4_netdev_by_hwid(struct pci_dev *pdev, unsigned int id)
2405 {
2406         const struct adapter *adap = pci_get_drvdata(pdev);
2407
2408         if (!adap || id >= NCHAN)
2409                 return NULL;
2410         id = adap->chan_map[id];
2411         return id < MAX_NPORTS ? adap->port[id] : NULL;
2412 }
2413 EXPORT_SYMBOL(cxgb4_netdev_by_hwid);
2414
2415 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
2416                          struct tp_tcp_stats *v6)
2417 {
2418         struct adapter *adap = pci_get_drvdata(pdev);
2419
2420         spin_lock(&adap->stats_lock);
2421         t4_tp_get_tcp_stats(adap, v4, v6);
2422         spin_unlock(&adap->stats_lock);
2423 }
2424 EXPORT_SYMBOL(cxgb4_get_tcp_stats);
2425
2426 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
2427                       const unsigned int *pgsz_order)
2428 {
2429         struct adapter *adap = netdev2adap(dev);
2430
2431         t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
2432         t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
2433                      HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
2434                      HPZ3(pgsz_order[3]));
2435 }
2436 EXPORT_SYMBOL(cxgb4_iscsi_init);
2437
2438 static struct pci_driver cxgb4_driver;
2439
2440 static void check_neigh_update(struct neighbour *neigh)
2441 {
2442         const struct device *parent;
2443         const struct net_device *netdev = neigh->dev;
2444
2445         if (netdev->priv_flags & IFF_802_1Q_VLAN)
2446                 netdev = vlan_dev_real_dev(netdev);
2447         parent = netdev->dev.parent;
2448         if (parent && parent->driver == &cxgb4_driver.driver)
2449                 t4_l2t_update(dev_get_drvdata(parent), neigh);
2450 }
2451
2452 static int netevent_cb(struct notifier_block *nb, unsigned long event,
2453                        void *data)
2454 {
2455         switch (event) {
2456         case NETEVENT_NEIGH_UPDATE:
2457                 check_neigh_update(data);
2458                 break;
2459         case NETEVENT_PMTU_UPDATE:
2460         case NETEVENT_REDIRECT:
2461         default:
2462                 break;
2463         }
2464         return 0;
2465 }
2466
2467 static bool netevent_registered;
2468 static struct notifier_block cxgb4_netevent_nb = {
2469         .notifier_call = netevent_cb
2470 };
2471
2472 static void uld_attach(struct adapter *adap, unsigned int uld)
2473 {
2474         void *handle;
2475         struct cxgb4_lld_info lli;
2476
2477         lli.pdev = adap->pdev;
2478         lli.l2t = adap->l2t;
2479         lli.tids = &adap->tids;
2480         lli.ports = adap->port;
2481         lli.vr = &adap->vres;
2482         lli.mtus = adap->params.mtus;
2483         if (uld == CXGB4_ULD_RDMA) {
2484                 lli.rxq_ids = adap->sge.rdma_rxq;
2485                 lli.nrxq = adap->sge.rdmaqs;
2486         } else if (uld == CXGB4_ULD_ISCSI) {
2487                 lli.rxq_ids = adap->sge.ofld_rxq;
2488                 lli.nrxq = adap->sge.ofldqsets;
2489         }
2490         lli.ntxq = adap->sge.ofldqsets;
2491         lli.nchan = adap->params.nports;
2492         lli.nports = adap->params.nports;
2493         lli.wr_cred = adap->params.ofldq_wr_cred;
2494         lli.adapter_type = adap->params.rev;
2495         lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
2496         lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
2497                         t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF));
2498         lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
2499                         t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF));
2500         lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
2501         lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
2502         lli.fw_vers = adap->params.fw_vers;
2503
2504         handle = ulds[uld].add(&lli);
2505         if (IS_ERR(handle)) {
2506                 dev_warn(adap->pdev_dev,
2507                          "could not attach to the %s driver, error %ld\n",
2508                          uld_str[uld], PTR_ERR(handle));
2509                 return;
2510         }
2511
2512         adap->uld_handle[uld] = handle;
2513
2514         if (!netevent_registered) {
2515                 register_netevent_notifier(&cxgb4_netevent_nb);
2516                 netevent_registered = true;
2517         }
2518
2519         if (adap->flags & FULL_INIT_DONE)
2520                 ulds[uld].state_change(handle, CXGB4_STATE_UP);
2521 }
2522
2523 static void attach_ulds(struct adapter *adap)
2524 {
2525         unsigned int i;
2526
2527         mutex_lock(&uld_mutex);
2528         list_add_tail(&adap->list_node, &adapter_list);
2529         for (i = 0; i < CXGB4_ULD_MAX; i++)
2530                 if (ulds[i].add)
2531                         uld_attach(adap, i);
2532         mutex_unlock(&uld_mutex);
2533 }
2534
2535 static void detach_ulds(struct adapter *adap)
2536 {
2537         unsigned int i;
2538
2539         mutex_lock(&uld_mutex);
2540         list_del(&adap->list_node);
2541         for (i = 0; i < CXGB4_ULD_MAX; i++)
2542                 if (adap->uld_handle[i]) {
2543                         ulds[i].state_change(adap->uld_handle[i],
2544                                              CXGB4_STATE_DETACH);
2545                         adap->uld_handle[i] = NULL;
2546                 }
2547         if (netevent_registered && list_empty(&adapter_list)) {
2548                 unregister_netevent_notifier(&cxgb4_netevent_nb);
2549                 netevent_registered = false;
2550         }
2551         mutex_unlock(&uld_mutex);
2552 }
2553
2554 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
2555 {
2556         unsigned int i;
2557
2558         mutex_lock(&uld_mutex);
2559         for (i = 0; i < CXGB4_ULD_MAX; i++)
2560                 if (adap->uld_handle[i])
2561                         ulds[i].state_change(adap->uld_handle[i], new_state);
2562         mutex_unlock(&uld_mutex);
2563 }
2564
2565 /**
2566  *      cxgb4_register_uld - register an upper-layer driver
2567  *      @type: the ULD type
2568  *      @p: the ULD methods
2569  *
2570  *      Registers an upper-layer driver with this driver and notifies the ULD
2571  *      about any presently available devices that support its type.  Returns
2572  *      %-EBUSY if a ULD of the same type is already registered.
2573  */
2574 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
2575 {
2576         int ret = 0;
2577         struct adapter *adap;
2578
2579         if (type >= CXGB4_ULD_MAX)
2580                 return -EINVAL;
2581         mutex_lock(&uld_mutex);
2582         if (ulds[type].add) {
2583                 ret = -EBUSY;
2584                 goto out;
2585         }
2586         ulds[type] = *p;
2587         list_for_each_entry(adap, &adapter_list, list_node)
2588                 uld_attach(adap, type);
2589 out:    mutex_unlock(&uld_mutex);
2590         return ret;
2591 }
2592 EXPORT_SYMBOL(cxgb4_register_uld);
2593
2594 /**
2595  *      cxgb4_unregister_uld - unregister an upper-layer driver
2596  *      @type: the ULD type
2597  *
2598  *      Unregisters an existing upper-layer driver.
2599  */
2600 int cxgb4_unregister_uld(enum cxgb4_uld type)
2601 {
2602         struct adapter *adap;
2603
2604         if (type >= CXGB4_ULD_MAX)
2605                 return -EINVAL;
2606         mutex_lock(&uld_mutex);
2607         list_for_each_entry(adap, &adapter_list, list_node)
2608                 adap->uld_handle[type] = NULL;
2609         ulds[type].add = NULL;
2610         mutex_unlock(&uld_mutex);
2611         return 0;
2612 }
2613 EXPORT_SYMBOL(cxgb4_unregister_uld);
2614
2615 /**
2616  *      cxgb_up - enable the adapter
2617  *      @adap: adapter being enabled
2618  *
2619  *      Called when the first port is enabled, this function performs the
2620  *      actions necessary to make an adapter operational, such as completing
2621  *      the initialization of HW modules, and enabling interrupts.
2622  *
2623  *      Must be called with the rtnl lock held.
2624  */
2625 static int cxgb_up(struct adapter *adap)
2626 {
2627         int err;
2628
2629         err = setup_sge_queues(adap);
2630         if (err)
2631                 goto out;
2632         err = setup_rss(adap);
2633         if (err)
2634                 goto freeq;
2635
2636         if (adap->flags & USING_MSIX) {
2637                 name_msix_vecs(adap);
2638                 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
2639                                   adap->msix_info[0].desc, adap);
2640                 if (err)
2641                         goto irq_err;
2642
2643                 err = request_msix_queue_irqs(adap);
2644                 if (err) {
2645                         free_irq(adap->msix_info[0].vec, adap);
2646                         goto irq_err;
2647                 }
2648         } else {
2649                 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2650                                   (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
2651                                   adap->name, adap);
2652                 if (err)
2653                         goto irq_err;
2654         }
2655         enable_rx(adap);
2656         t4_sge_start(adap);
2657         t4_intr_enable(adap);
2658         adap->flags |= FULL_INIT_DONE;
2659         notify_ulds(adap, CXGB4_STATE_UP);
2660  out:
2661         return err;
2662  irq_err:
2663         dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
2664  freeq:
2665         t4_free_sge_resources(adap);
2666         goto out;
2667 }
2668
2669 static void cxgb_down(struct adapter *adapter)
2670 {
2671         t4_intr_disable(adapter);
2672         cancel_work_sync(&adapter->tid_release_task);
2673         adapter->tid_release_task_busy = false;
2674         adapter->tid_release_head = NULL;
2675
2676         if (adapter->flags & USING_MSIX) {
2677                 free_msix_queue_irqs(adapter);
2678                 free_irq(adapter->msix_info[0].vec, adapter);
2679         } else
2680                 free_irq(adapter->pdev->irq, adapter);
2681         quiesce_rx(adapter);
2682         t4_sge_stop(adapter);
2683         t4_free_sge_resources(adapter);
2684         adapter->flags &= ~FULL_INIT_DONE;
2685 }
2686
2687 /*
2688  * net_device operations
2689  */
2690 static int cxgb_open(struct net_device *dev)
2691 {
2692         int err;
2693         struct port_info *pi = netdev_priv(dev);
2694         struct adapter *adapter = pi->adapter;
2695
2696         if (!(adapter->flags & FULL_INIT_DONE)) {
2697                 err = cxgb_up(adapter);
2698                 if (err < 0)
2699                         return err;
2700         }
2701
2702         dev->real_num_tx_queues = pi->nqsets;
2703         err = link_start(dev);
2704         if (!err)
2705                 netif_tx_start_all_queues(dev);
2706         return err;
2707 }
2708
2709 static int cxgb_close(struct net_device *dev)
2710 {
2711         struct port_info *pi = netdev_priv(dev);
2712         struct adapter *adapter = pi->adapter;
2713
2714         netif_tx_stop_all_queues(dev);
2715         netif_carrier_off(dev);
2716         return t4_enable_vi(adapter, 0, pi->viid, false, false);
2717 }
2718
2719 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
2720                                                 struct rtnl_link_stats64 *ns)
2721 {
2722         struct port_stats stats;
2723         struct port_info *p = netdev_priv(dev);
2724         struct adapter *adapter = p->adapter;
2725
2726         spin_lock(&adapter->stats_lock);
2727         t4_get_port_stats(adapter, p->tx_chan, &stats);
2728         spin_unlock(&adapter->stats_lock);
2729
2730         ns->tx_bytes   = stats.tx_octets;
2731         ns->tx_packets = stats.tx_frames;
2732         ns->rx_bytes   = stats.rx_octets;
2733         ns->rx_packets = stats.rx_frames;
2734         ns->multicast  = stats.rx_mcast_frames;
2735
2736         /* detailed rx_errors */
2737         ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
2738                                stats.rx_runt;
2739         ns->rx_over_errors   = 0;
2740         ns->rx_crc_errors    = stats.rx_fcs_err;
2741         ns->rx_frame_errors  = stats.rx_symbol_err;
2742         ns->rx_fifo_errors   = stats.rx_ovflow0 + stats.rx_ovflow1 +
2743                                stats.rx_ovflow2 + stats.rx_ovflow3 +
2744                                stats.rx_trunc0 + stats.rx_trunc1 +
2745                                stats.rx_trunc2 + stats.rx_trunc3;
2746         ns->rx_missed_errors = 0;
2747
2748         /* detailed tx_errors */
2749         ns->tx_aborted_errors   = 0;
2750         ns->tx_carrier_errors   = 0;
2751         ns->tx_fifo_errors      = 0;
2752         ns->tx_heartbeat_errors = 0;
2753         ns->tx_window_errors    = 0;
2754
2755         ns->tx_errors = stats.tx_error_frames;
2756         ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
2757                 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
2758         return ns;
2759 }
2760
2761 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2762 {
2763         int ret = 0, prtad, devad;
2764         struct port_info *pi = netdev_priv(dev);
2765         struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
2766
2767         switch (cmd) {
2768         case SIOCGMIIPHY:
2769                 if (pi->mdio_addr < 0)
2770                         return -EOPNOTSUPP;
2771                 data->phy_id = pi->mdio_addr;
2772                 break;
2773         case SIOCGMIIREG:
2774         case SIOCSMIIREG:
2775                 if (mdio_phy_id_is_c45(data->phy_id)) {
2776                         prtad = mdio_phy_id_prtad(data->phy_id);
2777                         devad = mdio_phy_id_devad(data->phy_id);
2778                 } else if (data->phy_id < 32) {
2779                         prtad = data->phy_id;
2780                         devad = 0;
2781                         data->reg_num &= 0x1f;
2782                 } else
2783                         return -EINVAL;
2784
2785                 if (cmd == SIOCGMIIREG)
2786                         ret = t4_mdio_rd(pi->adapter, 0, prtad, devad,
2787                                          data->reg_num, &data->val_out);
2788                 else
2789                         ret = t4_mdio_wr(pi->adapter, 0, prtad, devad,
2790                                          data->reg_num, data->val_in);
2791                 break;
2792         default:
2793                 return -EOPNOTSUPP;
2794         }
2795         return ret;
2796 }
2797
2798 static void cxgb_set_rxmode(struct net_device *dev)
2799 {
2800         /* unfortunately we can't return errors to the stack */
2801         set_rxmode(dev, -1, false);
2802 }
2803
2804 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2805 {
2806         int ret;
2807         struct port_info *pi = netdev_priv(dev);
2808
2809         if (new_mtu < 81 || new_mtu > MAX_MTU)         /* accommodate SACK */
2810                 return -EINVAL;
2811         ret = t4_set_rxmode(pi->adapter, 0, pi->viid, new_mtu, -1, -1, -1, -1,
2812                             true);
2813         if (!ret)
2814                 dev->mtu = new_mtu;
2815         return ret;
2816 }
2817
2818 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2819 {
2820         int ret;
2821         struct sockaddr *addr = p;
2822         struct port_info *pi = netdev_priv(dev);
2823
2824         if (!is_valid_ether_addr(addr->sa_data))
2825                 return -EINVAL;
2826
2827         ret = t4_change_mac(pi->adapter, 0, pi->viid, pi->xact_addr_filt,
2828                             addr->sa_data, true, true);
2829         if (ret < 0)
2830                 return ret;
2831
2832         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2833         pi->xact_addr_filt = ret;
2834         return 0;
2835 }
2836
2837 static void vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
2838 {
2839         struct port_info *pi = netdev_priv(dev);
2840
2841         pi->vlan_grp = grp;
2842         t4_set_rxmode(pi->adapter, 0, pi->viid, -1, -1, -1, -1, grp != NULL,
2843                       true);
2844 }
2845
2846 #ifdef CONFIG_NET_POLL_CONTROLLER
2847 static void cxgb_netpoll(struct net_device *dev)
2848 {
2849         struct port_info *pi = netdev_priv(dev);
2850         struct adapter *adap = pi->adapter;
2851
2852         if (adap->flags & USING_MSIX) {
2853                 int i;
2854                 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
2855
2856                 for (i = pi->nqsets; i; i--, rx++)
2857                         t4_sge_intr_msix(0, &rx->rspq);
2858         } else
2859                 t4_intr_handler(adap)(0, adap);
2860 }
2861 #endif
2862
2863 static const struct net_device_ops cxgb4_netdev_ops = {
2864         .ndo_open             = cxgb_open,
2865         .ndo_stop             = cxgb_close,
2866         .ndo_start_xmit       = t4_eth_xmit,
2867         .ndo_get_stats64      = cxgb_get_stats,
2868         .ndo_set_rx_mode      = cxgb_set_rxmode,
2869         .ndo_set_mac_address  = cxgb_set_mac_addr,
2870         .ndo_validate_addr    = eth_validate_addr,
2871         .ndo_do_ioctl         = cxgb_ioctl,
2872         .ndo_change_mtu       = cxgb_change_mtu,
2873         .ndo_vlan_rx_register = vlan_rx_register,
2874 #ifdef CONFIG_NET_POLL_CONTROLLER
2875         .ndo_poll_controller  = cxgb_netpoll,
2876 #endif
2877 };
2878
2879 void t4_fatal_err(struct adapter *adap)
2880 {
2881         t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
2882         t4_intr_disable(adap);
2883         dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
2884 }
2885
2886 static void setup_memwin(struct adapter *adap)
2887 {
2888         u32 bar0;
2889
2890         bar0 = pci_resource_start(adap->pdev, 0);  /* truncation intentional */
2891         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
2892                      (bar0 + MEMWIN0_BASE) | BIR(0) |
2893                      WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
2894         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
2895                      (bar0 + MEMWIN1_BASE) | BIR(0) |
2896                      WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
2897         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
2898                      (bar0 + MEMWIN2_BASE) | BIR(0) |
2899                      WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
2900 }
2901
2902 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
2903 {
2904         u32 v;
2905         int ret;
2906
2907         /* get device capabilities */
2908         memset(c, 0, sizeof(*c));
2909         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2910                                FW_CMD_REQUEST | FW_CMD_READ);
2911         c->retval_len16 = htonl(FW_LEN16(*c));
2912         ret = t4_wr_mbox(adap, 0, c, sizeof(*c), c);
2913         if (ret < 0)
2914                 return ret;
2915
2916         /* select capabilities we'll be using */
2917         if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
2918                 if (!vf_acls)
2919                         c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
2920                 else
2921                         c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
2922         } else if (vf_acls) {
2923                 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
2924                 return ret;
2925         }
2926         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2927                                FW_CMD_REQUEST | FW_CMD_WRITE);
2928         ret = t4_wr_mbox(adap, 0, c, sizeof(*c), NULL);
2929         if (ret < 0)
2930                 return ret;
2931
2932         ret = t4_config_glbl_rss(adap, 0,
2933                                  FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
2934                                  FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
2935                                  FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
2936         if (ret < 0)
2937                 return ret;
2938
2939         ret = t4_cfg_pfvf(adap, 0, 0, 0, MAX_EGRQ, 64, MAX_INGQ, 0, 0, 4,
2940                           0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
2941         if (ret < 0)
2942                 return ret;
2943
2944         t4_sge_init(adap);
2945
2946         /* get basic stuff going */
2947         ret = t4_early_init(adap, 0);
2948         if (ret < 0)
2949                 return ret;
2950
2951         /* tweak some settings */
2952         t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
2953         t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
2954         t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
2955         v = t4_read_reg(adap, TP_PIO_DATA);
2956         t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
2957         setup_memwin(adap);
2958         return 0;
2959 }
2960
2961 /*
2962  * Max # of ATIDs.  The absolute HW max is 16K but we keep it lower.
2963  */
2964 #define MAX_ATIDS 8192U
2965
2966 /*
2967  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
2968  */
2969 static int adap_init0(struct adapter *adap)
2970 {
2971         int ret;
2972         u32 v, port_vec;
2973         enum dev_state state;
2974         u32 params[7], val[7];
2975         struct fw_caps_config_cmd c;
2976
2977         ret = t4_check_fw_version(adap);
2978         if (ret == -EINVAL || ret > 0) {
2979                 if (upgrade_fw(adap) >= 0)             /* recache FW version */
2980                         ret = t4_check_fw_version(adap);
2981         }
2982         if (ret < 0)
2983                 return ret;
2984
2985         /* contact FW, request master */
2986         ret = t4_fw_hello(adap, 0, 0, MASTER_MUST, &state);
2987         if (ret < 0) {
2988                 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
2989                         ret);
2990                 return ret;
2991         }
2992
2993         /* reset device */
2994         ret = t4_fw_reset(adap, 0, PIORSTMODE | PIORST);
2995         if (ret < 0)
2996                 goto bye;
2997
2998         for (v = 0; v < SGE_NTIMERS - 1; v++)
2999                 adap->sge.timer_val[v] = min(intr_holdoff[v], MAX_SGE_TIMERVAL);
3000         adap->sge.timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
3001         adap->sge.counter_val[0] = 1;
3002         for (v = 1; v < SGE_NCOUNTERS; v++)
3003                 adap->sge.counter_val[v] = min(intr_cnt[v - 1],
3004                                                THRESHOLD_3_MASK);
3005 #define FW_PARAM_DEV(param) \
3006         (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
3007          FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
3008
3009         params[0] = FW_PARAM_DEV(CCLK);
3010         ret = t4_query_params(adap, 0, 0, 0, 1, params, val);
3011         if (ret < 0)
3012                 goto bye;
3013         adap->params.vpd.cclk = val[0];
3014
3015         ret = adap_init1(adap, &c);
3016         if (ret < 0)
3017                 goto bye;
3018
3019 #define FW_PARAM_PFVF(param) \
3020         (FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
3021          FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
3022
3023         params[0] = FW_PARAM_DEV(PORTVEC);
3024         params[1] = FW_PARAM_PFVF(L2T_START);
3025         params[2] = FW_PARAM_PFVF(L2T_END);
3026         params[3] = FW_PARAM_PFVF(FILTER_START);
3027         params[4] = FW_PARAM_PFVF(FILTER_END);
3028         ret = t4_query_params(adap, 0, 0, 0, 5, params, val);
3029         if (ret < 0)
3030                 goto bye;
3031         port_vec = val[0];
3032         adap->tids.ftid_base = val[3];
3033         adap->tids.nftids = val[4] - val[3] + 1;
3034
3035         if (c.ofldcaps) {
3036                 /* query offload-related parameters */
3037                 params[0] = FW_PARAM_DEV(NTID);
3038                 params[1] = FW_PARAM_PFVF(SERVER_START);
3039                 params[2] = FW_PARAM_PFVF(SERVER_END);
3040                 params[3] = FW_PARAM_PFVF(TDDP_START);
3041                 params[4] = FW_PARAM_PFVF(TDDP_END);
3042                 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3043                 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
3044                 if (ret < 0)
3045                         goto bye;
3046                 adap->tids.ntids = val[0];
3047                 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
3048                 adap->tids.stid_base = val[1];
3049                 adap->tids.nstids = val[2] - val[1] + 1;
3050                 adap->vres.ddp.start = val[3];
3051                 adap->vres.ddp.size = val[4] - val[3] + 1;
3052                 adap->params.ofldq_wr_cred = val[5];
3053                 adap->params.offload = 1;
3054         }
3055         if (c.rdmacaps) {
3056                 params[0] = FW_PARAM_PFVF(STAG_START);
3057                 params[1] = FW_PARAM_PFVF(STAG_END);
3058                 params[2] = FW_PARAM_PFVF(RQ_START);
3059                 params[3] = FW_PARAM_PFVF(RQ_END);
3060                 params[4] = FW_PARAM_PFVF(PBL_START);
3061                 params[5] = FW_PARAM_PFVF(PBL_END);
3062                 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
3063                 if (ret < 0)
3064                         goto bye;
3065                 adap->vres.stag.start = val[0];
3066                 adap->vres.stag.size = val[1] - val[0] + 1;
3067                 adap->vres.rq.start = val[2];
3068                 adap->vres.rq.size = val[3] - val[2] + 1;
3069                 adap->vres.pbl.start = val[4];
3070                 adap->vres.pbl.size = val[5] - val[4] + 1;
3071
3072                 params[0] = FW_PARAM_PFVF(SQRQ_START);
3073                 params[1] = FW_PARAM_PFVF(SQRQ_END);
3074                 params[2] = FW_PARAM_PFVF(CQ_START);
3075                 params[3] = FW_PARAM_PFVF(CQ_END);
3076                 ret = t4_query_params(adap, 0, 0, 0, 4, params, val);
3077                 if (ret < 0)
3078                         goto bye;
3079                 adap->vres.qp.start = val[0];
3080                 adap->vres.qp.size = val[1] - val[0] + 1;
3081                 adap->vres.cq.start = val[2];
3082                 adap->vres.cq.size = val[3] - val[2] + 1;
3083         }
3084         if (c.iscsicaps) {
3085                 params[0] = FW_PARAM_PFVF(ISCSI_START);
3086                 params[1] = FW_PARAM_PFVF(ISCSI_END);
3087                 ret = t4_query_params(adap, 0, 0, 0, 2, params, val);
3088                 if (ret < 0)
3089                         goto bye;
3090                 adap->vres.iscsi.start = val[0];
3091                 adap->vres.iscsi.size = val[1] - val[0] + 1;
3092         }
3093 #undef FW_PARAM_PFVF
3094 #undef FW_PARAM_DEV
3095
3096         adap->params.nports = hweight32(port_vec);
3097         adap->params.portvec = port_vec;
3098         adap->flags |= FW_OK;
3099
3100         /* These are finalized by FW initialization, load their values now */
3101         v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
3102         adap->params.tp.tre = TIMERRESOLUTION_GET(v);
3103         t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
3104         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3105                      adap->params.b_wnd);
3106
3107 #ifdef CONFIG_PCI_IOV
3108         /*
3109          * Provision resource limits for Virtual Functions.  We currently
3110          * grant them all the same static resource limits except for the Port
3111          * Access Rights Mask which we're assigning based on the PF.  All of
3112          * the static provisioning stuff for both the PF and VF really needs
3113          * to be managed in a persistent manner for each device which the
3114          * firmware controls.
3115          */
3116         {
3117                 int pf, vf;
3118
3119                 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
3120                         if (num_vf[pf] <= 0)
3121                                 continue;
3122
3123                         /* VF numbering starts at 1! */
3124                         for (vf = 1; vf <= num_vf[pf]; vf++) {
3125                                 ret = t4_cfg_pfvf(adap, 0, pf, vf,
3126                                                   VFRES_NEQ, VFRES_NETHCTRL,
3127                                                   VFRES_NIQFLINT, VFRES_NIQ,
3128                                                   VFRES_TC, VFRES_NVI,
3129                                                   FW_PFVF_CMD_CMASK_MASK,
3130                                                   pfvfres_pmask(adap, pf, vf),
3131                                                   VFRES_NEXACTF,
3132                                                   VFRES_R_CAPS, VFRES_WX_CAPS);
3133                                 if (ret < 0)
3134                                         dev_warn(adap->pdev_dev, "failed to "
3135                                                  "provision pf/vf=%d/%d; "
3136                                                  "err=%d\n", pf, vf, ret);
3137                         }
3138                 }
3139         }
3140 #endif
3141
3142         return 0;
3143
3144         /*
3145          * If a command timed out or failed with EIO FW does not operate within
3146          * its spec or something catastrophic happened to HW/FW, stop issuing
3147          * commands.
3148          */
3149 bye:    if (ret != -ETIMEDOUT && ret != -EIO)
3150                 t4_fw_bye(adap, 0);
3151         return ret;
3152 }
3153
3154 /* EEH callbacks */
3155
3156 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
3157                                          pci_channel_state_t state)
3158 {
3159         int i;
3160         struct adapter *adap = pci_get_drvdata(pdev);
3161
3162         if (!adap)
3163                 goto out;
3164
3165         rtnl_lock();
3166         adap->flags &= ~FW_OK;
3167         notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
3168         for_each_port(adap, i) {
3169                 struct net_device *dev = adap->port[i];
3170
3171                 netif_device_detach(dev);
3172                 netif_carrier_off(dev);
3173         }
3174         if (adap->flags & FULL_INIT_DONE)
3175                 cxgb_down(adap);
3176         rtnl_unlock();
3177         pci_disable_device(pdev);
3178 out:    return state == pci_channel_io_perm_failure ?
3179                 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
3180 }
3181
3182 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
3183 {
3184         int i, ret;
3185         struct fw_caps_config_cmd c;
3186         struct adapter *adap = pci_get_drvdata(pdev);
3187
3188         if (!adap) {
3189                 pci_restore_state(pdev);
3190                 pci_save_state(pdev);
3191                 return PCI_ERS_RESULT_RECOVERED;
3192         }
3193
3194         if (pci_enable_device(pdev)) {
3195                 dev_err(&pdev->dev, "cannot reenable PCI device after reset\n");
3196                 return PCI_ERS_RESULT_DISCONNECT;
3197         }
3198
3199         pci_set_master(pdev);
3200         pci_restore_state(pdev);
3201         pci_save_state(pdev);
3202         pci_cleanup_aer_uncorrect_error_status(pdev);
3203
3204         if (t4_wait_dev_ready(adap) < 0)
3205                 return PCI_ERS_RESULT_DISCONNECT;
3206         if (t4_fw_hello(adap, 0, 0, MASTER_MUST, NULL))
3207                 return PCI_ERS_RESULT_DISCONNECT;
3208         adap->flags |= FW_OK;
3209         if (adap_init1(adap, &c))
3210                 return PCI_ERS_RESULT_DISCONNECT;
3211
3212         for_each_port(adap, i) {
3213                 struct port_info *p = adap2pinfo(adap, i);
3214
3215                 ret = t4_alloc_vi(adap, 0, p->tx_chan, 0, 0, 1, NULL, NULL);
3216                 if (ret < 0)
3217                         return PCI_ERS_RESULT_DISCONNECT;
3218                 p->viid = ret;
3219                 p->xact_addr_filt = -1;
3220         }
3221
3222         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3223                      adap->params.b_wnd);
3224         if (cxgb_up(adap))
3225                 return PCI_ERS_RESULT_DISCONNECT;
3226         return PCI_ERS_RESULT_RECOVERED;
3227 }
3228
3229 static void eeh_resume(struct pci_dev *pdev)
3230 {
3231         int i;
3232         struct adapter *adap = pci_get_drvdata(pdev);
3233
3234         if (!adap)
3235                 return;
3236
3237         rtnl_lock();
3238         for_each_port(adap, i) {
3239                 struct net_device *dev = adap->port[i];
3240
3241                 if (netif_running(dev)) {
3242                         link_start(dev);
3243                         cxgb_set_rxmode(dev);
3244                 }
3245                 netif_device_attach(dev);
3246         }
3247         rtnl_unlock();
3248 }
3249
3250 static struct pci_error_handlers cxgb4_eeh = {
3251         .error_detected = eeh_err_detected,
3252         .slot_reset     = eeh_slot_reset,
3253         .resume         = eeh_resume,
3254 };
3255
3256 static inline bool is_10g_port(const struct link_config *lc)
3257 {
3258         return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
3259 }
3260
3261 static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
3262                              unsigned int size, unsigned int iqe_size)
3263 {
3264         q->intr_params = QINTR_TIMER_IDX(timer_idx) |
3265                          (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
3266         q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
3267         q->iqe_len = iqe_size;
3268         q->size = size;
3269 }
3270
3271 /*
3272  * Perform default configuration of DMA queues depending on the number and type
3273  * of ports we found and the number of available CPUs.  Most settings can be
3274  * modified by the admin prior to actual use.
3275  */
3276 static void __devinit cfg_queues(struct adapter *adap)
3277 {
3278         struct sge *s = &adap->sge;
3279         int i, q10g = 0, n10g = 0, qidx = 0;
3280
3281         for_each_port(adap, i)
3282                 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
3283
3284         /*
3285          * We default to 1 queue per non-10G port and up to # of cores queues
3286          * per 10G port.
3287          */
3288         if (n10g)
3289                 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
3290         if (q10g > num_online_cpus())
3291                 q10g = num_online_cpus();
3292
3293         for_each_port(adap, i) {
3294                 struct port_info *pi = adap2pinfo(adap, i);
3295
3296                 pi->first_qset = qidx;
3297                 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
3298                 qidx += pi->nqsets;
3299         }
3300
3301         s->ethqsets = qidx;
3302         s->max_ethqsets = qidx;   /* MSI-X may lower it later */
3303
3304         if (is_offload(adap)) {
3305                 /*
3306                  * For offload we use 1 queue/channel if all ports are up to 1G,
3307                  * otherwise we divide all available queues amongst the channels
3308                  * capped by the number of available cores.
3309                  */
3310                 if (n10g) {
3311                         i = min_t(int, ARRAY_SIZE(s->ofldrxq),
3312                                   num_online_cpus());
3313                         s->ofldqsets = roundup(i, adap->params.nports);
3314                 } else
3315                         s->ofldqsets = adap->params.nports;
3316                 /* For RDMA one Rx queue per channel suffices */
3317                 s->rdmaqs = adap->params.nports;
3318         }
3319
3320         for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
3321                 struct sge_eth_rxq *r = &s->ethrxq[i];
3322
3323                 init_rspq(&r->rspq, 0, 0, 1024, 64);
3324                 r->fl.size = 72;
3325         }
3326
3327         for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
3328                 s->ethtxq[i].q.size = 1024;
3329
3330         for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
3331                 s->ctrlq[i].q.size = 512;
3332
3333         for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
3334                 s->ofldtxq[i].q.size = 1024;
3335
3336         for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
3337                 struct sge_ofld_rxq *r = &s->ofldrxq[i];
3338
3339                 init_rspq(&r->rspq, 0, 0, 1024, 64);
3340                 r->rspq.uld = CXGB4_ULD_ISCSI;
3341                 r->fl.size = 72;
3342         }
3343
3344         for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
3345                 struct sge_ofld_rxq *r = &s->rdmarxq[i];
3346
3347                 init_rspq(&r->rspq, 0, 0, 511, 64);
3348                 r->rspq.uld = CXGB4_ULD_RDMA;
3349                 r->fl.size = 72;
3350         }
3351
3352         init_rspq(&s->fw_evtq, 6, 0, 512, 64);
3353         init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
3354 }
3355
3356 /*
3357  * Reduce the number of Ethernet queues across all ports to at most n.
3358  * n provides at least one queue per port.
3359  */
3360 static void __devinit reduce_ethqs(struct adapter *adap, int n)
3361 {
3362         int i;
3363         struct port_info *pi;
3364
3365         while (n < adap->sge.ethqsets)
3366                 for_each_port(adap, i) {
3367                         pi = adap2pinfo(adap, i);
3368                         if (pi->nqsets > 1) {
3369                                 pi->nqsets--;
3370                                 adap->sge.ethqsets--;
3371                                 if (adap->sge.ethqsets <= n)
3372                                         break;
3373                         }
3374                 }
3375
3376         n = 0;
3377         for_each_port(adap, i) {
3378                 pi = adap2pinfo(adap, i);
3379                 pi->first_qset = n;
3380                 n += pi->nqsets;
3381         }
3382 }
3383
3384 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
3385 #define EXTRA_VECS 2
3386
3387 static int __devinit enable_msix(struct adapter *adap)
3388 {
3389         int ofld_need = 0;
3390         int i, err, want, need;
3391         struct sge *s = &adap->sge;
3392         unsigned int nchan = adap->params.nports;
3393         struct msix_entry entries[MAX_INGQ + 1];
3394
3395         for (i = 0; i < ARRAY_SIZE(entries); ++i)
3396                 entries[i].entry = i;
3397
3398         want = s->max_ethqsets + EXTRA_VECS;
3399         if (is_offload(adap)) {
3400                 want += s->rdmaqs + s->ofldqsets;
3401                 /* need nchan for each possible ULD */
3402                 ofld_need = 2 * nchan;
3403         }
3404         need = adap->params.nports + EXTRA_VECS + ofld_need;
3405
3406         while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
3407                 want = err;
3408
3409         if (!err) {
3410                 /*
3411                  * Distribute available vectors to the various queue groups.
3412                  * Every group gets its minimum requirement and NIC gets top
3413                  * priority for leftovers.
3414                  */
3415                 i = want - EXTRA_VECS - ofld_need;
3416                 if (i < s->max_ethqsets) {
3417                         s->max_ethqsets = i;
3418                         if (i < s->ethqsets)
3419                                 reduce_ethqs(adap, i);
3420                 }
3421                 if (is_offload(adap)) {
3422                         i = want - EXTRA_VECS - s->max_ethqsets;
3423                         i -= ofld_need - nchan;
3424                         s->ofldqsets = (i / nchan) * nchan;  /* round down */
3425                 }
3426                 for (i = 0; i < want; ++i)
3427                         adap->msix_info[i].vec = entries[i].vector;
3428         } else if (err > 0)
3429                 dev_info(adap->pdev_dev,
3430                          "only %d MSI-X vectors left, not using MSI-X\n", err);
3431         return err;
3432 }
3433
3434 #undef EXTRA_VECS
3435
3436 static int __devinit init_rss(struct adapter *adap)
3437 {
3438         unsigned int i, j;
3439
3440         for_each_port(adap, i) {
3441                 struct port_info *pi = adap2pinfo(adap, i);
3442
3443                 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
3444                 if (!pi->rss)
3445                         return -ENOMEM;
3446                 for (j = 0; j < pi->rss_size; j++)
3447                         pi->rss[j] = j % pi->nqsets;
3448         }
3449         return 0;
3450 }
3451
3452 static void __devinit print_port_info(struct adapter *adap)
3453 {
3454         static const char *base[] = {
3455                 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
3456                 "KX", "KR", "KR SFP+", "KR FEC"
3457         };
3458
3459         int i;
3460         char buf[80];
3461         const char *spd = "";
3462
3463         if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
3464                 spd = " 2.5 GT/s";
3465         else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
3466                 spd = " 5 GT/s";
3467
3468         for_each_port(adap, i) {
3469                 struct net_device *dev = adap->port[i];
3470                 const struct port_info *pi = netdev_priv(dev);
3471                 char *bufp = buf;
3472
3473                 if (!test_bit(i, &adap->registered_device_map))
3474                         continue;
3475
3476                 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
3477                         bufp += sprintf(bufp, "100/");
3478                 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
3479                         bufp += sprintf(bufp, "1000/");
3480                 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
3481                         bufp += sprintf(bufp, "10G/");
3482                 if (bufp != buf)
3483                         --bufp;
3484                 sprintf(bufp, "BASE-%s", base[pi->port_type]);
3485
3486                 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
3487                             adap->params.vpd.id, adap->params.rev,
3488                             buf, is_offload(adap) ? "R" : "",
3489                             adap->params.pci.width, spd,
3490                             (adap->flags & USING_MSIX) ? " MSI-X" :
3491                             (adap->flags & USING_MSI) ? " MSI" : "");
3492                 if (adap->name == dev->name)
3493                         netdev_info(dev, "S/N: %s, E/C: %s\n",
3494                                     adap->params.vpd.sn, adap->params.vpd.ec);
3495         }
3496 }
3497
3498 /*
3499  * Free the following resources:
3500  * - memory used for tables
3501  * - MSI/MSI-X
3502  * - net devices
3503  * - resources FW is holding for us
3504  */
3505 static void free_some_resources(struct adapter *adapter)
3506 {
3507         unsigned int i;
3508
3509         t4_free_mem(adapter->l2t);
3510         t4_free_mem(adapter->tids.tid_tab);
3511         disable_msi(adapter);
3512
3513         for_each_port(adapter, i)
3514                 if (adapter->port[i]) {
3515                         kfree(adap2pinfo(adapter, i)->rss);
3516                         free_netdev(adapter->port[i]);
3517                 }
3518         if (adapter->flags & FW_OK)
3519                 t4_fw_bye(adapter, 0);
3520 }
3521
3522 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO | NETIF_F_TSO6 |\
3523                    NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3524
3525 static int __devinit init_one(struct pci_dev *pdev,
3526                               const struct pci_device_id *ent)
3527 {
3528         int func, i, err;
3529         struct port_info *pi;
3530         unsigned int highdma = 0;
3531         struct adapter *adapter = NULL;
3532
3533         printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3534
3535         err = pci_request_regions(pdev, KBUILD_MODNAME);
3536         if (err) {
3537                 /* Just info, some other driver may have claimed the device. */
3538                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3539                 return err;
3540         }
3541
3542         /* We control everything through PF 0 */
3543         func = PCI_FUNC(pdev->devfn);
3544         if (func > 0) {
3545                 pci_save_state(pdev);        /* to restore SR-IOV later */
3546                 goto sriov;
3547         }
3548
3549         err = pci_enable_device(pdev);
3550         if (err) {
3551                 dev_err(&pdev->dev, "cannot enable PCI device\n");
3552                 goto out_release_regions;
3553         }
3554
3555         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3556                 highdma = NETIF_F_HIGHDMA;
3557                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3558                 if (err) {
3559                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3560                                 "coherent allocations\n");
3561                         goto out_disable_device;
3562                 }
3563         } else {
3564                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3565                 if (err) {
3566                         dev_err(&pdev->dev, "no usable DMA configuration\n");
3567                         goto out_disable_device;
3568                 }
3569         }
3570
3571         pci_enable_pcie_error_reporting(pdev);
3572         pci_set_master(pdev);
3573         pci_save_state(pdev);
3574
3575         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3576         if (!adapter) {
3577                 err = -ENOMEM;
3578                 goto out_disable_device;
3579         }
3580
3581         adapter->regs = pci_ioremap_bar(pdev, 0);
3582         if (!adapter->regs) {
3583                 dev_err(&pdev->dev, "cannot map device registers\n");
3584                 err = -ENOMEM;
3585                 goto out_free_adapter;
3586         }
3587
3588         adapter->pdev = pdev;
3589         adapter->pdev_dev = &pdev->dev;
3590         adapter->name = pci_name(pdev);
3591         adapter->msg_enable = dflt_msg_enable;
3592         memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
3593
3594         spin_lock_init(&adapter->stats_lock);
3595         spin_lock_init(&adapter->tid_release_lock);
3596
3597         INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
3598
3599         err = t4_prep_adapter(adapter);
3600         if (err)
3601                 goto out_unmap_bar;
3602         err = adap_init0(adapter);
3603         if (err)
3604                 goto out_unmap_bar;
3605
3606         for_each_port(adapter, i) {
3607                 struct net_device *netdev;
3608
3609                 netdev = alloc_etherdev_mq(sizeof(struct port_info),
3610                                            MAX_ETH_QSETS);
3611                 if (!netdev) {
3612                         err = -ENOMEM;
3613                         goto out_free_dev;
3614                 }
3615
3616                 SET_NETDEV_DEV(netdev, &pdev->dev);
3617
3618                 adapter->port[i] = netdev;
3619                 pi = netdev_priv(netdev);
3620                 pi->adapter = adapter;
3621                 pi->xact_addr_filt = -1;
3622                 pi->rx_offload = RX_CSO;
3623                 pi->port_id = i;
3624                 netif_carrier_off(netdev);
3625                 netif_tx_stop_all_queues(netdev);
3626                 netdev->irq = pdev->irq;
3627
3628                 netdev->features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6;
3629                 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
3630                 netdev->features |= NETIF_F_GRO | NETIF_F_RXHASH | highdma;
3631                 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
3632                 netdev->vlan_features = netdev->features & VLAN_FEAT;
3633
3634                 netdev->netdev_ops = &cxgb4_netdev_ops;
3635                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3636         }
3637
3638         pci_set_drvdata(pdev, adapter);
3639
3640         if (adapter->flags & FW_OK) {
3641                 err = t4_port_init(adapter, 0, 0, 0);
3642                 if (err)
3643                         goto out_free_dev;
3644         }
3645
3646         /*
3647          * Configure queues and allocate tables now, they can be needed as
3648          * soon as the first register_netdev completes.
3649          */
3650         cfg_queues(adapter);
3651
3652         adapter->l2t = t4_init_l2t();
3653         if (!adapter->l2t) {
3654                 /* We tolerate a lack of L2T, giving up some functionality */
3655                 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
3656                 adapter->params.offload = 0;
3657         }
3658
3659         if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
3660                 dev_warn(&pdev->dev, "could not allocate TID table, "
3661                          "continuing\n");
3662                 adapter->params.offload = 0;
3663         }
3664
3665         /* See what interrupts we'll be using */
3666         if (msi > 1 && enable_msix(adapter) == 0)
3667                 adapter->flags |= USING_MSIX;
3668         else if (msi > 0 && pci_enable_msi(pdev) == 0)
3669                 adapter->flags |= USING_MSI;
3670
3671         err = init_rss(adapter);
3672         if (err)
3673                 goto out_free_dev;
3674
3675         /*
3676          * The card is now ready to go.  If any errors occur during device
3677          * registration we do not fail the whole card but rather proceed only
3678          * with the ports we manage to register successfully.  However we must
3679          * register at least one net device.
3680          */
3681         for_each_port(adapter, i) {
3682                 err = register_netdev(adapter->port[i]);
3683                 if (err)
3684                         dev_warn(&pdev->dev,
3685                                  "cannot register net device %s, skipping\n",
3686                                  adapter->port[i]->name);
3687                 else {
3688                         /*
3689                          * Change the name we use for messages to the name of
3690                          * the first successfully registered interface.
3691                          */
3692                         if (!adapter->registered_device_map)
3693                                 adapter->name = adapter->port[i]->name;
3694
3695                         __set_bit(i, &adapter->registered_device_map);
3696                         adapter->chan_map[adap2pinfo(adapter, i)->tx_chan] = i;
3697                 }
3698         }
3699         if (!adapter->registered_device_map) {
3700                 dev_err(&pdev->dev, "could not register any net devices\n");
3701                 goto out_free_dev;
3702         }
3703
3704         if (cxgb4_debugfs_root) {
3705                 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
3706                                                            cxgb4_debugfs_root);
3707                 setup_debugfs(adapter);
3708         }
3709
3710         if (is_offload(adapter))
3711                 attach_ulds(adapter);
3712
3713         print_port_info(adapter);
3714
3715 sriov:
3716 #ifdef CONFIG_PCI_IOV
3717         if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
3718                 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
3719                         dev_info(&pdev->dev,
3720                                  "instantiated %u virtual functions\n",
3721                                  num_vf[func]);
3722 #endif
3723         return 0;
3724
3725  out_free_dev:
3726         free_some_resources(adapter);
3727  out_unmap_bar:
3728         iounmap(adapter->regs);
3729  out_free_adapter:
3730         kfree(adapter);
3731  out_disable_device:
3732         pci_disable_pcie_error_reporting(pdev);
3733         pci_disable_device(pdev);
3734  out_release_regions:
3735         pci_release_regions(pdev);
3736         pci_set_drvdata(pdev, NULL);
3737         return err;
3738 }
3739
3740 static void __devexit remove_one(struct pci_dev *pdev)
3741 {
3742         struct adapter *adapter = pci_get_drvdata(pdev);
3743
3744         pci_disable_sriov(pdev);
3745
3746         if (adapter) {
3747                 int i;
3748
3749                 if (is_offload(adapter))
3750                         detach_ulds(adapter);
3751
3752                 for_each_port(adapter, i)
3753                         if (test_bit(i, &adapter->registered_device_map))
3754                                 unregister_netdev(adapter->port[i]);
3755
3756                 if (adapter->debugfs_root)
3757                         debugfs_remove_recursive(adapter->debugfs_root);
3758
3759                 if (adapter->flags & FULL_INIT_DONE)
3760                         cxgb_down(adapter);
3761
3762                 free_some_resources(adapter);
3763                 iounmap(adapter->regs);
3764                 kfree(adapter);
3765                 pci_disable_pcie_error_reporting(pdev);
3766                 pci_disable_device(pdev);
3767                 pci_release_regions(pdev);
3768                 pci_set_drvdata(pdev, NULL);
3769         } else if (PCI_FUNC(pdev->devfn) > 0)
3770                 pci_release_regions(pdev);
3771 }
3772
3773 static struct pci_driver cxgb4_driver = {
3774         .name     = KBUILD_MODNAME,
3775         .id_table = cxgb4_pci_tbl,
3776         .probe    = init_one,
3777         .remove   = __devexit_p(remove_one),
3778         .err_handler = &cxgb4_eeh,
3779 };
3780
3781 static int __init cxgb4_init_module(void)
3782 {
3783         int ret;
3784
3785         /* Debugfs support is optional, just warn if this fails */
3786         cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
3787         if (!cxgb4_debugfs_root)
3788                 pr_warning("could not create debugfs entry, continuing\n");
3789
3790         ret = pci_register_driver(&cxgb4_driver);
3791         if (ret < 0)
3792                 debugfs_remove(cxgb4_debugfs_root);
3793         return ret;
3794 }
3795
3796 static void __exit cxgb4_cleanup_module(void)
3797 {
3798         pci_unregister_driver(&cxgb4_driver);
3799         debugfs_remove(cxgb4_debugfs_root);  /* NULL ok */
3800 }
3801
3802 module_init(cxgb4_init_module);
3803 module_exit(cxgb4_cleanup_module);