drivers/net/tg3.c

   1 /*
   2  * tg3.c: Broadcom Tigon3 ethernet driver.
   3  *
   4  * Copyright (C) 2001, 2002, 2003, 2004 David S. Miller (davem@redhat.com)
   5  * Copyright (C) 2001, 2002, 2003 Jeff Garzik (jgarzik@pobox.com)
   6  * Copyright (C) 2004 Sun Microsystems Inc.
   7  * Copyright (C) 2005 Broadcom Corporation.
   8  *
   9  * Firmware is:
  10  *      Derived from proprietary unpublished source code,
  11  *      Copyright (C) 2000-2003 Broadcom Corporation.
  12  *
  13  *      Permission is hereby granted for the distribution of this firmware
  14  *      data in hexadecimal or equivalent format, provided this copyright
  15  *      notice is accompanying it.
  16  */
  17
  18
  19 #include <linux/module.h>
  20 #include <linux/moduleparam.h>
  21 #include <linux/kernel.h>
  22 #include <linux/types.h>
  23 #include <linux/compiler.h>
  24 #include <linux/slab.h>
  25 #include <linux/delay.h>
  26 #include <linux/in.h>
  27 #include <linux/init.h>
  28 #include <linux/ioport.h>
  29 #include <linux/pci.h>
  30 #include <linux/netdevice.h>
  31 #include <linux/etherdevice.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/ethtool.h>
  34 #include <linux/mii.h>
  35 #include <linux/if_vlan.h>
  36 #include <linux/ip.h>
  37 #include <linux/tcp.h>
  38 #include <linux/workqueue.h>
  39 #include <linux/prefetch.h>
  40 #include <linux/dma-mapping.h>
  41
  42 #include <net/checksum.h>
  43
  44 #include <asm/system.h>
  45 #include <asm/io.h>
  46 #include <asm/byteorder.h>
  47 #include <asm/uaccess.h>
  48
  49 #ifdef CONFIG_SPARC64
  50 #include <asm/idprom.h>
  51 #include <asm/oplib.h>
  52 #include <asm/pbm.h>
  53 #endif
  54
  55 #if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
  56 #define TG3_VLAN_TAG_USED 1
  57 #else
  58 #define TG3_VLAN_TAG_USED 0
  59 #endif
  60
  61 #ifdef NETIF_F_TSO
  62 #define TG3_TSO_SUPPORT 1
  63 #else
  64 #define TG3_TSO_SUPPORT 0
  65 #endif
  66
  67 #include "tg3.h"
  68
  69 #define DRV_MODULE_NAME         "tg3"
  70 #define PFX DRV_MODULE_NAME     ": "
  71 #define DRV_MODULE_VERSION      "3.70"
  72 #define DRV_MODULE_RELDATE      "December 1, 2006"
  73
  74 #define TG3_DEF_MAC_MODE        0
  75 #define TG3_DEF_RX_MODE         0
  76 #define TG3_DEF_TX_MODE         0
  77 #define TG3_DEF_MSG_ENABLE        \
  78         (NETIF_MSG_DRV          | \
  79          NETIF_MSG_PROBE        | \
  80          NETIF_MSG_LINK         | \
  81          NETIF_MSG_TIMER        | \
  82          NETIF_MSG_IFDOWN       | \
  83          NETIF_MSG_IFUP         | \
  84          NETIF_MSG_RX_ERR       | \
  85          NETIF_MSG_TX_ERR)
  86
  87 /* length of time before we decide the hardware is borked,
  88  * and dev->tx_timeout() should be called to fix the problem
  89  */
  90 #define TG3_TX_TIMEOUT                  (5 * HZ)
  91
  92 /* hardware minimum and maximum for a single frame's data payload */
  93 #define TG3_MIN_MTU                     60
  94 #define TG3_MAX_MTU(tp) \
  95         ((tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) ? 9000 : 1500)
  96
  97 /* These numbers seem to be hard coded in the NIC firmware somehow.
  98  * You can't change the ring sizes, but you can change where you place
  99  * them in the NIC onboard memory.
 100  */
 101 #define TG3_RX_RING_SIZE                512
 102 #define TG3_DEF_RX_RING_PENDING         200
 103 #define TG3_RX_JUMBO_RING_SIZE          256
 104 #define TG3_DEF_RX_JUMBO_RING_PENDING   100
 105
 106 /* Do not place this n-ring entries value into the tp struct itself,
 107  * we really want to expose these constants to GCC so that modulo et
 108  * al.  operations are done with shifts and masks instead of with
 109  * hw multiply/modulo instructions.  Another solution would be to
 110  * replace things like '% foo' with '& (foo - 1)'.
 111  */
 112 #define TG3_RX_RCB_RING_SIZE(tp)        \
 113         ((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ?  512 : 1024)
 114
 115 #define TG3_TX_RING_SIZE                512
 116 #define TG3_DEF_TX_RING_PENDING         (TG3_TX_RING_SIZE - 1)
 117
 118 #define TG3_RX_RING_BYTES       (sizeof(struct tg3_rx_buffer_desc) * \
 119                                  TG3_RX_RING_SIZE)
 120 #define TG3_RX_JUMBO_RING_BYTES (sizeof(struct tg3_rx_buffer_desc) * \
 121                                  TG3_RX_JUMBO_RING_SIZE)
 122 #define TG3_RX_RCB_RING_BYTES(tp) (sizeof(struct tg3_rx_buffer_desc) * \
 123                                    TG3_RX_RCB_RING_SIZE(tp))
 124 #define TG3_TX_RING_BYTES       (sizeof(struct tg3_tx_buffer_desc) * \
 125                                  TG3_TX_RING_SIZE)
 126 #define NEXT_TX(N)              (((N) + 1) & (TG3_TX_RING_SIZE - 1))
 127
 128 #define RX_PKT_BUF_SZ           (1536 + tp->rx_offset + 64)
 129 #define RX_JUMBO_PKT_BUF_SZ     (9046 + tp->rx_offset + 64)
 130
 131 /* minimum number of free TX descriptors required to wake up TX process */
 132 #define TG3_TX_WAKEUP_THRESH(tp)                ((tp)->tx_pending / 4)
 133
 134 /* number of ETHTOOL_GSTATS u64's */
 135 #define TG3_NUM_STATS           (sizeof(struct tg3_ethtool_stats)/sizeof(u64))
 136
 137 #define TG3_NUM_TEST            6
 138
 139 static char version[] __devinitdata =
 140         DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
 141
 142 MODULE_AUTHOR("David S. Miller (davem@redhat.com) and Jeff Garzik (jgarzik@pobox.com)");
 143 MODULE_DESCRIPTION("Broadcom Tigon3 ethernet driver");
 144 MODULE_LICENSE("GPL");
 145 MODULE_VERSION(DRV_MODULE_VERSION);
 146
 147 static int tg3_debug = -1;      /* -1 == use TG3_DEF_MSG_ENABLE as value */
 148 module_param(tg3_debug, int, 0);
 149 MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");
 150
 151 static struct pci_device_id tg3_pci_tbl[] = {
 152         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700)},
 153         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701)},
 154         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702)},
 155         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703)},
 156         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704)},
 157         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE)},
 158         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705)},
 159         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2)},
 160         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M)},
 161         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2)},
 162         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X)},
 163         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X)},
 164         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S)},
 165         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3)},
 166         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3)},
 167         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782)},
 168         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788)},
 169         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789)},
 170         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901)},
 171         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2)},
 172         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2)},
 173         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F)},
 174         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5720)},
 175         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721)},
 176         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5722)},
 177         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750)},
 178         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751)},
 179         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750M)},
 180         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M)},
 181         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F)},
 182         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752)},
 183         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M)},
 184         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753)},
 185         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M)},
 186         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F)},
 187         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754)},
 188         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M)},
 189         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755)},
 190         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M)},
 191         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5756)},
 192         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786)},
 193         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787)},
 194         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M)},
 195         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787F)},
 196         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714)},
 197         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S)},
 198         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715)},
 199         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S)},
 200         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780)},
 201         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S)},
 202         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781)},
 203         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906)},
 204         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906M)},
 205         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX)},
 206         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX)},
 207         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000)},
 208         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001)},
 209         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003)},
 210         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100)},
 211         {PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3)},
 212         {}
 213 };
 214
 215 MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);
 216
 217 static const struct {
 218         const char string[ETH_GSTRING_LEN];
 219 } ethtool_stats_keys[TG3_NUM_STATS] = {
 220         { "rx_octets" },
 221         { "rx_fragments" },
 222         { "rx_ucast_packets" },
 223         { "rx_mcast_packets" },
 224         { "rx_bcast_packets" },
 225         { "rx_fcs_errors" },
 226         { "rx_align_errors" },
 227         { "rx_xon_pause_rcvd" },
 228         { "rx_xoff_pause_rcvd" },
 229         { "rx_mac_ctrl_rcvd" },
 230         { "rx_xoff_entered" },
 231         { "rx_frame_too_long_errors" },
 232         { "rx_jabbers" },
 233         { "rx_undersize_packets" },
 234         { "rx_in_length_errors" },
 235         { "rx_out_length_errors" },
 236         { "rx_64_or_less_octet_packets" },
 237         { "rx_65_to_127_octet_packets" },
 238         { "rx_128_to_255_octet_packets" },
 239         { "rx_256_to_511_octet_packets" },
 240         { "rx_512_to_1023_octet_packets" },
 241         { "rx_1024_to_1522_octet_packets" },
 242         { "rx_1523_to_2047_octet_packets" },
 243         { "rx_2048_to_4095_octet_packets" },
 244         { "rx_4096_to_8191_octet_packets" },
 245         { "rx_8192_to_9022_octet_packets" },
 246
 247         { "tx_octets" },
 248         { "tx_collisions" },
 249
 250         { "tx_xon_sent" },
 251         { "tx_xoff_sent" },
 252         { "tx_flow_control" },
 253         { "tx_mac_errors" },
 254         { "tx_single_collisions" },
 255         { "tx_mult_collisions" },
 256         { "tx_deferred" },
 257         { "tx_excessive_collisions" },
 258         { "tx_late_collisions" },
 259         { "tx_collide_2times" },
 260         { "tx_collide_3times" },
 261         { "tx_collide_4times" },
 262         { "tx_collide_5times" },
 263         { "tx_collide_6times" },
 264         { "tx_collide_7times" },
 265         { "tx_collide_8times" },
 266         { "tx_collide_9times" },
 267         { "tx_collide_10times" },
 268         { "tx_collide_11times" },
 269         { "tx_collide_12times" },
 270         { "tx_collide_13times" },
 271         { "tx_collide_14times" },
 272         { "tx_collide_15times" },
 273         { "tx_ucast_packets" },
 274         { "tx_mcast_packets" },
 275         { "tx_bcast_packets" },
 276         { "tx_carrier_sense_errors" },
 277         { "tx_discards" },
 278         { "tx_errors" },
 279
 280         { "dma_writeq_full" },
 281         { "dma_write_prioq_full" },
 282         { "rxbds_empty" },
 283         { "rx_discards" },
 284         { "rx_errors" },
 285         { "rx_threshold_hit" },
 286
 287         { "dma_readq_full" },
 288         { "dma_read_prioq_full" },
 289         { "tx_comp_queue_full" },
 290
 291         { "ring_set_send_prod_index" },
 292         { "ring_status_update" },
 293         { "nic_irqs" },
 294         { "nic_avoided_irqs" },
 295         { "nic_tx_threshold_hit" }
 296 };
 297
 298 static const struct {
 299         const char string[ETH_GSTRING_LEN];
 300 } ethtool_test_keys[TG3_NUM_TEST] = {
 301         { "nvram test     (online) " },
 302         { "link test      (online) " },
 303         { "register test  (offline)" },
 304         { "memory test    (offline)" },
 305         { "loopback test  (offline)" },
 306         { "interrupt test (offline)" },
 307 };
 308
 309 static void tg3_write32(struct tg3 *tp, u32 off, u32 val)
 310 {
 311         writel(val, tp->regs + off);
 312 }
 313
 314 static u32 tg3_read32(struct tg3 *tp, u32 off)
 315 {
 316         return (readl(tp->regs + off));
 317 }
 318
 319 static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val)
 320 {
 321         unsigned long flags;
 322
 323         spin_lock_irqsave(&tp->indirect_lock, flags);
 324         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
 325         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
 326         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 327 }
 328
 329 static void tg3_write_flush_reg32(struct tg3 *tp, u32 off, u32 val)
 330 {
 331         writel(val, tp->regs + off);
 332         readl(tp->regs + off);
 333 }
 334
 335 static u32 tg3_read_indirect_reg32(struct tg3 *tp, u32 off)
 336 {
 337         unsigned long flags;
 338         u32 val;
 339
 340         spin_lock_irqsave(&tp->indirect_lock, flags);
 341         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
 342         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
 343         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 344         return val;
 345 }
 346
 347 static void tg3_write_indirect_mbox(struct tg3 *tp, u32 off, u32 val)
 348 {
 349         unsigned long flags;
 350
 351         if (off == (MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW)) {
 352                 pci_write_config_dword(tp->pdev, TG3PCI_RCV_RET_RING_CON_IDX +
 353                                        TG3_64BIT_REG_LOW, val);
 354                 return;
 355         }
 356         if (off == (MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW)) {
 357                 pci_write_config_dword(tp->pdev, TG3PCI_STD_RING_PROD_IDX +
 358                                        TG3_64BIT_REG_LOW, val);
 359                 return;
 360         }
 361
 362         spin_lock_irqsave(&tp->indirect_lock, flags);
 363         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
 364         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
 365         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 366
 367         /* In indirect mode when disabling interrupts, we also need
 368          * to clear the interrupt bit in the GRC local ctrl register.
 369          */
 370         if ((off == (MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW)) &&
 371             (val == 0x1)) {
 372                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_LOCAL_CTRL,
 373                                        tp->grc_local_ctrl|GRC_LCLCTRL_CLEARINT);
 374         }
 375 }
 376
 377 static u32 tg3_read_indirect_mbox(struct tg3 *tp, u32 off)
 378 {
 379         unsigned long flags;
 380         u32 val;
 381
 382         spin_lock_irqsave(&tp->indirect_lock, flags);
 383         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
 384         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
 385         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 386         return val;
 387 }
 388
 389 /* usec_wait specifies the wait time in usec when writing to certain registers
 390  * where it is unsafe to read back the register without some delay.
 391  * GRC_LOCAL_CTRL is one example if the GPIOs are toggled to switch power.
 392  * TG3PCI_CLOCK_CTRL is another example if the clock frequencies are changed.
 393  */
 394 static void _tw32_flush(struct tg3 *tp, u32 off, u32 val, u32 usec_wait)
 395 {
 396         if ((tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) ||
 397             (tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
 398                 /* Non-posted methods */
 399                 tp->write32(tp, off, val);
 400         else {
 401                 /* Posted method */
 402                 tg3_write32(tp, off, val);
 403                 if (usec_wait)
 404                         udelay(usec_wait);
 405                 tp->read32(tp, off);
 406         }
 407         /* Wait again after the read for the posted method to guarantee that
 408          * the wait time is met.
 409          */
 410         if (usec_wait)
 411                 udelay(usec_wait);
 412 }
 413
 414 static inline void tw32_mailbox_flush(struct tg3 *tp, u32 off, u32 val)
 415 {
 416         tp->write32_mbox(tp, off, val);
 417         if (!(tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) &&
 418             !(tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
 419                 tp->read32_mbox(tp, off);
 420 }
 421
 422 static void tg3_write32_tx_mbox(struct tg3 *tp, u32 off, u32 val)
 423 {
 424         void __iomem *mbox = tp->regs + off;
 425         writel(val, mbox);
 426         if (tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG)
 427                 writel(val, mbox);
 428         if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
 429                 readl(mbox);
 430 }
 431
 432 static u32 tg3_read32_mbox_5906(struct tg3 *tp, u32 off)
 433 {
 434         return (readl(tp->regs + off + GRCMBOX_BASE));
 435 }
 436
 437 static void tg3_write32_mbox_5906(struct tg3 *tp, u32 off, u32 val)
 438 {
 439         writel(val, tp->regs + off + GRCMBOX_BASE);
 440 }
 441
 442 #define tw32_mailbox(reg, val)  tp->write32_mbox(tp, reg, val)
 443 #define tw32_mailbox_f(reg, val)        tw32_mailbox_flush(tp, (reg), (val))
 444 #define tw32_rx_mbox(reg, val)  tp->write32_rx_mbox(tp, reg, val)
 445 #define tw32_tx_mbox(reg, val)  tp->write32_tx_mbox(tp, reg, val)
 446 #define tr32_mailbox(reg)       tp->read32_mbox(tp, reg)
 447
 448 #define tw32(reg,val)           tp->write32(tp, reg, val)
 449 #define tw32_f(reg,val)         _tw32_flush(tp,(reg),(val), 0)
 450 #define tw32_wait_f(reg,val,us) _tw32_flush(tp,(reg),(val), (us))
 451 #define tr32(reg)               tp->read32(tp, reg)
 452
 453 static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val)
 454 {
 455         unsigned long flags;
 456
 457         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) &&
 458             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC))
 459                 return;
 460
 461         spin_lock_irqsave(&tp->indirect_lock, flags);
 462         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
 463                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
 464                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
 465
 466                 /* Always leave this as zero. */
 467                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
 468         } else {
 469                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
 470                 tw32_f(TG3PCI_MEM_WIN_DATA, val);
 471
 472                 /* Always leave this as zero. */
 473                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
 474         }
 475         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 476 }
 477
 478 static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val)
 479 {
 480         unsigned long flags;
 481
 482         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) &&
 483             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC)) {
 484                 *val = 0;
 485                 return;
 486         }
 487
 488         spin_lock_irqsave(&tp->indirect_lock, flags);
 489         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
 490                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
 491                 pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
 492
 493                 /* Always leave this as zero. */
 494                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
 495         } else {
 496                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
 497                 *val = tr32(TG3PCI_MEM_WIN_DATA);
 498
 499                 /* Always leave this as zero. */
 500                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
 501         }
 502         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 503 }
 504
 505 static void tg3_disable_ints(struct tg3 *tp)
 506 {
 507         tw32(TG3PCI_MISC_HOST_CTRL,
 508              (tp->misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT));
 509         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
 510 }
 511
 512 static inline void tg3_cond_int(struct tg3 *tp)
 513 {
 514         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
 515             (tp->hw_status->status & SD_STATUS_UPDATED))
 516                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
 517         else
 518                 tw32(HOSTCC_MODE, tp->coalesce_mode |
 519                      (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
 520 }
 521
 522 static void tg3_enable_ints(struct tg3 *tp)
 523 {
 524         tp->irq_sync = 0;
 525         wmb();
 526
 527         tw32(TG3PCI_MISC_HOST_CTRL,
 528              (tp->misc_host_ctrl & ~MISC_HOST_CTRL_MASK_PCI_INT));
 529         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 530                        (tp->last_tag << 24));
 531         if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)
 532                 tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 533                                (tp->last_tag << 24));
 534         tg3_cond_int(tp);
 535 }
 536
 537 static inline unsigned int tg3_has_work(struct tg3 *tp)
 538 {
 539         struct tg3_hw_status *sblk = tp->hw_status;
 540         unsigned int work_exists = 0;
 541
 542         /* check for phy events */
 543         if (!(tp->tg3_flags &
 544               (TG3_FLAG_USE_LINKCHG_REG |
 545                TG3_FLAG_POLL_SERDES))) {
 546                 if (sblk->status & SD_STATUS_LINK_CHG)
 547                         work_exists = 1;
 548         }
 549         /* check for RX/TX work to do */
 550         if (sblk->idx[0].tx_consumer != tp->tx_cons ||
 551             sblk->idx[0].rx_producer != tp->rx_rcb_ptr)
 552                 work_exists = 1;
 553
 554         return work_exists;
 555 }
 556
 557 /* tg3_restart_ints
 558  *  similar to tg3_enable_ints, but it accurately determines whether there
 559  *  is new work pending and can return without flushing the PIO write
 560  *  which reenables interrupts
 561  */
 562 static void tg3_restart_ints(struct tg3 *tp)
 563 {
 564         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 565                      tp->last_tag << 24);
 566         mmiowb();
 567
 568         /* When doing tagged status, this work check is unnecessary.
 569          * The last_tag we write above tells the chip which piece of
 570          * work we've completed.
 571          */
 572         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
 573             tg3_has_work(tp))
 574                 tw32(HOSTCC_MODE, tp->coalesce_mode |
 575                      (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
 576 }
 577
 578 static inline void tg3_netif_stop(struct tg3 *tp)
 579 {
 580         tp->dev->trans_start = jiffies; /* prevent tx timeout */
 581         netif_poll_disable(tp->dev);
 582         netif_tx_disable(tp->dev);
 583 }
 584
 585 static inline void tg3_netif_start(struct tg3 *tp)
 586 {
 587         netif_wake_queue(tp->dev);
 588         /* NOTE: unconditional netif_wake_queue is only appropriate
 589          * so long as all callers are assured to have free tx slots
 590          * (such as after tg3_init_hw)
 591          */
 592         netif_poll_enable(tp->dev);
 593         tp->hw_status->status |= SD_STATUS_UPDATED;
 594         tg3_enable_ints(tp);
 595 }
 596
 597 static void tg3_switch_clocks(struct tg3 *tp)
 598 {
 599         u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL);
 600         u32 orig_clock_ctrl;
 601
 602         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
 603                 return;
 604
 605         orig_clock_ctrl = clock_ctrl;
 606         clock_ctrl &= (CLOCK_CTRL_FORCE_CLKRUN |
 607                        CLOCK_CTRL_CLKRUN_OENABLE |
 608                        0x1f);
 609         tp->pci_clock_ctrl = clock_ctrl;
 610
 611         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
 612                 if (orig_clock_ctrl & CLOCK_CTRL_625_CORE) {
 613                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
 614                                     clock_ctrl | CLOCK_CTRL_625_CORE, 40);
 615                 }
 616         } else if ((orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
 617                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
 618                             clock_ctrl |
 619                             (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK),
 620                             40);
 621                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
 622                             clock_ctrl | (CLOCK_CTRL_ALTCLK),
 623                             40);
 624         }
 625         tw32_wait_f(TG3PCI_CLOCK_CTRL, clock_ctrl, 40);
 626 }
 627
 628 #define PHY_BUSY_LOOPS  5000
 629
 630 static int tg3_readphy(struct tg3 *tp, int reg, u32 *val)
 631 {
 632         u32 frame_val;
 633         unsigned int loops;
 634         int ret;
 635
 636         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 637                 tw32_f(MAC_MI_MODE,
 638                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
 639                 udelay(80);
 640         }
 641
 642         *val = 0x0;
 643
 644         frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
 645                       MI_COM_PHY_ADDR_MASK);
 646         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
 647                       MI_COM_REG_ADDR_MASK);
 648         frame_val |= (MI_COM_CMD_READ | MI_COM_START);
 649
 650         tw32_f(MAC_MI_COM, frame_val);
 651
 652         loops = PHY_BUSY_LOOPS;
 653         while (loops != 0) {
 654                 udelay(10);
 655                 frame_val = tr32(MAC_MI_COM);
 656
 657                 if ((frame_val & MI_COM_BUSY) == 0) {
 658                         udelay(5);
 659                         frame_val = tr32(MAC_MI_COM);
 660                         break;
 661                 }
 662                 loops -= 1;
 663         }
 664
 665         ret = -EBUSY;
 666         if (loops != 0) {
 667                 *val = frame_val & MI_COM_DATA_MASK;
 668                 ret = 0;
 669         }
 670
 671         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 672                 tw32_f(MAC_MI_MODE, tp->mi_mode);
 673                 udelay(80);
 674         }
 675
 676         return ret;
 677 }
 678
 679 static int tg3_writephy(struct tg3 *tp, int reg, u32 val)
 680 {
 681         u32 frame_val;
 682         unsigned int loops;
 683         int ret;
 684
 685         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906 &&
 686             (reg == MII_TG3_CTRL || reg == MII_TG3_AUX_CTRL))
 687                 return 0;
 688
 689         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 690                 tw32_f(MAC_MI_MODE,
 691                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
 692                 udelay(80);
 693         }
 694
 695         frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
 696                       MI_COM_PHY_ADDR_MASK);
 697         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
 698                       MI_COM_REG_ADDR_MASK);
 699         frame_val |= (val & MI_COM_DATA_MASK);
 700         frame_val |= (MI_COM_CMD_WRITE | MI_COM_START);
 701
 702         tw32_f(MAC_MI_COM, frame_val);
 703
 704         loops = PHY_BUSY_LOOPS;
 705         while (loops != 0) {
 706                 udelay(10);
 707                 frame_val = tr32(MAC_MI_COM);
 708                 if ((frame_val & MI_COM_BUSY) == 0) {
 709                         udelay(5);
 710                         frame_val = tr32(MAC_MI_COM);
 711                         break;
 712                 }
 713                 loops -= 1;
 714         }
 715
 716         ret = -EBUSY;
 717         if (loops != 0)
 718                 ret = 0;
 719
 720         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 721                 tw32_f(MAC_MI_MODE, tp->mi_mode);
 722                 udelay(80);
 723         }
 724
 725         return ret;
 726 }
 727
 728 static void tg3_phy_set_wirespeed(struct tg3 *tp)
 729 {
 730         u32 val;
 731
 732         if (tp->tg3_flags2 & TG3_FLG2_NO_ETH_WIRE_SPEED)
 733                 return;
 734
 735         if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x7007) &&
 736             !tg3_readphy(tp, MII_TG3_AUX_CTRL, &val))
 737                 tg3_writephy(tp, MII_TG3_AUX_CTRL,
 738                              (val | (1 << 15) | (1 << 4)));
 739 }
 740
 741 static int tg3_bmcr_reset(struct tg3 *tp)
 742 {
 743         u32 phy_control;
 744         int limit, err;
 745
 746         /* OK, reset it, and poll the BMCR_RESET bit until it
 747          * clears or we time out.
 748          */
 749         phy_control = BMCR_RESET;
 750         err = tg3_writephy(tp, MII_BMCR, phy_control);
 751         if (err != 0)
 752                 return -EBUSY;
 753
 754         limit = 5000;
 755         while (limit--) {
 756                 err = tg3_readphy(tp, MII_BMCR, &phy_control);
 757                 if (err != 0)
 758                         return -EBUSY;
 759
 760                 if ((phy_control & BMCR_RESET) == 0) {
 761                         udelay(40);
 762                         break;
 763                 }
 764                 udelay(10);
 765         }
 766         if (limit <= 0)
 767                 return -EBUSY;
 768
 769         return 0;
 770 }
 771
 772 static int tg3_wait_macro_done(struct tg3 *tp)
 773 {
 774         int limit = 100;
 775
 776         while (limit--) {
 777                 u32 tmp32;
 778
 779                 if (!tg3_readphy(tp, 0x16, &tmp32)) {
 780                         if ((tmp32 & 0x1000) == 0)
 781                                 break;
 782                 }
 783         }
 784         if (limit <= 0)
 785                 return -EBUSY;
 786
 787         return 0;
 788 }
 789
 790 static int tg3_phy_write_and_check_testpat(struct tg3 *tp, int *resetp)
 791 {
 792         static const u32 test_pat[4][6] = {
 793         { 0x00005555, 0x00000005, 0x00002aaa, 0x0000000a, 0x00003456, 0x00000003 },
 794         { 0x00002aaa, 0x0000000a, 0x00003333, 0x00000003, 0x0000789a, 0x00000005 },
 795         { 0x00005a5a, 0x00000005, 0x00002a6a, 0x0000000a, 0x00001bcd, 0x00000003 },
 796         { 0x00002a5a, 0x0000000a, 0x000033c3, 0x00000003, 0x00002ef1, 0x00000005 }
 797         };
 798         int chan;
 799
 800         for (chan = 0; chan < 4; chan++) {
 801                 int i;
 802
 803                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 804                              (chan * 0x2000) | 0x0200);
 805                 tg3_writephy(tp, 0x16, 0x0002);
 806
 807                 for (i = 0; i < 6; i++)
 808                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT,
 809                                      test_pat[chan][i]);
 810
 811                 tg3_writephy(tp, 0x16, 0x0202);
 812                 if (tg3_wait_macro_done(tp)) {
 813                         *resetp = 1;
 814                         return -EBUSY;
 815                 }
 816
 817                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 818                              (chan * 0x2000) | 0x0200);
 819                 tg3_writephy(tp, 0x16, 0x0082);
 820                 if (tg3_wait_macro_done(tp)) {
 821                         *resetp = 1;
 822                         return -EBUSY;
 823                 }
 824
 825                 tg3_writephy(tp, 0x16, 0x0802);
 826                 if (tg3_wait_macro_done(tp)) {
 827                         *resetp = 1;
 828                         return -EBUSY;
 829                 }
 830
 831                 for (i = 0; i < 6; i += 2) {
 832                         u32 low, high;
 833
 834                         if (tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &low) ||
 835                             tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &high) ||
 836                             tg3_wait_macro_done(tp)) {
 837                                 *resetp = 1;
 838                                 return -EBUSY;
 839                         }
 840                         low &= 0x7fff;
 841                         high &= 0x000f;
 842                         if (low != test_pat[chan][i] ||
 843                             high != test_pat[chan][i+1]) {
 844                                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000b);
 845                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4001);
 846                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4005);
 847
 848                                 return -EBUSY;
 849                         }
 850                 }
 851         }
 852
 853         return 0;
 854 }
 855
 856 static int tg3_phy_reset_chanpat(struct tg3 *tp)
 857 {
 858         int chan;
 859
 860         for (chan = 0; chan < 4; chan++) {
 861                 int i;
 862
 863                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 864                              (chan * 0x2000) | 0x0200);
 865                 tg3_writephy(tp, 0x16, 0x0002);
 866                 for (i = 0; i < 6; i++)
 867                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x000);
 868                 tg3_writephy(tp, 0x16, 0x0202);
 869                 if (tg3_wait_macro_done(tp))
 870                         return -EBUSY;
 871         }
 872
 873         return 0;
 874 }
 875
 876 static int tg3_phy_reset_5703_4_5(struct tg3 *tp)
 877 {
 878         u32 reg32, phy9_orig;
 879         int retries, do_phy_reset, err;
 880
 881         retries = 10;
 882         do_phy_reset = 1;
 883         do {
 884                 if (do_phy_reset) {
 885                         err = tg3_bmcr_reset(tp);
 886                         if (err)
 887                                 return err;
 888                         do_phy_reset = 0;
 889                 }
 890
 891                 /* Disable transmitter and interrupt.  */
 892                 if (tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32))
 893                         continue;
 894
 895                 reg32 |= 0x3000;
 896                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
 897
 898                 /* Set full-duplex, 1000 mbps.  */
 899                 tg3_writephy(tp, MII_BMCR,
 900                              BMCR_FULLDPLX | TG3_BMCR_SPEED1000);
 901
 902                 /* Set to master mode.  */
 903                 if (tg3_readphy(tp, MII_TG3_CTRL, &phy9_orig))
 904                         continue;
 905
 906                 tg3_writephy(tp, MII_TG3_CTRL,
 907                              (MII_TG3_CTRL_AS_MASTER |
 908                               MII_TG3_CTRL_ENABLE_AS_MASTER));
 909
 910                 /* Enable SM_DSP_CLOCK and 6dB.  */
 911                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
 912
 913                 /* Block the PHY control access.  */
 914                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
 915                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0800);
 916
 917                 err = tg3_phy_write_and_check_testpat(tp, &do_phy_reset);
 918                 if (!err)
 919                         break;
 920         } while (--retries);
 921
 922         err = tg3_phy_reset_chanpat(tp);
 923         if (err)
 924                 return err;
 925
 926         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
 927         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0000);
 928
 929         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8200);
 930         tg3_writephy(tp, 0x16, 0x0000);
 931
 932         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
 933             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
 934                 /* Set Extended packet length bit for jumbo frames */
 935                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4400);
 936         }
 937         else {
 938                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
 939         }
 940
 941         tg3_writephy(tp, MII_TG3_CTRL, phy9_orig);
 942
 943         if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32)) {
 944                 reg32 &= ~0x3000;
 945                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
 946         } else if (!err)
 947                 err = -EBUSY;
 948
 949         return err;
 950 }
 951
 952 static void tg3_link_report(struct tg3 *);
 953
 954 /* This will reset the tigon3 PHY if there is no valid
 955  * link unless the FORCE argument is non-zero.
 956  */
 957 static int tg3_phy_reset(struct tg3 *tp)
 958 {
 959         u32 phy_status;
 960         int err;
 961
 962         err  = tg3_readphy(tp, MII_BMSR, &phy_status);
 963         err |= tg3_readphy(tp, MII_BMSR, &phy_status);
 964         if (err != 0)
 965                 return -EBUSY;
 966
 967         if (netif_running(tp->dev) && netif_carrier_ok(tp->dev)) {
 968                 netif_carrier_off(tp->dev);
 969                 tg3_link_report(tp);
 970         }
 971
 972         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
 973             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
 974             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
 975                 err = tg3_phy_reset_5703_4_5(tp);
 976                 if (err)
 977                         return err;
 978                 goto out;
 979         }
 980
 981         err = tg3_bmcr_reset(tp);
 982         if (err)
 983                 return err;
 984
 985 out:
 986         if (tp->tg3_flags2 & TG3_FLG2_PHY_ADC_BUG) {
 987                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
 988                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
 989                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x2aaa);
 990                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
 991                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0323);
 992                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
 993         }
 994         if (tp->tg3_flags2 & TG3_FLG2_PHY_5704_A0_BUG) {
 995                 tg3_writephy(tp, 0x1c, 0x8d68);
 996                 tg3_writephy(tp, 0x1c, 0x8d68);
 997         }
 998         if (tp->tg3_flags2 & TG3_FLG2_PHY_BER_BUG) {
 999                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1000                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1001                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x310b);
1002                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1003                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x9506);
1004                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x401f);
1005                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x14e2);
1006                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1007         }
1008         else if (tp->tg3_flags2 & TG3_FLG2_PHY_JITTER_BUG) {
1009                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1010                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1011                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x010b);
1012                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1013         }
1014         /* Set Extended packet length bit (bit 14) on all chips that */
1015         /* support jumbo frames */
1016         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
1017                 /* Cannot do read-modify-write on 5401 */
1018                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1019         } else if (tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) {
1020                 u32 phy_reg;
1021
1022                 /* Set bit 14 with read-modify-write to preserve other bits */
1023                 if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0007) &&
1024                     !tg3_readphy(tp, MII_TG3_AUX_CTRL, &phy_reg))
1025                         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy_reg | 0x4000);
1026         }
1027
1028         /* Set phy register 0x10 bit 0 to high fifo elasticity to support
1029          * jumbo frames transmission.
1030          */
1031         if (tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) {
1032                 u32 phy_reg;
1033
1034                 if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &phy_reg))
1035                     tg3_writephy(tp, MII_TG3_EXT_CTRL,
1036                                  phy_reg | MII_TG3_EXT_CTRL_FIFO_ELASTIC);
1037         }
1038
1039         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1040                 u32 phy_reg;
1041
1042                 /* adjust output voltage */
1043                 tg3_writephy(tp, MII_TG3_EPHY_PTEST, 0x12);
1044
1045                 if (!tg3_readphy(tp, MII_TG3_EPHY_TEST, &phy_reg)) {
1046                         u32 phy_reg2;
1047
1048                         tg3_writephy(tp, MII_TG3_EPHY_TEST,
1049                                      phy_reg | MII_TG3_EPHY_SHADOW_EN);
1050                         /* Enable auto-MDIX */
1051                         if (!tg3_readphy(tp, 0x10, &phy_reg2))
1052                                 tg3_writephy(tp, 0x10, phy_reg2 | 0x4000);
1053                         tg3_writephy(tp, MII_TG3_EPHY_TEST, phy_reg);
1054                 }
1055         }
1056
1057         tg3_phy_set_wirespeed(tp);
1058         return 0;
1059 }
1060
1061 static void tg3_frob_aux_power(struct tg3 *tp)
1062 {
1063         struct tg3 *tp_peer = tp;
1064
1065         if ((tp->tg3_flags2 & TG3_FLG2_IS_NIC) == 0)
1066                 return;
1067
1068         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
1069             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714)) {
1070                 struct net_device *dev_peer;
1071
1072                 dev_peer = pci_get_drvdata(tp->pdev_peer);
1073                 /* remove_one() may have been run on the peer. */
1074                 if (!dev_peer)
1075                         tp_peer = tp;
1076                 else
1077                         tp_peer = netdev_priv(dev_peer);
1078         }
1079
1080         if ((tp->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
1081             (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0 ||
1082             (tp_peer->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
1083             (tp_peer->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0) {
1084                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1085                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1086                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1087                                     (GRC_LCLCTRL_GPIO_OE0 |
1088                                      GRC_LCLCTRL_GPIO_OE1 |
1089                                      GRC_LCLCTRL_GPIO_OE2 |
1090                                      GRC_LCLCTRL_GPIO_OUTPUT0 |
1091                                      GRC_LCLCTRL_GPIO_OUTPUT1),
1092                                     100);
1093                 } else {
1094                         u32 no_gpio2;
1095                         u32 grc_local_ctrl = 0;
1096
1097                         if (tp_peer != tp &&
1098                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
1099                                 return;
1100
1101                         /* Workaround to prevent overdrawing Amps. */
1102                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
1103                             ASIC_REV_5714) {
1104                                 grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
1105                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1106                                             grc_local_ctrl, 100);
1107                         }
1108
1109                         /* On 5753 and variants, GPIO2 cannot be used. */
1110                         no_gpio2 = tp->nic_sram_data_cfg &
1111                                     NIC_SRAM_DATA_CFG_NO_GPIO2;
1112
1113                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
1114                                          GRC_LCLCTRL_GPIO_OE1 |
1115                                          GRC_LCLCTRL_GPIO_OE2 |
1116                                          GRC_LCLCTRL_GPIO_OUTPUT1 |
1117                                          GRC_LCLCTRL_GPIO_OUTPUT2;
1118                         if (no_gpio2) {
1119                                 grc_local_ctrl &= ~(GRC_LCLCTRL_GPIO_OE2 |
1120                                                     GRC_LCLCTRL_GPIO_OUTPUT2);
1121                         }
1122                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1123                                                     grc_local_ctrl, 100);
1124
1125                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT0;
1126
1127                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1128                                                     grc_local_ctrl, 100);
1129
1130                         if (!no_gpio2) {
1131                                 grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT2;
1132                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1133                                             grc_local_ctrl, 100);
1134                         }
1135                 }
1136         } else {
1137                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
1138                     GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
1139                         if (tp_peer != tp &&
1140                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
1141                                 return;
1142
1143                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1144                                     (GRC_LCLCTRL_GPIO_OE1 |
1145                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
1146
1147                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1148                                     GRC_LCLCTRL_GPIO_OE1, 100);
1149
1150                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1151                                     (GRC_LCLCTRL_GPIO_OE1 |
1152                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
1153                 }
1154         }
1155 }
1156
1157 static int tg3_setup_phy(struct tg3 *, int);
1158
1159 #define RESET_KIND_SHUTDOWN     0
1160 #define RESET_KIND_INIT         1
1161 #define RESET_KIND_SUSPEND      2
1162
1163 static void tg3_write_sig_post_reset(struct tg3 *, int);
1164 static int tg3_halt_cpu(struct tg3 *, u32);
1165 static int tg3_nvram_lock(struct tg3 *);
1166 static void tg3_nvram_unlock(struct tg3 *);
1167
1168 static void tg3_power_down_phy(struct tg3 *tp)
1169 {
1170         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
1171                 return;
1172
1173         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906) {
1174                 tg3_writephy(tp, MII_TG3_EXT_CTRL,
1175                              MII_TG3_EXT_CTRL_FORCE_LED_OFF);
1176                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x01b2);
1177         }
1178
1179         /* The PHY should not be powered down on some chips because
1180          * of bugs.
1181          */
1182         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1183             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1184             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 &&
1185              (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)))
1186                 return;
1187         tg3_writephy(tp, MII_BMCR, BMCR_PDOWN);
1188 }
1189
1190 static int tg3_set_power_state(struct tg3 *tp, pci_power_t state)
1191 {
1192         u32 misc_host_ctrl;
1193         u16 power_control, power_caps;
1194         int pm = tp->pm_cap;
1195
1196         /* Make sure register accesses (indirect or otherwise)
1197          * will function correctly.
1198          */
1199         pci_write_config_dword(tp->pdev,
1200                                TG3PCI_MISC_HOST_CTRL,
1201                                tp->misc_host_ctrl);
1202
1203         pci_read_config_word(tp->pdev,
1204                              pm + PCI_PM_CTRL,
1205                              &power_control);
1206         power_control |= PCI_PM_CTRL_PME_STATUS;
1207         power_control &= ~(PCI_PM_CTRL_STATE_MASK);
1208         switch (state) {
1209         case PCI_D0:
1210                 power_control |= 0;
1211                 pci_write_config_word(tp->pdev,
1212                                       pm + PCI_PM_CTRL,
1213                                       power_control);
1214                 udelay(100);    /* Delay after power state change */
1215
1216                 /* Switch out of Vaux if it is a NIC */
1217                 if (tp->tg3_flags2 & TG3_FLG2_IS_NIC)
1218                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl, 100);
1219
1220                 return 0;
1221
1222         case PCI_D1:
1223                 power_control |= 1;
1224                 break;
1225
1226         case PCI_D2:
1227                 power_control |= 2;
1228                 break;
1229
1230         case PCI_D3hot:
1231                 power_control |= 3;
1232                 break;
1233
1234         default:
1235                 printk(KERN_WARNING PFX "%s: Invalid power state (%d) "
1236                        "requested.\n",
1237                        tp->dev->name, state);
1238                 return -EINVAL;
1239         };
1240
1241         power_control |= PCI_PM_CTRL_PME_ENABLE;
1242
1243         misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
1244         tw32(TG3PCI_MISC_HOST_CTRL,
1245              misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT);
1246
1247         if (tp->link_config.phy_is_low_power == 0) {
1248                 tp->link_config.phy_is_low_power = 1;
1249                 tp->link_config.orig_speed = tp->link_config.speed;
1250                 tp->link_config.orig_duplex = tp->link_config.duplex;
1251                 tp->link_config.orig_autoneg = tp->link_config.autoneg;
1252         }
1253
1254         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)) {
1255                 tp->link_config.speed = SPEED_10;
1256                 tp->link_config.duplex = DUPLEX_HALF;
1257                 tp->link_config.autoneg = AUTONEG_ENABLE;
1258                 tg3_setup_phy(tp, 0);
1259         }
1260
1261         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1262                 u32 val;
1263
1264                 val = tr32(GRC_VCPU_EXT_CTRL);
1265                 tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_DISABLE_WOL);
1266         } else if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1267                 int i;
1268                 u32 val;
1269
1270                 for (i = 0; i < 200; i++) {
1271                         tg3_read_mem(tp, NIC_SRAM_FW_ASF_STATUS_MBOX, &val);
1272                         if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
1273                                 break;
1274                         msleep(1);
1275                 }
1276         }
1277         tg3_write_mem(tp, NIC_SRAM_WOL_MBOX, WOL_SIGNATURE |
1278                                              WOL_DRV_STATE_SHUTDOWN |
1279                                              WOL_DRV_WOL | WOL_SET_MAGIC_PKT);
1280
1281         pci_read_config_word(tp->pdev, pm + PCI_PM_PMC, &power_caps);
1282
1283         if (tp->tg3_flags & TG3_FLAG_WOL_ENABLE) {
1284                 u32 mac_mode;
1285
1286                 if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
1287                         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x5a);
1288                         udelay(40);
1289
1290                         if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
1291                                 mac_mode = MAC_MODE_PORT_MODE_GMII;
1292                         else
1293                                 mac_mode = MAC_MODE_PORT_MODE_MII;
1294
1295                         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 ||
1296                             !(tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB))
1297                                 mac_mode |= MAC_MODE_LINK_POLARITY;
1298                 } else {
1299                         mac_mode = MAC_MODE_PORT_MODE_TBI;
1300                 }
1301
1302                 if (!(tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
1303                         tw32(MAC_LED_CTRL, tp->led_ctrl);
1304
1305                 if (((power_caps & PCI_PM_CAP_PME_D3cold) &&
1306                      (tp->tg3_flags & TG3_FLAG_WOL_ENABLE)))
1307                         mac_mode |= MAC_MODE_MAGIC_PKT_ENABLE;
1308
1309                 tw32_f(MAC_MODE, mac_mode);
1310                 udelay(100);
1311
1312                 tw32_f(MAC_RX_MODE, RX_MODE_ENABLE);
1313                 udelay(10);
1314         }
1315
1316         if (!(tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB) &&
1317             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1318              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
1319                 u32 base_val;
1320
1321                 base_val = tp->pci_clock_ctrl;
1322                 base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
1323                              CLOCK_CTRL_TXCLK_DISABLE);
1324
1325                 tw32_wait_f(TG3PCI_CLOCK_CTRL, base_val | CLOCK_CTRL_ALTCLK |
1326                             CLOCK_CTRL_PWRDOWN_PLL133, 40);
1327         } else if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
1328                 /* do nothing */
1329         } else if (!((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
1330                      (tp->tg3_flags & TG3_FLAG_ENABLE_ASF))) {
1331                 u32 newbits1, newbits2;
1332
1333                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1334                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1335                         newbits1 = (CLOCK_CTRL_RXCLK_DISABLE |
1336                                     CLOCK_CTRL_TXCLK_DISABLE |
1337                                     CLOCK_CTRL_ALTCLK);
1338                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
1339                 } else if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
1340                         newbits1 = CLOCK_CTRL_625_CORE;
1341                         newbits2 = newbits1 | CLOCK_CTRL_ALTCLK;
1342                 } else {
1343                         newbits1 = CLOCK_CTRL_ALTCLK;
1344                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
1345                 }
1346
1347                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits1,
1348                             40);
1349
1350                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits2,
1351                             40);
1352
1353                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
1354                         u32 newbits3;
1355
1356                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1357                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1358                                 newbits3 = (CLOCK_CTRL_RXCLK_DISABLE |
1359                                             CLOCK_CTRL_TXCLK_DISABLE |
1360                                             CLOCK_CTRL_44MHZ_CORE);
1361                         } else {
1362                                 newbits3 = CLOCK_CTRL_44MHZ_CORE;
1363                         }
1364
1365                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
1366                                     tp->pci_clock_ctrl | newbits3, 40);
1367                 }
1368         }
1369
1370         if (!(tp->tg3_flags & TG3_FLAG_WOL_ENABLE) &&
1371             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF))
1372                 tg3_power_down_phy(tp);
1373
1374         tg3_frob_aux_power(tp);
1375
1376         /* Workaround for unstable PLL clock */
1377         if ((GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX) ||
1378             (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX)) {
1379                 u32 val = tr32(0x7d00);
1380
1381                 val &= ~((1 << 16) | (1 << 4) | (1 << 2) | (1 << 1) | 1);
1382                 tw32(0x7d00, val);
1383                 if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1384                         int err;
1385
1386                         err = tg3_nvram_lock(tp);
1387                         tg3_halt_cpu(tp, RX_CPU_BASE);
1388                         if (!err)
1389                                 tg3_nvram_unlock(tp);
1390                 }
1391         }
1392
1393         tg3_write_sig_post_reset(tp, RESET_KIND_SHUTDOWN);
1394
1395         /* Finally, set the new power state. */
1396         pci_write_config_word(tp->pdev, pm + PCI_PM_CTRL, power_control);
1397         udelay(100);    /* Delay after power state change */
1398
1399         return 0;
1400 }
1401
1402 static void tg3_link_report(struct tg3 *tp)
1403 {
1404         if (!netif_carrier_ok(tp->dev)) {
1405                 if (netif_msg_link(tp))
1406                         printk(KERN_INFO PFX "%s: Link is down.\n",
1407                                tp->dev->name);
1408         } else if (netif_msg_link(tp)) {
1409                 printk(KERN_INFO PFX "%s: Link is up at %d Mbps, %s duplex.\n",
1410                        tp->dev->name,
1411                        (tp->link_config.active_speed == SPEED_1000 ?
1412                         1000 :
1413                         (tp->link_config.active_speed == SPEED_100 ?
1414                          100 : 10)),
1415                        (tp->link_config.active_duplex == DUPLEX_FULL ?
1416                         "full" : "half"));
1417
1418                 printk(KERN_INFO PFX "%s: Flow control is %s for TX and "
1419                        "%s for RX.\n",
1420                        tp->dev->name,
1421                        (tp->tg3_flags & TG3_FLAG_TX_PAUSE) ? "on" : "off",
1422                        (tp->tg3_flags & TG3_FLAG_RX_PAUSE) ? "on" : "off");
1423         }
1424 }
1425
1426 static void tg3_setup_flow_control(struct tg3 *tp, u32 local_adv, u32 remote_adv)
1427 {
1428         u32 new_tg3_flags = 0;
1429         u32 old_rx_mode = tp->rx_mode;
1430         u32 old_tx_mode = tp->tx_mode;
1431
1432         if (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG) {
1433
1434                 /* Convert 1000BaseX flow control bits to 1000BaseT
1435                  * bits before resolving flow control.
1436                  */
1437                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
1438                         local_adv &= ~(ADVERTISE_PAUSE_CAP |
1439                                        ADVERTISE_PAUSE_ASYM);
1440                         remote_adv &= ~(LPA_PAUSE_CAP | LPA_PAUSE_ASYM);
1441
1442                         if (local_adv & ADVERTISE_1000XPAUSE)
1443                                 local_adv |= ADVERTISE_PAUSE_CAP;
1444                         if (local_adv & ADVERTISE_1000XPSE_ASYM)
1445                                 local_adv |= ADVERTISE_PAUSE_ASYM;
1446                         if (remote_adv & LPA_1000XPAUSE)
1447                                 remote_adv |= LPA_PAUSE_CAP;
1448                         if (remote_adv & LPA_1000XPAUSE_ASYM)
1449                                 remote_adv |= LPA_PAUSE_ASYM;
1450                 }
1451
1452                 if (local_adv & ADVERTISE_PAUSE_CAP) {
1453                         if (local_adv & ADVERTISE_PAUSE_ASYM) {
1454                                 if (remote_adv & LPA_PAUSE_CAP)
1455                                         new_tg3_flags |=
1456                                                 (TG3_FLAG_RX_PAUSE |
1457                                                 TG3_FLAG_TX_PAUSE);
1458                                 else if (remote_adv & LPA_PAUSE_ASYM)
1459                                         new_tg3_flags |=
1460                                                 (TG3_FLAG_RX_PAUSE);
1461                         } else {
1462                                 if (remote_adv & LPA_PAUSE_CAP)
1463                                         new_tg3_flags |=
1464                                                 (TG3_FLAG_RX_PAUSE |
1465                                                 TG3_FLAG_TX_PAUSE);
1466                         }
1467                 } else if (local_adv & ADVERTISE_PAUSE_ASYM) {
1468                         if ((remote_adv & LPA_PAUSE_CAP) &&
1469                         (remote_adv & LPA_PAUSE_ASYM))
1470                                 new_tg3_flags |= TG3_FLAG_TX_PAUSE;
1471                 }
1472
1473                 tp->tg3_flags &= ~(TG3_FLAG_RX_PAUSE | TG3_FLAG_TX_PAUSE);
1474                 tp->tg3_flags |= new_tg3_flags;
1475         } else {
1476                 new_tg3_flags = tp->tg3_flags;
1477         }
1478
1479         if (new_tg3_flags & TG3_FLAG_RX_PAUSE)
1480                 tp->rx_mode |= RX_MODE_FLOW_CTRL_ENABLE;
1481         else
1482                 tp->rx_mode &= ~RX_MODE_FLOW_CTRL_ENABLE;
1483
1484         if (old_rx_mode != tp->rx_mode) {
1485                 tw32_f(MAC_RX_MODE, tp->rx_mode);
1486         }
1487
1488         if (new_tg3_flags & TG3_FLAG_TX_PAUSE)
1489                 tp->tx_mode |= TX_MODE_FLOW_CTRL_ENABLE;
1490         else
1491                 tp->tx_mode &= ~TX_MODE_FLOW_CTRL_ENABLE;
1492
1493         if (old_tx_mode != tp->tx_mode) {
1494                 tw32_f(MAC_TX_MODE, tp->tx_mode);
1495         }
1496 }
1497
1498 static void tg3_aux_stat_to_speed_duplex(struct tg3 *tp, u32 val, u16 *speed, u8 *duplex)
1499 {
1500         switch (val & MII_TG3_AUX_STAT_SPDMASK) {
1501         case MII_TG3_AUX_STAT_10HALF:
1502                 *speed = SPEED_10;
1503                 *duplex = DUPLEX_HALF;
1504                 break;
1505
1506         case MII_TG3_AUX_STAT_10FULL:
1507                 *speed = SPEED_10;
1508                 *duplex = DUPLEX_FULL;
1509                 break;
1510
1511         case MII_TG3_AUX_STAT_100HALF:
1512                 *speed = SPEED_100;
1513                 *duplex = DUPLEX_HALF;
1514                 break;
1515
1516         case MII_TG3_AUX_STAT_100FULL:
1517                 *speed = SPEED_100;
1518                 *duplex = DUPLEX_FULL;
1519                 break;
1520
1521         case MII_TG3_AUX_STAT_1000HALF:
1522                 *speed = SPEED_1000;
1523                 *duplex = DUPLEX_HALF;
1524                 break;
1525
1526         case MII_TG3_AUX_STAT_1000FULL:
1527                 *speed = SPEED_1000;
1528                 *duplex = DUPLEX_FULL;
1529                 break;
1530
1531         default:
1532                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1533                         *speed = (val & MII_TG3_AUX_STAT_100) ? SPEED_100 :
1534                                  SPEED_10;
1535                         *duplex = (val & MII_TG3_AUX_STAT_FULL) ? DUPLEX_FULL :
1536                                   DUPLEX_HALF;
1537                         break;
1538                 }
1539                 *speed = SPEED_INVALID;
1540                 *duplex = DUPLEX_INVALID;
1541                 break;
1542         };
1543 }
1544
1545 static void tg3_phy_copper_begin(struct tg3 *tp)
1546 {
1547         u32 new_adv;
1548         int i;
1549
1550         if (tp->link_config.phy_is_low_power) {
1551                 /* Entering low power mode.  Disable gigabit and
1552                  * 100baseT advertisements.
1553                  */
1554                 tg3_writephy(tp, MII_TG3_CTRL, 0);
1555
1556                 new_adv = (ADVERTISE_10HALF | ADVERTISE_10FULL |
1557                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
1558                 if (tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB)
1559                         new_adv |= (ADVERTISE_100HALF | ADVERTISE_100FULL);
1560
1561                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
1562         } else if (tp->link_config.speed == SPEED_INVALID) {
1563                 if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
1564                         tp->link_config.advertising &=
1565                                 ~(ADVERTISED_1000baseT_Half |
1566                                   ADVERTISED_1000baseT_Full);
1567
1568                 new_adv = (ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
1569                 if (tp->link_config.advertising & ADVERTISED_10baseT_Half)
1570                         new_adv |= ADVERTISE_10HALF;
1571                 if (tp->link_config.advertising & ADVERTISED_10baseT_Full)
1572                         new_adv |= ADVERTISE_10FULL;
1573                 if (tp->link_config.advertising & ADVERTISED_100baseT_Half)
1574                         new_adv |= ADVERTISE_100HALF;
1575                 if (tp->link_config.advertising & ADVERTISED_100baseT_Full)
1576                         new_adv |= ADVERTISE_100FULL;
1577                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
1578
1579                 if (tp->link_config.advertising &
1580                     (ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full)) {
1581                         new_adv = 0;
1582                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
1583                                 new_adv |= MII_TG3_CTRL_ADV_1000_HALF;
1584                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
1585                                 new_adv |= MII_TG3_CTRL_ADV_1000_FULL;
1586                         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY) &&
1587                             (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1588                              tp->pci_chip_rev_id == CHIPREV_ID_5701_B0))
1589                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
1590                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
1591                         tg3_writephy(tp, MII_TG3_CTRL, new_adv);
1592                 } else {
1593                         tg3_writephy(tp, MII_TG3_CTRL, 0);
1594                 }
1595         } else {
1596                 /* Asking for a specific link mode. */
1597                 if (tp->link_config.speed == SPEED_1000) {
1598                         new_adv = ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP;
1599                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
1600
1601                         if (tp->link_config.duplex == DUPLEX_FULL)
1602                                 new_adv = MII_TG3_CTRL_ADV_1000_FULL;
1603                         else
1604                                 new_adv = MII_TG3_CTRL_ADV_1000_HALF;
1605                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1606                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
1607                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
1608                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
1609                         tg3_writephy(tp, MII_TG3_CTRL, new_adv);
1610                 } else {
1611                         tg3_writephy(tp, MII_TG3_CTRL, 0);
1612
1613                         new_adv = ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP;
1614                         if (tp->link_config.speed == SPEED_100) {
1615                                 if (tp->link_config.duplex == DUPLEX_FULL)
1616                                         new_adv |= ADVERTISE_100FULL;
1617                                 else
1618                                         new_adv |= ADVERTISE_100HALF;
1619                         } else {
1620                                 if (tp->link_config.duplex == DUPLEX_FULL)
1621                                         new_adv |= ADVERTISE_10FULL;
1622                                 else
1623                                         new_adv |= ADVERTISE_10HALF;
1624                         }
1625                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
1626                 }
1627         }
1628
1629         if (tp->link_config.autoneg == AUTONEG_DISABLE &&
1630             tp->link_config.speed != SPEED_INVALID) {
1631                 u32 bmcr, orig_bmcr;
1632
1633                 tp->link_config.active_speed = tp->link_config.speed;
1634                 tp->link_config.active_duplex = tp->link_config.duplex;
1635
1636                 bmcr = 0;
1637                 switch (tp->link_config.speed) {
1638                 default:
1639                 case SPEED_10:
1640                         break;
1641
1642                 case SPEED_100:
1643                         bmcr |= BMCR_SPEED100;
1644                         break;
1645
1646                 case SPEED_1000:
1647                         bmcr |= TG3_BMCR_SPEED1000;
1648                         break;
1649                 };
1650
1651                 if (tp->link_config.duplex == DUPLEX_FULL)
1652                         bmcr |= BMCR_FULLDPLX;
1653
1654                 if (!tg3_readphy(tp, MII_BMCR, &orig_bmcr) &&
1655                     (bmcr != orig_bmcr)) {
1656                         tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK);
1657                         for (i = 0; i < 1500; i++) {
1658                                 u32 tmp;
1659
1660                                 udelay(10);
1661                                 if (tg3_readphy(tp, MII_BMSR, &tmp) ||
1662                                     tg3_readphy(tp, MII_BMSR, &tmp))
1663                                         continue;
1664                                 if (!(tmp & BMSR_LSTATUS)) {
1665                                         udelay(40);
1666                                         break;
1667                                 }
1668                         }
1669                         tg3_writephy(tp, MII_BMCR, bmcr);
1670                         udelay(40);
1671                 }
1672         } else {
1673                 tg3_writephy(tp, MII_BMCR,
1674                              BMCR_ANENABLE | BMCR_ANRESTART);
1675         }
1676 }
1677
1678 static int tg3_init_5401phy_dsp(struct tg3 *tp)
1679 {
1680         int err;
1681
1682         /* Turn off tap power management. */
1683         /* Set Extended packet length bit */
1684         err  = tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1685
1686         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0012);
1687         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1804);
1688
1689         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0013);
1690         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1204);
1691
1692         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
1693         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0132);
1694
1695         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
1696         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0232);
1697
1698         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1699         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0a20);
1700
1701         udelay(40);
1702
1703         return err;
1704 }
1705
1706 static int tg3_copper_is_advertising_all(struct tg3 *tp, u32 mask)
1707 {
1708         u32 adv_reg, all_mask = 0;
1709
1710         if (mask & ADVERTISED_10baseT_Half)
1711                 all_mask |= ADVERTISE_10HALF;
1712         if (mask & ADVERTISED_10baseT_Full)
1713                 all_mask |= ADVERTISE_10FULL;
1714         if (mask & ADVERTISED_100baseT_Half)
1715                 all_mask |= ADVERTISE_100HALF;
1716         if (mask & ADVERTISED_100baseT_Full)
1717                 all_mask |= ADVERTISE_100FULL;
1718
1719         if (tg3_readphy(tp, MII_ADVERTISE, &adv_reg))
1720                 return 0;
1721
1722         if ((adv_reg & all_mask) != all_mask)
1723                 return 0;
1724         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
1725                 u32 tg3_ctrl;
1726
1727                 all_mask = 0;
1728                 if (mask & ADVERTISED_1000baseT_Half)
1729                         all_mask |= ADVERTISE_1000HALF;
1730                 if (mask & ADVERTISED_1000baseT_Full)
1731                         all_mask |= ADVERTISE_1000FULL;
1732
1733                 if (tg3_readphy(tp, MII_TG3_CTRL, &tg3_ctrl))
1734                         return 0;
1735
1736                 if ((tg3_ctrl & all_mask) != all_mask)
1737                         return 0;
1738         }
1739         return 1;
1740 }
1741
1742 static int tg3_setup_copper_phy(struct tg3 *tp, int force_reset)
1743 {
1744         int current_link_up;
1745         u32 bmsr, dummy;
1746         u16 current_speed;
1747         u8 current_duplex;
1748         int i, err;
1749
1750         tw32(MAC_EVENT, 0);
1751
1752         tw32_f(MAC_STATUS,
1753              (MAC_STATUS_SYNC_CHANGED |
1754               MAC_STATUS_CFG_CHANGED |
1755               MAC_STATUS_MI_COMPLETION |
1756               MAC_STATUS_LNKSTATE_CHANGED));
1757         udelay(40);
1758
1759         tp->mi_mode = MAC_MI_MODE_BASE;
1760         tw32_f(MAC_MI_MODE, tp->mi_mode);
1761         udelay(80);
1762
1763         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x02);
1764
1765         /* Some third-party PHYs need to be reset on link going
1766          * down.
1767          */
1768         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1769              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1770              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
1771             netif_carrier_ok(tp->dev)) {
1772                 tg3_readphy(tp, MII_BMSR, &bmsr);
1773                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
1774                     !(bmsr & BMSR_LSTATUS))
1775                         force_reset = 1;
1776         }
1777         if (force_reset)
1778                 tg3_phy_reset(tp);
1779
1780         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
1781                 tg3_readphy(tp, MII_BMSR, &bmsr);
1782                 if (tg3_readphy(tp, MII_BMSR, &bmsr) ||
1783                     !(tp->tg3_flags & TG3_FLAG_INIT_COMPLETE))
1784                         bmsr = 0;
1785
1786                 if (!(bmsr & BMSR_LSTATUS)) {
1787                         err = tg3_init_5401phy_dsp(tp);
1788                         if (err)
1789                                 return err;
1790
1791                         tg3_readphy(tp, MII_BMSR, &bmsr);
1792                         for (i = 0; i < 1000; i++) {
1793                                 udelay(10);
1794                                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
1795                                     (bmsr & BMSR_LSTATUS)) {
1796                                         udelay(40);
1797                                         break;
1798                                 }
1799                         }
1800
1801                         if ((tp->phy_id & PHY_ID_REV_MASK) == PHY_REV_BCM5401_B0 &&
1802                             !(bmsr & BMSR_LSTATUS) &&
1803                             tp->link_config.active_speed == SPEED_1000) {
1804                                 err = tg3_phy_reset(tp);
1805                                 if (!err)
1806                                         err = tg3_init_5401phy_dsp(tp);
1807                                 if (err)
1808                                         return err;
1809                         }
1810                 }
1811         } else if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1812                    tp->pci_chip_rev_id == CHIPREV_ID_5701_B0) {
1813                 /* 5701 {A0,B0} CRC bug workaround */
1814                 tg3_writephy(tp, 0x15, 0x0a75);
1815                 tg3_writephy(tp, 0x1c, 0x8c68);
1816                 tg3_writephy(tp, 0x1c, 0x8d68);
1817                 tg3_writephy(tp, 0x1c, 0x8c68);
1818         }
1819
1820         /* Clear pending interrupts... */
1821         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
1822         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
1823
1824         if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT)
1825                 tg3_writephy(tp, MII_TG3_IMASK, ~MII_TG3_INT_LINKCHG);
1826         else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906)
1827                 tg3_writephy(tp, MII_TG3_IMASK, ~0);
1828
1829         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1830             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1831                 if (tp->led_ctrl == LED_CTRL_MODE_PHY_1)
1832                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
1833                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
1834                 else
1835                         tg3_writephy(tp, MII_TG3_EXT_CTRL, 0);
1836         }
1837
1838         current_link_up = 0;
1839         current_speed = SPEED_INVALID;
1840         current_duplex = DUPLEX_INVALID;
1841
1842         if (tp->tg3_flags2 & TG3_FLG2_CAPACITIVE_COUPLING) {
1843                 u32 val;
1844
1845                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4007);
1846                 tg3_readphy(tp, MII_TG3_AUX_CTRL, &val);
1847                 if (!(val & (1 << 10))) {
1848                         val |= (1 << 10);
1849                         tg3_writephy(tp, MII_TG3_AUX_CTRL, val);
1850                         goto relink;
1851                 }
1852         }
1853
1854         bmsr = 0;
1855         for (i = 0; i < 100; i++) {
1856                 tg3_readphy(tp, MII_BMSR, &bmsr);
1857                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
1858                     (bmsr & BMSR_LSTATUS))
1859                         break;
1860                 udelay(40);
1861         }
1862
1863         if (bmsr & BMSR_LSTATUS) {
1864                 u32 aux_stat, bmcr;
1865
1866                 tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
1867                 for (i = 0; i < 2000; i++) {
1868                         udelay(10);
1869                         if (!tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat) &&
1870                             aux_stat)
1871                                 break;
1872                 }
1873
1874                 tg3_aux_stat_to_speed_duplex(tp, aux_stat,
1875                                              &current_speed,
1876                                              &current_duplex);
1877
1878                 bmcr = 0;
1879                 for (i = 0; i < 200; i++) {
1880                         tg3_readphy(tp, MII_BMCR, &bmcr);
1881                         if (tg3_readphy(tp, MII_BMCR, &bmcr))
1882                                 continue;
1883                         if (bmcr && bmcr != 0x7fff)
1884                                 break;
1885                         udelay(10);
1886                 }
1887
1888                 if (tp->link_config.autoneg == AUTONEG_ENABLE) {
1889                         if (bmcr & BMCR_ANENABLE) {
1890                                 current_link_up = 1;
1891
1892                                 /* Force autoneg restart if we are exiting
1893                                  * low power mode.
1894                                  */
1895                                 if (!tg3_copper_is_advertising_all(tp,
1896                                                 tp->link_config.advertising))
1897                                         current_link_up = 0;
1898                         } else {
1899                                 current_link_up = 0;
1900                         }
1901                 } else {
1902                         if (!(bmcr & BMCR_ANENABLE) &&
1903                             tp->link_config.speed == current_speed &&
1904                             tp->link_config.duplex == current_duplex) {
1905                                 current_link_up = 1;
1906                         } else {
1907                                 current_link_up = 0;
1908                         }
1909                 }
1910
1911                 tp->link_config.active_speed = current_speed;
1912                 tp->link_config.active_duplex = current_duplex;
1913         }
1914
1915         if (current_link_up == 1 &&
1916             (tp->link_config.active_duplex == DUPLEX_FULL) &&
1917             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
1918                 u32 local_adv, remote_adv;
1919
1920                 if (tg3_readphy(tp, MII_ADVERTISE, &local_adv))
1921                         local_adv = 0;
1922                 local_adv &= (ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM);
1923
1924                 if (tg3_readphy(tp, MII_LPA, &remote_adv))
1925                         remote_adv = 0;
1926
1927                 remote_adv &= (LPA_PAUSE_CAP | LPA_PAUSE_ASYM);
1928
1929                 /* If we are not advertising full pause capability,
1930                  * something is wrong.  Bring the link down and reconfigure.
1931                  */
1932                 if (local_adv != ADVERTISE_PAUSE_CAP) {
1933                         current_link_up = 0;
1934                 } else {
1935                         tg3_setup_flow_control(tp, local_adv, remote_adv);
1936                 }
1937         }
1938 relink:
1939         if (current_link_up == 0 || tp->link_config.phy_is_low_power) {
1940                 u32 tmp;
1941
1942                 tg3_phy_copper_begin(tp);
1943
1944                 tg3_readphy(tp, MII_BMSR, &tmp);
1945                 if (!tg3_readphy(tp, MII_BMSR, &tmp) &&
1946                     (tmp & BMSR_LSTATUS))
1947                         current_link_up = 1;
1948         }
1949
1950         tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
1951         if (current_link_up == 1) {
1952                 if (tp->link_config.active_speed == SPEED_100 ||
1953                     tp->link_config.active_speed == SPEED_10)
1954                         tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
1955                 else
1956                         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
1957         } else
1958                 tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
1959
1960         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
1961         if (tp->link_config.active_duplex == DUPLEX_HALF)
1962                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
1963
1964         tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
1965         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
1966                 if ((tp->led_ctrl == LED_CTRL_MODE_PHY_2) ||
1967                     (current_link_up == 1 &&
1968                      tp->link_config.active_speed == SPEED_10))
1969                         tp->mac_mode |= MAC_MODE_LINK_POLARITY;
1970         } else {
1971                 if (current_link_up == 1)
1972                         tp->mac_mode |= MAC_MODE_LINK_POLARITY;
1973         }
1974
1975         /* ??? Without this setting Netgear GA302T PHY does not
1976          * ??? send/receive packets...
1977          */
1978         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411 &&
1979             tp->pci_chip_rev_id == CHIPREV_ID_5700_ALTIMA) {
1980                 tp->mi_mode |= MAC_MI_MODE_AUTO_POLL;
1981                 tw32_f(MAC_MI_MODE, tp->mi_mode);
1982                 udelay(80);
1983         }
1984
1985         tw32_f(MAC_MODE, tp->mac_mode);
1986         udelay(40);
1987
1988         if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
1989                 /* Polled via timer. */
1990                 tw32_f(MAC_EVENT, 0);
1991         } else {
1992                 tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
1993         }
1994         udelay(40);
1995
1996         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 &&
1997             current_link_up == 1 &&
1998             tp->link_config.active_speed == SPEED_1000 &&
1999             ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) ||
2000              (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED))) {
2001                 udelay(120);
2002                 tw32_f(MAC_STATUS,
2003                      (MAC_STATUS_SYNC_CHANGED |
2004                       MAC_STATUS_CFG_CHANGED));
2005                 udelay(40);
2006                 tg3_write_mem(tp,
2007                               NIC_SRAM_FIRMWARE_MBOX,
2008                               NIC_SRAM_FIRMWARE_MBOX_MAGIC2);
2009         }
2010
2011         if (current_link_up != netif_carrier_ok(tp->dev)) {
2012                 if (current_link_up)
2013                         netif_carrier_on(tp->dev);
2014                 else
2015                         netif_carrier_off(tp->dev);
2016                 tg3_link_report(tp);
2017         }
2018
2019         return 0;
2020 }
2021
2022 struct tg3_fiber_aneginfo {
2023         int state;
2024 #define ANEG_STATE_UNKNOWN              0
2025 #define ANEG_STATE_AN_ENABLE            1
2026 #define ANEG_STATE_RESTART_INIT         2
2027 #define ANEG_STATE_RESTART              3
2028 #define ANEG_STATE_DISABLE_LINK_OK      4
2029 #define ANEG_STATE_ABILITY_DETECT_INIT  5
2030 #define ANEG_STATE_ABILITY_DETECT       6
2031 #define ANEG_STATE_ACK_DETECT_INIT      7
2032 #define ANEG_STATE_ACK_DETECT           8
2033 #define ANEG_STATE_COMPLETE_ACK_INIT    9
2034 #define ANEG_STATE_COMPLETE_ACK         10
2035 #define ANEG_STATE_IDLE_DETECT_INIT     11
2036 #define ANEG_STATE_IDLE_DETECT          12
2037 #define ANEG_STATE_LINK_OK              13
2038 #define ANEG_STATE_NEXT_PAGE_WAIT_INIT  14
2039 #define ANEG_STATE_NEXT_PAGE_WAIT       15
2040
2041         u32 flags;
2042 #define MR_AN_ENABLE            0x00000001
2043 #define MR_RESTART_AN           0x00000002
2044 #define MR_AN_COMPLETE          0x00000004
2045 #define MR_PAGE_RX              0x00000008
2046 #define MR_NP_LOADED            0x00000010
2047 #define MR_TOGGLE_TX            0x00000020
2048 #define MR_LP_ADV_FULL_DUPLEX   0x00000040
2049 #define MR_LP_ADV_HALF_DUPLEX   0x00000080
2050 #define MR_LP_ADV_SYM_PAUSE     0x00000100
2051 #define MR_LP_ADV_ASYM_PAUSE    0x00000200
2052 #define MR_LP_ADV_REMOTE_FAULT1 0x00000400
2053 #define MR_LP_ADV_REMOTE_FAULT2 0x00000800
2054 #define MR_LP_ADV_NEXT_PAGE     0x00001000
2055 #define MR_TOGGLE_RX            0x00002000
2056 #define MR_NP_RX                0x00004000
2057
2058 #define MR_LINK_OK              0x80000000
2059
2060         unsigned long link_time, cur_time;
2061
2062         u32 ability_match_cfg;
2063         int ability_match_count;
2064
2065         char ability_match, idle_match, ack_match;
2066
2067         u32 txconfig, rxconfig;
2068 #define ANEG_CFG_NP             0x00000080
2069 #define ANEG_CFG_ACK            0x00000040
2070 #define ANEG_CFG_RF2            0x00000020
2071 #define ANEG_CFG_RF1            0x00000010
2072 #define ANEG_CFG_PS2            0x00000001
2073 #define ANEG_CFG_PS1            0x00008000
2074 #define ANEG_CFG_HD             0x00004000
2075 #define ANEG_CFG_FD             0x00002000
2076 #define ANEG_CFG_INVAL          0x00001f06
2077
2078 };
2079 #define ANEG_OK         0
2080 #define ANEG_DONE       1
2081 #define ANEG_TIMER_ENAB 2
2082 #define ANEG_FAILED     -1
2083
2084 #define ANEG_STATE_SETTLE_TIME  10000
2085
2086 static int tg3_fiber_aneg_smachine(struct tg3 *tp,
2087                                    struct tg3_fiber_aneginfo *ap)
2088 {
2089         unsigned long delta;
2090         u32 rx_cfg_reg;
2091         int ret;
2092
2093         if (ap->state == ANEG_STATE_UNKNOWN) {
2094                 ap->rxconfig = 0;
2095                 ap->link_time = 0;
2096                 ap->cur_time = 0;
2097                 ap->ability_match_cfg = 0;
2098                 ap->ability_match_count = 0;
2099                 ap->ability_match = 0;
2100                 ap->idle_match = 0;
2101                 ap->ack_match = 0;
2102         }
2103         ap->cur_time++;
2104
2105         if (tr32(MAC_STATUS) & MAC_STATUS_RCVD_CFG) {
2106                 rx_cfg_reg = tr32(MAC_RX_AUTO_NEG);
2107
2108                 if (rx_cfg_reg != ap->ability_match_cfg) {
2109                         ap->ability_match_cfg = rx_cfg_reg;
2110                         ap->ability_match = 0;
2111                         ap->ability_match_count = 0;
2112                 } else {
2113                         if (++ap->ability_match_count > 1) {
2114                                 ap->ability_match = 1;
2115                                 ap->ability_match_cfg = rx_cfg_reg;
2116                         }
2117                 }
2118                 if (rx_cfg_reg & ANEG_CFG_ACK)
2119                         ap->ack_match = 1;
2120                 else
2121                         ap->ack_match = 0;
2122
2123                 ap->idle_match = 0;
2124         } else {
2125                 ap->idle_match = 1;
2126                 ap->ability_match_cfg = 0;
2127                 ap->ability_match_count = 0;
2128                 ap->ability_match = 0;
2129                 ap->ack_match = 0;
2130
2131                 rx_cfg_reg = 0;
2132         }
2133
2134         ap->rxconfig = rx_cfg_reg;
2135         ret = ANEG_OK;
2136
2137         switch(ap->state) {
2138         case ANEG_STATE_UNKNOWN:
2139                 if (ap->flags & (MR_AN_ENABLE | MR_RESTART_AN))
2140                         ap->state = ANEG_STATE_AN_ENABLE;
2141
2142                 /* fallthru */
2143         case ANEG_STATE_AN_ENABLE:
2144                 ap->flags &= ~(MR_AN_COMPLETE | MR_PAGE_RX);
2145                 if (ap->flags & MR_AN_ENABLE) {
2146                         ap->link_time = 0;
2147                         ap->cur_time = 0;
2148                         ap->ability_match_cfg = 0;
2149                         ap->ability_match_count = 0;
2150                         ap->ability_match = 0;
2151                         ap->idle_match = 0;
2152                         ap->ack_match = 0;
2153
2154                         ap->state = ANEG_STATE_RESTART_INIT;
2155                 } else {
2156                         ap->state = ANEG_STATE_DISABLE_LINK_OK;
2157                 }
2158                 break;
2159
2160         case ANEG_STATE_RESTART_INIT:
2161                 ap->link_time = ap->cur_time;
2162                 ap->flags &= ~(MR_NP_LOADED);
2163                 ap->txconfig = 0;
2164                 tw32(MAC_TX_AUTO_NEG, 0);
2165                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2166                 tw32_f(MAC_MODE, tp->mac_mode);
2167                 udelay(40);
2168
2169                 ret = ANEG_TIMER_ENAB;
2170                 ap->state = ANEG_STATE_RESTART;
2171
2172                 /* fallthru */
2173         case ANEG_STATE_RESTART:
2174                 delta = ap->cur_time - ap->link_time;
2175                 if (delta > ANEG_STATE_SETTLE_TIME) {
2176                         ap->state = ANEG_STATE_ABILITY_DETECT_INIT;
2177                 } else {
2178                         ret = ANEG_TIMER_ENAB;
2179                 }
2180                 break;
2181
2182         case ANEG_STATE_DISABLE_LINK_OK:
2183                 ret = ANEG_DONE;
2184                 break;
2185
2186         case ANEG_STATE_ABILITY_DETECT_INIT:
2187                 ap->flags &= ~(MR_TOGGLE_TX);
2188                 ap->txconfig = (ANEG_CFG_FD | ANEG_CFG_PS1);
2189                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
2190                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2191                 tw32_f(MAC_MODE, tp->mac_mode);
2192                 udelay(40);
2193
2194                 ap->state = ANEG_STATE_ABILITY_DETECT;
2195                 break;
2196
2197         case ANEG_STATE_ABILITY_DETECT:
2198                 if (ap->ability_match != 0 && ap->rxconfig != 0) {
2199                         ap->state = ANEG_STATE_ACK_DETECT_INIT;
2200                 }
2201                 break;
2202
2203         case ANEG_STATE_ACK_DETECT_INIT:
2204                 ap->txconfig |= ANEG_CFG_ACK;
2205                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
2206                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2207                 tw32_f(MAC_MODE, tp->mac_mode);
2208                 udelay(40);
2209
2210                 ap->state = ANEG_STATE_ACK_DETECT;
2211
2212                 /* fallthru */
2213         case ANEG_STATE_ACK_DETECT:
2214                 if (ap->ack_match != 0) {
2215                         if ((ap->rxconfig & ~ANEG_CFG_ACK) ==
2216                             (ap->ability_match_cfg & ~ANEG_CFG_ACK)) {
2217                                 ap->state = ANEG_STATE_COMPLETE_ACK_INIT;
2218                         } else {
2219                                 ap->state = ANEG_STATE_AN_ENABLE;
2220                         }
2221                 } else if (ap->ability_match != 0 &&
2222                            ap->rxconfig == 0) {
2223                         ap->state = ANEG_STATE_AN_ENABLE;
2224                 }
2225                 break;
2226
2227         case ANEG_STATE_COMPLETE_ACK_INIT:
2228                 if (ap->rxconfig & ANEG_CFG_INVAL) {
2229                         ret = ANEG_FAILED;
2230                         break;
2231                 }
2232                 ap->flags &= ~(MR_LP_ADV_FULL_DUPLEX |
2233                                MR_LP_ADV_HALF_DUPLEX |
2234                                MR_LP_ADV_SYM_PAUSE |
2235                                MR_LP_ADV_ASYM_PAUSE |
2236                                MR_LP_ADV_REMOTE_FAULT1 |
2237                                MR_LP_ADV_REMOTE_FAULT2 |
2238                                MR_LP_ADV_NEXT_PAGE |
2239                                MR_TOGGLE_RX |
2240                                MR_NP_RX);
2241                 if (ap->rxconfig & ANEG_CFG_FD)
2242                         ap->flags |= MR_LP_ADV_FULL_DUPLEX;
2243                 if (ap->rxconfig & ANEG_CFG_HD)
2244                         ap->flags |= MR_LP_ADV_HALF_DUPLEX;
2245                 if (ap->rxconfig & ANEG_CFG_PS1)
2246                         ap->flags |= MR_LP_ADV_SYM_PAUSE;
2247                 if (ap->rxconfig & ANEG_CFG_PS2)
2248                         ap->flags |= MR_LP_ADV_ASYM_PAUSE;
2249                 if (ap->rxconfig & ANEG_CFG_RF1)
2250                         ap->flags |= MR_LP_ADV_REMOTE_FAULT1;
2251                 if (ap->rxconfig & ANEG_CFG_RF2)
2252                         ap->flags |= MR_LP_ADV_REMOTE_FAULT2;
2253                 if (ap->rxconfig & ANEG_CFG_NP)
2254                         ap->flags |= MR_LP_ADV_NEXT_PAGE;
2255
2256                 ap->link_time = ap->cur_time;
2257
2258                 ap->flags ^= (MR_TOGGLE_TX);
2259                 if (ap->rxconfig & 0x0008)
2260                         ap->flags |= MR_TOGGLE_RX;
2261                 if (ap->rxconfig & ANEG_CFG_NP)
2262                         ap->flags |= MR_NP_RX;
2263                 ap->flags |= MR_PAGE_RX;
2264
2265                 ap->state = ANEG_STATE_COMPLETE_ACK;
2266                 ret = ANEG_TIMER_ENAB;
2267                 break;
2268
2269         case ANEG_STATE_COMPLETE_ACK:
2270                 if (ap->ability_match != 0 &&
2271                     ap->rxconfig == 0) {
2272                         ap->state = ANEG_STATE_AN_ENABLE;
2273                         break;
2274                 }
2275                 delta = ap->cur_time - ap->link_time;
2276                 if (delta > ANEG_STATE_SETTLE_TIME) {
2277                         if (!(ap->flags & (MR_LP_ADV_NEXT_PAGE))) {
2278                                 ap->state = ANEG_STATE_IDLE_DETECT_INIT;
2279                         } else {
2280                                 if ((ap->txconfig & ANEG_CFG_NP) == 0 &&
2281                                     !(ap->flags & MR_NP_RX)) {
2282                                         ap->state = ANEG_STATE_IDLE_DETECT_INIT;
2283                                 } else {
2284                                         ret = ANEG_FAILED;
2285                                 }
2286                         }
2287                 }
2288                 break;
2289
2290         case ANEG_STATE_IDLE_DETECT_INIT:
2291                 ap->link_time = ap->cur_time;
2292                 tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
2293                 tw32_f(MAC_MODE, tp->mac_mode);
2294                 udelay(40);
2295
2296                 ap->state = ANEG_STATE_IDLE_DETECT;
2297                 ret = ANEG_TIMER_ENAB;
2298                 break;
2299
2300         case ANEG_STATE_IDLE_DETECT:
2301                 if (ap->ability_match != 0 &&
2302                     ap->rxconfig == 0) {
2303                         ap->state = ANEG_STATE_AN_ENABLE;
2304                         break;
2305                 }
2306                 delta = ap->cur_time - ap->link_time;
2307                 if (delta > ANEG_STATE_SETTLE_TIME) {
2308                         /* XXX another gem from the Broadcom driver :( */
2309                         ap->state = ANEG_STATE_LINK_OK;
2310                 }
2311                 break;
2312
2313         case ANEG_STATE_LINK_OK:
2314                 ap->flags |= (MR_AN_COMPLETE | MR_LINK_OK);
2315                 ret = ANEG_DONE;
2316                 break;
2317
2318         case ANEG_STATE_NEXT_PAGE_WAIT_INIT:
2319                 /* ??? unimplemented */
2320                 break;
2321
2322         case ANEG_STATE_NEXT_PAGE_WAIT:
2323                 /* ??? unimplemented */
2324                 break;
2325
2326         default:
2327                 ret = ANEG_FAILED;
2328                 break;
2329         };
2330
2331         return ret;
2332 }
2333
2334 static int fiber_autoneg(struct tg3 *tp, u32 *flags)
2335 {
2336         int res = 0;
2337         struct tg3_fiber_aneginfo aninfo;
2338         int status = ANEG_FAILED;
2339         unsigned int tick;
2340         u32 tmp;
2341
2342         tw32_f(MAC_TX_AUTO_NEG, 0);
2343
2344         tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
2345         tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
2346         udelay(40);
2347
2348         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
2349         udelay(40);
2350
2351         memset(&aninfo, 0, sizeof(aninfo));
2352         aninfo.flags |= MR_AN_ENABLE;
2353         aninfo.state = ANEG_STATE_UNKNOWN;
2354         aninfo.cur_time = 0;
2355         tick = 0;
2356         while (++tick < 195000) {
2357                 status = tg3_fiber_aneg_smachine(tp, &aninfo);
2358                 if (status == ANEG_DONE || status == ANEG_FAILED)
2359                         break;
2360
2361                 udelay(1);
2362         }
2363
2364         tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
2365         tw32_f(MAC_MODE, tp->mac_mode);
2366         udelay(40);
2367
2368         *flags = aninfo.flags;
2369
2370         if (status == ANEG_DONE &&
2371             (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
2372                              MR_LP_ADV_FULL_DUPLEX)))
2373                 res = 1;
2374
2375         return res;
2376 }
2377
2378 static void tg3_init_bcm8002(struct tg3 *tp)
2379 {
2380         u32 mac_status = tr32(MAC_STATUS);
2381         int i;
2382
2383         /* Reset when initting first time or we have a link. */
2384         if ((tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) &&
2385             !(mac_status & MAC_STATUS_PCS_SYNCED))
2386                 return;
2387
2388         /* Set PLL lock range. */
2389         tg3_writephy(tp, 0x16, 0x8007);
2390
2391         /* SW reset */
2392         tg3_writephy(tp, MII_BMCR, BMCR_RESET);
2393
2394         /* Wait for reset to complete. */
2395         /* XXX schedule_timeout() ... */
2396         for (i = 0; i < 500; i++)
2397                 udelay(10);
2398
2399         /* Config mode; select PMA/Ch 1 regs. */
2400         tg3_writephy(tp, 0x10, 0x8411);
2401
2402         /* Enable auto-lock and comdet, select txclk for tx. */
2403         tg3_writephy(tp, 0x11, 0x0a10);
2404
2405         tg3_writephy(tp, 0x18, 0x00a0);
2406         tg3_writephy(tp, 0x16, 0x41ff);
2407
2408         /* Assert and deassert POR. */
2409         tg3_writephy(tp, 0x13, 0x0400);
2410         udelay(40);
2411         tg3_writephy(tp, 0x13, 0x0000);
2412
2413         tg3_writephy(tp, 0x11, 0x0a50);
2414         udelay(40);
2415         tg3_writephy(tp, 0x11, 0x0a10);
2416
2417         /* Wait for signal to stabilize */
2418         /* XXX schedule_timeout() ... */
2419         for (i = 0; i < 15000; i++)
2420                 udelay(10);
2421
2422         /* Deselect the channel register so we can read the PHYID
2423          * later.
2424          */
2425         tg3_writephy(tp, 0x10, 0x8011);
2426 }
2427
2428 static int tg3_setup_fiber_hw_autoneg(struct tg3 *tp, u32 mac_status)
2429 {
2430         u32 sg_dig_ctrl, sg_dig_status;
2431         u32 serdes_cfg, expected_sg_dig_ctrl;
2432         int workaround, port_a;
2433         int current_link_up;
2434
2435         serdes_cfg = 0;
2436         expected_sg_dig_ctrl = 0;
2437         workaround = 0;
2438         port_a = 1;
2439         current_link_up = 0;
2440
2441         if (tp->pci_chip_rev_id != CHIPREV_ID_5704_A0 &&
2442             tp->pci_chip_rev_id != CHIPREV_ID_5704_A1) {
2443                 workaround = 1;
2444                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
2445                         port_a = 0;
2446
2447                 /* preserve bits 0-11,13,14 for signal pre-emphasis */
2448                 /* preserve bits 20-23 for voltage regulator */
2449                 serdes_cfg = tr32(MAC_SERDES_CFG) & 0x00f06fff;
2450         }
2451
2452         sg_dig_ctrl = tr32(SG_DIG_CTRL);
2453
2454         if (tp->link_config.autoneg != AUTONEG_ENABLE) {
2455                 if (sg_dig_ctrl & (1 << 31)) {
2456                         if (workaround) {
2457                                 u32 val = serdes_cfg;
2458
2459                                 if (port_a)
2460                                         val |= 0xc010000;
2461                                 else
2462                                         val |= 0x4010000;
2463                                 tw32_f(MAC_SERDES_CFG, val);
2464                         }
2465                         tw32_f(SG_DIG_CTRL, 0x01388400);
2466                 }
2467                 if (mac_status & MAC_STATUS_PCS_SYNCED) {
2468                         tg3_setup_flow_control(tp, 0, 0);
2469                         current_link_up = 1;
2470                 }
2471                 goto out;
2472         }
2473
2474         /* Want auto-negotiation.  */
2475         expected_sg_dig_ctrl = 0x81388400;
2476
2477         /* Pause capability */
2478         expected_sg_dig_ctrl |= (1 << 11);
2479
2480         /* Asymettric pause */
2481         expected_sg_dig_ctrl |= (1 << 12);
2482
2483         if (sg_dig_ctrl != expected_sg_dig_ctrl) {
2484                 if ((tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT) &&
2485                     tp->serdes_counter &&
2486                     ((mac_status & (MAC_STATUS_PCS_SYNCED |
2487                                     MAC_STATUS_RCVD_CFG)) ==
2488                      MAC_STATUS_PCS_SYNCED)) {
2489                         tp->serdes_counter--;
2490                         current_link_up = 1;
2491                         goto out;
2492                 }
2493 restart_autoneg:
2494                 if (workaround)
2495                         tw32_f(MAC_SERDES_CFG, serdes_cfg | 0xc011000);
2496                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl | (1 << 30));
2497                 udelay(5);
2498                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl);
2499
2500                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
2501                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2502         } else if (mac_status & (MAC_STATUS_PCS_SYNCED |
2503                                  MAC_STATUS_SIGNAL_DET)) {
2504                 sg_dig_status = tr32(SG_DIG_STATUS);
2505                 mac_status = tr32(MAC_STATUS);
2506
2507                 if ((sg_dig_status & (1 << 1)) &&
2508                     (mac_status & MAC_STATUS_PCS_SYNCED)) {
2509                         u32 local_adv, remote_adv;
2510
2511                         local_adv = ADVERTISE_PAUSE_CAP;
2512                         remote_adv = 0;
2513                         if (sg_dig_status & (1 << 19))
2514                                 remote_adv |= LPA_PAUSE_CAP;
2515                         if (sg_dig_status & (1 << 20))
2516                                 remote_adv |= LPA_PAUSE_ASYM;
2517
2518                         tg3_setup_flow_control(tp, local_adv, remote_adv);
2519                         current_link_up = 1;
2520                         tp->serdes_counter = 0;
2521                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2522                 } else if (!(sg_dig_status & (1 << 1))) {
2523                         if (tp->serdes_counter)
2524                                 tp->serdes_counter--;
2525                         else {
2526                                 if (workaround) {
2527                                         u32 val = serdes_cfg;
2528
2529                                         if (port_a)
2530                                                 val |= 0xc010000;
2531                                         else
2532                                                 val |= 0x4010000;
2533
2534                                         tw32_f(MAC_SERDES_CFG, val);
2535                                 }
2536
2537                                 tw32_f(SG_DIG_CTRL, 0x01388400);
2538                                 udelay(40);
2539
2540                                 /* Link parallel detection - link is up */
2541                                 /* only if we have PCS_SYNC and not */
2542                                 /* receiving config code words */
2543                                 mac_status = tr32(MAC_STATUS);
2544                                 if ((mac_status & MAC_STATUS_PCS_SYNCED) &&
2545                                     !(mac_status & MAC_STATUS_RCVD_CFG)) {
2546                                         tg3_setup_flow_control(tp, 0, 0);
2547                                         current_link_up = 1;
2548                                         tp->tg3_flags2 |=
2549                                                 TG3_FLG2_PARALLEL_DETECT;
2550                                         tp->serdes_counter =
2551                                                 SERDES_PARALLEL_DET_TIMEOUT;
2552                                 } else
2553                                         goto restart_autoneg;
2554                         }
2555                 }
2556         } else {
2557                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
2558                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2559         }
2560
2561 out:
2562         return current_link_up;
2563 }
2564
2565 static int tg3_setup_fiber_by_hand(struct tg3 *tp, u32 mac_status)
2566 {
2567         int current_link_up = 0;
2568
2569         if (!(mac_status & MAC_STATUS_PCS_SYNCED)) {
2570                 tp->tg3_flags &= ~TG3_FLAG_GOT_SERDES_FLOWCTL;
2571                 goto out;
2572         }
2573
2574         if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2575                 u32 flags;
2576                 int i;
2577
2578                 if (fiber_autoneg(tp, &flags)) {
2579                         u32 local_adv, remote_adv;
2580
2581                         local_adv = ADVERTISE_PAUSE_CAP;
2582                         remote_adv = 0;
2583                         if (flags & MR_LP_ADV_SYM_PAUSE)
2584                                 remote_adv |= LPA_PAUSE_CAP;
2585                         if (flags & MR_LP_ADV_ASYM_PAUSE)
2586                                 remote_adv |= LPA_PAUSE_ASYM;
2587
2588                         tg3_setup_flow_control(tp, local_adv, remote_adv);
2589
2590                         tp->tg3_flags |= TG3_FLAG_GOT_SERDES_FLOWCTL;
2591                         current_link_up = 1;
2592                 }
2593                 for (i = 0; i < 30; i++) {
2594                         udelay(20);
2595                         tw32_f(MAC_STATUS,
2596                                (MAC_STATUS_SYNC_CHANGED |
2597                                 MAC_STATUS_CFG_CHANGED));
2598                         udelay(40);
2599                         if ((tr32(MAC_STATUS) &
2600                              (MAC_STATUS_SYNC_CHANGED |
2601                               MAC_STATUS_CFG_CHANGED)) == 0)
2602                                 break;
2603                 }
2604
2605                 mac_status = tr32(MAC_STATUS);
2606                 if (current_link_up == 0 &&
2607                     (mac_status & MAC_STATUS_PCS_SYNCED) &&
2608                     !(mac_status & MAC_STATUS_RCVD_CFG))
2609                         current_link_up = 1;
2610         } else {
2611                 /* Forcing 1000FD link up. */
2612                 current_link_up = 1;
2613                 tp->tg3_flags |= TG3_FLAG_GOT_SERDES_FLOWCTL;
2614
2615                 tw32_f(MAC_MODE, (tp->mac_mode | MAC_MODE_SEND_CONFIGS));
2616                 udelay(40);
2617         }
2618
2619 out:
2620         return current_link_up;
2621 }
2622
2623 static int tg3_setup_fiber_phy(struct tg3 *tp, int force_reset)
2624 {
2625         u32 orig_pause_cfg;
2626         u16 orig_active_speed;
2627         u8 orig_active_duplex;
2628         u32 mac_status;
2629         int current_link_up;
2630         int i;
2631
2632         orig_pause_cfg =
2633                 (tp->tg3_flags & (TG3_FLAG_RX_PAUSE |
2634                                   TG3_FLAG_TX_PAUSE));
2635         orig_active_speed = tp->link_config.active_speed;
2636         orig_active_duplex = tp->link_config.active_duplex;
2637
2638         if (!(tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG) &&
2639             netif_carrier_ok(tp->dev) &&
2640             (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE)) {
2641                 mac_status = tr32(MAC_STATUS);
2642                 mac_status &= (MAC_STATUS_PCS_SYNCED |
2643                                MAC_STATUS_SIGNAL_DET |
2644                                MAC_STATUS_CFG_CHANGED |
2645                                MAC_STATUS_RCVD_CFG);
2646                 if (mac_status == (MAC_STATUS_PCS_SYNCED |
2647                                    MAC_STATUS_SIGNAL_DET)) {
2648                         tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
2649                                             MAC_STATUS_CFG_CHANGED));
2650                         return 0;
2651                 }
2652         }
2653
2654         tw32_f(MAC_TX_AUTO_NEG, 0);
2655
2656         tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
2657         tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
2658         tw32_f(MAC_MODE, tp->mac_mode);
2659         udelay(40);
2660
2661         if (tp->phy_id == PHY_ID_BCM8002)
2662                 tg3_init_bcm8002(tp);
2663
2664         /* Enable link change event even when serdes polling.  */
2665         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2666         udelay(40);
2667
2668         current_link_up = 0;
2669         mac_status = tr32(MAC_STATUS);
2670
2671         if (tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG)
2672                 current_link_up = tg3_setup_fiber_hw_autoneg(tp, mac_status);
2673         else
2674                 current_link_up = tg3_setup_fiber_by_hand(tp, mac_status);
2675
2676         tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
2677         tw32_f(MAC_MODE, tp->mac_mode);
2678         udelay(40);
2679
2680         tp->hw_status->status =
2681                 (SD_STATUS_UPDATED |
2682                  (tp->hw_status->status & ~SD_STATUS_LINK_CHG));
2683
2684         for (i = 0; i < 100; i++) {
2685                 tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
2686                                     MAC_STATUS_CFG_CHANGED));
2687                 udelay(5);
2688                 if ((tr32(MAC_STATUS) & (MAC_STATUS_SYNC_CHANGED |
2689                                          MAC_STATUS_CFG_CHANGED |
2690                                          MAC_STATUS_LNKSTATE_CHANGED)) == 0)
2691                         break;
2692         }
2693
2694         mac_status = tr32(MAC_STATUS);
2695         if ((mac_status & MAC_STATUS_PCS_SYNCED) == 0) {
2696                 current_link_up = 0;
2697                 if (tp->link_config.autoneg == AUTONEG_ENABLE &&
2698                     tp->serdes_counter == 0) {
2699                         tw32_f(MAC_MODE, (tp->mac_mode |
2700                                           MAC_MODE_SEND_CONFIGS));
2701                         udelay(1);
2702                         tw32_f(MAC_MODE, tp->mac_mode);
2703                 }
2704         }
2705
2706         if (current_link_up == 1) {
2707                 tp->link_config.active_speed = SPEED_1000;
2708                 tp->link_config.active_duplex = DUPLEX_FULL;
2709                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
2710                                     LED_CTRL_LNKLED_OVERRIDE |
2711                                     LED_CTRL_1000MBPS_ON));
2712         } else {
2713                 tp->link_config.active_speed = SPEED_INVALID;
2714                 tp->link_config.active_duplex = DUPLEX_INVALID;
2715                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
2716                                     LED_CTRL_LNKLED_OVERRIDE |
2717                                     LED_CTRL_TRAFFIC_OVERRIDE));
2718         }
2719
2720         if (current_link_up != netif_carrier_ok(tp->dev)) {
2721                 if (current_link_up)
2722                         netif_carrier_on(tp->dev);
2723                 else
2724                         netif_carrier_off(tp->dev);
2725                 tg3_link_report(tp);
2726         } else {
2727                 u32 now_pause_cfg =
2728                         tp->tg3_flags & (TG3_FLAG_RX_PAUSE |
2729                                          TG3_FLAG_TX_PAUSE);
2730                 if (orig_pause_cfg != now_pause_cfg ||
2731                     orig_active_speed != tp->link_config.active_speed ||
2732                     orig_active_duplex != tp->link_config.active_duplex)
2733                         tg3_link_report(tp);
2734         }
2735
2736         return 0;
2737 }
2738
2739 static int tg3_setup_fiber_mii_phy(struct tg3 *tp, int force_reset)
2740 {
2741         int current_link_up, err = 0;
2742         u32 bmsr, bmcr;
2743         u16 current_speed;
2744         u8 current_duplex;
2745
2746         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
2747         tw32_f(MAC_MODE, tp->mac_mode);
2748         udelay(40);
2749
2750         tw32(MAC_EVENT, 0);
2751
2752         tw32_f(MAC_STATUS,
2753              (MAC_STATUS_SYNC_CHANGED |
2754               MAC_STATUS_CFG_CHANGED |
2755               MAC_STATUS_MI_COMPLETION |
2756               MAC_STATUS_LNKSTATE_CHANGED));
2757         udelay(40);
2758
2759         if (force_reset)
2760                 tg3_phy_reset(tp);
2761
2762         current_link_up = 0;
2763         current_speed = SPEED_INVALID;
2764         current_duplex = DUPLEX_INVALID;
2765
2766         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2767         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2768         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
2769                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
2770                         bmsr |= BMSR_LSTATUS;
2771                 else
2772                         bmsr &= ~BMSR_LSTATUS;
2773         }
2774
2775         err |= tg3_readphy(tp, MII_BMCR, &bmcr);
2776
2777         if ((tp->link_config.autoneg == AUTONEG_ENABLE) && !force_reset &&
2778             (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
2779                 /* do nothing, just check for link up at the end */
2780         } else if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2781                 u32 adv, new_adv;
2782
2783                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
2784                 new_adv = adv & ~(ADVERTISE_1000XFULL | ADVERTISE_1000XHALF |
2785                                   ADVERTISE_1000XPAUSE |
2786                                   ADVERTISE_1000XPSE_ASYM |
2787                                   ADVERTISE_SLCT);
2788
2789                 /* Always advertise symmetric PAUSE just like copper */
2790                 new_adv |= ADVERTISE_1000XPAUSE;
2791
2792                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
2793                         new_adv |= ADVERTISE_1000XHALF;
2794                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
2795                         new_adv |= ADVERTISE_1000XFULL;
2796
2797                 if ((new_adv != adv) || !(bmcr & BMCR_ANENABLE)) {
2798                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
2799                         bmcr |= BMCR_ANENABLE | BMCR_ANRESTART;
2800                         tg3_writephy(tp, MII_BMCR, bmcr);
2801
2802                         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2803                         tp->serdes_counter = SERDES_AN_TIMEOUT_5714S;
2804                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2805
2806                         return err;
2807                 }
2808         } else {
2809                 u32 new_bmcr;
2810
2811                 bmcr &= ~BMCR_SPEED1000;
2812                 new_bmcr = bmcr & ~(BMCR_ANENABLE | BMCR_FULLDPLX);
2813
2814                 if (tp->link_config.duplex == DUPLEX_FULL)
2815                         new_bmcr |= BMCR_FULLDPLX;
2816
2817                 if (new_bmcr != bmcr) {
2818                         /* BMCR_SPEED1000 is a reserved bit that needs
2819                          * to be set on write.
2820                          */
2821                         new_bmcr |= BMCR_SPEED1000;
2822
2823                         /* Force a linkdown */
2824                         if (netif_carrier_ok(tp->dev)) {
2825                                 u32 adv;
2826
2827                                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
2828                                 adv &= ~(ADVERTISE_1000XFULL |
2829                                          ADVERTISE_1000XHALF |
2830                                          ADVERTISE_SLCT);
2831                                 tg3_writephy(tp, MII_ADVERTISE, adv);
2832                                 tg3_writephy(tp, MII_BMCR, bmcr |
2833                                                            BMCR_ANRESTART |
2834                                                            BMCR_ANENABLE);
2835                                 udelay(10);
2836                                 netif_carrier_off(tp->dev);
2837                         }
2838                         tg3_writephy(tp, MII_BMCR, new_bmcr);
2839                         bmcr = new_bmcr;
2840                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2841                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2842                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
2843                             ASIC_REV_5714) {
2844                                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
2845                                         bmsr |= BMSR_LSTATUS;
2846                                 else
2847                                         bmsr &= ~BMSR_LSTATUS;
2848                         }
2849                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2850                 }
2851         }
2852
2853         if (bmsr & BMSR_LSTATUS) {
2854                 current_speed = SPEED_1000;
2855                 current_link_up = 1;
2856                 if (bmcr & BMCR_FULLDPLX)
2857                         current_duplex = DUPLEX_FULL;
2858                 else
2859                         current_duplex = DUPLEX_HALF;
2860
2861                 if (bmcr & BMCR_ANENABLE) {
2862                         u32 local_adv, remote_adv, common;
2863
2864                         err |= tg3_readphy(tp, MII_ADVERTISE, &local_adv);
2865                         err |= tg3_readphy(tp, MII_LPA, &remote_adv);
2866                         common = local_adv & remote_adv;
2867                         if (common & (ADVERTISE_1000XHALF |
2868                                       ADVERTISE_1000XFULL)) {
2869                                 if (common & ADVERTISE_1000XFULL)
2870                                         current_duplex = DUPLEX_FULL;
2871                                 else
2872                                         current_duplex = DUPLEX_HALF;
2873
2874                                 tg3_setup_flow_control(tp, local_adv,
2875                                                        remote_adv);
2876                         }
2877                         else
2878                                 current_link_up = 0;
2879                 }
2880         }
2881
2882         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
2883         if (tp->link_config.active_duplex == DUPLEX_HALF)
2884                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
2885
2886         tw32_f(MAC_MODE, tp->mac_mode);
2887         udelay(40);
2888
2889         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2890
2891         tp->link_config.active_speed = current_speed;
2892         tp->link_config.active_duplex = current_duplex;
2893
2894         if (current_link_up != netif_carrier_ok(tp->dev)) {
2895                 if (current_link_up)
2896                         netif_carrier_on(tp->dev);
2897                 else {
2898                         netif_carrier_off(tp->dev);
2899                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2900                 }
2901                 tg3_link_report(tp);
2902         }
2903         return err;
2904 }
2905
2906 static void tg3_serdes_parallel_detect(struct tg3 *tp)
2907 {
2908         if (tp->serdes_counter) {
2909                 /* Give autoneg time to complete. */
2910                 tp->serdes_counter--;
2911                 return;
2912         }
2913         if (!netif_carrier_ok(tp->dev) &&
2914             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
2915                 u32 bmcr;
2916
2917                 tg3_readphy(tp, MII_BMCR, &bmcr);
2918                 if (bmcr & BMCR_ANENABLE) {
2919                         u32 phy1, phy2;
2920
2921                         /* Select shadow register 0x1f */
2922                         tg3_writephy(tp, 0x1c, 0x7c00);
2923                         tg3_readphy(tp, 0x1c, &phy1);
2924
2925                         /* Select expansion interrupt status register */
2926                         tg3_writephy(tp, 0x17, 0x0f01);
2927                         tg3_readphy(tp, 0x15, &phy2);
2928                         tg3_readphy(tp, 0x15, &phy2);
2929
2930                         if ((phy1 & 0x10) && !(phy2 & 0x20)) {
2931                                 /* We have signal detect and not receiving
2932                                  * config code words, link is up by parallel
2933                                  * detection.
2934                                  */
2935
2936                                 bmcr &= ~BMCR_ANENABLE;
2937                                 bmcr |= BMCR_SPEED1000 | BMCR_FULLDPLX;
2938                                 tg3_writephy(tp, MII_BMCR, bmcr);
2939                                 tp->tg3_flags2 |= TG3_FLG2_PARALLEL_DETECT;
2940                         }
2941                 }
2942         }
2943         else if (netif_carrier_ok(tp->dev) &&
2944                  (tp->link_config.autoneg == AUTONEG_ENABLE) &&
2945                  (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
2946                 u32 phy2;
2947
2948                 /* Select expansion interrupt status register */
2949                 tg3_writephy(tp, 0x17, 0x0f01);
2950                 tg3_readphy(tp, 0x15, &phy2);
2951                 if (phy2 & 0x20) {
2952                         u32 bmcr;
2953
2954                         /* Config code words received, turn on autoneg. */
2955                         tg3_readphy(tp, MII_BMCR, &bmcr);
2956                         tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANENABLE);
2957
2958                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2959
2960                 }
2961         }
2962 }
2963
2964 static int tg3_setup_phy(struct tg3 *tp, int force_reset)
2965 {
2966         int err;
2967
2968         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
2969                 err = tg3_setup_fiber_phy(tp, force_reset);
2970         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
2971                 err = tg3_setup_fiber_mii_phy(tp, force_reset);
2972         } else {
2973                 err = tg3_setup_copper_phy(tp, force_reset);
2974         }
2975
2976         if (tp->link_config.active_speed == SPEED_1000 &&
2977             tp->link_config.active_duplex == DUPLEX_HALF)
2978                 tw32(MAC_TX_LENGTHS,
2979                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
2980                       (6 << TX_LENGTHS_IPG_SHIFT) |
2981                       (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
2982         else
2983                 tw32(MAC_TX_LENGTHS,
2984                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
2985                       (6 << TX_LENGTHS_IPG_SHIFT) |
2986                       (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
2987
2988         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
2989                 if (netif_carrier_ok(tp->dev)) {
2990                         tw32(HOSTCC_STAT_COAL_TICKS,
2991                              tp->coal.stats_block_coalesce_usecs);
2992                 } else {
2993                         tw32(HOSTCC_STAT_COAL_TICKS, 0);
2994                 }
2995         }
2996
2997         return err;
2998 }
2999
3000 /* This is called whenever we suspect that the system chipset is re-
3001  * ordering the sequence of MMIO to the tx send mailbox. The symptom
3002  * is bogus tx completions. We try to recover by setting the
3003  * TG3_FLAG_MBOX_WRITE_REORDER flag and resetting the chip later
3004  * in the workqueue.
3005  */
3006 static void tg3_tx_recover(struct tg3 *tp)
3007 {
3008         BUG_ON((tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) ||
3009                tp->write32_tx_mbox == tg3_write_indirect_mbox);
3010
3011         printk(KERN_WARNING PFX "%s: The system may be re-ordering memory-"
3012                "mapped I/O cycles to the network device, attempting to "
3013                "recover. Please report the problem to the driver maintainer "
3014                "and include system chipset information.\n", tp->dev->name);
3015
3016         spin_lock(&tp->lock);
3017         tp->tg3_flags |= TG3_FLAG_TX_RECOVERY_PENDING;
3018         spin_unlock(&tp->lock);
3019 }
3020
3021 static inline u32 tg3_tx_avail(struct tg3 *tp)
3022 {
3023         smp_mb();
3024         return (tp->tx_pending -
3025                 ((tp->tx_prod - tp->tx_cons) & (TG3_TX_RING_SIZE - 1)));
3026 }
3027
3028 /* Tigon3 never reports partial packet sends.  So we do not
3029  * need special logic to handle SKBs that have not had all
3030  * of their frags sent yet, like SunGEM does.
3031  */
3032 static void tg3_tx(struct tg3 *tp)
3033 {
3034         u32 hw_idx = tp->hw_status->idx[0].tx_consumer;
3035         u32 sw_idx = tp->tx_cons;
3036
3037         while (sw_idx != hw_idx) {
3038                 struct tx_ring_info *ri = &tp->tx_buffers[sw_idx];
3039                 struct sk_buff *skb = ri->skb;
3040                 int i, tx_bug = 0;
3041
3042                 if (unlikely(skb == NULL)) {
3043                         tg3_tx_recover(tp);
3044                         return;
3045                 }
3046
3047                 pci_unmap_single(tp->pdev,
3048                                  pci_unmap_addr(ri, mapping),
3049                                  skb_headlen(skb),
3050                                  PCI_DMA_TODEVICE);
3051
3052                 ri->skb = NULL;
3053
3054                 sw_idx = NEXT_TX(sw_idx);
3055
3056                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3057                         ri = &tp->tx_buffers[sw_idx];
3058                         if (unlikely(ri->skb != NULL || sw_idx == hw_idx))
3059                                 tx_bug = 1;
3060
3061                         pci_unmap_page(tp->pdev,
3062                                        pci_unmap_addr(ri, mapping),
3063                                        skb_shinfo(skb)->frags[i].size,
3064                                        PCI_DMA_TODEVICE);
3065
3066                         sw_idx = NEXT_TX(sw_idx);
3067                 }
3068
3069                 dev_kfree_skb(skb);
3070
3071                 if (unlikely(tx_bug)) {
3072                         tg3_tx_recover(tp);
3073                         return;
3074                 }
3075         }
3076
3077         tp->tx_cons = sw_idx;
3078
3079         /* Need to make the tx_cons update visible to tg3_start_xmit()
3080          * before checking for netif_queue_stopped().  Without the
3081          * memory barrier, there is a small possibility that tg3_start_xmit()
3082          * will miss it and cause the queue to be stopped forever.
3083          */
3084         smp_mb();
3085
3086         if (unlikely(netif_queue_stopped(tp->dev) &&
3087                      (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp)))) {
3088                 netif_tx_lock(tp->dev);
3089                 if (netif_queue_stopped(tp->dev) &&
3090                     (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp)))
3091                         netif_wake_queue(tp->dev);
3092                 netif_tx_unlock(tp->dev);
3093         }
3094 }
3095
3096 /* Returns size of skb allocated or < 0 on error.
3097  *
3098  * We only need to fill in the address because the other members
3099  * of the RX descriptor are invariant, see tg3_init_rings.
3100  *
3101  * Note the purposeful assymetry of cpu vs. chip accesses.  For
3102  * posting buffers we only dirty the first cache line of the RX
3103  * descriptor (containing the address).  Whereas for the RX status
3104  * buffers the cpu only reads the last cacheline of the RX descriptor
3105  * (to fetch the error flags, vlan tag, checksum, and opaque cookie).
3106  */
3107 static int tg3_alloc_rx_skb(struct tg3 *tp, u32 opaque_key,
3108                             int src_idx, u32 dest_idx_unmasked)
3109 {
3110         struct tg3_rx_buffer_desc *desc;
3111         struct ring_info *map, *src_map;
3112         struct sk_buff *skb;
3113         dma_addr_t mapping;
3114         int skb_size, dest_idx;
3115
3116         src_map = NULL;
3117         switch (opaque_key) {
3118         case RXD_OPAQUE_RING_STD:
3119                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
3120                 desc = &tp->rx_std[dest_idx];
3121                 map = &tp->rx_std_buffers[dest_idx];
3122                 if (src_idx >= 0)
3123                         src_map = &tp->rx_std_buffers[src_idx];
3124                 skb_size = tp->rx_pkt_buf_sz;
3125                 break;
3126
3127         case RXD_OPAQUE_RING_JUMBO:
3128                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
3129                 desc = &tp->rx_jumbo[dest_idx];
3130                 map = &tp->rx_jumbo_buffers[dest_idx];
3131                 if (src_idx >= 0)
3132                         src_map = &tp->rx_jumbo_buffers[src_idx];
3133                 skb_size = RX_JUMBO_PKT_BUF_SZ;
3134                 break;
3135
3136         default:
3137                 return -EINVAL;
3138         };
3139
3140         /* Do not overwrite any of the map or rp information
3141          * until we are sure we can commit to a new buffer.
3142          *
3143          * Callers depend upon this behavior and assume that
3144          * we leave everything unchanged if we fail.
3145          */
3146         skb = netdev_alloc_skb(tp->dev, skb_size);
3147         if (skb == NULL)
3148                 return -ENOMEM;
3149
3150         skb_reserve(skb, tp->rx_offset);
3151
3152         mapping = pci_map_single(tp->pdev, skb->data,
3153                                  skb_size - tp->rx_offset,
3154                                  PCI_DMA_FROMDEVICE);
3155
3156         map->skb = skb;
3157         pci_unmap_addr_set(map, mapping, mapping);
3158
3159         if (src_map != NULL)
3160                 src_map->skb = NULL;
3161
3162         desc->addr_hi = ((u64)mapping >> 32);
3163         desc->addr_lo = ((u64)mapping & 0xffffffff);
3164
3165         return skb_size;
3166 }
3167
3168 /* We only need to move over in the address because the other
3169  * members of the RX descriptor are invariant.  See notes above
3170  * tg3_alloc_rx_skb for full details.
3171  */
3172 static void tg3_recycle_rx(struct tg3 *tp, u32 opaque_key,
3173                            int src_idx, u32 dest_idx_unmasked)
3174 {
3175         struct tg3_rx_buffer_desc *src_desc, *dest_desc;
3176         struct ring_info *src_map, *dest_map;
3177         int dest_idx;
3178
3179         switch (opaque_key) {
3180         case RXD_OPAQUE_RING_STD:
3181                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
3182                 dest_desc = &tp->rx_std[dest_idx];
3183                 dest_map = &tp->rx_std_buffers[dest_idx];
3184                 src_desc = &tp->rx_std[src_idx];
3185                 src_map = &tp->rx_std_buffers[src_idx];
3186                 break;
3187
3188         case RXD_OPAQUE_RING_JUMBO:
3189                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
3190                 dest_desc = &tp->rx_jumbo[dest_idx];
3191                 dest_map = &tp->rx_jumbo_buffers[dest_idx];
3192                 src_desc = &tp->rx_jumbo[src_idx];
3193                 src_map = &tp->rx_jumbo_buffers[src_idx];
3194                 break;
3195
3196         default:
3197                 return;
3198         };
3199
3200         dest_map->skb = src_map->skb;
3201         pci_unmap_addr_set(dest_map, mapping,
3202                            pci_unmap_addr(src_map, mapping));
3203         dest_desc->addr_hi = src_desc->addr_hi;
3204         dest_desc->addr_lo = src_desc->addr_lo;
3205
3206         src_map->skb = NULL;
3207 }
3208
3209 #if TG3_VLAN_TAG_USED
3210 static int tg3_vlan_rx(struct tg3 *tp, struct sk_buff *skb, u16 vlan_tag)
3211 {
3212         return vlan_hwaccel_receive_skb(skb, tp->vlgrp, vlan_tag);
3213 }
3214 #endif
3215
3216 /* The RX ring scheme is composed of multiple rings which post fresh
3217  * buffers to the chip, and one special ring the chip uses to report
3218  * status back to the host.
3219  *
3220  * The special ring reports the status of received packets to the
3221  * host.  The chip does not write into the original descriptor the
3222  * RX buffer was obtained from.  The chip simply takes the original
3223  * descriptor as provided by the host, updates the status and length
3224  * field, then writes this into the next status ring entry.
3225  *
3226  * Each ring the host uses to post buffers to the chip is described
3227  * by a TG3_BDINFO entry in the chips SRAM area.  When a packet arrives,
3228  * it is first placed into the on-chip ram.  When the packet's length
3229  * is known, it walks down the TG3_BDINFO entries to select the ring.
3230  * Each TG3_BDINFO specifies a MAXLEN field and the first TG3_BDINFO
3231  * which is within the range of the new packet's length is chosen.
3232  *
3233  * The "separate ring for rx status" scheme may sound queer, but it makes
3234  * sense from a cache coherency perspective.  If only the host writes
3235  * to the buffer post rings, and only the chip writes to the rx status
3236  * rings, then cache lines never move beyond shared-modified state.
3237  * If both the host and chip were to write into the same ring, cache line
3238  * eviction could occur since both entities want it in an exclusive state.
3239  */
3240 static int tg3_rx(struct tg3 *tp, int budget)
3241 {
3242         u32 work_mask, rx_std_posted = 0;
3243         u32 sw_idx = tp->rx_rcb_ptr;
3244         u16 hw_idx;
3245         int received;
3246
3247         hw_idx = tp->hw_status->idx[0].rx_producer;
3248         /*
3249          * We need to order the read of hw_idx and the read of
3250          * the opaque cookie.
3251          */
3252         rmb();
3253         work_mask = 0;
3254         received = 0;
3255         while (sw_idx != hw_idx && budget > 0) {
3256                 struct tg3_rx_buffer_desc *desc = &tp->rx_rcb[sw_idx];
3257                 unsigned int len;
3258                 struct sk_buff *skb;
3259                 dma_addr_t dma_addr;
3260                 u32 opaque_key, desc_idx, *post_ptr;
3261
3262                 desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
3263                 opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
3264                 if (opaque_key == RXD_OPAQUE_RING_STD) {
3265                         dma_addr = pci_unmap_addr(&tp->rx_std_buffers[desc_idx],
3266                                                   mapping);
3267                         skb = tp->rx_std_buffers[desc_idx].skb;
3268                         post_ptr = &tp->rx_std_ptr;
3269                         rx_std_posted++;
3270                 } else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
3271                         dma_addr = pci_unmap_addr(&tp->rx_jumbo_buffers[desc_idx],
3272                                                   mapping);
3273                         skb = tp->rx_jumbo_buffers[desc_idx].skb;
3274                         post_ptr = &tp->rx_jumbo_ptr;
3275                 }
3276                 else {
3277                         goto next_pkt_nopost;
3278                 }
3279
3280                 work_mask |= opaque_key;
3281
3282                 if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
3283                     (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII)) {
3284                 drop_it:
3285                         tg3_recycle_rx(tp, opaque_key,
3286                                        desc_idx, *post_ptr);
3287                 drop_it_no_recycle:
3288                         /* Other statistics kept track of by card. */
3289                         tp->net_stats.rx_dropped++;
3290                         goto next_pkt;
3291                 }
3292
3293                 len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) - 4; /* omit crc */
3294
3295                 if (len > RX_COPY_THRESHOLD
3296                         && tp->rx_offset == 2
3297                         /* rx_offset != 2 iff this is a 5701 card running
3298                          * in PCI-X mode [see tg3_get_invariants()] */
3299                 ) {
3300                         int skb_size;
3301
3302                         skb_size = tg3_alloc_rx_skb(tp, opaque_key,
3303                                                     desc_idx, *post_ptr);
3304                         if (skb_size < 0)
3305                                 goto drop_it;
3306
3307                         pci_unmap_single(tp->pdev, dma_addr,
3308                                          skb_size - tp->rx_offset,
3309                                          PCI_DMA_FROMDEVICE);
3310
3311                         skb_put(skb, len);
3312                 } else {
3313                         struct sk_buff *copy_skb;
3314
3315                         tg3_recycle_rx(tp, opaque_key,
3316                                        desc_idx, *post_ptr);
3317
3318                         copy_skb = netdev_alloc_skb(tp->dev, len + 2);
3319                         if (copy_skb == NULL)
3320                                 goto drop_it_no_recycle;
3321
3322                         skb_reserve(copy_skb, 2);
3323                         skb_put(copy_skb, len);
3324                         pci_dma_sync_single_for_cpu(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
3325                         memcpy(copy_skb->data, skb->data, len);
3326                         pci_dma_sync_single_for_device(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
3327
3328                         /* We'll reuse the original ring buffer. */
3329                         skb = copy_skb;
3330                 }
3331
3332                 if ((tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) &&
3333                     (desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
3334                     (((desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
3335                       >> RXD_TCPCSUM_SHIFT) == 0xffff))
3336                         skb->ip_summed = CHECKSUM_UNNECESSARY;
3337                 else
3338                         skb->ip_summed = CHECKSUM_NONE;
3339
3340                 skb->protocol = eth_type_trans(skb, tp->dev);
3341 #if TG3_VLAN_TAG_USED
3342                 if (tp->vlgrp != NULL &&
3343                     desc->type_flags & RXD_FLAG_VLAN) {
3344                         tg3_vlan_rx(tp, skb,
3345                                     desc->err_vlan & RXD_VLAN_MASK);
3346                 } else
3347 #endif
3348                         netif_receive_skb(skb);
3349
3350                 tp->dev->last_rx = jiffies;
3351                 received++;
3352                 budget--;
3353
3354 next_pkt:
3355                 (*post_ptr)++;
3356
3357                 if (unlikely(rx_std_posted >= tp->rx_std_max_post)) {
3358                         u32 idx = *post_ptr % TG3_RX_RING_SIZE;
3359
3360                         tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX +
3361                                      TG3_64BIT_REG_LOW, idx);
3362                         work_mask &= ~RXD_OPAQUE_RING_STD;
3363                         rx_std_posted = 0;
3364                 }
3365 next_pkt_nopost:
3366                 sw_idx++;
3367                 sw_idx %= TG3_RX_RCB_RING_SIZE(tp);
3368
3369                 /* Refresh hw_idx to see if there is new work */
3370                 if (sw_idx == hw_idx) {
3371                         hw_idx = tp->hw_status->idx[0].rx_producer;
3372                         rmb();
3373                 }
3374         }
3375
3376         /* ACK the status ring. */
3377         tp->rx_rcb_ptr = sw_idx;
3378         tw32_rx_mbox(MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW, sw_idx);
3379
3380         /* Refill RX ring(s). */
3381         if (work_mask & RXD_OPAQUE_RING_STD) {
3382                 sw_idx = tp->rx_std_ptr % TG3_RX_RING_SIZE;
3383                 tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW,
3384                              sw_idx);
3385         }
3386         if (work_mask & RXD_OPAQUE_RING_JUMBO) {
3387                 sw_idx = tp->rx_jumbo_ptr % TG3_RX_JUMBO_RING_SIZE;
3388                 tw32_rx_mbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW,
3389                              sw_idx);
3390         }
3391         mmiowb();
3392
3393         return received;
3394 }
3395
3396 static int tg3_poll(struct net_device *netdev, int *budget)
3397 {
3398         struct tg3 *tp = netdev_priv(netdev);
3399         struct tg3_hw_status *sblk = tp->hw_status;
3400         int done;
3401
3402         /* handle link change and other phy events */
3403         if (!(tp->tg3_flags &
3404               (TG3_FLAG_USE_LINKCHG_REG |
3405                TG3_FLAG_POLL_SERDES))) {
3406                 if (sblk->status & SD_STATUS_LINK_CHG) {
3407                         sblk->status = SD_STATUS_UPDATED |
3408                                 (sblk->status & ~SD_STATUS_LINK_CHG);
3409                         spin_lock(&tp->lock);
3410                         tg3_setup_phy(tp, 0);
3411                         spin_unlock(&tp->lock);
3412                 }
3413         }
3414
3415         /* run TX completion thread */
3416         if (sblk->idx[0].tx_consumer != tp->tx_cons) {
3417                 tg3_tx(tp);
3418                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING)) {
3419                         netif_rx_complete(netdev);
3420                         schedule_work(&tp->reset_task);
3421                         return 0;
3422                 }
3423         }
3424
3425         /* run RX thread, within the bounds set by NAPI.
3426          * All RX "locking" is done by ensuring outside
3427          * code synchronizes with dev->poll()
3428          */
3429         if (sblk->idx[0].rx_producer != tp->rx_rcb_ptr) {
3430                 int orig_budget = *budget;
3431                 int work_done;
3432
3433                 if (orig_budget > netdev->quota)
3434                         orig_budget = netdev->quota;
3435
3436                 work_done = tg3_rx(tp, orig_budget);
3437
3438                 *budget -= work_done;
3439                 netdev->quota -= work_done;
3440         }
3441
3442         if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) {
3443                 tp->last_tag = sblk->status_tag;
3444                 rmb();
3445         } else
3446                 sblk->status &= ~SD_STATUS_UPDATED;
3447
3448         /* if no more work, tell net stack and NIC we're done */
3449         done = !tg3_has_work(tp);
3450         if (done) {
3451                 netif_rx_complete(netdev);
3452                 tg3_restart_ints(tp);
3453         }
3454
3455         return (done ? 0 : 1);
3456 }
3457
3458 static void tg3_irq_quiesce(struct tg3 *tp)
3459 {
3460         BUG_ON(tp->irq_sync);
3461
3462         tp->irq_sync = 1;
3463         smp_mb();
3464
3465         synchronize_irq(tp->pdev->irq);
3466 }
3467
3468 static inline int tg3_irq_sync(struct tg3 *tp)
3469 {
3470         return tp->irq_sync;
3471 }
3472
3473 /* Fully shutdown all tg3 driver activity elsewhere in the system.
3474  * If irq_sync is non-zero, then the IRQ handler must be synchronized
3475  * with as well.  Most of the time, this is not necessary except when
3476  * shutting down the device.
3477  */
3478 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync)
3479 {
3480         if (irq_sync)
3481                 tg3_irq_quiesce(tp);
3482         spin_lock_bh(&tp->lock);
3483 }
3484
3485 static inline void tg3_full_unlock(struct tg3 *tp)
3486 {
3487         spin_unlock_bh(&tp->lock);
3488 }
3489
3490 /* One-shot MSI handler - Chip automatically disables interrupt
3491  * after sending MSI so driver doesn't have to do it.
3492  */
3493 static irqreturn_t tg3_msi_1shot(int irq, void *dev_id)
3494 {
3495         struct net_device *dev = dev_id;
3496         struct tg3 *tp = netdev_priv(dev);
3497
3498         prefetch(tp->hw_status);
3499         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3500
3501         if (likely(!tg3_irq_sync(tp)))
3502                 netif_rx_schedule(dev);         /* schedule NAPI poll */
3503
3504         return IRQ_HANDLED;
3505 }
3506
3507 /* MSI ISR - No need to check for interrupt sharing and no need to
3508  * flush status block and interrupt mailbox. PCI ordering rules
3509  * guarantee that MSI will arrive after the status block.
3510  */
3511 static irqreturn_t tg3_msi(int irq, void *dev_id)
3512 {
3513         struct net_device *dev = dev_id;
3514         struct tg3 *tp = netdev_priv(dev);
3515
3516         prefetch(tp->hw_status);
3517         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3518         /*
3519          * Writing any value to intr-mbox-0 clears PCI INTA# and
3520          * chip-internal interrupt pending events.
3521          * Writing non-zero to intr-mbox-0 additional tells the
3522          * NIC to stop sending us irqs, engaging "in-intr-handler"
3523          * event coalescing.
3524          */
3525         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
3526         if (likely(!tg3_irq_sync(tp)))
3527                 netif_rx_schedule(dev);         /* schedule NAPI poll */
3528
3529         return IRQ_RETVAL(1);
3530 }
3531
3532 static irqreturn_t tg3_interrupt(int irq, void *dev_id)
3533 {
3534         struct net_device *dev = dev_id;
3535         struct tg3 *tp = netdev_priv(dev);
3536         struct tg3_hw_status *sblk = tp->hw_status;
3537         unsigned int handled = 1;
3538
3539         /* In INTx mode, it is possible for the interrupt to arrive at
3540          * the CPU before the status block posted prior to the interrupt.
3541          * Reading the PCI State register will confirm whether the
3542          * interrupt is ours and will flush the status block.
3543          */
3544         if ((sblk->status & SD_STATUS_UPDATED) ||
3545             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3546                 /*
3547                  * Writing any value to intr-mbox-0 clears PCI INTA# and
3548                  * chip-internal interrupt pending events.
3549                  * Writing non-zero to intr-mbox-0 additional tells the
3550                  * NIC to stop sending us irqs, engaging "in-intr-handler"
3551                  * event coalescing.
3552                  */
3553                 tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3554                              0x00000001);
3555                 if (tg3_irq_sync(tp))
3556                         goto out;
3557                 sblk->status &= ~SD_STATUS_UPDATED;
3558                 if (likely(tg3_has_work(tp))) {
3559                         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3560                         netif_rx_schedule(dev);         /* schedule NAPI poll */
3561                 } else {
3562                         /* No work, shared interrupt perhaps?  re-enable
3563                          * interrupts, and flush that PCI write
3564                          */
3565                         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3566                                 0x00000000);
3567                 }
3568         } else {        /* shared interrupt */
3569                 handled = 0;
3570         }
3571 out:
3572         return IRQ_RETVAL(handled);
3573 }
3574
3575 static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id)
3576 {
3577         struct net_device *dev = dev_id;
3578         struct tg3 *tp = netdev_priv(dev);
3579         struct tg3_hw_status *sblk = tp->hw_status;
3580         unsigned int handled = 1;
3581
3582         /* In INTx mode, it is possible for the interrupt to arrive at
3583          * the CPU before the status block posted prior to the interrupt.
3584          * Reading the PCI State register will confirm whether the
3585          * interrupt is ours and will flush the status block.
3586          */
3587         if ((sblk->status_tag != tp->last_tag) ||
3588             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3589                 /*
3590                  * writing any value to intr-mbox-0 clears PCI INTA# and
3591                  * chip-internal interrupt pending events.
3592                  * writing non-zero to intr-mbox-0 additional tells the
3593                  * NIC to stop sending us irqs, engaging "in-intr-handler"
3594                  * event coalescing.
3595                  */
3596                 tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3597                              0x00000001);
3598                 if (tg3_irq_sync(tp))
3599                         goto out;
3600                 if (netif_rx_schedule_prep(dev)) {
3601                         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3602                         /* Update last_tag to mark that this status has been
3603                          * seen. Because interrupt may be shared, we may be
3604                          * racing with tg3_poll(), so only update last_tag
3605                          * if tg3_poll() is not scheduled.
3606                          */
3607                         tp->last_tag = sblk->status_tag;
3608                         __netif_rx_schedule(dev);
3609                 }
3610         } else {        /* shared interrupt */
3611                 handled = 0;
3612         }
3613 out:
3614         return IRQ_RETVAL(handled);
3615 }
3616
3617 /* ISR for interrupt test */
3618 static irqreturn_t tg3_test_isr(int irq, void *dev_id)
3619 {
3620         struct net_device *dev = dev_id;
3621         struct tg3 *tp = netdev_priv(dev);
3622         struct tg3_hw_status *sblk = tp->hw_status;
3623
3624         if ((sblk->status & SD_STATUS_UPDATED) ||
3625             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3626                 tg3_disable_ints(tp);
3627                 return IRQ_RETVAL(1);
3628         }
3629         return IRQ_RETVAL(0);
3630 }
3631
3632 static int tg3_init_hw(struct tg3 *, int);
3633 static int tg3_halt(struct tg3 *, int, int);
3634
3635 /* Restart hardware after configuration changes, self-test, etc.
3636  * Invoked with tp->lock held.
3637  */
3638 static int tg3_restart_hw(struct tg3 *tp, int reset_phy)
3639 {
3640         int err;
3641
3642         err = tg3_init_hw(tp, reset_phy);
3643         if (err) {
3644                 printk(KERN_ERR PFX "%s: Failed to re-initialize device, "
3645                        "aborting.\n", tp->dev->name);
3646                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
3647                 tg3_full_unlock(tp);
3648                 del_timer_sync(&tp->timer);
3649                 tp->irq_sync = 0;
3650                 netif_poll_enable(tp->dev);
3651                 dev_close(tp->dev);
3652                 tg3_full_lock(tp, 0);
3653         }
3654         return err;
3655 }
3656
3657 #ifdef CONFIG_NET_POLL_CONTROLLER
3658 static void tg3_poll_controller(struct net_device *dev)
3659 {
3660         struct tg3 *tp = netdev_priv(dev);
3661
3662         tg3_interrupt(tp->pdev->irq, dev);
3663 }
3664 #endif
3665
3666 static void tg3_reset_task(struct work_struct *work)
3667 {
3668         struct tg3 *tp = container_of(work, struct tg3, reset_task);
3669         unsigned int restart_timer;
3670
3671         tg3_full_lock(tp, 0);
3672         tp->tg3_flags |= TG3_FLAG_IN_RESET_TASK;
3673
3674         if (!netif_running(tp->dev)) {
3675                 tp->tg3_flags &= ~TG3_FLAG_IN_RESET_TASK;
3676                 tg3_full_unlock(tp);
3677                 return;
3678         }
3679
3680         tg3_full_unlock(tp);
3681
3682         tg3_netif_stop(tp);
3683
3684         tg3_full_lock(tp, 1);
3685
3686         restart_timer = tp->tg3_flags2 & TG3_FLG2_RESTART_TIMER;
3687         tp->tg3_flags2 &= ~TG3_FLG2_RESTART_TIMER;
3688
3689         if (tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING) {
3690                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
3691                 tp->write32_rx_mbox = tg3_write_flush_reg32;
3692                 tp->tg3_flags |= TG3_FLAG_MBOX_WRITE_REORDER;
3693                 tp->tg3_flags &= ~TG3_FLAG_TX_RECOVERY_PENDING;
3694         }
3695
3696         tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
3697         if (tg3_init_hw(tp, 1))
3698                 goto out;
3699
3700         tg3_netif_start(tp);
3701
3702         if (restart_timer)
3703                 mod_timer(&tp->timer, jiffies + 1);
3704
3705 out:
3706         tp->tg3_flags &= ~TG3_FLAG_IN_RESET_TASK;
3707
3708         tg3_full_unlock(tp);
3709 }
3710
3711 static void tg3_tx_timeout(struct net_device *dev)
3712 {
3713         struct tg3 *tp = netdev_priv(dev);
3714
3715         if (netif_msg_tx_err(tp))
3716                 printk(KERN_ERR PFX "%s: transmit timed out, resetting\n",
3717                        dev->name);
3718
3719         schedule_work(&tp->reset_task);
3720 }
3721
3722 /* Test for DMA buffers crossing any 4GB boundaries: 4G, 8G, etc */
3723 static inline int tg3_4g_overflow_test(dma_addr_t mapping, int len)
3724 {
3725         u32 base = (u32) mapping & 0xffffffff;
3726
3727         return ((base > 0xffffdcc0) &&
3728                 (base + len + 8 < base));
3729 }
3730
3731 /* Test for DMA addresses > 40-bit */
3732 static inline int tg3_40bit_overflow_test(struct tg3 *tp, dma_addr_t mapping,
3733                                           int len)
3734 {
3735 #if defined(CONFIG_HIGHMEM) && (BITS_PER_LONG == 64)
3736         if (tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG)
3737                 return (((u64) mapping + len) > DMA_40BIT_MASK);
3738         return 0;
3739 #else
3740         return 0;
3741 #endif
3742 }
3743
3744 static void tg3_set_txd(struct tg3 *, int, dma_addr_t, int, u32, u32);
3745
3746 /* Workaround 4GB and 40-bit hardware DMA bugs. */
3747 static int tigon3_dma_hwbug_workaround(struct tg3 *tp, struct sk_buff *skb,
3748                                        u32 last_plus_one, u32 *start,
3749                                        u32 base_flags, u32 mss)
3750 {
3751         struct sk_buff *new_skb = skb_copy(skb, GFP_ATOMIC);
3752         dma_addr_t new_addr = 0;
3753         u32 entry = *start;
3754         int i, ret = 0;
3755
3756         if (!new_skb) {
3757                 ret = -1;
3758         } else {
3759                 /* New SKB is guaranteed to be linear. */
3760                 entry = *start;
3761                 new_addr = pci_map_single(tp->pdev, new_skb->data, new_skb->len,
3762                                           PCI_DMA_TODEVICE);
3763                 /* Make sure new skb does not cross any 4G boundaries.
3764                  * Drop the packet if it does.
3765                  */
3766                 if (tg3_4g_overflow_test(new_addr, new_skb->len)) {
3767                         ret = -1;
3768                         dev_kfree_skb(new_skb);
3769                         new_skb = NULL;
3770                 } else {
3771                         tg3_set_txd(tp, entry, new_addr, new_skb->len,
3772                                     base_flags, 1 | (mss << 1));
3773                         *start = NEXT_TX(entry);
3774                 }
3775         }
3776
3777         /* Now clean up the sw ring entries. */
3778         i = 0;
3779         while (entry != last_plus_one) {
3780                 int len;
3781
3782                 if (i == 0)
3783                         len = skb_headlen(skb);
3784                 else
3785                         len = skb_shinfo(skb)->frags[i-1].size;
3786                 pci_unmap_single(tp->pdev,
3787                                  pci_unmap_addr(&tp->tx_buffers[entry], mapping),
3788                                  len, PCI_DMA_TODEVICE);
3789                 if (i == 0) {
3790                         tp->tx_buffers[entry].skb = new_skb;
3791                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, new_addr);
3792                 } else {
3793                         tp->tx_buffers[entry].skb = NULL;
3794                 }
3795                 entry = NEXT_TX(entry);
3796                 i++;
3797         }
3798
3799         dev_kfree_skb(skb);
3800
3801         return ret;
3802 }
3803
3804 static void tg3_set_txd(struct tg3 *tp, int entry,
3805                         dma_addr_t mapping, int len, u32 flags,
3806                         u32 mss_and_is_end)
3807 {
3808         struct tg3_tx_buffer_desc *txd = &tp->tx_ring[entry];
3809         int is_end = (mss_and_is_end & 0x1);
3810         u32 mss = (mss_and_is_end >> 1);
3811         u32 vlan_tag = 0;
3812
3813         if (is_end)
3814                 flags |= TXD_FLAG_END;
3815         if (flags & TXD_FLAG_VLAN) {
3816                 vlan_tag = flags >> 16;
3817                 flags &= 0xffff;
3818         }
3819         vlan_tag |= (mss << TXD_MSS_SHIFT);
3820
3821         txd->addr_hi = ((u64) mapping >> 32);
3822         txd->addr_lo = ((u64) mapping & 0xffffffff);
3823         txd->len_flags = (len << TXD_LEN_SHIFT) | flags;
3824         txd->vlan_tag = vlan_tag << TXD_VLAN_TAG_SHIFT;
3825 }
3826
3827 /* hard_start_xmit for devices that don't have any bugs and
3828  * support TG3_FLG2_HW_TSO_2 only.
3829  */
3830 static int tg3_start_xmit(struct sk_buff *skb, struct net_device *dev)
3831 {
3832         struct tg3 *tp = netdev_priv(dev);
3833         dma_addr_t mapping;
3834         u32 len, entry, base_flags, mss;
3835
3836         len = skb_headlen(skb);
3837
3838         /* We are running in BH disabled context with netif_tx_lock
3839          * and TX reclaim runs via tp->poll inside of a software
3840          * interrupt.  Furthermore, IRQ processing runs lockless so we have
3841          * no IRQ context deadlocks to worry about either.  Rejoice!
3842          */
3843         if (unlikely(tg3_tx_avail(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
3844                 if (!netif_queue_stopped(dev)) {
3845                         netif_stop_queue(dev);
3846
3847                         /* This is a hard error, log it. */
3848                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
3849                                "queue awake!\n", dev->name);
3850                 }
3851                 return NETDEV_TX_BUSY;
3852         }
3853
3854         entry = tp->tx_prod;
3855         base_flags = 0;
3856 #if TG3_TSO_SUPPORT != 0
3857         mss = 0;
3858         if (skb->len > (tp->dev->mtu + ETH_HLEN) &&
3859             (mss = skb_shinfo(skb)->gso_size) != 0) {
3860                 int tcp_opt_len, ip_tcp_len;
3861
3862                 if (skb_header_cloned(skb) &&
3863                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
3864                         dev_kfree_skb(skb);
3865                         goto out_unlock;
3866                 }
3867
3868                 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
3869                         mss |= (skb_headlen(skb) - ETH_HLEN) << 9;
3870                 else {
3871                         tcp_opt_len = ((skb->h.th->doff - 5) * 4);
3872                         ip_tcp_len = (skb->nh.iph->ihl * 4) +
3873                                      sizeof(struct tcphdr);
3874
3875                         skb->nh.iph->check = 0;
3876                         skb->nh.iph->tot_len = htons(mss + ip_tcp_len +
3877                                                      tcp_opt_len);
3878                         mss |= (ip_tcp_len + tcp_opt_len) << 9;
3879                 }
3880
3881                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
3882                                TXD_FLAG_CPU_POST_DMA);
3883
3884                 skb->h.th->check = 0;
3885
3886         }
3887         else if (skb->ip_summed == CHECKSUM_PARTIAL)
3888                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
3889 #else
3890         mss = 0;
3891         if (skb->ip_summed == CHECKSUM_PARTIAL)
3892                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
3893 #endif
3894 #if TG3_VLAN_TAG_USED
3895         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
3896                 base_flags |= (TXD_FLAG_VLAN |
3897                                (vlan_tx_tag_get(skb) << 16));
3898 #endif
3899
3900         /* Queue skb data, a.k.a. the main skb fragment. */
3901         mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
3902
3903         tp->tx_buffers[entry].skb = skb;
3904         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
3905
3906         tg3_set_txd(tp, entry, mapping, len, base_flags,
3907                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
3908
3909         entry = NEXT_TX(entry);
3910
3911         /* Now loop through additional data fragments, and queue them. */
3912         if (skb_shinfo(skb)->nr_frags > 0) {
3913                 unsigned int i, last;
3914
3915                 last = skb_shinfo(skb)->nr_frags - 1;
3916                 for (i = 0; i <= last; i++) {
3917                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3918
3919                         len = frag->size;
3920                         mapping = pci_map_page(tp->pdev,
3921                                                frag->page,
3922                                                frag->page_offset,
3923                                                len, PCI_DMA_TODEVICE);
3924
3925                         tp->tx_buffers[entry].skb = NULL;
3926                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
3927
3928                         tg3_set_txd(tp, entry, mapping, len,
3929                                     base_flags, (i == last) | (mss << 1));
3930
3931                         entry = NEXT_TX(entry);
3932                 }
3933         }
3934
3935         /* Packets are ready, update Tx producer idx local and on card. */
3936         tw32_tx_mbox((MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW), entry);
3937
3938         tp->tx_prod = entry;
3939         if (unlikely(tg3_tx_avail(tp) <= (MAX_SKB_FRAGS + 1))) {
3940                 netif_stop_queue(dev);
3941                 if (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp))
3942                         netif_wake_queue(tp->dev);
3943         }
3944
3945 out_unlock:
3946         mmiowb();
3947
3948         dev->trans_start = jiffies;
3949
3950         return NETDEV_TX_OK;
3951 }
3952
3953 #if TG3_TSO_SUPPORT != 0
3954 static int tg3_start_xmit_dma_bug(struct sk_buff *, struct net_device *);
3955
3956 /* Use GSO to workaround a rare TSO bug that may be triggered when the
3957  * TSO header is greater than 80 bytes.
3958  */
3959 static int tg3_tso_bug(struct tg3 *tp, struct sk_buff *skb)
3960 {
3961         struct sk_buff *segs, *nskb;
3962
3963         /* Estimate the number of fragments in the worst case */
3964         if (unlikely(tg3_tx_avail(tp) <= (skb_shinfo(skb)->gso_segs * 3))) {
3965                 netif_stop_queue(tp->dev);
3966                 return NETDEV_TX_BUSY;
3967         }
3968
3969         segs = skb_gso_segment(skb, tp->dev->features & ~NETIF_F_TSO);
3970         if (unlikely(IS_ERR(segs)))
3971                 goto tg3_tso_bug_end;
3972
3973         do {
3974                 nskb = segs;
3975                 segs = segs->next;
3976                 nskb->next = NULL;
3977                 tg3_start_xmit_dma_bug(nskb, tp->dev);
3978         } while (segs);
3979
3980 tg3_tso_bug_end:
3981         dev_kfree_skb(skb);
3982
3983         return NETDEV_TX_OK;
3984 }
3985 #endif
3986
3987 /* hard_start_xmit for devices that have the 4G bug and/or 40-bit bug and
3988  * support TG3_FLG2_HW_TSO_1 or firmware TSO only.
3989  */
3990 static int tg3_start_xmit_dma_bug(struct sk_buff *skb, struct net_device *dev)
3991 {
3992         struct tg3 *tp = netdev_priv(dev);
3993         dma_addr_t mapping;
3994         u32 len, entry, base_flags, mss;
3995         int would_hit_hwbug;
3996
3997         len = skb_headlen(skb);
3998
3999         /* We are running in BH disabled context with netif_tx_lock
4000          * and TX reclaim runs via tp->poll inside of a software
4001          * interrupt.  Furthermore, IRQ processing runs lockless so we have
4002          * no IRQ context deadlocks to worry about either.  Rejoice!
4003          */
4004         if (unlikely(tg3_tx_avail(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
4005                 if (!netif_queue_stopped(dev)) {
4006                         netif_stop_queue(dev);
4007
4008                         /* This is a hard error, log it. */
4009                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
4010                                "queue awake!\n", dev->name);
4011                 }
4012                 return NETDEV_TX_BUSY;
4013         }
4014
4015         entry = tp->tx_prod;
4016         base_flags = 0;
4017         if (skb->ip_summed == CHECKSUM_PARTIAL)
4018                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
4019 #if TG3_TSO_SUPPORT != 0
4020         mss = 0;
4021         if (skb->len > (tp->dev->mtu + ETH_HLEN) &&
4022             (mss = skb_shinfo(skb)->gso_size) != 0) {
4023                 int tcp_opt_len, ip_tcp_len, hdr_len;
4024
4025                 if (skb_header_cloned(skb) &&
4026                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
4027                         dev_kfree_skb(skb);
4028                         goto out_unlock;
4029                 }
4030
4031                 tcp_opt_len = ((skb->h.th->doff - 5) * 4);
4032                 ip_tcp_len = (skb->nh.iph->ihl * 4) + sizeof(struct tcphdr);
4033
4034                 hdr_len = ip_tcp_len + tcp_opt_len;
4035                 if (unlikely((ETH_HLEN + hdr_len) > 80) &&
4036                              (tp->tg3_flags2 & TG3_FLG2_HW_TSO_1_BUG))
4037                         return (tg3_tso_bug(tp, skb));
4038
4039                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
4040                                TXD_FLAG_CPU_POST_DMA);
4041
4042                 skb->nh.iph->check = 0;
4043                 skb->nh.iph->tot_len = htons(mss + hdr_len);
4044                 if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
4045                         skb->h.th->check = 0;
4046                         base_flags &= ~TXD_FLAG_TCPUDP_CSUM;
4047                 }
4048                 else {
4049                         skb->h.th->check =
4050                                 ~csum_tcpudp_magic(skb->nh.iph->saddr,
4051                                                    skb->nh.iph->daddr,
4052                                                    0, IPPROTO_TCP, 0);
4053                 }
4054
4055                 if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO) ||
4056                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705)) {
4057                         if (tcp_opt_len || skb->nh.iph->ihl > 5) {
4058                                 int tsflags;
4059
4060                                 tsflags = ((skb->nh.iph->ihl - 5) +
4061                                            (tcp_opt_len >> 2));
4062                                 mss |= (tsflags << 11);
4063                         }
4064                 } else {
4065                         if (tcp_opt_len || skb->nh.iph->ihl > 5) {
4066                                 int tsflags;
4067
4068                                 tsflags = ((skb->nh.iph->ihl - 5) +
4069                                            (tcp_opt_len >> 2));
4070                                 base_flags |= tsflags << 12;
4071                         }
4072                 }
4073         }
4074 #else
4075         mss = 0;
4076 #endif
4077 #if TG3_VLAN_TAG_USED
4078         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
4079                 base_flags |= (TXD_FLAG_VLAN |
4080                                (vlan_tx_tag_get(skb) << 16));
4081 #endif
4082
4083         /* Queue skb data, a.k.a. the main skb fragment. */
4084         mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
4085
4086         tp->tx_buffers[entry].skb = skb;
4087         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
4088
4089         would_hit_hwbug = 0;
4090
4091         if (tg3_4g_overflow_test(mapping, len))
4092                 would_hit_hwbug = 1;
4093
4094         tg3_set_txd(tp, entry, mapping, len, base_flags,
4095                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
4096
4097         entry = NEXT_TX(entry);
4098
4099         /* Now loop through additional data fragments, and queue them. */
4100         if (skb_shinfo(skb)->nr_frags > 0) {
4101                 unsigned int i, last;
4102
4103                 last = skb_shinfo(skb)->nr_frags - 1;
4104                 for (i = 0; i <= last; i++) {
4105                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4106
4107                         len = frag->size;
4108                         mapping = pci_map_page(tp->pdev,
4109                                                frag->page,
4110                                                frag->page_offset,
4111                                                len, PCI_DMA_TODEVICE);
4112
4113                         tp->tx_buffers[entry].skb = NULL;
4114                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
4115
4116                         if (tg3_4g_overflow_test(mapping, len))
4117                                 would_hit_hwbug = 1;
4118
4119                         if (tg3_40bit_overflow_test(tp, mapping, len))
4120                                 would_hit_hwbug = 1;
4121
4122                         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
4123                                 tg3_set_txd(tp, entry, mapping, len,
4124                                             base_flags, (i == last)|(mss << 1));
4125                         else
4126                                 tg3_set_txd(tp, entry, mapping, len,
4127                                             base_flags, (i == last));
4128
4129                         entry = NEXT_TX(entry);
4130                 }
4131         }
4132
4133         if (would_hit_hwbug) {
4134                 u32 last_plus_one = entry;
4135                 u32 start;
4136
4137                 start = entry - 1 - skb_shinfo(skb)->nr_frags;
4138                 start &= (TG3_TX_RING_SIZE - 1);
4139
4140                 /* If the workaround fails due to memory/mapping
4141                  * failure, silently drop this packet.
4142                  */
4143                 if (tigon3_dma_hwbug_workaround(tp, skb, last_plus_one,
4144                                                 &start, base_flags, mss))
4145                         goto out_unlock;
4146
4147                 entry = start;
4148         }
4149
4150         /* Packets are ready, update Tx producer idx local and on card. */
4151         tw32_tx_mbox((MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW), entry);
4152
4153         tp->tx_prod = entry;
4154         if (unlikely(tg3_tx_avail(tp) <= (MAX_SKB_FRAGS + 1))) {
4155                 netif_stop_queue(dev);
4156                 if (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp))
4157                         netif_wake_queue(tp->dev);
4158         }
4159
4160 out_unlock:
4161         mmiowb();
4162
4163         dev->trans_start = jiffies;
4164
4165         return NETDEV_TX_OK;
4166 }
4167
4168 static inline void tg3_set_mtu(struct net_device *dev, struct tg3 *tp,
4169                                int new_mtu)
4170 {
4171         dev->mtu = new_mtu;
4172
4173         if (new_mtu > ETH_DATA_LEN) {
4174                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
4175                         tp->tg3_flags2 &= ~TG3_FLG2_TSO_CAPABLE;
4176                         ethtool_op_set_tso(dev, 0);
4177                 }
4178                 else
4179                         tp->tg3_flags |= TG3_FLAG_JUMBO_RING_ENABLE;
4180         } else {
4181                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
4182                         tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
4183                 tp->tg3_flags &= ~TG3_FLAG_JUMBO_RING_ENABLE;
4184         }
4185 }
4186
4187 static int tg3_change_mtu(struct net_device *dev, int new_mtu)
4188 {
4189         struct tg3 *tp = netdev_priv(dev);
4190         int err;
4191
4192         if (new_mtu < TG3_MIN_MTU || new_mtu > TG3_MAX_MTU(tp))
4193                 return -EINVAL;
4194
4195         if (!netif_running(dev)) {
4196                 /* We'll just catch it later when the
4197                  * device is up'd.
4198                  */
4199                 tg3_set_mtu(dev, tp, new_mtu);
4200                 return 0;
4201         }
4202
4203         tg3_netif_stop(tp);
4204
4205         tg3_full_lock(tp, 1);
4206
4207         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
4208
4209         tg3_set_mtu(dev, tp, new_mtu);
4210
4211         err = tg3_restart_hw(tp, 0);
4212
4213         if (!err)
4214                 tg3_netif_start(tp);
4215
4216         tg3_full_unlock(tp);
4217
4218         return err;
4219 }
4220
4221 /* Free up pending packets in all rx/tx rings.
4222  *
4223  * The chip has been shut down and the driver detached from
4224  * the networking, so no interrupts or new tx packets will
4225  * end up in the driver.  tp->{tx,}lock is not held and we are not
4226  * in an interrupt context and thus may sleep.
4227  */
4228 static void tg3_free_rings(struct tg3 *tp)
4229 {
4230         struct ring_info *rxp;
4231         int i;
4232
4233         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
4234                 rxp = &tp->rx_std_buffers[i];
4235
4236                 if (rxp->skb == NULL)
4237                         continue;
4238                 pci_unmap_single(tp->pdev,
4239                                  pci_unmap_addr(rxp, mapping),
4240                                  tp->rx_pkt_buf_sz - tp->rx_offset,
4241                                  PCI_DMA_FROMDEVICE);
4242                 dev_kfree_skb_any(rxp->skb);
4243                 rxp->skb = NULL;
4244         }
4245
4246         for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
4247                 rxp = &tp->rx_jumbo_buffers[i];
4248
4249                 if (rxp->skb == NULL)
4250                         continue;
4251                 pci_unmap_single(tp->pdev,
4252                                  pci_unmap_addr(rxp, mapping),
4253                                  RX_JUMBO_PKT_BUF_SZ - tp->rx_offset,
4254                                  PCI_DMA_FROMDEVICE);
4255                 dev_kfree_skb_any(rxp->skb);
4256                 rxp->skb = NULL;
4257         }
4258
4259         for (i = 0; i < TG3_TX_RING_SIZE; ) {
4260                 struct tx_ring_info *txp;
4261                 struct sk_buff *skb;
4262                 int j;
4263
4264                 txp = &tp->tx_buffers[i];
4265                 skb = txp->skb;
4266
4267                 if (skb == NULL) {
4268                         i++;
4269                         continue;
4270                 }
4271
4272                 pci_unmap_single(tp->pdev,
4273                                  pci_unmap_addr(txp, mapping),
4274                                  skb_headlen(skb),
4275                                  PCI_DMA_TODEVICE);
4276                 txp->skb = NULL;
4277
4278                 i++;
4279
4280                 for (j = 0; j < skb_shinfo(skb)->nr_frags; j++) {
4281                         txp = &tp->tx_buffers[i & (TG3_TX_RING_SIZE - 1)];
4282                         pci_unmap_page(tp->pdev,
4283                                        pci_unmap_addr(txp, mapping),
4284                                        skb_shinfo(skb)->frags[j].size,
4285                                        PCI_DMA_TODEVICE);
4286                         i++;
4287                 }
4288
4289                 dev_kfree_skb_any(skb);
4290         }
4291 }
4292
4293 /* Initialize tx/rx rings for packet processing.
4294  *
4295  * The chip has been shut down and the driver detached from
4296  * the networking, so no interrupts or new tx packets will
4297  * end up in the driver.  tp->{tx,}lock are held and thus
4298  * we may not sleep.
4299  */
4300 static int tg3_init_rings(struct tg3 *tp)
4301 {
4302         u32 i;
4303
4304         /* Free up all the SKBs. */
4305         tg3_free_rings(tp);
4306
4307         /* Zero out all descriptors. */
4308         memset(tp->rx_std, 0, TG3_RX_RING_BYTES);
4309         memset(tp->rx_jumbo, 0, TG3_RX_JUMBO_RING_BYTES);
4310         memset(tp->rx_rcb, 0, TG3_RX_RCB_RING_BYTES(tp));
4311         memset(tp->tx_ring, 0, TG3_TX_RING_BYTES);
4312
4313         tp->rx_pkt_buf_sz = RX_PKT_BUF_SZ;
4314         if ((tp->tg3_flags2 & TG3_FLG2_5780_CLASS) &&
4315             (tp->dev->mtu > ETH_DATA_LEN))
4316                 tp->rx_pkt_buf_sz = RX_JUMBO_PKT_BUF_SZ;
4317
4318         /* Initialize invariants of the rings, we only set this
4319          * stuff once.  This works because the card does not
4320          * write into the rx buffer posting rings.
4321          */
4322         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
4323                 struct tg3_rx_buffer_desc *rxd;
4324
4325                 rxd = &tp->rx_std[i];
4326                 rxd->idx_len = (tp->rx_pkt_buf_sz - tp->rx_offset - 64)
4327                         << RXD_LEN_SHIFT;
4328                 rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT);
4329                 rxd->opaque = (RXD_OPAQUE_RING_STD |
4330                                (i << RXD_OPAQUE_INDEX_SHIFT));
4331         }
4332
4333         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
4334                 for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
4335                         struct tg3_rx_buffer_desc *rxd;
4336
4337                         rxd = &tp->rx_jumbo[i];
4338                         rxd->idx_len = (RX_JUMBO_PKT_BUF_SZ - tp->rx_offset - 64)
4339                                 << RXD_LEN_SHIFT;
4340                         rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT) |
4341                                 RXD_FLAG_JUMBO;
4342                         rxd->opaque = (RXD_OPAQUE_RING_JUMBO |
4343                                (i << RXD_OPAQUE_INDEX_SHIFT));
4344                 }
4345         }
4346
4347         /* Now allocate fresh SKBs for each rx ring. */
4348         for (i = 0; i < tp->rx_pending; i++) {
4349                 if (tg3_alloc_rx_skb(tp, RXD_OPAQUE_RING_STD, -1, i) < 0) {
4350                         printk(KERN_WARNING PFX
4351                                "%s: Using a smaller RX standard ring, "
4352                                "only %d out of %d buffers were allocated "
4353                                "successfully.\n",
4354                                tp->dev->name, i, tp->rx_pending);
4355                         if (i == 0)
4356                                 return -ENOMEM;
4357                         tp->rx_pending = i;
4358                         break;
4359                 }
4360         }
4361
4362         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
4363                 for (i = 0; i < tp->rx_jumbo_pending; i++) {
4364                         if (tg3_alloc_rx_skb(tp, RXD_OPAQUE_RING_JUMBO,
4365                                              -1, i) < 0) {
4366                                 printk(KERN_WARNING PFX
4367                                        "%s: Using a smaller RX jumbo ring, "
4368                                        "only %d out of %d buffers were "
4369                                        "allocated successfully.\n",
4370                                        tp->dev->name, i, tp->rx_jumbo_pending);
4371                                 if (i == 0) {
4372                                         tg3_free_rings(tp);
4373                                         return -ENOMEM;
4374                                 }
4375                                 tp->rx_jumbo_pending = i;
4376                                 break;
4377                         }
4378                 }
4379         }
4380         return 0;
4381 }
4382
4383 /*
4384  * Must not be invoked with interrupt sources disabled and
4385  * the hardware shutdown down.
4386  */
4387 static void tg3_free_consistent(struct tg3 *tp)
4388 {
4389         kfree(tp->rx_std_buffers);
4390         tp->rx_std_buffers = NULL;
4391         if (tp->rx_std) {
4392                 pci_free_consistent(tp->pdev, TG3_RX_RING_BYTES,
4393                                     tp->rx_std, tp->rx_std_mapping);
4394                 tp->rx_std = NULL;
4395         }
4396         if (tp->rx_jumbo) {
4397                 pci_free_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
4398                                     tp->rx_jumbo, tp->rx_jumbo_mapping);
4399                 tp->rx_jumbo = NULL;
4400         }
4401         if (tp->rx_rcb) {
4402                 pci_free_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
4403                                     tp->rx_rcb, tp->rx_rcb_mapping);
4404                 tp->rx_rcb = NULL;
4405         }
4406         if (tp->tx_ring) {
4407                 pci_free_consistent(tp->pdev, TG3_TX_RING_BYTES,
4408                         tp->tx_ring, tp->tx_desc_mapping);
4409                 tp->tx_ring = NULL;
4410         }
4411         if (tp->hw_status) {
4412                 pci_free_consistent(tp->pdev, TG3_HW_STATUS_SIZE,
4413                                     tp->hw_status, tp->status_mapping);
4414                 tp->hw_status = NULL;
4415         }
4416         if (tp->hw_stats) {
4417                 pci_free_consistent(tp->pdev, sizeof(struct tg3_hw_stats),
4418                                     tp->hw_stats, tp->stats_mapping);
4419                 tp->hw_stats = NULL;
4420         }
4421 }
4422
4423 /*
4424  * Must not be invoked with interrupt sources disabled and
4425  * the hardware shutdown down.  Can sleep.
4426  */
4427 static int tg3_alloc_consistent(struct tg3 *tp)
4428 {
4429         tp->rx_std_buffers = kmalloc((sizeof(struct ring_info) *
4430                                       (TG3_RX_RING_SIZE +
4431                                        TG3_RX_JUMBO_RING_SIZE)) +
4432                                      (sizeof(struct tx_ring_info) *
4433                                       TG3_TX_RING_SIZE),
4434                                      GFP_KERNEL);
4435         if (!tp->rx_std_buffers)
4436                 return -ENOMEM;
4437
4438         memset(tp->rx_std_buffers, 0,
4439                (sizeof(struct ring_info) *
4440                 (TG3_RX_RING_SIZE +
4441                  TG3_RX_JUMBO_RING_SIZE)) +
4442                (sizeof(struct tx_ring_info) *
4443                 TG3_TX_RING_SIZE));
4444
4445         tp->rx_jumbo_buffers = &tp->rx_std_buffers[TG3_RX_RING_SIZE];
4446         tp->tx_buffers = (struct tx_ring_info *)
4447                 &tp->rx_jumbo_buffers[TG3_RX_JUMBO_RING_SIZE];
4448
4449         tp->rx_std = pci_alloc_consistent(tp->pdev, TG3_RX_RING_BYTES,
4450                                           &tp->rx_std_mapping);
4451         if (!tp->rx_std)
4452                 goto err_out;
4453
4454         tp->rx_jumbo = pci_alloc_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
4455                                             &tp->rx_jumbo_mapping);
4456
4457         if (!tp->rx_jumbo)
4458                 goto err_out;
4459
4460         tp->rx_rcb = pci_alloc_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
4461                                           &tp->rx_rcb_mapping);
4462         if (!tp->rx_rcb)
4463                 goto err_out;
4464
4465         tp->tx_ring = pci_alloc_consistent(tp->pdev, TG3_TX_RING_BYTES,
4466                                            &tp->tx_desc_mapping);
4467         if (!tp->tx_ring)
4468                 goto err_out;
4469
4470         tp->hw_status = pci_alloc_consistent(tp->pdev,
4471                                              TG3_HW_STATUS_SIZE,
4472                                              &tp->status_mapping);
4473         if (!tp->hw_status)
4474                 goto err_out;
4475
4476         tp->hw_stats = pci_alloc_consistent(tp->pdev,
4477                                             sizeof(struct tg3_hw_stats),
4478                                             &tp->stats_mapping);
4479         if (!tp->hw_stats)
4480                 goto err_out;
4481
4482         memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
4483         memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
4484
4485         return 0;
4486
4487 err_out:
4488         tg3_free_consistent(tp);
4489         return -ENOMEM;
4490 }
4491
4492 #define MAX_WAIT_CNT 1000
4493
4494 /* To stop a block, clear the enable bit and poll till it
4495  * clears.  tp->lock is held.
4496  */
4497 static int tg3_stop_block(struct tg3 *tp, unsigned long ofs, u32 enable_bit, int silent)
4498 {
4499         unsigned int i;
4500         u32 val;
4501
4502         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
4503                 switch (ofs) {
4504                 case RCVLSC_MODE:
4505                 case DMAC_MODE:
4506                 case MBFREE_MODE:
4507                 case BUFMGR_MODE:
4508                 case MEMARB_MODE:
4509                         /* We can't enable/disable these bits of the
4510                          * 5705/5750, just say success.
4511                          */
4512                         return 0;
4513
4514                 default:
4515                         break;
4516                 };
4517         }
4518
4519         val = tr32(ofs);
4520         val &= ~enable_bit;
4521         tw32_f(ofs, val);
4522
4523         for (i = 0; i < MAX_WAIT_CNT; i++) {
4524                 udelay(100);
4525                 val = tr32(ofs);
4526                 if ((val & enable_bit) == 0)
4527                         break;
4528         }
4529
4530         if (i == MAX_WAIT_CNT && !silent) {
4531                 printk(KERN_ERR PFX "tg3_stop_block timed out, "
4532                        "ofs=%lx enable_bit=%x\n",
4533                        ofs, enable_bit);
4534                 return -ENODEV;
4535         }
4536
4537         return 0;
4538 }
4539
4540 /* tp->lock is held. */
4541 static int tg3_abort_hw(struct tg3 *tp, int silent)
4542 {
4543         int i, err;
4544
4545         tg3_disable_ints(tp);
4546
4547         tp->rx_mode &= ~RX_MODE_ENABLE;
4548         tw32_f(MAC_RX_MODE, tp->rx_mode);
4549         udelay(10);
4550
4551         err  = tg3_stop_block(tp, RCVBDI_MODE, RCVBDI_MODE_ENABLE, silent);
4552         err |= tg3_stop_block(tp, RCVLPC_MODE, RCVLPC_MODE_ENABLE, silent);
4553         err |= tg3_stop_block(tp, RCVLSC_MODE, RCVLSC_MODE_ENABLE, silent);
4554         err |= tg3_stop_block(tp, RCVDBDI_MODE, RCVDBDI_MODE_ENABLE, silent);
4555         err |= tg3_stop_block(tp, RCVDCC_MODE, RCVDCC_MODE_ENABLE, silent);
4556         err |= tg3_stop_block(tp, RCVCC_MODE, RCVCC_MODE_ENABLE, silent);
4557
4558         err |= tg3_stop_block(tp, SNDBDS_MODE, SNDBDS_MODE_ENABLE, silent);
4559         err |= tg3_stop_block(tp, SNDBDI_MODE, SNDBDI_MODE_ENABLE, silent);
4560         err |= tg3_stop_block(tp, SNDDATAI_MODE, SNDDATAI_MODE_ENABLE, silent);
4561         err |= tg3_stop_block(tp, RDMAC_MODE, RDMAC_MODE_ENABLE, silent);
4562         err |= tg3_stop_block(tp, SNDDATAC_MODE, SNDDATAC_MODE_ENABLE, silent);
4563         err |= tg3_stop_block(tp, DMAC_MODE, DMAC_MODE_ENABLE, silent);
4564         err |= tg3_stop_block(tp, SNDBDC_MODE, SNDBDC_MODE_ENABLE, silent);
4565
4566         tp->mac_mode &= ~MAC_MODE_TDE_ENABLE;
4567         tw32_f(MAC_MODE, tp->mac_mode);
4568         udelay(40);
4569
4570         tp->tx_mode &= ~TX_MODE_ENABLE;
4571         tw32_f(MAC_TX_MODE, tp->tx_mode);
4572
4573         for (i = 0; i < MAX_WAIT_CNT; i++) {
4574                 udelay(100);
4575                 if (!(tr32(MAC_TX_MODE) & TX_MODE_ENABLE))
4576                         break;
4577         }
4578         if (i >= MAX_WAIT_CNT) {
4579                 printk(KERN_ERR PFX "tg3_abort_hw timed out for %s, "
4580                        "TX_MODE_ENABLE will not clear MAC_TX_MODE=%08x\n",
4581                        tp->dev->name, tr32(MAC_TX_MODE));
4582                 err |= -ENODEV;
4583         }
4584
4585         err |= tg3_stop_block(tp, HOSTCC_MODE, HOSTCC_MODE_ENABLE, silent);
4586         err |= tg3_stop_block(tp, WDMAC_MODE, WDMAC_MODE_ENABLE, silent);
4587         err |= tg3_stop_block(tp, MBFREE_MODE, MBFREE_MODE_ENABLE, silent);
4588
4589         tw32(FTQ_RESET, 0xffffffff);
4590         tw32(FTQ_RESET, 0x00000000);
4591
4592         err |= tg3_stop_block(tp, BUFMGR_MODE, BUFMGR_MODE_ENABLE, silent);
4593         err |= tg3_stop_block(tp, MEMARB_MODE, MEMARB_MODE_ENABLE, silent);
4594
4595         if (tp->hw_status)
4596                 memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
4597         if (tp->hw_stats)
4598                 memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
4599
4600         return err;
4601 }
4602
4603 /* tp->lock is held. */
4604 static int tg3_nvram_lock(struct tg3 *tp)
4605 {
4606         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
4607                 int i;
4608
4609                 if (tp->nvram_lock_cnt == 0) {
4610                         tw32(NVRAM_SWARB, SWARB_REQ_SET1);
4611                         for (i = 0; i < 8000; i++) {
4612                                 if (tr32(NVRAM_SWARB) & SWARB_GNT1)
4613                                         break;
4614                                 udelay(20);
4615                         }
4616                         if (i == 8000) {
4617                                 tw32(NVRAM_SWARB, SWARB_REQ_CLR1);
4618                                 return -ENODEV;
4619                         }
4620                 }
4621                 tp->nvram_lock_cnt++;
4622         }
4623         return 0;
4624 }
4625
4626 /* tp->lock is held. */
4627 static void tg3_nvram_unlock(struct tg3 *tp)
4628 {
4629         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
4630                 if (tp->nvram_lock_cnt > 0)
4631                         tp->nvram_lock_cnt--;
4632                 if (tp->nvram_lock_cnt == 0)
4633                         tw32_f(NVRAM_SWARB, SWARB_REQ_CLR1);
4634         }
4635 }
4636
4637 /* tp->lock is held. */
4638 static void tg3_enable_nvram_access(struct tg3 *tp)
4639 {
4640         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
4641             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
4642                 u32 nvaccess = tr32(NVRAM_ACCESS);
4643
4644                 tw32(NVRAM_ACCESS, nvaccess | ACCESS_ENABLE);
4645         }
4646 }
4647
4648 /* tp->lock is held. */
4649 static void tg3_disable_nvram_access(struct tg3 *tp)
4650 {
4651         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
4652             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
4653                 u32 nvaccess = tr32(NVRAM_ACCESS);
4654
4655                 tw32(NVRAM_ACCESS, nvaccess & ~ACCESS_ENABLE);
4656         }
4657 }
4658
4659 /* tp->lock is held. */
4660 static void tg3_write_sig_pre_reset(struct tg3 *tp, int kind)
4661 {
4662         tg3_write_mem(tp, NIC_SRAM_FIRMWARE_MBOX,
4663                       NIC_SRAM_FIRMWARE_MBOX_MAGIC1);
4664
4665         if (tp->tg3_flags2 & TG3_FLG2_ASF_NEW_HANDSHAKE) {
4666                 switch (kind) {
4667                 case RESET_KIND_INIT:
4668                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4669                                       DRV_STATE_START);
4670                         break;
4671
4672                 case RESET_KIND_SHUTDOWN:
4673                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4674                                       DRV_STATE_UNLOAD);
4675                         break;
4676
4677                 case RESET_KIND_SUSPEND:
4678                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4679                                       DRV_STATE_SUSPEND);
4680                         break;
4681
4682                 default:
4683                         break;
4684                 };
4685         }
4686 }
4687
4688 /* tp->lock is held. */
4689 static void tg3_write_sig_post_reset(struct tg3 *tp, int kind)
4690 {
4691         if (tp->tg3_flags2 & TG3_FLG2_ASF_NEW_HANDSHAKE) {
4692                 switch (kind) {
4693                 case RESET_KIND_INIT:
4694                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4695                                       DRV_STATE_START_DONE);
4696                         break;
4697
4698                 case RESET_KIND_SHUTDOWN:
4699                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4700                                       DRV_STATE_UNLOAD_DONE);
4701                         break;
4702
4703                 default:
4704                         break;
4705                 };
4706         }
4707 }
4708
4709 /* tp->lock is held. */
4710 static void tg3_write_sig_legacy(struct tg3 *tp, int kind)
4711 {
4712         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
4713                 switch (kind) {
4714                 case RESET_KIND_INIT:
4715                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4716                                       DRV_STATE_START);
4717                         break;
4718
4719                 case RESET_KIND_SHUTDOWN:
4720                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4721                                       DRV_STATE_UNLOAD);
4722                         break;
4723
4724                 case RESET_KIND_SUSPEND:
4725                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4726                                       DRV_STATE_SUSPEND);
4727                         break;
4728
4729                 default:
4730                         break;
4731                 };
4732         }
4733 }
4734
4735 static int tg3_poll_fw(struct tg3 *tp)
4736 {
4737         int i;
4738         u32 val;
4739
4740         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
4741                 /* Wait up to 20ms for init done. */
4742                 for (i = 0; i < 200; i++) {
4743                         if (tr32(VCPU_STATUS) & VCPU_STATUS_INIT_DONE)
4744                                 return 0;
4745                         udelay(100);
4746                 }
4747                 return -ENODEV;
4748         }
4749
4750         /* Wait for firmware initialization to complete. */
4751         for (i = 0; i < 100000; i++) {
4752                 tg3_read_mem(tp, NIC_SRAM_FIRMWARE_MBOX, &val);
4753                 if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
4754                         break;
4755                 udelay(10);
4756         }
4757
4758         /* Chip might not be fitted with firmware.  Some Sun onboard
4759          * parts are configured like that.  So don't signal the timeout
4760          * of the above loop as an error, but do report the lack of
4761          * running firmware once.
4762          */
4763         if (i >= 100000 &&
4764             !(tp->tg3_flags2 & TG3_FLG2_NO_FWARE_REPORTED)) {
4765                 tp->tg3_flags2 |= TG3_FLG2_NO_FWARE_REPORTED;
4766
4767                 printk(KERN_INFO PFX "%s: No firmware running.\n",
4768                        tp->dev->name);
4769         }
4770
4771         return 0;
4772 }
4773
4774 static void tg3_stop_fw(struct tg3 *);
4775
4776 /* tp->lock is held. */
4777 static int tg3_chip_reset(struct tg3 *tp)
4778 {
4779         u32 val;
4780         void (*write_op)(struct tg3 *, u32, u32);
4781         int err;
4782
4783         tg3_nvram_lock(tp);
4784
4785         /* No matching tg3_nvram_unlock() after this because
4786          * chip reset below will undo the nvram lock.
4787          */
4788         tp->nvram_lock_cnt = 0;
4789
4790         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
4791             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
4792             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
4793                 tw32(GRC_FASTBOOT_PC, 0);
4794
4795         /*
4796          * We must avoid the readl() that normally takes place.
4797          * It locks machines, causes machine checks, and other
4798          * fun things.  So, temporarily disable the 5701
4799          * hardware workaround, while we do the reset.
4800          */
4801         write_op = tp->write32;
4802         if (write_op == tg3_write_flush_reg32)
4803                 tp->write32 = tg3_write32;
4804
4805         /* do the reset */
4806         val = GRC_MISC_CFG_CORECLK_RESET;
4807
4808         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
4809                 if (tr32(0x7e2c) == 0x60) {
4810                         tw32(0x7e2c, 0x20);
4811                 }
4812                 if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0) {
4813                         tw32(GRC_MISC_CFG, (1 << 29));
4814                         val |= (1 << 29);
4815                 }
4816         }
4817
4818         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
4819                 tw32(VCPU_STATUS, tr32(VCPU_STATUS) | VCPU_STATUS_DRV_RESET);
4820                 tw32(GRC_VCPU_EXT_CTRL,
4821                      tr32(GRC_VCPU_EXT_CTRL) & ~GRC_VCPU_EXT_CTRL_HALT_CPU);
4822         }
4823
4824         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
4825                 val |= GRC_MISC_CFG_KEEP_GPHY_POWER;
4826         tw32(GRC_MISC_CFG, val);
4827
4828         /* restore 5701 hardware bug workaround write method */
4829         tp->write32 = write_op;
4830
4831         /* Unfortunately, we have to delay before the PCI read back.
4832          * Some 575X chips even will not respond to a PCI cfg access
4833          * when the reset command is given to the chip.
4834          *
4835          * How do these hardware designers expect things to work
4836          * properly if the PCI write is posted for a long period
4837          * of time?  It is always necessary to have some method by
4838          * which a register read back can occur to push the write
4839          * out which does the reset.
4840          *
4841          * For most tg3 variants the trick below was working.
4842          * Ho hum...
4843          */
4844         udelay(120);
4845
4846         /* Flush PCI posted writes.  The normal MMIO registers
4847          * are inaccessible at this time so this is the only
4848          * way to make this reliably (actually, this is no longer
4849          * the case, see above).  I tried to use indirect
4850          * register read/write but this upset some 5701 variants.
4851          */
4852         pci_read_config_dword(tp->pdev, PCI_COMMAND, &val);
4853
4854         udelay(120);
4855
4856         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
4857                 if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A0) {
4858                         int i;
4859                         u32 cfg_val;
4860
4861                         /* Wait for link training to complete.  */
4862                         for (i = 0; i < 5000; i++)
4863                                 udelay(100);
4864
4865                         pci_read_config_dword(tp->pdev, 0xc4, &cfg_val);
4866                         pci_write_config_dword(tp->pdev, 0xc4,
4867                                                cfg_val | (1 << 15));
4868                 }
4869                 /* Set PCIE max payload size and clear error status.  */
4870                 pci_write_config_dword(tp->pdev, 0xd8, 0xf5000);
4871         }
4872
4873         /* Re-enable indirect register accesses. */
4874         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
4875                                tp->misc_host_ctrl);
4876
4877         /* Set MAX PCI retry to zero. */
4878         val = (PCISTATE_ROM_ENABLE | PCISTATE_ROM_RETRY_ENABLE);
4879         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0 &&
4880             (tp->tg3_flags & TG3_FLAG_PCIX_MODE))
4881                 val |= PCISTATE_RETRY_SAME_DMA;
4882         pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, val);
4883
4884         pci_restore_state(tp->pdev);
4885
4886         /* Make sure PCI-X relaxed ordering bit is clear. */
4887         pci_read_config_dword(tp->pdev, TG3PCI_X_CAPS, &val);
4888         val &= ~PCIX_CAPS_RELAXED_ORDERING;
4889         pci_write_config_dword(tp->pdev, TG3PCI_X_CAPS, val);
4890
4891         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
4892                 u32 val;
4893
4894                 /* Chip reset on 5780 will reset MSI enable bit,
4895                  * so need to restore it.
4896                  */
4897                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
4898                         u16 ctrl;
4899
4900                         pci_read_config_word(tp->pdev,
4901                                              tp->msi_cap + PCI_MSI_FLAGS,
4902                                              &ctrl);
4903                         pci_write_config_word(tp->pdev,
4904                                               tp->msi_cap + PCI_MSI_FLAGS,
4905                                               ctrl | PCI_MSI_FLAGS_ENABLE);
4906                         val = tr32(MSGINT_MODE);
4907                         tw32(MSGINT_MODE, val | MSGINT_MODE_ENABLE);
4908                 }
4909
4910                 val = tr32(MEMARB_MODE);
4911                 tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
4912
4913         } else
4914                 tw32(MEMARB_MODE, MEMARB_MODE_ENABLE);
4915
4916         if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A3) {
4917                 tg3_stop_fw(tp);
4918                 tw32(0x5000, 0x400);
4919         }
4920
4921         tw32(GRC_MODE, tp->grc_mode);
4922
4923         if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0) {
4924                 u32 val = tr32(0xc4);
4925
4926                 tw32(0xc4, val | (1 << 15));
4927         }
4928
4929         if ((tp->nic_sram_data_cfg & NIC_SRAM_DATA_CFG_MINI_PCI) != 0 &&
4930             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
4931                 tp->pci_clock_ctrl |= CLOCK_CTRL_CLKRUN_OENABLE;
4932                 if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0)
4933                         tp->pci_clock_ctrl |= CLOCK_CTRL_FORCE_CLKRUN;
4934                 tw32(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
4935         }
4936
4937         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
4938                 tp->mac_mode = MAC_MODE_PORT_MODE_TBI;
4939                 tw32_f(MAC_MODE, tp->mac_mode);
4940         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
4941                 tp->mac_mode = MAC_MODE_PORT_MODE_GMII;
4942                 tw32_f(MAC_MODE, tp->mac_mode);
4943         } else
4944                 tw32_f(MAC_MODE, 0);
4945         udelay(40);
4946
4947         err = tg3_poll_fw(tp);
4948         if (err)
4949                 return err;
4950
4951         if ((tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) &&
4952             tp->pci_chip_rev_id != CHIPREV_ID_5750_A0) {
4953                 u32 val = tr32(0x7c00);
4954
4955                 tw32(0x7c00, val | (1 << 25));
4956         }
4957
4958         /* Reprobe ASF enable state.  */
4959         tp->tg3_flags &= ~TG3_FLAG_ENABLE_ASF;
4960         tp->tg3_flags2 &= ~TG3_FLG2_ASF_NEW_HANDSHAKE;
4961         tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
4962         if (val == NIC_SRAM_DATA_SIG_MAGIC) {
4963                 u32 nic_cfg;
4964
4965                 tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
4966                 if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
4967                         tp->tg3_flags |= TG3_FLAG_ENABLE_ASF;
4968                         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
4969                                 tp->tg3_flags2 |= TG3_FLG2_ASF_NEW_HANDSHAKE;
4970                 }
4971         }
4972
4973         return 0;
4974 }
4975
4976 /* tp->lock is held. */
4977 static void tg3_stop_fw(struct tg3 *tp)
4978 {
4979         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
4980                 u32 val;
4981                 int i;
4982
4983                 tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_PAUSE_FW);
4984                 val = tr32(GRC_RX_CPU_EVENT);
4985                 val |= (1 << 14);
4986                 tw32(GRC_RX_CPU_EVENT, val);
4987
4988                 /* Wait for RX cpu to ACK the event.  */
4989                 for (i = 0; i < 100; i++) {
4990                         if (!(tr32(GRC_RX_CPU_EVENT) & (1 << 14)))
4991                                 break;
4992                         udelay(1);
4993                 }
4994         }
4995 }
4996
4997 /* tp->lock is held. */
4998 static int tg3_halt(struct tg3 *tp, int kind, int silent)
4999 {
5000         int err;
5001
5002         tg3_stop_fw(tp);
5003
5004         tg3_write_sig_pre_reset(tp, kind);
5005
5006         tg3_abort_hw(tp, silent);
5007         err = tg3_chip_reset(tp);
5008
5009         tg3_write_sig_legacy(tp, kind);
5010         tg3_write_sig_post_reset(tp, kind);
5011
5012         if (err)
5013                 return err;
5014
5015         return 0;
5016 }
5017
5018 #define TG3_FW_RELEASE_MAJOR    0x0
5019 #define TG3_FW_RELASE_MINOR     0x0
5020 #define TG3_FW_RELEASE_FIX      0x0
5021 #define TG3_FW_START_ADDR       0x08000000
5022 #define TG3_FW_TEXT_ADDR        0x08000000
5023 #define TG3_FW_TEXT_LEN         0x9c0
5024 #define TG3_FW_RODATA_ADDR      0x080009c0
5025 #define TG3_FW_RODATA_LEN       0x60
5026 #define TG3_FW_DATA_ADDR        0x08000a40
5027 #define TG3_FW_DATA_LEN         0x20
5028 #define TG3_FW_SBSS_ADDR        0x08000a60
5029 #define TG3_FW_SBSS_LEN         0xc
5030 #define TG3_FW_BSS_ADDR         0x08000a70
5031 #define TG3_FW_BSS_LEN          0x10
5032
5033 static const u32 tg3FwText[(TG3_FW_TEXT_LEN / sizeof(u32)) + 1] = {
5034         0x00000000, 0x10000003, 0x00000000, 0x0000000d, 0x0000000d, 0x3c1d0800,
5035         0x37bd3ffc, 0x03a0f021, 0x3c100800, 0x26100000, 0x0e000018, 0x00000000,
5036         0x0000000d, 0x3c1d0800, 0x37bd3ffc, 0x03a0f021, 0x3c100800, 0x26100034,
5037         0x0e00021c, 0x00000000, 0x0000000d, 0x00000000, 0x00000000, 0x00000000,
5038         0x27bdffe0, 0x3c1cc000, 0xafbf0018, 0xaf80680c, 0x0e00004c, 0x241b2105,
5039         0x97850000, 0x97870002, 0x9782002c, 0x9783002e, 0x3c040800, 0x248409c0,
5040         0xafa00014, 0x00021400, 0x00621825, 0x00052c00, 0xafa30010, 0x8f860010,
5041         0x00e52825, 0x0e000060, 0x24070102, 0x3c02ac00, 0x34420100, 0x3c03ac01,
5042         0x34630100, 0xaf820490, 0x3c02ffff, 0xaf820494, 0xaf830498, 0xaf82049c,
5043         0x24020001, 0xaf825ce0, 0x0e00003f, 0xaf825d00, 0x0e000140, 0x00000000,
5044         0x8fbf0018, 0x03e00008, 0x27bd0020, 0x2402ffff, 0xaf825404, 0x8f835400,
5045         0x34630400, 0xaf835400, 0xaf825404, 0x3c020800, 0x24420034, 0xaf82541c,
5046         0x03e00008, 0xaf805400, 0x00000000, 0x00000000, 0x3c020800, 0x34423000,
5047         0x3c030800, 0x34633000, 0x3c040800, 0x348437ff, 0x3c010800, 0xac220a64,
5048         0x24020040, 0x3c010800, 0xac220a68, 0x3c010800, 0xac200a60, 0xac600000,
5049         0x24630004, 0x0083102b, 0x5040fffd, 0xac600000, 0x03e00008, 0x00000000,
5050         0x00804821, 0x8faa0010, 0x3c020800, 0x8c420a60, 0x3c040800, 0x8c840a68,
5051         0x8fab0014, 0x24430001, 0x0044102b, 0x3c010800, 0xac230a60, 0x14400003,
5052         0x00004021, 0x3c010800, 0xac200a60, 0x3c020800, 0x8c420a60, 0x3c030800,
5053         0x8c630a64, 0x91240000, 0x00021140, 0x00431021, 0x00481021, 0x25080001,
5054         0xa0440000, 0x29020008, 0x1440fff4, 0x25290001, 0x3c020800, 0x8c420a60,
5055         0x3c030800, 0x8c630a64, 0x8f84680c, 0x00021140, 0x00431021, 0xac440008,
5056         0xac45000c, 0xac460010, 0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c,
5057         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5058         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5059         0, 0, 0, 0, 0, 0,
5060         0x02000008, 0x00000000, 0x0a0001e3, 0x3c0a0001, 0x0a0001e3, 0x3c0a0002,
5061         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5062         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5063         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5064         0x0a0001e3, 0x3c0a0007, 0x0a0001e3, 0x3c0a0008, 0x0a0001e3, 0x3c0a0009,
5065         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a000b,
5066         0x0a0001e3, 0x3c0a000c, 0x0a0001e3, 0x3c0a000d, 0x0a0001e3, 0x00000000,
5067         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a000e, 0x0a0001e3, 0x00000000,
5068         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5069         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5070         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a0013, 0x0a0001e3, 0x3c0a0014,
5071         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5072         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5073         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5074         0x27bdffe0, 0x00001821, 0x00001021, 0xafbf0018, 0xafb10014, 0xafb00010,
5075         0x3c010800, 0x00220821, 0xac200a70, 0x3c010800, 0x00220821, 0xac200a74,
5076         0x3c010800, 0x00220821, 0xac200a78, 0x24630001, 0x1860fff5, 0x2442000c,
5077         0x24110001, 0x8f906810, 0x32020004, 0x14400005, 0x24040001, 0x3c020800,
5078         0x8c420a78, 0x18400003, 0x00002021, 0x0e000182, 0x00000000, 0x32020001,
5079         0x10400003, 0x00000000, 0x0e000169, 0x00000000, 0x0a000153, 0xaf915028,
5080         0x8fbf0018, 0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0020, 0x3c050800,
5081         0x8ca50a70, 0x3c060800, 0x8cc60a80, 0x3c070800, 0x8ce70a78, 0x27bdffe0,
5082         0x3c040800, 0x248409d0, 0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014,
5083         0x0e00017b, 0x00002021, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x24020001,
5084         0x8f836810, 0x00821004, 0x00021027, 0x00621824, 0x03e00008, 0xaf836810,
5085         0x27bdffd8, 0xafbf0024, 0x1080002e, 0xafb00020, 0x8f825cec, 0xafa20018,
5086         0x8f825cec, 0x3c100800, 0x26100a78, 0xafa2001c, 0x34028000, 0xaf825cec,
5087         0x8e020000, 0x18400016, 0x00000000, 0x3c020800, 0x94420a74, 0x8fa3001c,
5088         0x000221c0, 0xac830004, 0x8fa2001c, 0x3c010800, 0x0e000201, 0xac220a74,
5089         0x10400005, 0x00000000, 0x8e020000, 0x24420001, 0x0a0001df, 0xae020000,
5090         0x3c020800, 0x8c420a70, 0x00021c02, 0x000321c0, 0x0a0001c5, 0xafa2001c,
5091         0x0e000201, 0x00000000, 0x1040001f, 0x00000000, 0x8e020000, 0x8fa3001c,
5092         0x24420001, 0x3c010800, 0xac230a70, 0x3c010800, 0xac230a74, 0x0a0001df,
5093         0xae020000, 0x3c100800, 0x26100a78, 0x8e020000, 0x18400028, 0x00000000,
5094         0x0e000201, 0x00000000, 0x14400024, 0x00000000, 0x8e020000, 0x3c030800,
5095         0x8c630a70, 0x2442ffff, 0xafa3001c, 0x18400006, 0xae020000, 0x00031402,
5096         0x000221c0, 0x8c820004, 0x3c010800, 0xac220a70, 0x97a2001e, 0x2442ff00,
5097         0x2c420300, 0x1440000b, 0x24024000, 0x3c040800, 0x248409dc, 0xafa00010,
5098         0xafa00014, 0x8fa6001c, 0x24050008, 0x0e000060, 0x00003821, 0x0a0001df,
5099         0x00000000, 0xaf825cf8, 0x3c020800, 0x8c420a40, 0x8fa3001c, 0x24420001,
5100         0xaf835cf8, 0x3c010800, 0xac220a40, 0x8fbf0024, 0x8fb00020, 0x03e00008,
5101         0x27bd0028, 0x27bdffe0, 0x3c040800, 0x248409e8, 0x00002821, 0x00003021,
5102         0x00003821, 0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014, 0x8fbf0018,
5103         0x03e00008, 0x27bd0020, 0x8f82680c, 0x8f85680c, 0x00021827, 0x0003182b,
5104         0x00031823, 0x00431024, 0x00441021, 0x00a2282b, 0x10a00006, 0x00000000,
5105         0x00401821, 0x8f82680c, 0x0043102b, 0x1440fffd, 0x00000000, 0x03e00008,
5106         0x00000000, 0x3c040800, 0x8c840000, 0x3c030800, 0x8c630a40, 0x0064102b,
5107         0x54400002, 0x00831023, 0x00641023, 0x2c420008, 0x03e00008, 0x38420001,
5108         0x27bdffe0, 0x00802821, 0x3c040800, 0x24840a00, 0x00003021, 0x00003821,
5109         0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014, 0x0a000216, 0x00000000,
5110         0x8fbf0018, 0x03e00008, 0x27bd0020, 0x00000000, 0x27bdffe0, 0x3c1cc000,
5111         0xafbf0018, 0x0e00004c, 0xaf80680c, 0x3c040800, 0x24840a10, 0x03802821,
5112         0x00003021, 0x00003821, 0xafa00010, 0x0e000060, 0xafa00014, 0x2402ffff,
5113         0xaf825404, 0x3c0200aa, 0x0e000234, 0xaf825434, 0x8fbf0018, 0x03e00008,
5114         0x27bd0020, 0x00000000, 0x00000000, 0x00000000, 0x27bdffe8, 0xafb00010,
5115         0x24100001, 0xafbf0014, 0x3c01c003, 0xac200000, 0x8f826810, 0x30422000,
5116         0x10400003, 0x00000000, 0x0e000246, 0x00000000, 0x0a00023a, 0xaf905428,
5117         0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x27bdfff8, 0x8f845d0c,
5118         0x3c0200ff, 0x3c030800, 0x8c630a50, 0x3442fff8, 0x00821024, 0x1043001e,
5119         0x3c0500ff, 0x34a5fff8, 0x3c06c003, 0x3c074000, 0x00851824, 0x8c620010,
5120         0x3c010800, 0xac230a50, 0x30420008, 0x10400005, 0x00871025, 0x8cc20000,
5121         0x24420001, 0xacc20000, 0x00871025, 0xaf825d0c, 0x8fa20000, 0x24420001,
5122         0xafa20000, 0x8fa20000, 0x8fa20000, 0x24420001, 0xafa20000, 0x8fa20000,
5123         0x8f845d0c, 0x3c030800, 0x8c630a50, 0x00851024, 0x1443ffe8, 0x00851824,
5124         0x27bd0008, 0x03e00008, 0x00000000, 0x00000000, 0x00000000
5125 };
5126
5127 static const u32 tg3FwRodata[(TG3_FW_RODATA_LEN / sizeof(u32)) + 1] = {
5128         0x35373031, 0x726c7341, 0x00000000, 0x00000000, 0x53774576, 0x656e7430,
5129         0x00000000, 0x726c7045, 0x76656e74, 0x31000000, 0x556e6b6e, 0x45766e74,
5130         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x66617461, 0x6c457272,
5131         0x00000000, 0x00000000, 0x4d61696e, 0x43707542, 0x00000000, 0x00000000,
5132         0x00000000
5133 };
5134
5135 #if 0 /* All zeros, don't eat up space with it. */
5136 u32 tg3FwData[(TG3_FW_DATA_LEN / sizeof(u32)) + 1] = {
5137         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
5138         0x00000000, 0x00000000, 0x00000000, 0x00000000
5139 };
5140 #endif
5141
5142 #define RX_CPU_SCRATCH_BASE     0x30000
5143 #define RX_CPU_SCRATCH_SIZE     0x04000
5144 #define TX_CPU_SCRATCH_BASE     0x34000
5145 #define TX_CPU_SCRATCH_SIZE     0x04000
5146
5147 /* tp->lock is held. */
5148 static int tg3_halt_cpu(struct tg3 *tp, u32 offset)
5149 {
5150         int i;
5151
5152         BUG_ON(offset == TX_CPU_BASE &&
5153             (tp->tg3_flags2 & TG3_FLG2_5705_PLUS));
5154
5155         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
5156                 u32 val = tr32(GRC_VCPU_EXT_CTRL);
5157
5158                 tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_HALT_CPU);
5159                 return 0;
5160         }
5161         if (offset == RX_CPU_BASE) {
5162                 for (i = 0; i < 10000; i++) {
5163                         tw32(offset + CPU_STATE, 0xffffffff);
5164                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
5165                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
5166                                 break;
5167                 }
5168
5169                 tw32(offset + CPU_STATE, 0xffffffff);
5170                 tw32_f(offset + CPU_MODE,  CPU_MODE_HALT);
5171                 udelay(10);
5172         } else {
5173                 for (i = 0; i < 10000; i++) {
5174                         tw32(offset + CPU_STATE, 0xffffffff);
5175                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
5176                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
5177                                 break;
5178                 }
5179         }
5180
5181         if (i >= 10000) {
5182                 printk(KERN_ERR PFX "tg3_reset_cpu timed out for %s, "
5183                        "and %s CPU\n",
5184                        tp->dev->name,
5185                        (offset == RX_CPU_BASE ? "RX" : "TX"));
5186                 return -ENODEV;
5187         }
5188
5189         /* Clear firmware's nvram arbitration. */
5190         if (tp->tg3_flags & TG3_FLAG_NVRAM)
5191                 tw32(NVRAM_SWARB, SWARB_REQ_CLR0);
5192         return 0;
5193 }
5194
5195 struct fw_info {
5196         unsigned int text_base;
5197         unsigned int text_len;
5198         const u32 *text_data;
5199         unsigned int rodata_base;
5200         unsigned int rodata_len;
5201         const u32 *rodata_data;
5202         unsigned int data_base;
5203         unsigned int data_len;
5204         const u32 *data_data;
5205 };
5206
5207 /* tp->lock is held. */
5208 static int tg3_load_firmware_cpu(struct tg3 *tp, u32 cpu_base, u32 cpu_scratch_base,
5209                                  int cpu_scratch_size, struct fw_info *info)
5210 {
5211         int err, lock_err, i;
5212         void (*write_op)(struct tg3 *, u32, u32);
5213
5214         if (cpu_base == TX_CPU_BASE &&
5215             (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
5216                 printk(KERN_ERR PFX "tg3_load_firmware_cpu: Trying to load "
5217                        "TX cpu firmware on %s which is 5705.\n",
5218                        tp->dev->name);
5219                 return -EINVAL;
5220         }
5221
5222         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
5223                 write_op = tg3_write_mem;
5224         else
5225                 write_op = tg3_write_indirect_reg32;
5226
5227         /* It is possible that bootcode is still loading at this point.
5228          * Get the nvram lock first before halting the cpu.
5229          */
5230         lock_err = tg3_nvram_lock(tp);
5231         err = tg3_halt_cpu(tp, cpu_base);
5232         if (!lock_err)
5233                 tg3_nvram_unlock(tp);
5234         if (err)
5235                 goto out;
5236
5237         for (i = 0; i < cpu_scratch_size; i += sizeof(u32))
5238                 write_op(tp, cpu_scratch_base + i, 0);
5239         tw32(cpu_base + CPU_STATE, 0xffffffff);
5240         tw32(cpu_base + CPU_MODE, tr32(cpu_base+CPU_MODE)|CPU_MODE_HALT);
5241         for (i = 0; i < (info->text_len / sizeof(u32)); i++)
5242                 write_op(tp, (cpu_scratch_base +
5243                               (info->text_base & 0xffff) +
5244                               (i * sizeof(u32))),
5245                          (info->text_data ?
5246                           info->text_data[i] : 0));
5247         for (i = 0; i < (info->rodata_len / sizeof(u32)); i++)
5248                 write_op(tp, (cpu_scratch_base +
5249                               (info->rodata_base & 0xffff) +
5250                               (i * sizeof(u32))),
5251                          (info->rodata_data ?
5252                           info->rodata_data[i] : 0));
5253         for (i = 0; i < (info->data_len / sizeof(u32)); i++)
5254                 write_op(tp, (cpu_scratch_base +
5255                               (info->data_base & 0xffff) +
5256                               (i * sizeof(u32))),
5257                          (info->data_data ?
5258                           info->data_data[i] : 0));
5259
5260         err = 0;
5261
5262 out:
5263         return err;
5264 }
5265
5266 /* tp->lock is held. */
5267 static int tg3_load_5701_a0_firmware_fix(struct tg3 *tp)
5268 {
5269         struct fw_info info;
5270         int err, i;
5271
5272         info.text_base = TG3_FW_TEXT_ADDR;
5273         info.text_len = TG3_FW_TEXT_LEN;
5274         info.text_data = &tg3FwText[0];
5275         info.rodata_base = TG3_FW_RODATA_ADDR;
5276         info.rodata_len = TG3_FW_RODATA_LEN;
5277         info.rodata_data = &tg3FwRodata[0];
5278         info.data_base = TG3_FW_DATA_ADDR;
5279         info.data_len = TG3_FW_DATA_LEN;
5280         info.data_data = NULL;
5281
5282         err = tg3_load_firmware_cpu(tp, RX_CPU_BASE,
5283                                     RX_CPU_SCRATCH_BASE, RX_CPU_SCRATCH_SIZE,
5284                                     &info);
5285         if (err)
5286                 return err;
5287
5288         err = tg3_load_firmware_cpu(tp, TX_CPU_BASE,
5289                                     TX_CPU_SCRATCH_BASE, TX_CPU_SCRATCH_SIZE,
5290                                     &info);
5291         if (err)
5292                 return err;
5293
5294         /* Now startup only the RX cpu. */
5295         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5296         tw32_f(RX_CPU_BASE + CPU_PC,    TG3_FW_TEXT_ADDR);
5297
5298         for (i = 0; i < 5; i++) {
5299                 if (tr32(RX_CPU_BASE + CPU_PC) == TG3_FW_TEXT_ADDR)
5300                         break;
5301                 tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5302                 tw32(RX_CPU_BASE + CPU_MODE,  CPU_MODE_HALT);
5303                 tw32_f(RX_CPU_BASE + CPU_PC,    TG3_FW_TEXT_ADDR);
5304                 udelay(1000);
5305         }
5306         if (i >= 5) {
5307                 printk(KERN_ERR PFX "tg3_load_firmware fails for %s "
5308                        "to set RX CPU PC, is %08x should be %08x\n",
5309                        tp->dev->name, tr32(RX_CPU_BASE + CPU_PC),
5310                        TG3_FW_TEXT_ADDR);
5311                 return -ENODEV;
5312         }
5313         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5314         tw32_f(RX_CPU_BASE + CPU_MODE,  0x00000000);
5315
5316         return 0;
5317 }
5318
5319 #if TG3_TSO_SUPPORT != 0
5320
5321 #define TG3_TSO_FW_RELEASE_MAJOR        0x1
5322 #define TG3_TSO_FW_RELASE_MINOR         0x6
5323 #define TG3_TSO_FW_RELEASE_FIX          0x0
5324 #define TG3_TSO_FW_START_ADDR           0x08000000
5325 #define TG3_TSO_FW_TEXT_ADDR            0x08000000
5326 #define TG3_TSO_FW_TEXT_LEN             0x1aa0
5327 #define TG3_TSO_FW_RODATA_ADDR          0x08001aa0
5328 #define TG3_TSO_FW_RODATA_LEN           0x60
5329 #define TG3_TSO_FW_DATA_ADDR            0x08001b20
5330 #define TG3_TSO_FW_DATA_LEN             0x30
5331 #define TG3_TSO_FW_SBSS_ADDR            0x08001b50
5332 #define TG3_TSO_FW_SBSS_LEN             0x2c
5333 #define TG3_TSO_FW_BSS_ADDR             0x08001b80
5334 #define TG3_TSO_FW_BSS_LEN              0x894
5335
5336 static const u32 tg3TsoFwText[(TG3_TSO_FW_TEXT_LEN / 4) + 1] = {
5337         0x0e000003, 0x00000000, 0x08001b24, 0x00000000, 0x10000003, 0x00000000,
5338         0x0000000d, 0x0000000d, 0x3c1d0800, 0x37bd4000, 0x03a0f021, 0x3c100800,
5339         0x26100000, 0x0e000010, 0x00000000, 0x0000000d, 0x27bdffe0, 0x3c04fefe,
5340         0xafbf0018, 0x0e0005d8, 0x34840002, 0x0e000668, 0x00000000, 0x3c030800,
5341         0x90631b68, 0x24020002, 0x3c040800, 0x24841aac, 0x14620003, 0x24050001,
5342         0x3c040800, 0x24841aa0, 0x24060006, 0x00003821, 0xafa00010, 0x0e00067c,
5343         0xafa00014, 0x8f625c50, 0x34420001, 0xaf625c50, 0x8f625c90, 0x34420001,
5344         0xaf625c90, 0x2402ffff, 0x0e000034, 0xaf625404, 0x8fbf0018, 0x03e00008,
5345         0x27bd0020, 0x00000000, 0x00000000, 0x00000000, 0x27bdffe0, 0xafbf001c,
5346         0xafb20018, 0xafb10014, 0x0e00005b, 0xafb00010, 0x24120002, 0x24110001,
5347         0x8f706820, 0x32020100, 0x10400003, 0x00000000, 0x0e0000bb, 0x00000000,
5348         0x8f706820, 0x32022000, 0x10400004, 0x32020001, 0x0e0001f0, 0x24040001,
5349         0x32020001, 0x10400003, 0x00000000, 0x0e0000a3, 0x00000000, 0x3c020800,
5350         0x90421b98, 0x14520003, 0x00000000, 0x0e0004c0, 0x00000000, 0x0a00003c,
5351         0xaf715028, 0x8fbf001c, 0x8fb20018, 0x8fb10014, 0x8fb00010, 0x03e00008,
5352         0x27bd0020, 0x27bdffe0, 0x3c040800, 0x24841ac0, 0x00002821, 0x00003021,
5353         0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014, 0x3c040800,
5354         0x248423d8, 0xa4800000, 0x3c010800, 0xa0201b98, 0x3c010800, 0xac201b9c,
5355         0x3c010800, 0xac201ba0, 0x3c010800, 0xac201ba4, 0x3c010800, 0xac201bac,
5356         0x3c010800, 0xac201bb8, 0x3c010800, 0xac201bbc, 0x8f624434, 0x3c010800,
5357         0xac221b88, 0x8f624438, 0x3c010800, 0xac221b8c, 0x8f624410, 0xac80f7a8,
5358         0x3c010800, 0xac201b84, 0x3c010800, 0xac2023e0, 0x3c010800, 0xac2023c8,
5359         0x3c010800, 0xac2023cc, 0x3c010800, 0xac202400, 0x3c010800, 0xac221b90,
5360         0x8f620068, 0x24030007, 0x00021702, 0x10430005, 0x00000000, 0x8f620068,
5361         0x00021702, 0x14400004, 0x24020001, 0x3c010800, 0x0a000097, 0xac20240c,
5362         0xac820034, 0x3c040800, 0x24841acc, 0x3c050800, 0x8ca5240c, 0x00003021,
5363         0x00003821, 0xafa00010, 0x0e00067c, 0xafa00014, 0x8fbf0018, 0x03e00008,
5364         0x27bd0020, 0x27bdffe0, 0x3c040800, 0x24841ad8, 0x00002821, 0x00003021,
5365         0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014, 0x0e00005b,
5366         0x00000000, 0x0e0000b4, 0x00002021, 0x8fbf0018, 0x03e00008, 0x27bd0020,
5367         0x24020001, 0x8f636820, 0x00821004, 0x00021027, 0x00621824, 0x03e00008,
5368         0xaf636820, 0x27bdffd0, 0xafbf002c, 0xafb60028, 0xafb50024, 0xafb40020,
5369         0xafb3001c, 0xafb20018, 0xafb10014, 0xafb00010, 0x8f675c5c, 0x3c030800,
5370         0x24631bbc, 0x8c620000, 0x14470005, 0x3c0200ff, 0x3c020800, 0x90421b98,
5371         0x14400119, 0x3c0200ff, 0x3442fff8, 0x00e28824, 0xac670000, 0x00111902,
5372         0x306300ff, 0x30e20003, 0x000211c0, 0x00622825, 0x00a04021, 0x00071602,
5373         0x3c030800, 0x90631b98, 0x3044000f, 0x14600036, 0x00804821, 0x24020001,
5374         0x3c010800, 0xa0221b98, 0x00051100, 0x00821025, 0x3c010800, 0xac201b9c,
5375         0x3c010800, 0xac201ba0, 0x3c010800, 0xac201ba4, 0x3c010800, 0xac201bac,
5376         0x3c010800, 0xac201bb8, 0x3c010800, 0xac201bb0, 0x3c010800, 0xac201bb4,
5377         0x3c010800, 0xa42223d8, 0x9622000c, 0x30437fff, 0x3c010800, 0xa4222410,
5378         0x30428000, 0x3c010800, 0xa4231bc6, 0x10400005, 0x24020001, 0x3c010800,
5379         0xac2223f4, 0x0a000102, 0x2406003e, 0x24060036, 0x3c010800, 0xac2023f4,
5380         0x9622000a, 0x3c030800, 0x94631bc6, 0x3c010800, 0xac2023f0, 0x3c010800,
5381         0xac2023f8, 0x00021302, 0x00021080, 0x00c21021, 0x00621821, 0x3c010800,
5382         0xa42223d0, 0x3c010800, 0x0a000115, 0xa4231b96, 0x9622000c, 0x3c010800,
5383         0xa42223ec, 0x3c040800, 0x24841b9c, 0x8c820000, 0x00021100, 0x3c010800,
5384         0x00220821, 0xac311bc8, 0x8c820000, 0x00021100, 0x3c010800, 0x00220821,
5385         0xac271bcc, 0x8c820000, 0x25030001, 0x306601ff, 0x00021100, 0x3c010800,
5386         0x00220821, 0xac261bd0, 0x8c820000, 0x00021100, 0x3c010800, 0x00220821,
5387         0xac291bd4, 0x96230008, 0x3c020800, 0x8c421bac, 0x00432821, 0x3c010800,
5388         0xac251bac, 0x9622000a, 0x30420004, 0x14400018, 0x00061100, 0x8f630c14,
5389         0x3063000f, 0x2c620002, 0x1440000b, 0x3c02c000, 0x8f630c14, 0x3c020800,
5390         0x8c421b40, 0x3063000f, 0x24420001, 0x3c010800, 0xac221b40, 0x2c620002,
5391         0x1040fff7, 0x3c02c000, 0x00e21825, 0xaf635c5c, 0x8f625c50, 0x30420002,
5392         0x10400014, 0x00000000, 0x0a000147, 0x00000000, 0x3c030800, 0x8c631b80,
5393         0x3c040800, 0x94841b94, 0x01221025, 0x3c010800, 0xa42223da, 0x24020001,
5394         0x3c010800, 0xac221bb8, 0x24630001, 0x0085202a, 0x3c010800, 0x10800003,
5395         0xac231b80, 0x3c010800, 0xa4251b94, 0x3c060800, 0x24c61b9c, 0x8cc20000,
5396         0x24420001, 0xacc20000, 0x28420080, 0x14400005, 0x00000000, 0x0e000656,
5397         0x24040002, 0x0a0001e6, 0x00000000, 0x3c020800, 0x8c421bb8, 0x10400078,
5398         0x24020001, 0x3c050800, 0x90a51b98, 0x14a20072, 0x00000000, 0x3c150800,
5399         0x96b51b96, 0x3c040800, 0x8c841bac, 0x32a3ffff, 0x0083102a, 0x1440006c,
5400         0x00000000, 0x14830003, 0x00000000, 0x3c010800, 0xac2523f0, 0x1060005c,
5401         0x00009021, 0x24d60004, 0x0060a021, 0x24d30014, 0x8ec20000, 0x00028100,
5402         0x3c110800, 0x02308821, 0x0e000625, 0x8e311bc8, 0x00402821, 0x10a00054,
5403         0x00000000, 0x9628000a, 0x31020040, 0x10400005, 0x2407180c, 0x8e22000c,
5404         0x2407188c, 0x00021400, 0xaca20018, 0x3c030800, 0x00701821, 0x8c631bd0,
5405         0x3c020800, 0x00501021, 0x8c421bd4, 0x00031d00, 0x00021400, 0x00621825,
5406         0xaca30014, 0x8ec30004, 0x96220008, 0x00432023, 0x3242ffff, 0x3083ffff,
5407         0x00431021, 0x0282102a, 0x14400002, 0x02b23023, 0x00803021, 0x8e620000,
5408         0x30c4ffff, 0x00441021, 0xae620000, 0x8e220000, 0xaca20000, 0x8e220004,
5409         0x8e63fff4, 0x00431021, 0xaca20004, 0xa4a6000e, 0x8e62fff4, 0x00441021,
5410         0xae62fff4, 0x96230008, 0x0043102a, 0x14400005, 0x02469021, 0x8e62fff0,
5411         0xae60fff4, 0x24420001, 0xae62fff0, 0xaca00008, 0x3242ffff, 0x14540008,
5412         0x24020305, 0x31020080, 0x54400001, 0x34e70010, 0x24020905, 0xa4a2000c,
5413         0x0a0001cb, 0x34e70020, 0xa4a2000c, 0x3c020800, 0x8c4223f0, 0x10400003,
5414         0x3c024b65, 0x0a0001d3, 0x34427654, 0x3c02b49a, 0x344289ab, 0xaca2001c,
5415         0x30e2ffff, 0xaca20010, 0x0e0005a2, 0x00a02021, 0x3242ffff, 0x0054102b,
5416         0x1440ffa9, 0x00000000, 0x24020002, 0x3c010800, 0x0a0001e6, 0xa0221b98,
5417         0x8ec2083c, 0x24420001, 0x0a0001e6, 0xaec2083c, 0x0e0004c0, 0x00000000,
5418         0x8fbf002c, 0x8fb60028, 0x8fb50024, 0x8fb40020, 0x8fb3001c, 0x8fb20018,
5419         0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0030, 0x27bdffd0, 0xafbf0028,
5420         0xafb30024, 0xafb20020, 0xafb1001c, 0xafb00018, 0x8f725c9c, 0x3c0200ff,
5421         0x3442fff8, 0x3c070800, 0x24e71bb4, 0x02428824, 0x9623000e, 0x8ce20000,
5422         0x00431021, 0xace20000, 0x8e220010, 0x30420020, 0x14400011, 0x00809821,
5423         0x0e00063b, 0x02202021, 0x3c02c000, 0x02421825, 0xaf635c9c, 0x8f625c90,
5424         0x30420002, 0x1040011e, 0x00000000, 0xaf635c9c, 0x8f625c90, 0x30420002,
5425         0x10400119, 0x00000000, 0x0a00020d, 0x00000000, 0x8e240008, 0x8e230014,
5426         0x00041402, 0x000231c0, 0x00031502, 0x304201ff, 0x2442ffff, 0x3042007f,
5427         0x00031942, 0x30637800, 0x00021100, 0x24424000, 0x00624821, 0x9522000a,
5428         0x3084ffff, 0x30420008, 0x104000b0, 0x000429c0, 0x3c020800, 0x8c422400,
5429         0x14400024, 0x24c50008, 0x94c20014, 0x3c010800, 0xa42223d0, 0x8cc40010,
5430         0x00041402, 0x3c010800, 0xa42223d2, 0x3c010800, 0xa42423d4, 0x94c2000e,
5431         0x3083ffff, 0x00431023, 0x3c010800, 0xac222408, 0x94c2001a, 0x3c010800,
5432         0xac262400, 0x3c010800, 0xac322404, 0x3c010800, 0xac2223fc, 0x3c02c000,
5433         0x02421825, 0xaf635c9c, 0x8f625c90, 0x30420002, 0x104000e5, 0x00000000,
5434         0xaf635c9c, 0x8f625c90, 0x30420002, 0x104000e0, 0x00000000, 0x0a000246,
5435         0x00000000, 0x94c2000e, 0x3c030800, 0x946323d4, 0x00434023, 0x3103ffff,
5436         0x2c620008, 0x1040001c, 0x00000000, 0x94c20014, 0x24420028, 0x00a22821,
5437         0x00031042, 0x1840000b, 0x00002021, 0x24e60848, 0x00403821, 0x94a30000,
5438         0x8cc20000, 0x24840001, 0x00431021, 0xacc20000, 0x0087102a, 0x1440fff9,
5439         0x24a50002, 0x31020001, 0x1040001f, 0x3c024000, 0x3c040800, 0x248423fc,
5440         0xa0a00001, 0x94a30000, 0x8c820000, 0x00431021, 0x0a000285, 0xac820000,
5441         0x8f626800, 0x3c030010, 0x00431024, 0x10400009, 0x00000000, 0x94c2001a,
5442         0x3c030800, 0x8c6323fc, 0x00431021, 0x3c010800, 0xac2223fc, 0x0a000286,
5443         0x3c024000, 0x94c2001a, 0x94c4001c, 0x3c030800, 0x8c6323fc, 0x00441023,
5444         0x00621821, 0x3c010800, 0xac2323fc, 0x3c024000, 0x02421825, 0xaf635c9c,
5445         0x8f625c90, 0x30420002, 0x1440fffc, 0x00000000, 0x9522000a, 0x30420010,
5446         0x1040009b, 0x00000000, 0x3c030800, 0x946323d4, 0x3c070800, 0x24e72400,
5447         0x8ce40000, 0x8f626800, 0x24630030, 0x00832821, 0x3c030010, 0x00431024,
5448         0x1440000a, 0x00000000, 0x94a20004, 0x3c040800, 0x8c842408, 0x3c030800,
5449         0x8c6323fc, 0x00441023, 0x00621821, 0x3c010800, 0xac2323fc, 0x3c040800,
5450         0x8c8423fc, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402, 0x00822021,
5451         0x00041027, 0xa4a20006, 0x3c030800, 0x8c632404, 0x3c0200ff, 0x3442fff8,
5452         0x00628824, 0x96220008, 0x24050001, 0x24034000, 0x000231c0, 0x00801021,
5453         0xa4c2001a, 0xa4c0001c, 0xace00000, 0x3c010800, 0xac251b60, 0xaf635cb8,
5454         0x8f625cb0, 0x30420002, 0x10400003, 0x00000000, 0x3c010800, 0xac201b60,
5455         0x8e220008, 0xaf625cb8, 0x8f625cb0, 0x30420002, 0x10400003, 0x00000000,
5456         0x3c010800, 0xac201b60, 0x3c020800, 0x8c421b60, 0x1040ffec, 0x00000000,
5457         0x3c040800, 0x0e00063b, 0x8c842404, 0x0a00032a, 0x00000000, 0x3c030800,
5458         0x90631b98, 0x24020002, 0x14620003, 0x3c034b65, 0x0a0002e1, 0x00008021,
5459         0x8e22001c, 0x34637654, 0x10430002, 0x24100002, 0x24100001, 0x00c02021,
5460         0x0e000350, 0x02003021, 0x24020003, 0x3c010800, 0xa0221b98, 0x24020002,
5461         0x1202000a, 0x24020001, 0x3c030800, 0x8c6323f0, 0x10620006, 0x00000000,
5462         0x3c020800, 0x944223d8, 0x00021400, 0x0a00031f, 0xae220014, 0x3c040800,
5463         0x248423da, 0x94820000, 0x00021400, 0xae220014, 0x3c020800, 0x8c421bbc,
5464         0x3c03c000, 0x3c010800, 0xa0201b98, 0x00431025, 0xaf625c5c, 0x8f625c50,
5465         0x30420002, 0x10400009, 0x00000000, 0x2484f7e2, 0x8c820000, 0x00431025,
5466         0xaf625c5c, 0x8f625c50, 0x30420002, 0x1440fffa, 0x00000000, 0x3c020800,
5467         0x24421b84, 0x8c430000, 0x24630001, 0xac430000, 0x8f630c14, 0x3063000f,
5468         0x2c620002, 0x1440000c, 0x3c024000, 0x8f630c14, 0x3c020800, 0x8c421b40,
5469         0x3063000f, 0x24420001, 0x3c010800, 0xac221b40, 0x2c620002, 0x1040fff7,
5470         0x00000000, 0x3c024000, 0x02421825, 0xaf635c9c, 0x8f625c90, 0x30420002,
5471         0x1440fffc, 0x00000000, 0x12600003, 0x00000000, 0x0e0004c0, 0x00000000,
5472         0x8fbf0028, 0x8fb30024, 0x8fb20020, 0x8fb1001c, 0x8fb00018, 0x03e00008,
5473         0x27bd0030, 0x8f634450, 0x3c040800, 0x24841b88, 0x8c820000, 0x00031c02,
5474         0x0043102b, 0x14400007, 0x3c038000, 0x8c840004, 0x8f624450, 0x00021c02,
5475         0x0083102b, 0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024,
5476         0x1440fffd, 0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3c024000,
5477         0x00822025, 0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00000000,
5478         0x03e00008, 0x00000000, 0x27bdffe0, 0x00805821, 0x14c00011, 0x256e0008,
5479         0x3c020800, 0x8c4223f4, 0x10400007, 0x24020016, 0x3c010800, 0xa42223d2,
5480         0x2402002a, 0x3c010800, 0x0a000364, 0xa42223d4, 0x8d670010, 0x00071402,
5481         0x3c010800, 0xa42223d2, 0x3c010800, 0xa42723d4, 0x3c040800, 0x948423d4,
5482         0x3c030800, 0x946323d2, 0x95cf0006, 0x3c020800, 0x944223d0, 0x00832023,
5483         0x01e2c023, 0x3065ffff, 0x24a20028, 0x01c24821, 0x3082ffff, 0x14c0001a,
5484         0x01226021, 0x9582000c, 0x3042003f, 0x3c010800, 0xa42223d6, 0x95820004,
5485         0x95830006, 0x3c010800, 0xac2023e4, 0x3c010800, 0xac2023e8, 0x00021400,
5486         0x00431025, 0x3c010800, 0xac221bc0, 0x95220004, 0x3c010800, 0xa4221bc4,
5487         0x95230002, 0x01e51023, 0x0043102a, 0x10400010, 0x24020001, 0x3c010800,
5488         0x0a000398, 0xac2223f8, 0x3c030800, 0x8c6323e8, 0x3c020800, 0x94421bc4,
5489         0x00431021, 0xa5220004, 0x3c020800, 0x94421bc0, 0xa5820004, 0x3c020800,
5490         0x8c421bc0, 0xa5820006, 0x3c020800, 0x8c4223f0, 0x3c0d0800, 0x8dad23e4,
5491         0x3c0a0800, 0x144000e5, 0x8d4a23e8, 0x3c020800, 0x94421bc4, 0x004a1821,
5492         0x3063ffff, 0x0062182b, 0x24020002, 0x10c2000d, 0x01435023, 0x3c020800,
5493         0x944223d6, 0x30420009, 0x10400008, 0x00000000, 0x9582000c, 0x3042fff6,
5494         0xa582000c, 0x3c020800, 0x944223d6, 0x30420009, 0x01a26823, 0x3c020800,
5495         0x8c4223f8, 0x1040004a, 0x01203821, 0x3c020800, 0x944223d2, 0x00004021,
5496         0xa520000a, 0x01e21023, 0xa5220002, 0x3082ffff, 0x00021042, 0x18400008,
5497         0x00003021, 0x00401821, 0x94e20000, 0x25080001, 0x00c23021, 0x0103102a,
5498         0x1440fffb, 0x24e70002, 0x00061c02, 0x30c2ffff, 0x00623021, 0x00061402,
5499         0x00c23021, 0x00c02821, 0x00061027, 0xa522000a, 0x00003021, 0x2527000c,
5500         0x00004021, 0x94e20000, 0x25080001, 0x00c23021, 0x2d020004, 0x1440fffb,
5501         0x24e70002, 0x95220002, 0x00004021, 0x91230009, 0x00442023, 0x01803821,
5502         0x3082ffff, 0xa4e00010, 0x00621821, 0x00021042, 0x18400010, 0x00c33021,
5503         0x00404821, 0x94e20000, 0x24e70002, 0x00c23021, 0x30e2007f, 0x14400006,
5504         0x25080001, 0x8d630000, 0x3c02007f, 0x3442ff80, 0x00625824, 0x25670008,
5505         0x0109102a, 0x1440fff3, 0x00000000, 0x30820001, 0x10400005, 0x00061c02,
5506         0xa0e00001, 0x94e20000, 0x00c23021, 0x00061c02, 0x30c2ffff, 0x00623021,
5507         0x00061402, 0x00c23021, 0x0a00047d, 0x30c6ffff, 0x24020002, 0x14c20081,
5508         0x00000000, 0x3c020800, 0x8c42240c, 0x14400007, 0x00000000, 0x3c020800,
5509         0x944223d2, 0x95230002, 0x01e21023, 0x10620077, 0x00000000, 0x3c020800,
5510         0x944223d2, 0x01e21023, 0xa5220002, 0x3c020800, 0x8c42240c, 0x1040001a,
5511         0x31e3ffff, 0x8dc70010, 0x3c020800, 0x94421b96, 0x00e04021, 0x00072c02,
5512         0x00aa2021, 0x00431023, 0x00823823, 0x00072402, 0x30e2ffff, 0x00823821,
5513         0x00071027, 0xa522000a, 0x3102ffff, 0x3c040800, 0x948423d4, 0x00453023,
5514         0x00e02821, 0x00641823, 0x006d1821, 0x00c33021, 0x00061c02, 0x30c2ffff,
5515         0x0a00047d, 0x00623021, 0x01203821, 0x00004021, 0x3082ffff, 0x00021042,
5516         0x18400008, 0x00003021, 0x00401821, 0x94e20000, 0x25080001, 0x00c23021,
5517         0x0103102a, 0x1440fffb, 0x24e70002, 0x00061c02, 0x30c2ffff, 0x00623021,
5518         0x00061402, 0x00c23021, 0x00c02821, 0x00061027, 0xa522000a, 0x00003021,
5519         0x2527000c, 0x00004021, 0x94e20000, 0x25080001, 0x00c23021, 0x2d020004,
5520         0x1440fffb, 0x24e70002, 0x95220002, 0x00004021, 0x91230009, 0x00442023,
5521         0x01803821, 0x3082ffff, 0xa4e00010, 0x3c040800, 0x948423d4, 0x00621821,
5522         0x00c33021, 0x00061c02, 0x30c2ffff, 0x00623021, 0x00061c02, 0x3c020800,
5523         0x944223d0, 0x00c34821, 0x00441023, 0x00021fc2, 0x00431021, 0x00021043,
5524         0x18400010, 0x00003021, 0x00402021, 0x94e20000, 0x24e70002, 0x00c23021,
5525         0x30e2007f, 0x14400006, 0x25080001, 0x8d630000, 0x3c02007f, 0x3442ff80,
5526         0x00625824, 0x25670008, 0x0104102a, 0x1440fff3, 0x00000000, 0x3c020800,
5527         0x944223ec, 0x00c23021, 0x3122ffff, 0x00c23021, 0x00061c02, 0x30c2ffff,
5528         0x00623021, 0x00061402, 0x00c23021, 0x00c04021, 0x00061027, 0xa5820010,
5529         0xadc00014, 0x0a00049d, 0xadc00000, 0x8dc70010, 0x00e04021, 0x11400007,
5530         0x00072c02, 0x00aa3021, 0x00061402, 0x30c3ffff, 0x00433021, 0x00061402,
5531         0x00c22821, 0x00051027, 0xa522000a, 0x3c030800, 0x946323d4, 0x3102ffff,
5532         0x01e21021, 0x00433023, 0x00cd3021, 0x00061c02, 0x30c2ffff, 0x00623021,
5533         0x00061402, 0x00c23021, 0x00c04021, 0x00061027, 0xa5820010, 0x3102ffff,
5534         0x00051c00, 0x00431025, 0xadc20010, 0x3c020800, 0x8c4223f4, 0x10400005,
5535         0x2de205eb, 0x14400002, 0x25e2fff2, 0x34028870, 0xa5c20034, 0x3c030800,
5536         0x246323e8, 0x8c620000, 0x24420001, 0xac620000, 0x3c040800, 0x8c8423e4,
5537         0x3c020800, 0x8c421bc0, 0x3303ffff, 0x00832021, 0x00431821, 0x0062102b,
5538         0x3c010800, 0xac2423e4, 0x10400003, 0x2482ffff, 0x3c010800, 0xac2223e4,
5539         0x3c010800, 0xac231bc0, 0x03e00008, 0x27bd0020, 0x27bdffb8, 0x3c050800,
5540         0x24a51b96, 0xafbf0044, 0xafbe0040, 0xafb7003c, 0xafb60038, 0xafb50034,
5541         0xafb40030, 0xafb3002c, 0xafb20028, 0xafb10024, 0xafb00020, 0x94a90000,
5542         0x3c020800, 0x944223d0, 0x3c030800, 0x8c631bb0, 0x3c040800, 0x8c841bac,
5543         0x01221023, 0x0064182a, 0xa7a9001e, 0x106000be, 0xa7a20016, 0x24be0022,
5544         0x97b6001e, 0x24b3001a, 0x24b70016, 0x8fc20000, 0x14400008, 0x00000000,
5545         0x8fc2fff8, 0x97a30016, 0x8fc4fff4, 0x00431021, 0x0082202a, 0x148000b0,
5546         0x00000000, 0x97d50818, 0x32a2ffff, 0x104000a3, 0x00009021, 0x0040a021,
5547         0x00008821, 0x0e000625, 0x00000000, 0x00403021, 0x14c00007, 0x00000000,
5548         0x3c020800, 0x8c4223dc, 0x24420001, 0x3c010800, 0x0a000596, 0xac2223dc,
5549         0x3c100800, 0x02118021, 0x8e101bc8, 0x9608000a, 0x31020040, 0x10400005,
5550         0x2407180c, 0x8e02000c, 0x2407188c, 0x00021400, 0xacc20018, 0x31020080,
5551         0x54400001, 0x34e70010, 0x3c020800, 0x00511021, 0x8c421bd0, 0x3c030800,
5552         0x00711821, 0x8c631bd4, 0x00021500, 0x00031c00, 0x00431025, 0xacc20014,
5553         0x96040008, 0x3242ffff, 0x00821021, 0x0282102a, 0x14400002, 0x02b22823,
5554         0x00802821, 0x8e020000, 0x02459021, 0xacc20000, 0x8e020004, 0x00c02021,
5555         0x26310010, 0xac820004, 0x30e2ffff, 0xac800008, 0xa485000e, 0xac820010,
5556         0x24020305, 0x0e0005a2, 0xa482000c, 0x3242ffff, 0x0054102b, 0x1440ffc5,
5557         0x3242ffff, 0x0a00058e, 0x00000000, 0x8e620000, 0x8e63fffc, 0x0043102a,
5558         0x10400067, 0x00000000, 0x8e62fff0, 0x00028900, 0x3c100800, 0x02118021,
5559         0x0e000625, 0x8e101bc8, 0x00403021, 0x14c00005, 0x00000000, 0x8e62082c,
5560         0x24420001, 0x0a000596, 0xae62082c, 0x9608000a, 0x31020040, 0x10400005,
5561         0x2407180c, 0x8e02000c, 0x2407188c, 0x00021400, 0xacc20018, 0x3c020800,
5562         0x00511021, 0x8c421bd0, 0x3c030800, 0x00711821, 0x8c631bd4, 0x00021500,
5563         0x00031c00, 0x00431025, 0xacc20014, 0x8e63fff4, 0x96020008, 0x00432023,
5564         0x3242ffff, 0x3083ffff, 0x00431021, 0x02c2102a, 0x10400003, 0x00802821,
5565         0x97a9001e, 0x01322823, 0x8e620000, 0x30a4ffff, 0x00441021, 0xae620000,
5566         0xa4c5000e, 0x8e020000, 0xacc20000, 0x8e020004, 0x8e63fff4, 0x00431021,
5567         0xacc20004, 0x8e63fff4, 0x96020008, 0x00641821, 0x0062102a, 0x14400006,
5568         0x02459021, 0x8e62fff0, 0xae60fff4, 0x24420001, 0x0a000571, 0xae62fff0,
5569         0xae63fff4, 0xacc00008, 0x3242ffff, 0x10560003, 0x31020004, 0x10400006,
5570         0x24020305, 0x31020080, 0x54400001, 0x34e70010, 0x34e70020, 0x24020905,
5571         0xa4c2000c, 0x8ee30000, 0x8ee20004, 0x14620007, 0x3c02b49a, 0x8ee20860,
5572         0x54400001, 0x34e70400, 0x3c024b65, 0x0a000588, 0x34427654, 0x344289ab,
5573         0xacc2001c, 0x30e2ffff, 0xacc20010, 0x0e0005a2, 0x00c02021, 0x3242ffff,
5574         0x0056102b, 0x1440ff9b, 0x00000000, 0x8e620000, 0x8e63fffc, 0x0043102a,
5575         0x1440ff48, 0x00000000, 0x8fbf0044, 0x8fbe0040, 0x8fb7003c, 0x8fb60038,
5576         0x8fb50034, 0x8fb40030, 0x8fb3002c, 0x8fb20028, 0x8fb10024, 0x8fb00020,
5577         0x03e00008, 0x27bd0048, 0x27bdffe8, 0xafbf0014, 0xafb00010, 0x8f624450,
5578         0x8f634410, 0x0a0005b1, 0x00808021, 0x8f626820, 0x30422000, 0x10400003,
5579         0x00000000, 0x0e0001f0, 0x00002021, 0x8f624450, 0x8f634410, 0x3042ffff,
5580         0x0043102b, 0x1440fff5, 0x00000000, 0x8f630c14, 0x3063000f, 0x2c620002,
5581         0x1440000b, 0x00000000, 0x8f630c14, 0x3c020800, 0x8c421b40, 0x3063000f,
5582         0x24420001, 0x3c010800, 0xac221b40, 0x2c620002, 0x1040fff7, 0x00000000,
5583         0xaf705c18, 0x8f625c10, 0x30420002, 0x10400009, 0x00000000, 0x8f626820,
5584         0x30422000, 0x1040fff8, 0x00000000, 0x0e0001f0, 0x00002021, 0x0a0005c4,
5585         0x00000000, 0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x00000000,
5586         0x00000000, 0x00000000, 0x27bdffe8, 0x3c1bc000, 0xafbf0014, 0xafb00010,
5587         0xaf60680c, 0x8f626804, 0x34420082, 0xaf626804, 0x8f634000, 0x24020b50,
5588         0x3c010800, 0xac221b54, 0x24020b78, 0x3c010800, 0xac221b64, 0x34630002,
5589         0xaf634000, 0x0e000605, 0x00808021, 0x3c010800, 0xa0221b68, 0x304200ff,
5590         0x24030002, 0x14430005, 0x00000000, 0x3c020800, 0x8c421b54, 0x0a0005f8,
5591         0xac5000c0, 0x3c020800, 0x8c421b54, 0xac5000bc, 0x8f624434, 0x8f634438,
5592         0x8f644410, 0x3c010800, 0xac221b5c, 0x3c010800, 0xac231b6c, 0x3c010800,
5593         0xac241b58, 0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x3c040800,
5594         0x8c870000, 0x3c03aa55, 0x3463aa55, 0x3c06c003, 0xac830000, 0x8cc20000,
5595         0x14430007, 0x24050002, 0x3c0355aa, 0x346355aa, 0xac830000, 0x8cc20000,
5596         0x50430001, 0x24050001, 0x3c020800, 0xac470000, 0x03e00008, 0x00a01021,
5597         0x27bdfff8, 0x18800009, 0x00002821, 0x8f63680c, 0x8f62680c, 0x1043fffe,
5598         0x00000000, 0x24a50001, 0x00a4102a, 0x1440fff9, 0x00000000, 0x03e00008,
5599         0x27bd0008, 0x8f634450, 0x3c020800, 0x8c421b5c, 0x00031c02, 0x0043102b,
5600         0x14400008, 0x3c038000, 0x3c040800, 0x8c841b6c, 0x8f624450, 0x00021c02,
5601         0x0083102b, 0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024,
5602         0x1440fffd, 0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3082ffff,
5603         0x2442e000, 0x2c422001, 0x14400003, 0x3c024000, 0x0a000648, 0x2402ffff,
5604         0x00822025, 0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00001021,
5605         0x03e00008, 0x00000000, 0x8f624450, 0x3c030800, 0x8c631b58, 0x0a000651,
5606         0x3042ffff, 0x8f624450, 0x3042ffff, 0x0043102b, 0x1440fffc, 0x00000000,
5607         0x03e00008, 0x00000000, 0x27bdffe0, 0x00802821, 0x3c040800, 0x24841af0,
5608         0x00003021, 0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014,
5609         0x0a000660, 0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x00000000,
5610         0x00000000, 0x00000000, 0x3c020800, 0x34423000, 0x3c030800, 0x34633000,
5611         0x3c040800, 0x348437ff, 0x3c010800, 0xac221b74, 0x24020040, 0x3c010800,
5612         0xac221b78, 0x3c010800, 0xac201b70, 0xac600000, 0x24630004, 0x0083102b,
5613         0x5040fffd, 0xac600000, 0x03e00008, 0x00000000, 0x00804821, 0x8faa0010,
5614         0x3c020800, 0x8c421b70, 0x3c040800, 0x8c841b78, 0x8fab0014, 0x24430001,
5615         0x0044102b, 0x3c010800, 0xac231b70, 0x14400003, 0x00004021, 0x3c010800,
5616         0xac201b70, 0x3c020800, 0x8c421b70, 0x3c030800, 0x8c631b74, 0x91240000,
5617         0x00021140, 0x00431021, 0x00481021, 0x25080001, 0xa0440000, 0x29020008,
5618         0x1440fff4, 0x25290001, 0x3c020800, 0x8c421b70, 0x3c030800, 0x8c631b74,
5619         0x8f64680c, 0x00021140, 0x00431021, 0xac440008, 0xac45000c, 0xac460010,
5620         0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c, 0x00000000, 0x00000000,
5621 };
5622
5623 static const u32 tg3TsoFwRodata[] = {
5624         0x4d61696e, 0x43707542, 0x00000000, 0x4d61696e, 0x43707541, 0x00000000,
5625         0x00000000, 0x00000000, 0x73746b6f, 0x66666c64, 0x496e0000, 0x73746b6f,
5626         0x66662a2a, 0x00000000, 0x53774576, 0x656e7430, 0x00000000, 0x00000000,
5627         0x00000000, 0x00000000, 0x66617461, 0x6c457272, 0x00000000, 0x00000000,
5628         0x00000000,
5629 };
5630
5631 static const u32 tg3TsoFwData[] = {
5632         0x00000000, 0x73746b6f, 0x66666c64, 0x5f76312e, 0x362e3000, 0x00000000,
5633         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
5634         0x00000000,
5635 };
5636
5637 /* 5705 needs a special version of the TSO firmware.  */
5638 #define TG3_TSO5_FW_RELEASE_MAJOR       0x1
5639 #define TG3_TSO5_FW_RELASE_MINOR        0x2
5640 #define TG3_TSO5_FW_RELEASE_FIX         0x0
5641 #define TG3_TSO5_FW_START_ADDR          0x00010000
5642 #define TG3_TSO5_FW_TEXT_ADDR           0x00010000
5643 #define TG3_TSO5_FW_TEXT_LEN  &