clocksource: improve sh_cmt clocksource overflow handling
[linux-2.6.git] / drivers / net / tg3.c
1 /*
2  * tg3.c: Broadcom Tigon3 ethernet driver.
3  *
4  * Copyright (C) 2001, 2002, 2003, 2004 David S. Miller (davem@redhat.com)
5  * Copyright (C) 2001, 2002, 2003 Jeff Garzik (jgarzik@pobox.com)
6  * Copyright (C) 2004 Sun Microsystems Inc.
7  * Copyright (C) 2005-2009 Broadcom Corporation.
8  *
9  * Firmware is:
10  *      Derived from proprietary unpublished source code,
11  *      Copyright (C) 2000-2003 Broadcom Corporation.
12  *
13  *      Permission is hereby granted for the distribution of this firmware
14  *      data in hexadecimal or equivalent format, provided this copyright
15  *      notice is accompanying it.
16  */
17
18
19 #include <linux/module.h>
20 #include <linux/moduleparam.h>
21 #include <linux/kernel.h>
22 #include <linux/types.h>
23 #include <linux/compiler.h>
24 #include <linux/slab.h>
25 #include <linux/delay.h>
26 #include <linux/in.h>
27 #include <linux/init.h>
28 #include <linux/ioport.h>
29 #include <linux/pci.h>
30 #include <linux/netdevice.h>
31 #include <linux/etherdevice.h>
32 #include <linux/skbuff.h>
33 #include <linux/ethtool.h>
34 #include <linux/mii.h>
35 #include <linux/phy.h>
36 #include <linux/brcmphy.h>
37 #include <linux/if_vlan.h>
38 #include <linux/ip.h>
39 #include <linux/tcp.h>
40 #include <linux/workqueue.h>
41 #include <linux/prefetch.h>
42 #include <linux/dma-mapping.h>
43 #include <linux/firmware.h>
44
45 #include <net/checksum.h>
46 #include <net/ip.h>
47
48 #include <asm/system.h>
49 #include <asm/io.h>
50 #include <asm/byteorder.h>
51 #include <asm/uaccess.h>
52
53 #ifdef CONFIG_SPARC
54 #include <asm/idprom.h>
55 #include <asm/prom.h>
56 #endif
57
58 #define BAR_0   0
59 #define BAR_2   2
60
61 #if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
62 #define TG3_VLAN_TAG_USED 1
63 #else
64 #define TG3_VLAN_TAG_USED 0
65 #endif
66
67 #include "tg3.h"
68
69 #define DRV_MODULE_NAME         "tg3"
70 #define PFX DRV_MODULE_NAME     ": "
71 #define DRV_MODULE_VERSION      "3.98"
72 #define DRV_MODULE_RELDATE      "February 25, 2009"
73
74 #define TG3_DEF_MAC_MODE        0
75 #define TG3_DEF_RX_MODE         0
76 #define TG3_DEF_TX_MODE         0
77 #define TG3_DEF_MSG_ENABLE        \
78         (NETIF_MSG_DRV          | \
79          NETIF_MSG_PROBE        | \
80          NETIF_MSG_LINK         | \
81          NETIF_MSG_TIMER        | \
82          NETIF_MSG_IFDOWN       | \
83          NETIF_MSG_IFUP         | \
84          NETIF_MSG_RX_ERR       | \
85          NETIF_MSG_TX_ERR)
86
87 /* length of time before we decide the hardware is borked,
88  * and dev->tx_timeout() should be called to fix the problem
89  */
90 #define TG3_TX_TIMEOUT                  (5 * HZ)
91
92 /* hardware minimum and maximum for a single frame's data payload */
93 #define TG3_MIN_MTU                     60
94 #define TG3_MAX_MTU(tp) \
95         ((tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) ? 9000 : 1500)
96
97 /* These numbers seem to be hard coded in the NIC firmware somehow.
98  * You can't change the ring sizes, but you can change where you place
99  * them in the NIC onboard memory.
100  */
101 #define TG3_RX_RING_SIZE                512
102 #define TG3_DEF_RX_RING_PENDING         200
103 #define TG3_RX_JUMBO_RING_SIZE          256
104 #define TG3_DEF_RX_JUMBO_RING_PENDING   100
105
106 /* Do not place this n-ring entries value into the tp struct itself,
107  * we really want to expose these constants to GCC so that modulo et
108  * al.  operations are done with shifts and masks instead of with
109  * hw multiply/modulo instructions.  Another solution would be to
110  * replace things like '% foo' with '& (foo - 1)'.
111  */
112 #define TG3_RX_RCB_RING_SIZE(tp)        \
113         ((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ?  512 : 1024)
114
115 #define TG3_TX_RING_SIZE                512
116 #define TG3_DEF_TX_RING_PENDING         (TG3_TX_RING_SIZE - 1)
117
118 #define TG3_RX_RING_BYTES       (sizeof(struct tg3_rx_buffer_desc) * \
119                                  TG3_RX_RING_SIZE)
120 #define TG3_RX_JUMBO_RING_BYTES (sizeof(struct tg3_rx_buffer_desc) * \
121                                  TG3_RX_JUMBO_RING_SIZE)
122 #define TG3_RX_RCB_RING_BYTES(tp) (sizeof(struct tg3_rx_buffer_desc) * \
123                                    TG3_RX_RCB_RING_SIZE(tp))
124 #define TG3_TX_RING_BYTES       (sizeof(struct tg3_tx_buffer_desc) * \
125                                  TG3_TX_RING_SIZE)
126 #define NEXT_TX(N)              (((N) + 1) & (TG3_TX_RING_SIZE - 1))
127
128 #define RX_PKT_BUF_SZ           (1536 + tp->rx_offset + 64)
129 #define RX_JUMBO_PKT_BUF_SZ     (9046 + tp->rx_offset + 64)
130
131 /* minimum number of free TX descriptors required to wake up TX process */
132 #define TG3_TX_WAKEUP_THRESH(tp)                ((tp)->tx_pending / 4)
133
134 #define TG3_RAW_IP_ALIGN 2
135
136 /* number of ETHTOOL_GSTATS u64's */
137 #define TG3_NUM_STATS           (sizeof(struct tg3_ethtool_stats)/sizeof(u64))
138
139 #define TG3_NUM_TEST            6
140
141 #define FIRMWARE_TG3            "tigon/tg3.bin"
142 #define FIRMWARE_TG3TSO         "tigon/tg3_tso.bin"
143 #define FIRMWARE_TG3TSO5        "tigon/tg3_tso5.bin"
144
145 static char version[] __devinitdata =
146         DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
147
148 MODULE_AUTHOR("David S. Miller (davem@redhat.com) and Jeff Garzik (jgarzik@pobox.com)");
149 MODULE_DESCRIPTION("Broadcom Tigon3 ethernet driver");
150 MODULE_LICENSE("GPL");
151 MODULE_VERSION(DRV_MODULE_VERSION);
152 MODULE_FIRMWARE(FIRMWARE_TG3);
153 MODULE_FIRMWARE(FIRMWARE_TG3TSO);
154 MODULE_FIRMWARE(FIRMWARE_TG3TSO5);
155
156
157 static int tg3_debug = -1;      /* -1 == use TG3_DEF_MSG_ENABLE as value */
158 module_param(tg3_debug, int, 0);
159 MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");
160
161 static struct pci_device_id tg3_pci_tbl[] = {
162         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700)},
163         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701)},
164         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702)},
165         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703)},
166         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704)},
167         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE)},
168         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705)},
169         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2)},
170         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M)},
171         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2)},
172         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X)},
173         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X)},
174         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S)},
175         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3)},
176         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3)},
177         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782)},
178         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788)},
179         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789)},
180         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901)},
181         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2)},
182         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2)},
183         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F)},
184         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5720)},
185         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721)},
186         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5722)},
187         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750)},
188         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751)},
189         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750M)},
190         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M)},
191         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F)},
192         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752)},
193         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M)},
194         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753)},
195         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M)},
196         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F)},
197         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754)},
198         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M)},
199         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755)},
200         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M)},
201         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5756)},
202         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786)},
203         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787)},
204         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M)},
205         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787F)},
206         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714)},
207         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S)},
208         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715)},
209         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S)},
210         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780)},
211         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S)},
212         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781)},
213         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906)},
214         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906M)},
215         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5784)},
216         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5764)},
217         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5723)},
218         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761)},
219         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761E)},
220         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5761S)},
221         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5761SE)},
222         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5785)},
223         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57780)},
224         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57760)},
225         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57790)},
226         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57720)},
227         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX)},
228         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX)},
229         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000)},
230         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001)},
231         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003)},
232         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100)},
233         {PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3)},
234         {}
235 };
236
237 MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);
238
239 static const struct {
240         const char string[ETH_GSTRING_LEN];
241 } ethtool_stats_keys[TG3_NUM_STATS] = {
242         { "rx_octets" },
243         { "rx_fragments" },
244         { "rx_ucast_packets" },
245         { "rx_mcast_packets" },
246         { "rx_bcast_packets" },
247         { "rx_fcs_errors" },
248         { "rx_align_errors" },
249         { "rx_xon_pause_rcvd" },
250         { "rx_xoff_pause_rcvd" },
251         { "rx_mac_ctrl_rcvd" },
252         { "rx_xoff_entered" },
253         { "rx_frame_too_long_errors" },
254         { "rx_jabbers" },
255         { "rx_undersize_packets" },
256         { "rx_in_length_errors" },
257         { "rx_out_length_errors" },
258         { "rx_64_or_less_octet_packets" },
259         { "rx_65_to_127_octet_packets" },
260         { "rx_128_to_255_octet_packets" },
261         { "rx_256_to_511_octet_packets" },
262         { "rx_512_to_1023_octet_packets" },
263         { "rx_1024_to_1522_octet_packets" },
264         { "rx_1523_to_2047_octet_packets" },
265         { "rx_2048_to_4095_octet_packets" },
266         { "rx_4096_to_8191_octet_packets" },
267         { "rx_8192_to_9022_octet_packets" },
268
269         { "tx_octets" },
270         { "tx_collisions" },
271
272         { "tx_xon_sent" },
273         { "tx_xoff_sent" },
274         { "tx_flow_control" },
275         { "tx_mac_errors" },
276         { "tx_single_collisions" },
277         { "tx_mult_collisions" },
278         { "tx_deferred" },
279         { "tx_excessive_collisions" },
280         { "tx_late_collisions" },
281         { "tx_collide_2times" },
282         { "tx_collide_3times" },
283         { "tx_collide_4times" },
284         { "tx_collide_5times" },
285         { "tx_collide_6times" },
286         { "tx_collide_7times" },
287         { "tx_collide_8times" },
288         { "tx_collide_9times" },
289         { "tx_collide_10times" },
290         { "tx_collide_11times" },
291         { "tx_collide_12times" },
292         { "tx_collide_13times" },
293         { "tx_collide_14times" },
294         { "tx_collide_15times" },
295         { "tx_ucast_packets" },
296         { "tx_mcast_packets" },
297         { "tx_bcast_packets" },
298         { "tx_carrier_sense_errors" },
299         { "tx_discards" },
300         { "tx_errors" },
301
302         { "dma_writeq_full" },
303         { "dma_write_prioq_full" },
304         { "rxbds_empty" },
305         { "rx_discards" },
306         { "rx_errors" },
307         { "rx_threshold_hit" },
308
309         { "dma_readq_full" },
310         { "dma_read_prioq_full" },
311         { "tx_comp_queue_full" },
312
313         { "ring_set_send_prod_index" },
314         { "ring_status_update" },
315         { "nic_irqs" },
316         { "nic_avoided_irqs" },
317         { "nic_tx_threshold_hit" }
318 };
319
320 static const struct {
321         const char string[ETH_GSTRING_LEN];
322 } ethtool_test_keys[TG3_NUM_TEST] = {
323         { "nvram test     (online) " },
324         { "link test      (online) " },
325         { "register test  (offline)" },
326         { "memory test    (offline)" },
327         { "loopback test  (offline)" },
328         { "interrupt test (offline)" },
329 };
330
331 static void tg3_write32(struct tg3 *tp, u32 off, u32 val)
332 {
333         writel(val, tp->regs + off);
334 }
335
336 static u32 tg3_read32(struct tg3 *tp, u32 off)
337 {
338         return (readl(tp->regs + off));
339 }
340
341 static void tg3_ape_write32(struct tg3 *tp, u32 off, u32 val)
342 {
343         writel(val, tp->aperegs + off);
344 }
345
346 static u32 tg3_ape_read32(struct tg3 *tp, u32 off)
347 {
348         return (readl(tp->aperegs + off));
349 }
350
351 static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val)
352 {
353         unsigned long flags;
354
355         spin_lock_irqsave(&tp->indirect_lock, flags);
356         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
357         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
358         spin_unlock_irqrestore(&tp->indirect_lock, flags);
359 }
360
361 static void tg3_write_flush_reg32(struct tg3 *tp, u32 off, u32 val)
362 {
363         writel(val, tp->regs + off);
364         readl(tp->regs + off);
365 }
366
367 static u32 tg3_read_indirect_reg32(struct tg3 *tp, u32 off)
368 {
369         unsigned long flags;
370         u32 val;
371
372         spin_lock_irqsave(&tp->indirect_lock, flags);
373         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
374         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
375         spin_unlock_irqrestore(&tp->indirect_lock, flags);
376         return val;
377 }
378
379 static void tg3_write_indirect_mbox(struct tg3 *tp, u32 off, u32 val)
380 {
381         unsigned long flags;
382
383         if (off == (MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW)) {
384                 pci_write_config_dword(tp->pdev, TG3PCI_RCV_RET_RING_CON_IDX +
385                                        TG3_64BIT_REG_LOW, val);
386                 return;
387         }
388         if (off == (MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW)) {
389                 pci_write_config_dword(tp->pdev, TG3PCI_STD_RING_PROD_IDX +
390                                        TG3_64BIT_REG_LOW, val);
391                 return;
392         }
393
394         spin_lock_irqsave(&tp->indirect_lock, flags);
395         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
396         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
397         spin_unlock_irqrestore(&tp->indirect_lock, flags);
398
399         /* In indirect mode when disabling interrupts, we also need
400          * to clear the interrupt bit in the GRC local ctrl register.
401          */
402         if ((off == (MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW)) &&
403             (val == 0x1)) {
404                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_LOCAL_CTRL,
405                                        tp->grc_local_ctrl|GRC_LCLCTRL_CLEARINT);
406         }
407 }
408
409 static u32 tg3_read_indirect_mbox(struct tg3 *tp, u32 off)
410 {
411         unsigned long flags;
412         u32 val;
413
414         spin_lock_irqsave(&tp->indirect_lock, flags);
415         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
416         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
417         spin_unlock_irqrestore(&tp->indirect_lock, flags);
418         return val;
419 }
420
421 /* usec_wait specifies the wait time in usec when writing to certain registers
422  * where it is unsafe to read back the register without some delay.
423  * GRC_LOCAL_CTRL is one example if the GPIOs are toggled to switch power.
424  * TG3PCI_CLOCK_CTRL is another example if the clock frequencies are changed.
425  */
426 static void _tw32_flush(struct tg3 *tp, u32 off, u32 val, u32 usec_wait)
427 {
428         if ((tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) ||
429             (tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
430                 /* Non-posted methods */
431                 tp->write32(tp, off, val);
432         else {
433                 /* Posted method */
434                 tg3_write32(tp, off, val);
435                 if (usec_wait)
436                         udelay(usec_wait);
437                 tp->read32(tp, off);
438         }
439         /* Wait again after the read for the posted method to guarantee that
440          * the wait time is met.
441          */
442         if (usec_wait)
443                 udelay(usec_wait);
444 }
445
446 static inline void tw32_mailbox_flush(struct tg3 *tp, u32 off, u32 val)
447 {
448         tp->write32_mbox(tp, off, val);
449         if (!(tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) &&
450             !(tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
451                 tp->read32_mbox(tp, off);
452 }
453
454 static void tg3_write32_tx_mbox(struct tg3 *tp, u32 off, u32 val)
455 {
456         void __iomem *mbox = tp->regs + off;
457         writel(val, mbox);
458         if (tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG)
459                 writel(val, mbox);
460         if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
461                 readl(mbox);
462 }
463
464 static u32 tg3_read32_mbox_5906(struct tg3 *tp, u32 off)
465 {
466         return (readl(tp->regs + off + GRCMBOX_BASE));
467 }
468
469 static void tg3_write32_mbox_5906(struct tg3 *tp, u32 off, u32 val)
470 {
471         writel(val, tp->regs + off + GRCMBOX_BASE);
472 }
473
474 #define tw32_mailbox(reg, val)  tp->write32_mbox(tp, reg, val)
475 #define tw32_mailbox_f(reg, val)        tw32_mailbox_flush(tp, (reg), (val))
476 #define tw32_rx_mbox(reg, val)  tp->write32_rx_mbox(tp, reg, val)
477 #define tw32_tx_mbox(reg, val)  tp->write32_tx_mbox(tp, reg, val)
478 #define tr32_mailbox(reg)       tp->read32_mbox(tp, reg)
479
480 #define tw32(reg,val)           tp->write32(tp, reg, val)
481 #define tw32_f(reg,val)         _tw32_flush(tp,(reg),(val), 0)
482 #define tw32_wait_f(reg,val,us) _tw32_flush(tp,(reg),(val), (us))
483 #define tr32(reg)               tp->read32(tp, reg)
484
485 static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val)
486 {
487         unsigned long flags;
488
489         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) &&
490             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC))
491                 return;
492
493         spin_lock_irqsave(&tp->indirect_lock, flags);
494         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
495                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
496                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
497
498                 /* Always leave this as zero. */
499                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
500         } else {
501                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
502                 tw32_f(TG3PCI_MEM_WIN_DATA, val);
503
504                 /* Always leave this as zero. */
505                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
506         }
507         spin_unlock_irqrestore(&tp->indirect_lock, flags);
508 }
509
510 static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val)
511 {
512         unsigned long flags;
513
514         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) &&
515             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC)) {
516                 *val = 0;
517                 return;
518         }
519
520         spin_lock_irqsave(&tp->indirect_lock, flags);
521         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
522                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
523                 pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
524
525                 /* Always leave this as zero. */
526                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
527         } else {
528                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
529                 *val = tr32(TG3PCI_MEM_WIN_DATA);
530
531                 /* Always leave this as zero. */
532                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
533         }
534         spin_unlock_irqrestore(&tp->indirect_lock, flags);
535 }
536
537 static void tg3_ape_lock_init(struct tg3 *tp)
538 {
539         int i;
540
541         /* Make sure the driver hasn't any stale locks. */
542         for (i = 0; i < 8; i++)
543                 tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + 4 * i,
544                                 APE_LOCK_GRANT_DRIVER);
545 }
546
547 static int tg3_ape_lock(struct tg3 *tp, int locknum)
548 {
549         int i, off;
550         int ret = 0;
551         u32 status;
552
553         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
554                 return 0;
555
556         switch (locknum) {
557                 case TG3_APE_LOCK_GRC:
558                 case TG3_APE_LOCK_MEM:
559                         break;
560                 default:
561                         return -EINVAL;
562         }
563
564         off = 4 * locknum;
565
566         tg3_ape_write32(tp, TG3_APE_LOCK_REQ + off, APE_LOCK_REQ_DRIVER);
567
568         /* Wait for up to 1 millisecond to acquire lock. */
569         for (i = 0; i < 100; i++) {
570                 status = tg3_ape_read32(tp, TG3_APE_LOCK_GRANT + off);
571                 if (status == APE_LOCK_GRANT_DRIVER)
572                         break;
573                 udelay(10);
574         }
575
576         if (status != APE_LOCK_GRANT_DRIVER) {
577                 /* Revoke the lock request. */
578                 tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + off,
579                                 APE_LOCK_GRANT_DRIVER);
580
581                 ret = -EBUSY;
582         }
583
584         return ret;
585 }
586
587 static void tg3_ape_unlock(struct tg3 *tp, int locknum)
588 {
589         int off;
590
591         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
592                 return;
593
594         switch (locknum) {
595                 case TG3_APE_LOCK_GRC:
596                 case TG3_APE_LOCK_MEM:
597                         break;
598                 default:
599                         return;
600         }
601
602         off = 4 * locknum;
603         tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + off, APE_LOCK_GRANT_DRIVER);
604 }
605
606 static void tg3_disable_ints(struct tg3 *tp)
607 {
608         tw32(TG3PCI_MISC_HOST_CTRL,
609              (tp->misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT));
610         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
611 }
612
613 static inline void tg3_cond_int(struct tg3 *tp)
614 {
615         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
616             (tp->hw_status->status & SD_STATUS_UPDATED))
617                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
618         else
619                 tw32(HOSTCC_MODE, tp->coalesce_mode |
620                      (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
621 }
622
623 static void tg3_enable_ints(struct tg3 *tp)
624 {
625         tp->irq_sync = 0;
626         wmb();
627
628         tw32(TG3PCI_MISC_HOST_CTRL,
629              (tp->misc_host_ctrl & ~MISC_HOST_CTRL_MASK_PCI_INT));
630         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
631                        (tp->last_tag << 24));
632         if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)
633                 tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
634                                (tp->last_tag << 24));
635         tg3_cond_int(tp);
636 }
637
638 static inline unsigned int tg3_has_work(struct tg3 *tp)
639 {
640         struct tg3_hw_status *sblk = tp->hw_status;
641         unsigned int work_exists = 0;
642
643         /* check for phy events */
644         if (!(tp->tg3_flags &
645               (TG3_FLAG_USE_LINKCHG_REG |
646                TG3_FLAG_POLL_SERDES))) {
647                 if (sblk->status & SD_STATUS_LINK_CHG)
648                         work_exists = 1;
649         }
650         /* check for RX/TX work to do */
651         if (sblk->idx[0].tx_consumer != tp->tx_cons ||
652             sblk->idx[0].rx_producer != tp->rx_rcb_ptr)
653                 work_exists = 1;
654
655         return work_exists;
656 }
657
658 /* tg3_restart_ints
659  *  similar to tg3_enable_ints, but it accurately determines whether there
660  *  is new work pending and can return without flushing the PIO write
661  *  which reenables interrupts
662  */
663 static void tg3_restart_ints(struct tg3 *tp)
664 {
665         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
666                      tp->last_tag << 24);
667         mmiowb();
668
669         /* When doing tagged status, this work check is unnecessary.
670          * The last_tag we write above tells the chip which piece of
671          * work we've completed.
672          */
673         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
674             tg3_has_work(tp))
675                 tw32(HOSTCC_MODE, tp->coalesce_mode |
676                      (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
677 }
678
679 static inline void tg3_netif_stop(struct tg3 *tp)
680 {
681         tp->dev->trans_start = jiffies; /* prevent tx timeout */
682         napi_disable(&tp->napi);
683         netif_tx_disable(tp->dev);
684 }
685
686 static inline void tg3_netif_start(struct tg3 *tp)
687 {
688         netif_wake_queue(tp->dev);
689         /* NOTE: unconditional netif_wake_queue is only appropriate
690          * so long as all callers are assured to have free tx slots
691          * (such as after tg3_init_hw)
692          */
693         napi_enable(&tp->napi);
694         tp->hw_status->status |= SD_STATUS_UPDATED;
695         tg3_enable_ints(tp);
696 }
697
698 static void tg3_switch_clocks(struct tg3 *tp)
699 {
700         u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL);
701         u32 orig_clock_ctrl;
702
703         if ((tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) ||
704             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
705                 return;
706
707         orig_clock_ctrl = clock_ctrl;
708         clock_ctrl &= (CLOCK_CTRL_FORCE_CLKRUN |
709                        CLOCK_CTRL_CLKRUN_OENABLE |
710                        0x1f);
711         tp->pci_clock_ctrl = clock_ctrl;
712
713         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
714                 if (orig_clock_ctrl & CLOCK_CTRL_625_CORE) {
715                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
716                                     clock_ctrl | CLOCK_CTRL_625_CORE, 40);
717                 }
718         } else if ((orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
719                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
720                             clock_ctrl |
721                             (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK),
722                             40);
723                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
724                             clock_ctrl | (CLOCK_CTRL_ALTCLK),
725                             40);
726         }
727         tw32_wait_f(TG3PCI_CLOCK_CTRL, clock_ctrl, 40);
728 }
729
730 #define PHY_BUSY_LOOPS  5000
731
732 static int tg3_readphy(struct tg3 *tp, int reg, u32 *val)
733 {
734         u32 frame_val;
735         unsigned int loops;
736         int ret;
737
738         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
739                 tw32_f(MAC_MI_MODE,
740                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
741                 udelay(80);
742         }
743
744         *val = 0x0;
745
746         frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
747                       MI_COM_PHY_ADDR_MASK);
748         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
749                       MI_COM_REG_ADDR_MASK);
750         frame_val |= (MI_COM_CMD_READ | MI_COM_START);
751
752         tw32_f(MAC_MI_COM, frame_val);
753
754         loops = PHY_BUSY_LOOPS;
755         while (loops != 0) {
756                 udelay(10);
757                 frame_val = tr32(MAC_MI_COM);
758
759                 if ((frame_val & MI_COM_BUSY) == 0) {
760                         udelay(5);
761                         frame_val = tr32(MAC_MI_COM);
762                         break;
763                 }
764                 loops -= 1;
765         }
766
767         ret = -EBUSY;
768         if (loops != 0) {
769                 *val = frame_val & MI_COM_DATA_MASK;
770                 ret = 0;
771         }
772
773         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
774                 tw32_f(MAC_MI_MODE, tp->mi_mode);
775                 udelay(80);
776         }
777
778         return ret;
779 }
780
781 static int tg3_writephy(struct tg3 *tp, int reg, u32 val)
782 {
783         u32 frame_val;
784         unsigned int loops;
785         int ret;
786
787         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906 &&
788             (reg == MII_TG3_CTRL || reg == MII_TG3_AUX_CTRL))
789                 return 0;
790
791         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
792                 tw32_f(MAC_MI_MODE,
793                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
794                 udelay(80);
795         }
796
797         frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
798                       MI_COM_PHY_ADDR_MASK);
799         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
800                       MI_COM_REG_ADDR_MASK);
801         frame_val |= (val & MI_COM_DATA_MASK);
802         frame_val |= (MI_COM_CMD_WRITE | MI_COM_START);
803
804         tw32_f(MAC_MI_COM, frame_val);
805
806         loops = PHY_BUSY_LOOPS;
807         while (loops != 0) {
808                 udelay(10);
809                 frame_val = tr32(MAC_MI_COM);
810                 if ((frame_val & MI_COM_BUSY) == 0) {
811                         udelay(5);
812                         frame_val = tr32(MAC_MI_COM);
813                         break;
814                 }
815                 loops -= 1;
816         }
817
818         ret = -EBUSY;
819         if (loops != 0)
820                 ret = 0;
821
822         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
823                 tw32_f(MAC_MI_MODE, tp->mi_mode);
824                 udelay(80);
825         }
826
827         return ret;
828 }
829
830 static int tg3_bmcr_reset(struct tg3 *tp)
831 {
832         u32 phy_control;
833         int limit, err;
834
835         /* OK, reset it, and poll the BMCR_RESET bit until it
836          * clears or we time out.
837          */
838         phy_control = BMCR_RESET;
839         err = tg3_writephy(tp, MII_BMCR, phy_control);
840         if (err != 0)
841                 return -EBUSY;
842
843         limit = 5000;
844         while (limit--) {
845                 err = tg3_readphy(tp, MII_BMCR, &phy_control);
846                 if (err != 0)
847                         return -EBUSY;
848
849                 if ((phy_control & BMCR_RESET) == 0) {
850                         udelay(40);
851                         break;
852                 }
853                 udelay(10);
854         }
855         if (limit < 0)
856                 return -EBUSY;
857
858         return 0;
859 }
860
861 static int tg3_mdio_read(struct mii_bus *bp, int mii_id, int reg)
862 {
863         struct tg3 *tp = bp->priv;
864         u32 val;
865
866         if (tp->tg3_flags3 & TG3_FLG3_MDIOBUS_PAUSED)
867                 return -EAGAIN;
868
869         if (tg3_readphy(tp, reg, &val))
870                 return -EIO;
871
872         return val;
873 }
874
875 static int tg3_mdio_write(struct mii_bus *bp, int mii_id, int reg, u16 val)
876 {
877         struct tg3 *tp = bp->priv;
878
879         if (tp->tg3_flags3 & TG3_FLG3_MDIOBUS_PAUSED)
880                 return -EAGAIN;
881
882         if (tg3_writephy(tp, reg, val))
883                 return -EIO;
884
885         return 0;
886 }
887
888 static int tg3_mdio_reset(struct mii_bus *bp)
889 {
890         return 0;
891 }
892
893 static void tg3_mdio_config_5785(struct tg3 *tp)
894 {
895         u32 val;
896         struct phy_device *phydev;
897
898         phydev = tp->mdio_bus->phy_map[PHY_ADDR];
899         switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) {
900         case TG3_PHY_ID_BCM50610:
901                 val = MAC_PHYCFG2_50610_LED_MODES;
902                 break;
903         case TG3_PHY_ID_BCMAC131:
904                 val = MAC_PHYCFG2_AC131_LED_MODES;
905                 break;
906         case TG3_PHY_ID_RTL8211C:
907                 val = MAC_PHYCFG2_RTL8211C_LED_MODES;
908                 break;
909         case TG3_PHY_ID_RTL8201E:
910                 val = MAC_PHYCFG2_RTL8201E_LED_MODES;
911                 break;
912         default:
913                 return;
914         }
915
916         if (phydev->interface != PHY_INTERFACE_MODE_RGMII) {
917                 tw32(MAC_PHYCFG2, val);
918
919                 val = tr32(MAC_PHYCFG1);
920                 val &= ~MAC_PHYCFG1_RGMII_INT;
921                 tw32(MAC_PHYCFG1, val);
922
923                 return;
924         }
925
926         if (!(tp->tg3_flags3 & TG3_FLG3_RGMII_STD_IBND_DISABLE))
927                 val |= MAC_PHYCFG2_EMODE_MASK_MASK |
928                        MAC_PHYCFG2_FMODE_MASK_MASK |
929                        MAC_PHYCFG2_GMODE_MASK_MASK |
930                        MAC_PHYCFG2_ACT_MASK_MASK   |
931                        MAC_PHYCFG2_QUAL_MASK_MASK |
932                        MAC_PHYCFG2_INBAND_ENABLE;
933
934         tw32(MAC_PHYCFG2, val);
935
936         val = tr32(MAC_PHYCFG1) & ~(MAC_PHYCFG1_RGMII_EXT_RX_DEC |
937                                     MAC_PHYCFG1_RGMII_SND_STAT_EN);
938         if (tp->tg3_flags3 & TG3_FLG3_RGMII_STD_IBND_DISABLE) {
939                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_RX_EN)
940                         val |= MAC_PHYCFG1_RGMII_EXT_RX_DEC;
941                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_TX_EN)
942                         val |= MAC_PHYCFG1_RGMII_SND_STAT_EN;
943         }
944         tw32(MAC_PHYCFG1, val | MAC_PHYCFG1_RGMII_INT | MAC_PHYCFG1_TXC_DRV);
945
946         val = tr32(MAC_EXT_RGMII_MODE);
947         val &= ~(MAC_RGMII_MODE_RX_INT_B |
948                  MAC_RGMII_MODE_RX_QUALITY |
949                  MAC_RGMII_MODE_RX_ACTIVITY |
950                  MAC_RGMII_MODE_RX_ENG_DET |
951                  MAC_RGMII_MODE_TX_ENABLE |
952                  MAC_RGMII_MODE_TX_LOWPWR |
953                  MAC_RGMII_MODE_TX_RESET);
954         if (!(tp->tg3_flags3 & TG3_FLG3_RGMII_STD_IBND_DISABLE)) {
955                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_RX_EN)
956                         val |= MAC_RGMII_MODE_RX_INT_B |
957                                MAC_RGMII_MODE_RX_QUALITY |
958                                MAC_RGMII_MODE_RX_ACTIVITY |
959                                MAC_RGMII_MODE_RX_ENG_DET;
960                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_TX_EN)
961                         val |= MAC_RGMII_MODE_TX_ENABLE |
962                                MAC_RGMII_MODE_TX_LOWPWR |
963                                MAC_RGMII_MODE_TX_RESET;
964         }
965         tw32(MAC_EXT_RGMII_MODE, val);
966 }
967
968 static void tg3_mdio_start(struct tg3 *tp)
969 {
970         if (tp->tg3_flags3 & TG3_FLG3_MDIOBUS_INITED) {
971                 mutex_lock(&tp->mdio_bus->mdio_lock);
972                 tp->tg3_flags3 &= ~TG3_FLG3_MDIOBUS_PAUSED;
973                 mutex_unlock(&tp->mdio_bus->mdio_lock);
974         }
975
976         tp->mi_mode &= ~MAC_MI_MODE_AUTO_POLL;
977         tw32_f(MAC_MI_MODE, tp->mi_mode);
978         udelay(80);
979
980         if ((tp->tg3_flags3 & TG3_FLG3_MDIOBUS_INITED) &&
981             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785)
982                 tg3_mdio_config_5785(tp);
983 }
984
985 static void tg3_mdio_stop(struct tg3 *tp)
986 {
987         if (tp->tg3_flags3 & TG3_FLG3_MDIOBUS_INITED) {
988                 mutex_lock(&tp->mdio_bus->mdio_lock);
989                 tp->tg3_flags3 |= TG3_FLG3_MDIOBUS_PAUSED;
990                 mutex_unlock(&tp->mdio_bus->mdio_lock);
991         }
992 }
993
994 static int tg3_mdio_init(struct tg3 *tp)
995 {
996         int i;
997         u32 reg;
998         struct phy_device *phydev;
999
1000         tg3_mdio_start(tp);
1001
1002         if (!(tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) ||
1003             (tp->tg3_flags3 & TG3_FLG3_MDIOBUS_INITED))
1004                 return 0;
1005
1006         tp->mdio_bus = mdiobus_alloc();
1007         if (tp->mdio_bus == NULL)
1008                 return -ENOMEM;
1009
1010         tp->mdio_bus->name     = "tg3 mdio bus";
1011         snprintf(tp->mdio_bus->id, MII_BUS_ID_SIZE, "%x",
1012                  (tp->pdev->bus->number << 8) | tp->pdev->devfn);
1013         tp->mdio_bus->priv     = tp;
1014         tp->mdio_bus->parent   = &tp->pdev->dev;
1015         tp->mdio_bus->read     = &tg3_mdio_read;
1016         tp->mdio_bus->write    = &tg3_mdio_write;
1017         tp->mdio_bus->reset    = &tg3_mdio_reset;
1018         tp->mdio_bus->phy_mask = ~(1 << PHY_ADDR);
1019         tp->mdio_bus->irq      = &tp->mdio_irq[0];
1020
1021         for (i = 0; i < PHY_MAX_ADDR; i++)
1022                 tp->mdio_bus->irq[i] = PHY_POLL;
1023
1024         /* The bus registration will look for all the PHYs on the mdio bus.
1025          * Unfortunately, it does not ensure the PHY is powered up before
1026          * accessing the PHY ID registers.  A chip reset is the
1027          * quickest way to bring the device back to an operational state..
1028          */
1029         if (tg3_readphy(tp, MII_BMCR, &reg) || (reg & BMCR_PDOWN))
1030                 tg3_bmcr_reset(tp);
1031
1032         i = mdiobus_register(tp->mdio_bus);
1033         if (i) {
1034                 printk(KERN_WARNING "%s: mdiobus_reg failed (0x%x)\n",
1035                         tp->dev->name, i);
1036                 mdiobus_free(tp->mdio_bus);
1037                 return i;
1038         }
1039
1040         phydev = tp->mdio_bus->phy_map[PHY_ADDR];
1041
1042         if (!phydev || !phydev->drv) {
1043                 printk(KERN_WARNING "%s: No PHY devices\n", tp->dev->name);
1044                 mdiobus_unregister(tp->mdio_bus);
1045                 mdiobus_free(tp->mdio_bus);
1046                 return -ENODEV;
1047         }
1048
1049         switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) {
1050         case TG3_PHY_ID_BCM57780:
1051                 phydev->interface = PHY_INTERFACE_MODE_GMII;
1052                 break;
1053         case TG3_PHY_ID_BCM50610:
1054                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_STD_IBND_DISABLE)
1055                         phydev->dev_flags |= PHY_BRCM_STD_IBND_DISABLE;
1056                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_RX_EN)
1057                         phydev->dev_flags |= PHY_BRCM_EXT_IBND_RX_ENABLE;
1058                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_TX_EN)
1059                         phydev->dev_flags |= PHY_BRCM_EXT_IBND_TX_ENABLE;
1060                 /* fallthru */
1061         case TG3_PHY_ID_RTL8211C:
1062                 phydev->interface = PHY_INTERFACE_MODE_RGMII;
1063                 break;
1064         case TG3_PHY_ID_RTL8201E:
1065         case TG3_PHY_ID_BCMAC131:
1066                 phydev->interface = PHY_INTERFACE_MODE_MII;
1067                 break;
1068         }
1069
1070         tp->tg3_flags3 |= TG3_FLG3_MDIOBUS_INITED;
1071
1072         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785)
1073                 tg3_mdio_config_5785(tp);
1074
1075         return 0;
1076 }
1077
1078 static void tg3_mdio_fini(struct tg3 *tp)
1079 {
1080         if (tp->tg3_flags3 & TG3_FLG3_MDIOBUS_INITED) {
1081                 tp->tg3_flags3 &= ~TG3_FLG3_MDIOBUS_INITED;
1082                 mdiobus_unregister(tp->mdio_bus);
1083                 mdiobus_free(tp->mdio_bus);
1084                 tp->tg3_flags3 &= ~TG3_FLG3_MDIOBUS_PAUSED;
1085         }
1086 }
1087
1088 /* tp->lock is held. */
1089 static inline void tg3_generate_fw_event(struct tg3 *tp)
1090 {
1091         u32 val;
1092
1093         val = tr32(GRC_RX_CPU_EVENT);
1094         val |= GRC_RX_CPU_DRIVER_EVENT;
1095         tw32_f(GRC_RX_CPU_EVENT, val);
1096
1097         tp->last_event_jiffies = jiffies;
1098 }
1099
1100 #define TG3_FW_EVENT_TIMEOUT_USEC 2500
1101
1102 /* tp->lock is held. */
1103 static void tg3_wait_for_event_ack(struct tg3 *tp)
1104 {
1105         int i;
1106         unsigned int delay_cnt;
1107         long time_remain;
1108
1109         /* If enough time has passed, no wait is necessary. */
1110         time_remain = (long)(tp->last_event_jiffies + 1 +
1111                       usecs_to_jiffies(TG3_FW_EVENT_TIMEOUT_USEC)) -
1112                       (long)jiffies;
1113         if (time_remain < 0)
1114                 return;
1115
1116         /* Check if we can shorten the wait time. */
1117         delay_cnt = jiffies_to_usecs(time_remain);
1118         if (delay_cnt > TG3_FW_EVENT_TIMEOUT_USEC)
1119                 delay_cnt = TG3_FW_EVENT_TIMEOUT_USEC;
1120         delay_cnt = (delay_cnt >> 3) + 1;
1121
1122         for (i = 0; i < delay_cnt; i++) {
1123                 if (!(tr32(GRC_RX_CPU_EVENT) & GRC_RX_CPU_DRIVER_EVENT))
1124                         break;
1125                 udelay(8);
1126         }
1127 }
1128
1129 /* tp->lock is held. */
1130 static void tg3_ump_link_report(struct tg3 *tp)
1131 {
1132         u32 reg;
1133         u32 val;
1134
1135         if (!(tp->tg3_flags2 & TG3_FLG2_5780_CLASS) ||
1136             !(tp->tg3_flags  & TG3_FLAG_ENABLE_ASF))
1137                 return;
1138
1139         tg3_wait_for_event_ack(tp);
1140
1141         tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_LINK_UPDATE);
1142
1143         tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 14);
1144
1145         val = 0;
1146         if (!tg3_readphy(tp, MII_BMCR, &reg))
1147                 val = reg << 16;
1148         if (!tg3_readphy(tp, MII_BMSR, &reg))
1149                 val |= (reg & 0xffff);
1150         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX, val);
1151
1152         val = 0;
1153         if (!tg3_readphy(tp, MII_ADVERTISE, &reg))
1154                 val = reg << 16;
1155         if (!tg3_readphy(tp, MII_LPA, &reg))
1156                 val |= (reg & 0xffff);
1157         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 4, val);
1158
1159         val = 0;
1160         if (!(tp->tg3_flags2 & TG3_FLG2_MII_SERDES)) {
1161                 if (!tg3_readphy(tp, MII_CTRL1000, &reg))
1162                         val = reg << 16;
1163                 if (!tg3_readphy(tp, MII_STAT1000, &reg))
1164                         val |= (reg & 0xffff);
1165         }
1166         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 8, val);
1167
1168         if (!tg3_readphy(tp, MII_PHYADDR, &reg))
1169                 val = reg << 16;
1170         else
1171                 val = 0;
1172         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 12, val);
1173
1174         tg3_generate_fw_event(tp);
1175 }
1176
1177 static void tg3_link_report(struct tg3 *tp)
1178 {
1179         if (!netif_carrier_ok(tp->dev)) {
1180                 if (netif_msg_link(tp))
1181                         printk(KERN_INFO PFX "%s: Link is down.\n",
1182                                tp->dev->name);
1183                 tg3_ump_link_report(tp);
1184         } else if (netif_msg_link(tp)) {
1185                 printk(KERN_INFO PFX "%s: Link is up at %d Mbps, %s duplex.\n",
1186                        tp->dev->name,
1187                        (tp->link_config.active_speed == SPEED_1000 ?
1188                         1000 :
1189                         (tp->link_config.active_speed == SPEED_100 ?
1190                          100 : 10)),
1191                        (tp->link_config.active_duplex == DUPLEX_FULL ?
1192                         "full" : "half"));
1193
1194                 printk(KERN_INFO PFX
1195                        "%s: Flow control is %s for TX and %s for RX.\n",
1196                        tp->dev->name,
1197                        (tp->link_config.active_flowctrl & FLOW_CTRL_TX) ?
1198                        "on" : "off",
1199                        (tp->link_config.active_flowctrl & FLOW_CTRL_RX) ?
1200                        "on" : "off");
1201                 tg3_ump_link_report(tp);
1202         }
1203 }
1204
1205 static u16 tg3_advert_flowctrl_1000T(u8 flow_ctrl)
1206 {
1207         u16 miireg;
1208
1209         if ((flow_ctrl & FLOW_CTRL_TX) && (flow_ctrl & FLOW_CTRL_RX))
1210                 miireg = ADVERTISE_PAUSE_CAP;
1211         else if (flow_ctrl & FLOW_CTRL_TX)
1212                 miireg = ADVERTISE_PAUSE_ASYM;
1213         else if (flow_ctrl & FLOW_CTRL_RX)
1214                 miireg = ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM;
1215         else
1216                 miireg = 0;
1217
1218         return miireg;
1219 }
1220
1221 static u16 tg3_advert_flowctrl_1000X(u8 flow_ctrl)
1222 {
1223         u16 miireg;
1224
1225         if ((flow_ctrl & FLOW_CTRL_TX) && (flow_ctrl & FLOW_CTRL_RX))
1226                 miireg = ADVERTISE_1000XPAUSE;
1227         else if (flow_ctrl & FLOW_CTRL_TX)
1228                 miireg = ADVERTISE_1000XPSE_ASYM;
1229         else if (flow_ctrl & FLOW_CTRL_RX)
1230                 miireg = ADVERTISE_1000XPAUSE | ADVERTISE_1000XPSE_ASYM;
1231         else
1232                 miireg = 0;
1233
1234         return miireg;
1235 }
1236
1237 static u8 tg3_resolve_flowctrl_1000X(u16 lcladv, u16 rmtadv)
1238 {
1239         u8 cap = 0;
1240
1241         if (lcladv & ADVERTISE_1000XPAUSE) {
1242                 if (lcladv & ADVERTISE_1000XPSE_ASYM) {
1243                         if (rmtadv & LPA_1000XPAUSE)
1244                                 cap = FLOW_CTRL_TX | FLOW_CTRL_RX;
1245                         else if (rmtadv & LPA_1000XPAUSE_ASYM)
1246                                 cap = FLOW_CTRL_RX;
1247                 } else {
1248                         if (rmtadv & LPA_1000XPAUSE)
1249                                 cap = FLOW_CTRL_TX | FLOW_CTRL_RX;
1250                 }
1251         } else if (lcladv & ADVERTISE_1000XPSE_ASYM) {
1252                 if ((rmtadv & LPA_1000XPAUSE) && (rmtadv & LPA_1000XPAUSE_ASYM))
1253                         cap = FLOW_CTRL_TX;
1254         }
1255
1256         return cap;
1257 }
1258
1259 static void tg3_setup_flow_control(struct tg3 *tp, u32 lcladv, u32 rmtadv)
1260 {
1261         u8 autoneg;
1262         u8 flowctrl = 0;
1263         u32 old_rx_mode = tp->rx_mode;
1264         u32 old_tx_mode = tp->tx_mode;
1265
1266         if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB)
1267                 autoneg = tp->mdio_bus->phy_map[PHY_ADDR]->autoneg;
1268         else
1269                 autoneg = tp->link_config.autoneg;
1270
1271         if (autoneg == AUTONEG_ENABLE &&
1272             (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG)) {
1273                 if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
1274                         flowctrl = tg3_resolve_flowctrl_1000X(lcladv, rmtadv);
1275                 else
1276                         flowctrl = mii_resolve_flowctrl_fdx(lcladv, rmtadv);
1277         } else
1278                 flowctrl = tp->link_config.flowctrl;
1279
1280         tp->link_config.active_flowctrl = flowctrl;
1281
1282         if (flowctrl & FLOW_CTRL_RX)
1283                 tp->rx_mode |= RX_MODE_FLOW_CTRL_ENABLE;
1284         else
1285                 tp->rx_mode &= ~RX_MODE_FLOW_CTRL_ENABLE;
1286
1287         if (old_rx_mode != tp->rx_mode)
1288                 tw32_f(MAC_RX_MODE, tp->rx_mode);
1289
1290         if (flowctrl & FLOW_CTRL_TX)
1291                 tp->tx_mode |= TX_MODE_FLOW_CTRL_ENABLE;
1292         else
1293                 tp->tx_mode &= ~TX_MODE_FLOW_CTRL_ENABLE;
1294
1295         if (old_tx_mode != tp->tx_mode)
1296                 tw32_f(MAC_TX_MODE, tp->tx_mode);
1297 }
1298
1299 static void tg3_adjust_link(struct net_device *dev)
1300 {
1301         u8 oldflowctrl, linkmesg = 0;
1302         u32 mac_mode, lcl_adv, rmt_adv;
1303         struct tg3 *tp = netdev_priv(dev);
1304         struct phy_device *phydev = tp->mdio_bus->phy_map[PHY_ADDR];
1305
1306         spin_lock(&tp->lock);
1307
1308         mac_mode = tp->mac_mode & ~(MAC_MODE_PORT_MODE_MASK |
1309                                     MAC_MODE_HALF_DUPLEX);
1310
1311         oldflowctrl = tp->link_config.active_flowctrl;
1312
1313         if (phydev->link) {
1314                 lcl_adv = 0;
1315                 rmt_adv = 0;
1316
1317                 if (phydev->speed == SPEED_100 || phydev->speed == SPEED_10)
1318                         mac_mode |= MAC_MODE_PORT_MODE_MII;
1319                 else
1320                         mac_mode |= MAC_MODE_PORT_MODE_GMII;
1321
1322                 if (phydev->duplex == DUPLEX_HALF)
1323                         mac_mode |= MAC_MODE_HALF_DUPLEX;
1324                 else {
1325                         lcl_adv = tg3_advert_flowctrl_1000T(
1326                                   tp->link_config.flowctrl);
1327
1328                         if (phydev->pause)
1329                                 rmt_adv = LPA_PAUSE_CAP;
1330                         if (phydev->asym_pause)
1331                                 rmt_adv |= LPA_PAUSE_ASYM;
1332                 }
1333
1334                 tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
1335         } else
1336                 mac_mode |= MAC_MODE_PORT_MODE_GMII;
1337
1338         if (mac_mode != tp->mac_mode) {
1339                 tp->mac_mode = mac_mode;
1340                 tw32_f(MAC_MODE, tp->mac_mode);
1341                 udelay(40);
1342         }
1343
1344         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785) {
1345                 if (phydev->speed == SPEED_10)
1346                         tw32(MAC_MI_STAT,
1347                              MAC_MI_STAT_10MBPS_MODE |
1348                              MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
1349                 else
1350                         tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
1351         }
1352
1353         if (phydev->speed == SPEED_1000 && phydev->duplex == DUPLEX_HALF)
1354                 tw32(MAC_TX_LENGTHS,
1355                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
1356                       (6 << TX_LENGTHS_IPG_SHIFT) |
1357                       (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
1358         else
1359                 tw32(MAC_TX_LENGTHS,
1360                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
1361                       (6 << TX_LENGTHS_IPG_SHIFT) |
1362                       (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
1363
1364         if ((phydev->link && tp->link_config.active_speed == SPEED_INVALID) ||
1365             (!phydev->link && tp->link_config.active_speed != SPEED_INVALID) ||
1366             phydev->speed != tp->link_config.active_speed ||
1367             phydev->duplex != tp->link_config.active_duplex ||
1368             oldflowctrl != tp->link_config.active_flowctrl)
1369             linkmesg = 1;
1370
1371         tp->link_config.active_speed = phydev->speed;
1372         tp->link_config.active_duplex = phydev->duplex;
1373
1374         spin_unlock(&tp->lock);
1375
1376         if (linkmesg)
1377                 tg3_link_report(tp);
1378 }
1379
1380 static int tg3_phy_init(struct tg3 *tp)
1381 {
1382         struct phy_device *phydev;
1383
1384         if (tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED)
1385                 return 0;
1386
1387         /* Bring the PHY back to a known state. */
1388         tg3_bmcr_reset(tp);
1389
1390         phydev = tp->mdio_bus->phy_map[PHY_ADDR];
1391
1392         /* Attach the MAC to the PHY. */
1393         phydev = phy_connect(tp->dev, dev_name(&phydev->dev), tg3_adjust_link,
1394                              phydev->dev_flags, phydev->interface);
1395         if (IS_ERR(phydev)) {
1396                 printk(KERN_ERR "%s: Could not attach to PHY\n", tp->dev->name);
1397                 return PTR_ERR(phydev);
1398         }
1399
1400         /* Mask with MAC supported features. */
1401         switch (phydev->interface) {
1402         case PHY_INTERFACE_MODE_GMII:
1403         case PHY_INTERFACE_MODE_RGMII:
1404                 if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
1405                         phydev->supported &= (PHY_GBIT_FEATURES |
1406                                               SUPPORTED_Pause |
1407                                               SUPPORTED_Asym_Pause);
1408                         break;
1409                 }
1410                 /* fallthru */
1411         case PHY_INTERFACE_MODE_MII:
1412                 phydev->supported &= (PHY_BASIC_FEATURES |
1413                                       SUPPORTED_Pause |
1414                                       SUPPORTED_Asym_Pause);
1415                 break;
1416         default:
1417                 phy_disconnect(tp->mdio_bus->phy_map[PHY_ADDR]);
1418                 return -EINVAL;
1419         }
1420
1421         tp->tg3_flags3 |= TG3_FLG3_PHY_CONNECTED;
1422
1423         phydev->advertising = phydev->supported;
1424
1425         return 0;
1426 }
1427
1428 static void tg3_phy_start(struct tg3 *tp)
1429 {
1430         struct phy_device *phydev;
1431
1432         if (!(tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED))
1433                 return;
1434
1435         phydev = tp->mdio_bus->phy_map[PHY_ADDR];
1436
1437         if (tp->link_config.phy_is_low_power) {
1438                 tp->link_config.phy_is_low_power = 0;
1439                 phydev->speed = tp->link_config.orig_speed;
1440                 phydev->duplex = tp->link_config.orig_duplex;
1441                 phydev->autoneg = tp->link_config.orig_autoneg;
1442                 phydev->advertising = tp->link_config.orig_advertising;
1443         }
1444
1445         phy_start(phydev);
1446
1447         phy_start_aneg(phydev);
1448 }
1449
1450 static void tg3_phy_stop(struct tg3 *tp)
1451 {
1452         if (!(tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED))
1453                 return;
1454
1455         phy_stop(tp->mdio_bus->phy_map[PHY_ADDR]);
1456 }
1457
1458 static void tg3_phy_fini(struct tg3 *tp)
1459 {
1460         if (tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED) {
1461                 phy_disconnect(tp->mdio_bus->phy_map[PHY_ADDR]);
1462                 tp->tg3_flags3 &= ~TG3_FLG3_PHY_CONNECTED;
1463         }
1464 }
1465
1466 static void tg3_phydsp_write(struct tg3 *tp, u32 reg, u32 val)
1467 {
1468         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, reg);
1469         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, val);
1470 }
1471
1472 static void tg3_phy_toggle_apd(struct tg3 *tp, bool enable)
1473 {
1474         u32 reg;
1475
1476         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ||
1477             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
1478                 return;
1479
1480         reg = MII_TG3_MISC_SHDW_WREN |
1481               MII_TG3_MISC_SHDW_SCR5_SEL |
1482               MII_TG3_MISC_SHDW_SCR5_LPED |
1483               MII_TG3_MISC_SHDW_SCR5_DLPTLM |
1484               MII_TG3_MISC_SHDW_SCR5_SDTL |
1485               MII_TG3_MISC_SHDW_SCR5_C125OE;
1486         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5784 || !enable)
1487                 reg |= MII_TG3_MISC_SHDW_SCR5_DLLAPD;
1488
1489         tg3_writephy(tp, MII_TG3_MISC_SHDW, reg);
1490
1491
1492         reg = MII_TG3_MISC_SHDW_WREN |
1493               MII_TG3_MISC_SHDW_APD_SEL |
1494               MII_TG3_MISC_SHDW_APD_WKTM_84MS;
1495         if (enable)
1496                 reg |= MII_TG3_MISC_SHDW_APD_ENABLE;
1497
1498         tg3_writephy(tp, MII_TG3_MISC_SHDW, reg);
1499 }
1500
1501 static void tg3_phy_toggle_automdix(struct tg3 *tp, int enable)
1502 {
1503         u32 phy;
1504
1505         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ||
1506             (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES))
1507                 return;
1508
1509         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1510                 u32 ephy;
1511
1512                 if (!tg3_readphy(tp, MII_TG3_EPHY_TEST, &ephy)) {
1513                         tg3_writephy(tp, MII_TG3_EPHY_TEST,
1514                                      ephy | MII_TG3_EPHY_SHADOW_EN);
1515                         if (!tg3_readphy(tp, MII_TG3_EPHYTST_MISCCTRL, &phy)) {
1516                                 if (enable)
1517                                         phy |= MII_TG3_EPHYTST_MISCCTRL_MDIX;
1518                                 else
1519                                         phy &= ~MII_TG3_EPHYTST_MISCCTRL_MDIX;
1520                                 tg3_writephy(tp, MII_TG3_EPHYTST_MISCCTRL, phy);
1521                         }
1522                         tg3_writephy(tp, MII_TG3_EPHY_TEST, ephy);
1523                 }
1524         } else {
1525                 phy = MII_TG3_AUXCTL_MISC_RDSEL_MISC |
1526                       MII_TG3_AUXCTL_SHDWSEL_MISC;
1527                 if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, phy) &&
1528                     !tg3_readphy(tp, MII_TG3_AUX_CTRL, &phy)) {
1529                         if (enable)
1530                                 phy |= MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
1531                         else
1532                                 phy &= ~MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
1533                         phy |= MII_TG3_AUXCTL_MISC_WREN;
1534                         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy);
1535                 }
1536         }
1537 }
1538
1539 static void tg3_phy_set_wirespeed(struct tg3 *tp)
1540 {
1541         u32 val;
1542
1543         if (tp->tg3_flags2 & TG3_FLG2_NO_ETH_WIRE_SPEED)
1544                 return;
1545
1546         if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x7007) &&
1547             !tg3_readphy(tp, MII_TG3_AUX_CTRL, &val))
1548                 tg3_writephy(tp, MII_TG3_AUX_CTRL,
1549                              (val | (1 << 15) | (1 << 4)));
1550 }
1551
1552 static void tg3_phy_apply_otp(struct tg3 *tp)
1553 {
1554         u32 otp, phy;
1555
1556         if (!tp->phy_otp)
1557                 return;
1558
1559         otp = tp->phy_otp;
1560
1561         /* Enable SM_DSP clock and tx 6dB coding. */
1562         phy = MII_TG3_AUXCTL_SHDWSEL_AUXCTL |
1563               MII_TG3_AUXCTL_ACTL_SMDSP_ENA |
1564               MII_TG3_AUXCTL_ACTL_TX_6DB;
1565         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy);
1566
1567         phy = ((otp & TG3_OTP_AGCTGT_MASK) >> TG3_OTP_AGCTGT_SHIFT);
1568         phy |= MII_TG3_DSP_TAP1_AGCTGT_DFLT;
1569         tg3_phydsp_write(tp, MII_TG3_DSP_TAP1, phy);
1570
1571         phy = ((otp & TG3_OTP_HPFFLTR_MASK) >> TG3_OTP_HPFFLTR_SHIFT) |
1572               ((otp & TG3_OTP_HPFOVER_MASK) >> TG3_OTP_HPFOVER_SHIFT);
1573         tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH0, phy);
1574
1575         phy = ((otp & TG3_OTP_LPFDIS_MASK) >> TG3_OTP_LPFDIS_SHIFT);
1576         phy |= MII_TG3_DSP_AADJ1CH3_ADCCKADJ;
1577         tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH3, phy);
1578
1579         phy = ((otp & TG3_OTP_VDAC_MASK) >> TG3_OTP_VDAC_SHIFT);
1580         tg3_phydsp_write(tp, MII_TG3_DSP_EXP75, phy);
1581
1582         phy = ((otp & TG3_OTP_10BTAMP_MASK) >> TG3_OTP_10BTAMP_SHIFT);
1583         tg3_phydsp_write(tp, MII_TG3_DSP_EXP96, phy);
1584
1585         phy = ((otp & TG3_OTP_ROFF_MASK) >> TG3_OTP_ROFF_SHIFT) |
1586               ((otp & TG3_OTP_RCOFF_MASK) >> TG3_OTP_RCOFF_SHIFT);
1587         tg3_phydsp_write(tp, MII_TG3_DSP_EXP97, phy);
1588
1589         /* Turn off SM_DSP clock. */
1590         phy = MII_TG3_AUXCTL_SHDWSEL_AUXCTL |
1591               MII_TG3_AUXCTL_ACTL_TX_6DB;
1592         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy);
1593 }
1594
1595 static int tg3_wait_macro_done(struct tg3 *tp)
1596 {
1597         int limit = 100;
1598
1599         while (limit--) {
1600                 u32 tmp32;
1601
1602                 if (!tg3_readphy(tp, 0x16, &tmp32)) {
1603                         if ((tmp32 & 0x1000) == 0)
1604                                 break;
1605                 }
1606         }
1607         if (limit < 0)
1608                 return -EBUSY;
1609
1610         return 0;
1611 }
1612
1613 static int tg3_phy_write_and_check_testpat(struct tg3 *tp, int *resetp)
1614 {
1615         static const u32 test_pat[4][6] = {
1616         { 0x00005555, 0x00000005, 0x00002aaa, 0x0000000a, 0x00003456, 0x00000003 },
1617         { 0x00002aaa, 0x0000000a, 0x00003333, 0x00000003, 0x0000789a, 0x00000005 },
1618         { 0x00005a5a, 0x00000005, 0x00002a6a, 0x0000000a, 0x00001bcd, 0x00000003 },
1619         { 0x00002a5a, 0x0000000a, 0x000033c3, 0x00000003, 0x00002ef1, 0x00000005 }
1620         };
1621         int chan;
1622
1623         for (chan = 0; chan < 4; chan++) {
1624                 int i;
1625
1626                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
1627                              (chan * 0x2000) | 0x0200);
1628                 tg3_writephy(tp, 0x16, 0x0002);
1629
1630                 for (i = 0; i < 6; i++)
1631                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT,
1632                                      test_pat[chan][i]);
1633
1634                 tg3_writephy(tp, 0x16, 0x0202);
1635                 if (tg3_wait_macro_done(tp)) {
1636                         *resetp = 1;
1637                         return -EBUSY;
1638                 }
1639
1640                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
1641                              (chan * 0x2000) | 0x0200);
1642                 tg3_writephy(tp, 0x16, 0x0082);
1643                 if (tg3_wait_macro_done(tp)) {
1644                         *resetp = 1;
1645                         return -EBUSY;
1646                 }
1647
1648                 tg3_writephy(tp, 0x16, 0x0802);
1649                 if (tg3_wait_macro_done(tp)) {
1650                         *resetp = 1;
1651                         return -EBUSY;
1652                 }
1653
1654                 for (i = 0; i < 6; i += 2) {
1655                         u32 low, high;
1656
1657                         if (tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &low) ||
1658                             tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &high) ||
1659                             tg3_wait_macro_done(tp)) {
1660                                 *resetp = 1;
1661                                 return -EBUSY;
1662                         }
1663                         low &= 0x7fff;
1664                         high &= 0x000f;
1665                         if (low != test_pat[chan][i] ||
1666                             high != test_pat[chan][i+1]) {
1667                                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000b);
1668                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4001);
1669                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4005);
1670
1671                                 return -EBUSY;
1672                         }
1673                 }
1674         }
1675
1676         return 0;
1677 }
1678
1679 static int tg3_phy_reset_chanpat(struct tg3 *tp)
1680 {
1681         int chan;
1682
1683         for (chan = 0; chan < 4; chan++) {
1684                 int i;
1685
1686                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
1687                              (chan * 0x2000) | 0x0200);
1688                 tg3_writephy(tp, 0x16, 0x0002);
1689                 for (i = 0; i < 6; i++)
1690                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x000);
1691                 tg3_writephy(tp, 0x16, 0x0202);
1692                 if (tg3_wait_macro_done(tp))
1693                         return -EBUSY;
1694         }
1695
1696         return 0;
1697 }
1698
1699 static int tg3_phy_reset_5703_4_5(struct tg3 *tp)
1700 {
1701         u32 reg32, phy9_orig;
1702         int retries, do_phy_reset, err;
1703
1704         retries = 10;
1705         do_phy_reset = 1;
1706         do {
1707                 if (do_phy_reset) {
1708                         err = tg3_bmcr_reset(tp);
1709                         if (err)
1710                                 return err;
1711                         do_phy_reset = 0;
1712                 }
1713
1714                 /* Disable transmitter and interrupt.  */
1715                 if (tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32))
1716                         continue;
1717
1718                 reg32 |= 0x3000;
1719                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
1720
1721                 /* Set full-duplex, 1000 mbps.  */
1722                 tg3_writephy(tp, MII_BMCR,
1723                              BMCR_FULLDPLX | TG3_BMCR_SPEED1000);
1724
1725                 /* Set to master mode.  */
1726                 if (tg3_readphy(tp, MII_TG3_CTRL, &phy9_orig))
1727                         continue;
1728
1729                 tg3_writephy(tp, MII_TG3_CTRL,
1730                              (MII_TG3_CTRL_AS_MASTER |
1731                               MII_TG3_CTRL_ENABLE_AS_MASTER));
1732
1733                 /* Enable SM_DSP_CLOCK and 6dB.  */
1734                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1735
1736                 /* Block the PHY control access.  */
1737                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
1738                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0800);
1739
1740                 err = tg3_phy_write_and_check_testpat(tp, &do_phy_reset);
1741                 if (!err)
1742                         break;
1743         } while (--retries);
1744
1745         err = tg3_phy_reset_chanpat(tp);
1746         if (err)
1747                 return err;
1748
1749         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
1750         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0000);
1751
1752         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8200);
1753         tg3_writephy(tp, 0x16, 0x0000);
1754
1755         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1756             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
1757                 /* Set Extended packet length bit for jumbo frames */
1758                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4400);
1759         }
1760         else {
1761                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1762         }
1763
1764         tg3_writephy(tp, MII_TG3_CTRL, phy9_orig);
1765
1766         if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32)) {
1767                 reg32 &= ~0x3000;
1768                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
1769         } else if (!err)
1770                 err = -EBUSY;
1771
1772         return err;
1773 }
1774
1775 /* This will reset the tigon3 PHY if there is no valid
1776  * link unless the FORCE argument is non-zero.
1777  */
1778 static int tg3_phy_reset(struct tg3 *tp)
1779 {
1780         u32 cpmuctrl;
1781         u32 phy_status;
1782         int err;
1783
1784         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1785                 u32 val;
1786
1787                 val = tr32(GRC_MISC_CFG);
1788                 tw32_f(GRC_MISC_CFG, val & ~GRC_MISC_CFG_EPHY_IDDQ);
1789                 udelay(40);
1790         }
1791         err  = tg3_readphy(tp, MII_BMSR, &phy_status);
1792         err |= tg3_readphy(tp, MII_BMSR, &phy_status);
1793         if (err != 0)
1794                 return -EBUSY;
1795
1796         if (netif_running(tp->dev) && netif_carrier_ok(tp->dev)) {
1797                 netif_carrier_off(tp->dev);
1798                 tg3_link_report(tp);
1799         }
1800
1801         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1802             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1803             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
1804                 err = tg3_phy_reset_5703_4_5(tp);
1805                 if (err)
1806                         return err;
1807                 goto out;
1808         }
1809
1810         cpmuctrl = 0;
1811         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 &&
1812             GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5784_AX) {
1813                 cpmuctrl = tr32(TG3_CPMU_CTRL);
1814                 if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY)
1815                         tw32(TG3_CPMU_CTRL,
1816                              cpmuctrl & ~CPMU_CTRL_GPHY_10MB_RXONLY);
1817         }
1818
1819         err = tg3_bmcr_reset(tp);
1820         if (err)
1821                 return err;
1822
1823         if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY) {
1824                 u32 phy;
1825
1826                 phy = MII_TG3_DSP_EXP8_AEDW | MII_TG3_DSP_EXP8_REJ2MHz;
1827                 tg3_phydsp_write(tp, MII_TG3_DSP_EXP8, phy);
1828
1829                 tw32(TG3_CPMU_CTRL, cpmuctrl);
1830         }
1831
1832         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX ||
1833             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5761_AX) {
1834                 u32 val;
1835
1836                 val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
1837                 if ((val & CPMU_LSPD_1000MB_MACCLK_MASK) ==
1838                     CPMU_LSPD_1000MB_MACCLK_12_5) {
1839                         val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
1840                         udelay(40);
1841                         tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
1842                 }
1843         }
1844
1845         tg3_phy_apply_otp(tp);
1846
1847         if (tp->tg3_flags3 & TG3_FLG3_PHY_ENABLE_APD)
1848                 tg3_phy_toggle_apd(tp, true);
1849         else
1850                 tg3_phy_toggle_apd(tp, false);
1851
1852 out:
1853         if (tp->tg3_flags2 & TG3_FLG2_PHY_ADC_BUG) {
1854                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1855                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1856                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x2aaa);
1857                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1858                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0323);
1859                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1860         }
1861         if (tp->tg3_flags2 & TG3_FLG2_PHY_5704_A0_BUG) {
1862                 tg3_writephy(tp, 0x1c, 0x8d68);
1863                 tg3_writephy(tp, 0x1c, 0x8d68);
1864         }
1865         if (tp->tg3_flags2 & TG3_FLG2_PHY_BER_BUG) {
1866                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1867                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1868                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x310b);
1869                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1870                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x9506);
1871                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x401f);
1872                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x14e2);
1873                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1874         }
1875         else if (tp->tg3_flags2 & TG3_FLG2_PHY_JITTER_BUG) {
1876                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1877                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1878                 if (tp->tg3_flags2 & TG3_FLG2_PHY_ADJUST_TRIM) {
1879                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x110b);
1880                         tg3_writephy(tp, MII_TG3_TEST1,
1881                                      MII_TG3_TEST1_TRIM_EN | 0x4);
1882                 } else
1883                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x010b);
1884                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1885         }
1886         /* Set Extended packet length bit (bit 14) on all chips that */
1887         /* support jumbo frames */
1888         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
1889                 /* Cannot do read-modify-write on 5401 */
1890                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1891         } else if (tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) {
1892                 u32 phy_reg;
1893
1894                 /* Set bit 14 with read-modify-write to preserve other bits */
1895                 if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0007) &&
1896                     !tg3_readphy(tp, MII_TG3_AUX_CTRL, &phy_reg))
1897                         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy_reg | 0x4000);
1898         }
1899
1900         /* Set phy register 0x10 bit 0 to high fifo elasticity to support
1901          * jumbo frames transmission.
1902          */
1903         if (tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) {
1904                 u32 phy_reg;
1905
1906                 if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &phy_reg))
1907                     tg3_writephy(tp, MII_TG3_EXT_CTRL,
1908                                  phy_reg | MII_TG3_EXT_CTRL_FIFO_ELASTIC);
1909         }
1910
1911         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1912                 /* adjust output voltage */
1913                 tg3_writephy(tp, MII_TG3_EPHY_PTEST, 0x12);
1914         }
1915
1916         tg3_phy_toggle_automdix(tp, 1);
1917         tg3_phy_set_wirespeed(tp);
1918         return 0;
1919 }
1920
1921 static void tg3_frob_aux_power(struct tg3 *tp)
1922 {
1923         struct tg3 *tp_peer = tp;
1924
1925         if ((tp->tg3_flags2 & TG3_FLG2_IS_NIC) == 0)
1926                 return;
1927
1928         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
1929             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714)) {
1930                 struct net_device *dev_peer;
1931
1932                 dev_peer = pci_get_drvdata(tp->pdev_peer);
1933                 /* remove_one() may have been run on the peer. */
1934                 if (!dev_peer)
1935                         tp_peer = tp;
1936                 else
1937                         tp_peer = netdev_priv(dev_peer);
1938         }
1939
1940         if ((tp->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
1941             (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0 ||
1942             (tp_peer->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
1943             (tp_peer->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0) {
1944                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1945                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1946                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1947                                     (GRC_LCLCTRL_GPIO_OE0 |
1948                                      GRC_LCLCTRL_GPIO_OE1 |
1949                                      GRC_LCLCTRL_GPIO_OE2 |
1950                                      GRC_LCLCTRL_GPIO_OUTPUT0 |
1951                                      GRC_LCLCTRL_GPIO_OUTPUT1),
1952                                     100);
1953                 } else if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761) {
1954                         /* The 5761 non-e device swaps GPIO 0 and GPIO 2. */
1955                         u32 grc_local_ctrl = GRC_LCLCTRL_GPIO_OE0 |
1956                                              GRC_LCLCTRL_GPIO_OE1 |
1957                                              GRC_LCLCTRL_GPIO_OE2 |
1958                                              GRC_LCLCTRL_GPIO_OUTPUT0 |
1959                                              GRC_LCLCTRL_GPIO_OUTPUT1 |
1960                                              tp->grc_local_ctrl;
1961                         tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl, 100);
1962
1963                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT2;
1964                         tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl, 100);
1965
1966                         grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT0;
1967                         tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl, 100);
1968                 } else {
1969                         u32 no_gpio2;
1970                         u32 grc_local_ctrl = 0;
1971
1972                         if (tp_peer != tp &&
1973                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
1974                                 return;
1975
1976                         /* Workaround to prevent overdrawing Amps. */
1977                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
1978                             ASIC_REV_5714) {
1979                                 grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
1980                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1981                                             grc_local_ctrl, 100);
1982                         }
1983
1984                         /* On 5753 and variants, GPIO2 cannot be used. */
1985                         no_gpio2 = tp->nic_sram_data_cfg &
1986                                     NIC_SRAM_DATA_CFG_NO_GPIO2;
1987
1988                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
1989                                          GRC_LCLCTRL_GPIO_OE1 |
1990                                          GRC_LCLCTRL_GPIO_OE2 |
1991                                          GRC_LCLCTRL_GPIO_OUTPUT1 |
1992                                          GRC_LCLCTRL_GPIO_OUTPUT2;
1993                         if (no_gpio2) {
1994                                 grc_local_ctrl &= ~(GRC_LCLCTRL_GPIO_OE2 |
1995                                                     GRC_LCLCTRL_GPIO_OUTPUT2);
1996                         }
1997                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1998                                                     grc_local_ctrl, 100);
1999
2000                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT0;
2001
2002                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2003                                                     grc_local_ctrl, 100);
2004
2005                         if (!no_gpio2) {
2006                                 grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT2;
2007                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2008                                             grc_local_ctrl, 100);
2009                         }
2010                 }
2011         } else {
2012                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
2013                     GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
2014                         if (tp_peer != tp &&
2015                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
2016                                 return;
2017
2018                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2019                                     (GRC_LCLCTRL_GPIO_OE1 |
2020                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
2021
2022                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2023                                     GRC_LCLCTRL_GPIO_OE1, 100);
2024
2025                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2026                                     (GRC_LCLCTRL_GPIO_OE1 |
2027                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
2028                 }
2029         }
2030 }
2031
2032 static int tg3_5700_link_polarity(struct tg3 *tp, u32 speed)
2033 {
2034         if (tp->led_ctrl == LED_CTRL_MODE_PHY_2)
2035                 return 1;
2036         else if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411) {
2037                 if (speed != SPEED_10)
2038                         return 1;
2039         } else if (speed == SPEED_10)
2040                 return 1;
2041
2042         return 0;
2043 }
2044
2045 static int tg3_setup_phy(struct tg3 *, int);
2046
2047 #define RESET_KIND_SHUTDOWN     0
2048 #define RESET_KIND_INIT         1
2049 #define RESET_KIND_SUSPEND      2
2050
2051 static void tg3_write_sig_post_reset(struct tg3 *, int);
2052 static int tg3_halt_cpu(struct tg3 *, u32);
2053
2054 static void tg3_power_down_phy(struct tg3 *tp, bool do_low_power)
2055 {
2056         u32 val;
2057
2058         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
2059                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
2060                         u32 sg_dig_ctrl = tr32(SG_DIG_CTRL);
2061                         u32 serdes_cfg = tr32(MAC_SERDES_CFG);
2062
2063                         sg_dig_ctrl |=
2064                                 SG_DIG_USING_HW_AUTONEG | SG_DIG_SOFT_RESET;
2065                         tw32(SG_DIG_CTRL, sg_dig_ctrl);
2066                         tw32(MAC_SERDES_CFG, serdes_cfg | (1 << 15));
2067                 }
2068                 return;
2069         }
2070
2071         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2072                 tg3_bmcr_reset(tp);
2073                 val = tr32(GRC_MISC_CFG);
2074                 tw32_f(GRC_MISC_CFG, val | GRC_MISC_CFG_EPHY_IDDQ);
2075                 udelay(40);
2076                 return;
2077         } else if (do_low_power) {
2078                 tg3_writephy(tp, MII_TG3_EXT_CTRL,
2079                              MII_TG3_EXT_CTRL_FORCE_LED_OFF);
2080
2081                 tg3_writephy(tp, MII_TG3_AUX_CTRL,
2082                              MII_TG3_AUXCTL_SHDWSEL_PWRCTL |
2083                              MII_TG3_AUXCTL_PCTL_100TX_LPWR |
2084                              MII_TG3_AUXCTL_PCTL_SPR_ISOLATE |
2085                              MII_TG3_AUXCTL_PCTL_VREG_11V);
2086         }
2087
2088         /* The PHY should not be powered down on some chips because
2089          * of bugs.
2090          */
2091         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2092             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
2093             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 &&
2094              (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)))
2095                 return;
2096
2097         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX ||
2098             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5761_AX) {
2099                 val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
2100                 val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
2101                 val |= CPMU_LSPD_1000MB_MACCLK_12_5;
2102                 tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
2103         }
2104
2105         tg3_writephy(tp, MII_BMCR, BMCR_PDOWN);
2106 }
2107
2108 /* tp->lock is held. */
2109 static int tg3_nvram_lock(struct tg3 *tp)
2110 {
2111         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
2112                 int i;
2113
2114                 if (tp->nvram_lock_cnt == 0) {
2115                         tw32(NVRAM_SWARB, SWARB_REQ_SET1);
2116                         for (i = 0; i < 8000; i++) {
2117                                 if (tr32(NVRAM_SWARB) & SWARB_GNT1)
2118                                         break;
2119                                 udelay(20);
2120                         }
2121                         if (i == 8000) {
2122                                 tw32(NVRAM_SWARB, SWARB_REQ_CLR1);
2123                                 return -ENODEV;
2124                         }
2125                 }
2126                 tp->nvram_lock_cnt++;
2127         }
2128         return 0;
2129 }
2130
2131 /* tp->lock is held. */
2132 static void tg3_nvram_unlock(struct tg3 *tp)
2133 {
2134         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
2135                 if (tp->nvram_lock_cnt > 0)
2136                         tp->nvram_lock_cnt--;
2137                 if (tp->nvram_lock_cnt == 0)
2138                         tw32_f(NVRAM_SWARB, SWARB_REQ_CLR1);
2139         }
2140 }
2141
2142 /* tp->lock is held. */
2143 static void tg3_enable_nvram_access(struct tg3 *tp)
2144 {
2145         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
2146             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
2147                 u32 nvaccess = tr32(NVRAM_ACCESS);
2148
2149                 tw32(NVRAM_ACCESS, nvaccess | ACCESS_ENABLE);
2150         }
2151 }
2152
2153 /* tp->lock is held. */
2154 static void tg3_disable_nvram_access(struct tg3 *tp)
2155 {
2156         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
2157             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
2158                 u32 nvaccess = tr32(NVRAM_ACCESS);
2159
2160                 tw32(NVRAM_ACCESS, nvaccess & ~ACCESS_ENABLE);
2161         }
2162 }
2163
2164 static int tg3_nvram_read_using_eeprom(struct tg3 *tp,
2165                                         u32 offset, u32 *val)
2166 {
2167         u32 tmp;
2168         int i;
2169
2170         if (offset > EEPROM_ADDR_ADDR_MASK || (offset % 4) != 0)
2171                 return -EINVAL;
2172
2173         tmp = tr32(GRC_EEPROM_ADDR) & ~(EEPROM_ADDR_ADDR_MASK |
2174                                         EEPROM_ADDR_DEVID_MASK |
2175                                         EEPROM_ADDR_READ);
2176         tw32(GRC_EEPROM_ADDR,
2177              tmp |
2178              (0 << EEPROM_ADDR_DEVID_SHIFT) |
2179              ((offset << EEPROM_ADDR_ADDR_SHIFT) &
2180               EEPROM_ADDR_ADDR_MASK) |
2181              EEPROM_ADDR_READ | EEPROM_ADDR_START);
2182
2183         for (i = 0; i < 1000; i++) {
2184                 tmp = tr32(GRC_EEPROM_ADDR);
2185
2186                 if (tmp & EEPROM_ADDR_COMPLETE)
2187                         break;
2188                 msleep(1);
2189         }
2190         if (!(tmp & EEPROM_ADDR_COMPLETE))
2191                 return -EBUSY;
2192
2193         tmp = tr32(GRC_EEPROM_DATA);
2194
2195         /*
2196          * The data will always be opposite the native endian
2197          * format.  Perform a blind byteswap to compensate.
2198          */
2199         *val = swab32(tmp);
2200
2201         return 0;
2202 }
2203
2204 #define NVRAM_CMD_TIMEOUT 10000
2205
2206 static int tg3_nvram_exec_cmd(struct tg3 *tp, u32 nvram_cmd)
2207 {
2208         int i;
2209
2210         tw32(NVRAM_CMD, nvram_cmd);
2211         for (i = 0; i < NVRAM_CMD_TIMEOUT; i++) {
2212                 udelay(10);
2213                 if (tr32(NVRAM_CMD) & NVRAM_CMD_DONE) {
2214                         udelay(10);
2215                         break;
2216                 }
2217         }
2218
2219         if (i == NVRAM_CMD_TIMEOUT)
2220                 return -EBUSY;
2221
2222         return 0;
2223 }
2224
2225 static u32 tg3_nvram_phys_addr(struct tg3 *tp, u32 addr)
2226 {
2227         if ((tp->tg3_flags & TG3_FLAG_NVRAM) &&
2228             (tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) &&
2229             (tp->tg3_flags2 & TG3_FLG2_FLASH) &&
2230            !(tp->tg3_flags3 & TG3_FLG3_NO_NVRAM_ADDR_TRANS) &&
2231             (tp->nvram_jedecnum == JEDEC_ATMEL))
2232
2233                 addr = ((addr / tp->nvram_pagesize) <<
2234                         ATMEL_AT45DB0X1B_PAGE_POS) +
2235                        (addr % tp->nvram_pagesize);
2236
2237         return addr;
2238 }
2239
2240 static u32 tg3_nvram_logical_addr(struct tg3 *tp, u32 addr)
2241 {
2242         if ((tp->tg3_flags & TG3_FLAG_NVRAM) &&
2243             (tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) &&
2244             (tp->tg3_flags2 & TG3_FLG2_FLASH) &&
2245            !(tp->tg3_flags3 & TG3_FLG3_NO_NVRAM_ADDR_TRANS) &&
2246             (tp->nvram_jedecnum == JEDEC_ATMEL))
2247
2248                 addr = ((addr >> ATMEL_AT45DB0X1B_PAGE_POS) *
2249                         tp->nvram_pagesize) +
2250                        (addr & ((1 << ATMEL_AT45DB0X1B_PAGE_POS) - 1));
2251
2252         return addr;
2253 }
2254
2255 /* NOTE: Data read in from NVRAM is byteswapped according to
2256  * the byteswapping settings for all other register accesses.
2257  * tg3 devices are BE devices, so on a BE machine, the data
2258  * returned will be exactly as it is seen in NVRAM.  On a LE
2259  * machine, the 32-bit value will be byteswapped.
2260  */
2261 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val)
2262 {
2263         int ret;
2264
2265         if (!(tp->tg3_flags & TG3_FLAG_NVRAM))
2266                 return tg3_nvram_read_using_eeprom(tp, offset, val);
2267
2268         offset = tg3_nvram_phys_addr(tp, offset);
2269
2270         if (offset > NVRAM_ADDR_MSK)
2271                 return -EINVAL;
2272
2273         ret = tg3_nvram_lock(tp);
2274         if (ret)
2275                 return ret;
2276
2277         tg3_enable_nvram_access(tp);
2278
2279         tw32(NVRAM_ADDR, offset);
2280         ret = tg3_nvram_exec_cmd(tp, NVRAM_CMD_RD | NVRAM_CMD_GO |
2281                 NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_DONE);
2282
2283         if (ret == 0)
2284                 *val = tr32(NVRAM_RDDATA);
2285
2286         tg3_disable_nvram_access(tp);
2287
2288         tg3_nvram_unlock(tp);
2289
2290         return ret;
2291 }
2292
2293 /* Ensures NVRAM data is in bytestream format. */
2294 static int tg3_nvram_read_be32(struct tg3 *tp, u32 offset, __be32 *val)
2295 {
2296         u32 v;
2297         int res = tg3_nvram_read(tp, offset, &v);
2298         if (!res)
2299                 *val = cpu_to_be32(v);
2300         return res;
2301 }
2302
2303 /* tp->lock is held. */
2304 static void __tg3_set_mac_addr(struct tg3 *tp, int skip_mac_1)
2305 {
2306         u32 addr_high, addr_low;
2307         int i;
2308
2309         addr_high = ((tp->dev->dev_addr[0] << 8) |
2310                      tp->dev->dev_addr[1]);
2311         addr_low = ((tp->dev->dev_addr[2] << 24) |
2312                     (tp->dev->dev_addr[3] << 16) |
2313                     (tp->dev->dev_addr[4] <<  8) |
2314                     (tp->dev->dev_addr[5] <<  0));
2315         for (i = 0; i < 4; i++) {
2316                 if (i == 1 && skip_mac_1)
2317                         continue;
2318                 tw32(MAC_ADDR_0_HIGH + (i * 8), addr_high);
2319                 tw32(MAC_ADDR_0_LOW + (i * 8), addr_low);
2320         }
2321
2322         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
2323             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
2324                 for (i = 0; i < 12; i++) {
2325                         tw32(MAC_EXTADDR_0_HIGH + (i * 8), addr_high);
2326                         tw32(MAC_EXTADDR_0_LOW + (i * 8), addr_low);
2327                 }
2328         }
2329
2330         addr_high = (tp->dev->dev_addr[0] +
2331                      tp->dev->dev_addr[1] +
2332                      tp->dev->dev_addr[2] +
2333                      tp->dev->dev_addr[3] +
2334                      tp->dev->dev_addr[4] +
2335                      tp->dev->dev_addr[5]) &
2336                 TX_BACKOFF_SEED_MASK;
2337         tw32(MAC_TX_BACKOFF_SEED, addr_high);
2338 }
2339
2340 static int tg3_set_power_state(struct tg3 *tp, pci_power_t state)
2341 {
2342         u32 misc_host_ctrl;
2343         bool device_should_wake, do_low_power;
2344
2345         /* Make sure register accesses (indirect or otherwise)
2346          * will function correctly.
2347          */
2348         pci_write_config_dword(tp->pdev,
2349                                TG3PCI_MISC_HOST_CTRL,
2350                                tp->misc_host_ctrl);
2351
2352         switch (state) {
2353         case PCI_D0:
2354                 pci_enable_wake(tp->pdev, state, false);
2355                 pci_set_power_state(tp->pdev, PCI_D0);
2356
2357                 /* Switch out of Vaux if it is a NIC */
2358                 if (tp->tg3_flags2 & TG3_FLG2_IS_NIC)
2359                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl, 100);
2360
2361                 return 0;
2362
2363         case PCI_D1:
2364         case PCI_D2:
2365         case PCI_D3hot:
2366                 break;
2367
2368         default:
2369                 printk(KERN_ERR PFX "%s: Invalid power state (D%d) requested\n",
2370                         tp->dev->name, state);
2371                 return -EINVAL;
2372         }
2373
2374         /* Restore the CLKREQ setting. */
2375         if (tp->tg3_flags3 & TG3_FLG3_CLKREQ_BUG) {
2376                 u16 lnkctl;
2377
2378                 pci_read_config_word(tp->pdev,
2379                                      tp->pcie_cap + PCI_EXP_LNKCTL,
2380                                      &lnkctl);
2381                 lnkctl |= PCI_EXP_LNKCTL_CLKREQ_EN;
2382                 pci_write_config_word(tp->pdev,
2383                                       tp->pcie_cap + PCI_EXP_LNKCTL,
2384                                       lnkctl);
2385         }
2386
2387         misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
2388         tw32(TG3PCI_MISC_HOST_CTRL,
2389              misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT);
2390
2391         device_should_wake = pci_pme_capable(tp->pdev, state) &&
2392                              device_may_wakeup(&tp->pdev->dev) &&
2393                              (tp->tg3_flags & TG3_FLAG_WOL_ENABLE);
2394
2395         if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) {
2396                 do_low_power = false;
2397                 if ((tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED) &&
2398                     !tp->link_config.phy_is_low_power) {
2399                         struct phy_device *phydev;
2400                         u32 phyid, advertising;
2401
2402                         phydev = tp->mdio_bus->phy_map[PHY_ADDR];
2403
2404                         tp->link_config.phy_is_low_power = 1;
2405
2406                         tp->link_config.orig_speed = phydev->speed;
2407                         tp->link_config.orig_duplex = phydev->duplex;
2408                         tp->link_config.orig_autoneg = phydev->autoneg;
2409                         tp->link_config.orig_advertising = phydev->advertising;
2410
2411                         advertising = ADVERTISED_TP |
2412                                       ADVERTISED_Pause |
2413                                       ADVERTISED_Autoneg |
2414                                       ADVERTISED_10baseT_Half;
2415
2416                         if ((tp->tg3_flags & TG3_FLAG_ENABLE_ASF) ||
2417                             device_should_wake) {
2418                                 if (tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB)
2419                                         advertising |=
2420                                                 ADVERTISED_100baseT_Half |
2421                                                 ADVERTISED_100baseT_Full |
2422                                                 ADVERTISED_10baseT_Full;
2423                                 else
2424                                         advertising |= ADVERTISED_10baseT_Full;
2425                         }
2426
2427                         phydev->advertising = advertising;
2428
2429                         phy_start_aneg(phydev);
2430
2431                         phyid = phydev->drv->phy_id & phydev->drv->phy_id_mask;
2432                         if (phyid != TG3_PHY_ID_BCMAC131) {
2433                                 phyid &= TG3_PHY_OUI_MASK;
2434                                 if (phyid == TG3_PHY_OUI_1 ||
2435                                     phyid == TG3_PHY_OUI_2 ||
2436                                     phyid == TG3_PHY_OUI_3)
2437                                         do_low_power = true;
2438                         }
2439                 }
2440         } else {
2441                 do_low_power = true;
2442
2443                 if (tp->link_config.phy_is_low_power == 0) {
2444                         tp->link_config.phy_is_low_power = 1;
2445                         tp->link_config.orig_speed = tp->link_config.speed;
2446                         tp->link_config.orig_duplex = tp->link_config.duplex;
2447                         tp->link_config.orig_autoneg = tp->link_config.autoneg;
2448                 }
2449
2450                 if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)) {
2451                         tp->link_config.speed = SPEED_10;
2452                         tp->link_config.duplex = DUPLEX_HALF;
2453                         tp->link_config.autoneg = AUTONEG_ENABLE;
2454                         tg3_setup_phy(tp, 0);
2455                 }
2456         }
2457
2458         __tg3_set_mac_addr(tp, 0);
2459
2460         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2461                 u32 val;
2462
2463                 val = tr32(GRC_VCPU_EXT_CTRL);
2464                 tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_DISABLE_WOL);
2465         } else if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
2466                 int i;
2467                 u32 val;
2468
2469                 for (i = 0; i < 200; i++) {
2470                         tg3_read_mem(tp, NIC_SRAM_FW_ASF_STATUS_MBOX, &val);
2471                         if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
2472                                 break;
2473                         msleep(1);
2474                 }
2475         }
2476         if (tp->tg3_flags & TG3_FLAG_WOL_CAP)
2477                 tg3_write_mem(tp, NIC_SRAM_WOL_MBOX, WOL_SIGNATURE |
2478                                                      WOL_DRV_STATE_SHUTDOWN |
2479                                                      WOL_DRV_WOL |
2480                                                      WOL_SET_MAGIC_PKT);
2481
2482         if (device_should_wake) {
2483                 u32 mac_mode;
2484
2485                 if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
2486                         if (do_low_power) {
2487                                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x5a);
2488                                 udelay(40);
2489                         }
2490
2491                         if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
2492                                 mac_mode = MAC_MODE_PORT_MODE_GMII;
2493                         else
2494                                 mac_mode = MAC_MODE_PORT_MODE_MII;
2495
2496                         mac_mode |= tp->mac_mode & MAC_MODE_LINK_POLARITY;
2497                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
2498                             ASIC_REV_5700) {
2499                                 u32 speed = (tp->tg3_flags &
2500                                              TG3_FLAG_WOL_SPEED_100MB) ?
2501                                              SPEED_100 : SPEED_10;
2502                                 if (tg3_5700_link_polarity(tp, speed))
2503                                         mac_mode |= MAC_MODE_LINK_POLARITY;
2504                                 else
2505                                         mac_mode &= ~MAC_MODE_LINK_POLARITY;
2506                         }
2507                 } else {
2508                         mac_mode = MAC_MODE_PORT_MODE_TBI;
2509                 }
2510
2511                 if (!(tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
2512                         tw32(MAC_LED_CTRL, tp->led_ctrl);
2513
2514                 mac_mode |= MAC_MODE_MAGIC_PKT_ENABLE;
2515                 if (((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) &&
2516                     !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) &&
2517                     ((tp->tg3_flags & TG3_FLAG_ENABLE_ASF) ||
2518                      (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)))
2519                         mac_mode |= MAC_MODE_KEEP_FRAME_IN_WOL;
2520
2521                 if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) {
2522                         mac_mode |= tp->mac_mode &
2523                                     (MAC_MODE_APE_TX_EN | MAC_MODE_APE_RX_EN);
2524                         if (mac_mode & MAC_MODE_APE_TX_EN)
2525                                 mac_mode |= MAC_MODE_TDE_ENABLE;
2526                 }
2527
2528                 tw32_f(MAC_MODE, mac_mode);
2529                 udelay(100);
2530
2531                 tw32_f(MAC_RX_MODE, RX_MODE_ENABLE);
2532                 udelay(10);
2533         }
2534
2535         if (!(tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB) &&
2536             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2537              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
2538                 u32 base_val;
2539
2540                 base_val = tp->pci_clock_ctrl;
2541                 base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
2542                              CLOCK_CTRL_TXCLK_DISABLE);
2543
2544                 tw32_wait_f(TG3PCI_CLOCK_CTRL, base_val | CLOCK_CTRL_ALTCLK |
2545                             CLOCK_CTRL_PWRDOWN_PLL133, 40);
2546         } else if ((tp->tg3_flags2 & TG3_FLG2_5780_CLASS) ||
2547                    (tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) ||
2548                    (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)) {
2549                 /* do nothing */
2550         } else if (!((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
2551                      (tp->tg3_flags & TG3_FLAG_ENABLE_ASF))) {
2552                 u32 newbits1, newbits2;
2553
2554                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2555                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2556                         newbits1 = (CLOCK_CTRL_RXCLK_DISABLE |
2557                                     CLOCK_CTRL_TXCLK_DISABLE |
2558                                     CLOCK_CTRL_ALTCLK);
2559                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
2560                 } else if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
2561                         newbits1 = CLOCK_CTRL_625_CORE;
2562                         newbits2 = newbits1 | CLOCK_CTRL_ALTCLK;
2563                 } else {
2564                         newbits1 = CLOCK_CTRL_ALTCLK;
2565                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
2566                 }
2567
2568                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits1,
2569                             40);
2570
2571                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits2,
2572                             40);
2573
2574                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
2575                         u32 newbits3;
2576
2577                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2578                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2579                                 newbits3 = (CLOCK_CTRL_RXCLK_DISABLE |
2580                                             CLOCK_CTRL_TXCLK_DISABLE |
2581                                             CLOCK_CTRL_44MHZ_CORE);
2582                         } else {
2583                                 newbits3 = CLOCK_CTRL_44MHZ_CORE;
2584                         }
2585
2586                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
2587                                     tp->pci_clock_ctrl | newbits3, 40);
2588                 }
2589         }
2590
2591         if (!(device_should_wake) &&
2592             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF))
2593                 tg3_power_down_phy(tp, do_low_power);
2594
2595         tg3_frob_aux_power(tp);
2596
2597         /* Workaround for unstable PLL clock */
2598         if ((GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX) ||
2599             (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX)) {
2600                 u32 val = tr32(0x7d00);
2601
2602                 val &= ~((1 << 16) | (1 << 4) | (1 << 2) | (1 << 1) | 1);
2603                 tw32(0x7d00, val);
2604                 if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
2605                         int err;
2606
2607                         err = tg3_nvram_lock(tp);
2608                         tg3_halt_cpu(tp, RX_CPU_BASE);
2609                         if (!err)
2610                                 tg3_nvram_unlock(tp);
2611                 }
2612         }
2613
2614         tg3_write_sig_post_reset(tp, RESET_KIND_SHUTDOWN);
2615
2616         if (device_should_wake)
2617                 pci_enable_wake(tp->pdev, state, true);
2618
2619         /* Finally, set the new power state. */
2620         pci_set_power_state(tp->pdev, state);
2621
2622         return 0;
2623 }
2624
2625 static void tg3_aux_stat_to_speed_duplex(struct tg3 *tp, u32 val, u16 *speed, u8 *duplex)
2626 {
2627         switch (val & MII_TG3_AUX_STAT_SPDMASK) {
2628         case MII_TG3_AUX_STAT_10HALF:
2629                 *speed = SPEED_10;
2630                 *duplex = DUPLEX_HALF;
2631                 break;
2632
2633         case MII_TG3_AUX_STAT_10FULL:
2634                 *speed = SPEED_10;
2635                 *duplex = DUPLEX_FULL;
2636                 break;
2637
2638         case MII_TG3_AUX_STAT_100HALF:
2639                 *speed = SPEED_100;
2640                 *duplex = DUPLEX_HALF;
2641                 break;
2642
2643         case MII_TG3_AUX_STAT_100FULL:
2644                 *speed = SPEED_100;
2645                 *duplex = DUPLEX_FULL;
2646                 break;
2647
2648         case MII_TG3_AUX_STAT_1000HALF:
2649                 *speed = SPEED_1000;
2650                 *duplex = DUPLEX_HALF;
2651                 break;
2652
2653         case MII_TG3_AUX_STAT_1000FULL:
2654                 *speed = SPEED_1000;
2655                 *duplex = DUPLEX_FULL;
2656                 break;
2657
2658         default:
2659                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2660                         *speed = (val & MII_TG3_AUX_STAT_100) ? SPEED_100 :
2661                                  SPEED_10;
2662                         *duplex = (val & MII_TG3_AUX_STAT_FULL) ? DUPLEX_FULL :
2663                                   DUPLEX_HALF;
2664                         break;
2665                 }
2666                 *speed = SPEED_INVALID;
2667                 *duplex = DUPLEX_INVALID;
2668                 break;
2669         }
2670 }
2671
2672 static void tg3_phy_copper_begin(struct tg3 *tp)
2673 {
2674         u32 new_adv;
2675         int i;
2676
2677         if (tp->link_config.phy_is_low_power) {
2678                 /* Entering low power mode.  Disable gigabit and
2679                  * 100baseT advertisements.
2680                  */
2681                 tg3_writephy(tp, MII_TG3_CTRL, 0);
2682
2683                 new_adv = (ADVERTISE_10HALF | ADVERTISE_10FULL |
2684                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
2685                 if (tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB)
2686                         new_adv |= (ADVERTISE_100HALF | ADVERTISE_100FULL);
2687
2688                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
2689         } else if (tp->link_config.speed == SPEED_INVALID) {
2690                 if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
2691                         tp->link_config.advertising &=
2692                                 ~(ADVERTISED_1000baseT_Half |
2693                                   ADVERTISED_1000baseT_Full);
2694
2695                 new_adv = ADVERTISE_CSMA;
2696                 if (tp->link_config.advertising & ADVERTISED_10baseT_Half)
2697                         new_adv |= ADVERTISE_10HALF;
2698                 if (tp->link_config.advertising & ADVERTISED_10baseT_Full)
2699                         new_adv |= ADVERTISE_10FULL;
2700                 if (tp->link_config.advertising & ADVERTISED_100baseT_Half)
2701                         new_adv |= ADVERTISE_100HALF;
2702                 if (tp->link_config.advertising & ADVERTISED_100baseT_Full)
2703                         new_adv |= ADVERTISE_100FULL;
2704
2705                 new_adv |= tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
2706
2707                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
2708
2709                 if (tp->link_config.advertising &
2710                     (ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full)) {
2711                         new_adv = 0;
2712                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
2713                                 new_adv |= MII_TG3_CTRL_ADV_1000_HALF;
2714                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
2715                                 new_adv |= MII_TG3_CTRL_ADV_1000_FULL;
2716                         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY) &&
2717                             (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
2718                              tp->pci_chip_rev_id == CHIPREV_ID_5701_B0))
2719                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
2720                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
2721                         tg3_writephy(tp, MII_TG3_CTRL, new_adv);
2722                 } else {
2723                         tg3_writephy(tp, MII_TG3_CTRL, 0);
2724                 }
2725         } else {
2726                 new_adv = tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
2727                 new_adv |= ADVERTISE_CSMA;
2728
2729                 /* Asking for a specific link mode. */
2730                 if (tp->link_config.speed == SPEED_1000) {
2731                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
2732
2733                         if (tp->link_config.duplex == DUPLEX_FULL)
2734                                 new_adv = MII_TG3_CTRL_ADV_1000_FULL;
2735                         else
2736                                 new_adv = MII_TG3_CTRL_ADV_1000_HALF;
2737                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
2738                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
2739                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
2740                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
2741                 } else {
2742                         if (tp->link_config.speed == SPEED_100) {
2743                                 if (tp->link_config.duplex == DUPLEX_FULL)
2744                                         new_adv |= ADVERTISE_100FULL;
2745                                 else
2746                                         new_adv |= ADVERTISE_100HALF;
2747                         } else {
2748                                 if (tp->link_config.duplex == DUPLEX_FULL)
2749                                         new_adv |= ADVERTISE_10FULL;
2750                                 else
2751                                         new_adv |= ADVERTISE_10HALF;
2752                         }
2753                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
2754
2755                         new_adv = 0;
2756                 }
2757
2758                 tg3_writephy(tp, MII_TG3_CTRL, new_adv);
2759         }
2760
2761         if (tp->link_config.autoneg == AUTONEG_DISABLE &&
2762             tp->link_config.speed != SPEED_INVALID) {
2763                 u32 bmcr, orig_bmcr;
2764
2765                 tp->link_config.active_speed = tp->link_config.speed;
2766                 tp->link_config.active_duplex = tp->link_config.duplex;
2767
2768                 bmcr = 0;
2769                 switch (tp->link_config.speed) {
2770                 default:
2771                 case SPEED_10:
2772                         break;
2773
2774                 case SPEED_100:
2775                         bmcr |= BMCR_SPEED100;
2776                         break;
2777
2778                 case SPEED_1000:
2779                         bmcr |= TG3_BMCR_SPEED1000;
2780                         break;
2781                 }
2782
2783                 if (tp->link_config.duplex == DUPLEX_FULL)
2784                         bmcr |= BMCR_FULLDPLX;
2785
2786                 if (!tg3_readphy(tp, MII_BMCR, &orig_bmcr) &&
2787                     (bmcr != orig_bmcr)) {
2788                         tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK);
2789                         for (i = 0; i < 1500; i++) {
2790                                 u32 tmp;
2791
2792                                 udelay(10);
2793                                 if (tg3_readphy(tp, MII_BMSR, &tmp) ||
2794                                     tg3_readphy(tp, MII_BMSR, &tmp))
2795                                         continue;
2796                                 if (!(tmp & BMSR_LSTATUS)) {
2797                                         udelay(40);
2798                                         break;
2799                                 }
2800                         }
2801                         tg3_writephy(tp, MII_BMCR, bmcr);
2802                         udelay(40);
2803                 }
2804         } else {
2805                 tg3_writephy(tp, MII_BMCR,
2806                              BMCR_ANENABLE | BMCR_ANRESTART);
2807         }
2808 }
2809
2810 static int tg3_init_5401phy_dsp(struct tg3 *tp)
2811 {
2812         int err;
2813
2814         /* Turn off tap power management. */
2815         /* Set Extended packet length bit */
2816         err  = tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
2817
2818         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0012);
2819         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1804);
2820
2821         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0013);
2822         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1204);
2823
2824         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
2825         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0132);
2826
2827         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
2828         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0232);
2829
2830         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
2831         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0a20);
2832
2833         udelay(40);
2834
2835         return err;
2836 }
2837
2838 static int tg3_copper_is_advertising_all(struct tg3 *tp, u32 mask)
2839 {
2840         u32 adv_reg, all_mask = 0;
2841
2842         if (mask & ADVERTISED_10baseT_Half)
2843                 all_mask |= ADVERTISE_10HALF;
2844         if (mask & ADVERTISED_10baseT_Full)
2845                 all_mask |= ADVERTISE_10FULL;
2846         if (mask & ADVERTISED_100baseT_Half)
2847                 all_mask |= ADVERTISE_100HALF;
2848         if (mask & ADVERTISED_100baseT_Full)
2849                 all_mask |= ADVERTISE_100FULL;
2850
2851         if (tg3_readphy(tp, MII_ADVERTISE, &adv_reg))
2852                 return 0;
2853
2854         if ((adv_reg & all_mask) != all_mask)
2855                 return 0;
2856         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
2857                 u32 tg3_ctrl;
2858
2859                 all_mask = 0;
2860                 if (mask & ADVERTISED_1000baseT_Half)
2861                         all_mask |= ADVERTISE_1000HALF;
2862                 if (mask & ADVERTISED_1000baseT_Full)
2863                         all_mask |= ADVERTISE_1000FULL;
2864
2865                 if (tg3_readphy(tp, MII_TG3_CTRL, &tg3_ctrl))
2866                         return 0;
2867
2868                 if ((tg3_ctrl & all_mask) != all_mask)
2869                         return 0;
2870         }
2871         return 1;
2872 }
2873
2874 static int tg3_adv_1000T_flowctrl_ok(struct tg3 *tp, u32 *lcladv, u32 *rmtadv)
2875 {
2876         u32 curadv, reqadv;
2877
2878         if (tg3_readphy(tp, MII_ADVERTISE, lcladv))
2879                 return 1;
2880
2881         curadv = *lcladv & (ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM);
2882         reqadv = tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
2883
2884         if (tp->link_config.active_duplex == DUPLEX_FULL) {
2885                 if (curadv != reqadv)
2886                         return 0;
2887
2888                 if (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG)
2889                         tg3_readphy(tp, MII_LPA, rmtadv);
2890         } else {
2891                 /* Reprogram the advertisement register, even if it
2892                  * does not affect the current link.  If the link
2893                  * gets renegotiated in the future, we can save an
2894                  * additional renegotiation cycle by advertising
2895                  * it correctly in the first place.
2896                  */
2897                 if (curadv != reqadv) {
2898                         *lcladv &= ~(ADVERTISE_PAUSE_CAP |
2899                                      ADVERTISE_PAUSE_ASYM);
2900                         tg3_writephy(tp, MII_ADVERTISE, *lcladv | reqadv);
2901                 }
2902         }
2903
2904         return 1;
2905 }
2906
2907 static int tg3_setup_copper_phy(struct tg3 *tp, int force_reset)
2908 {
2909         int current_link_up;
2910         u32 bmsr, dummy;
2911         u32 lcl_adv, rmt_adv;
2912         u16 current_speed;
2913         u8 current_duplex;
2914         int i, err;
2915
2916         tw32(MAC_EVENT, 0);
2917
2918         tw32_f(MAC_STATUS,
2919              (MAC_STATUS_SYNC_CHANGED |
2920               MAC_STATUS_CFG_CHANGED |
2921               MAC_STATUS_MI_COMPLETION |
2922               MAC_STATUS_LNKSTATE_CHANGED));
2923         udelay(40);
2924
2925         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
2926                 tw32_f(MAC_MI_MODE,
2927                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
2928                 udelay(80);
2929         }
2930
2931         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x02);
2932
2933         /* Some third-party PHYs need to be reset on link going
2934          * down.
2935          */
2936         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
2937              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
2938              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
2939             netif_carrier_ok(tp->dev)) {
2940                 tg3_readphy(tp, MII_BMSR, &bmsr);
2941                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
2942                     !(bmsr & BMSR_LSTATUS))
2943                         force_reset = 1;
2944         }
2945         if (force_reset)
2946                 tg3_phy_reset(tp);
2947
2948         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
2949                 tg3_readphy(tp, MII_BMSR, &bmsr);
2950                 if (tg3_readphy(tp, MII_BMSR, &bmsr) ||
2951                     !(tp->tg3_flags & TG3_FLAG_INIT_COMPLETE))
2952                         bmsr = 0;
2953
2954                 if (!(bmsr & BMSR_LSTATUS)) {
2955                         err = tg3_init_5401phy_dsp(tp);
2956                         if (err)
2957                                 return err;
2958
2959                         tg3_readphy(tp, MII_BMSR, &bmsr);
2960                         for (i = 0; i < 1000; i++) {
2961                                 udelay(10);
2962                                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
2963                                     (bmsr & BMSR_LSTATUS)) {
2964                                         udelay(40);
2965                                         break;
2966                                 }
2967                         }
2968
2969                         if ((tp->phy_id & PHY_ID_REV_MASK) == PHY_REV_BCM5401_B0 &&
2970                             !(bmsr & BMSR_LSTATUS) &&
2971                             tp->link_config.active_speed == SPEED_1000) {
2972                                 err = tg3_phy_reset(tp);
2973                                 if (!err)
2974                                         err = tg3_init_5401phy_dsp(tp);
2975                                 if (err)
2976                                         return err;
2977                         }
2978                 }
2979         } else if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
2980                    tp->pci_chip_rev_id == CHIPREV_ID_5701_B0) {
2981                 /* 5701 {A0,B0} CRC bug workaround */
2982                 tg3_writephy(tp, 0x15, 0x0a75);
2983                 tg3_writephy(tp, 0x1c, 0x8c68);
2984                 tg3_writephy(tp, 0x1c, 0x8d68);
2985                 tg3_writephy(tp, 0x1c, 0x8c68);
2986         }
2987
2988         /* Clear pending interrupts... */
2989         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
2990         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
2991
2992         if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT)
2993                 tg3_writephy(tp, MII_TG3_IMASK, ~MII_TG3_INT_LINKCHG);
2994         else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906)
2995                 tg3_writephy(tp, MII_TG3_IMASK, ~0);
2996
2997         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2998             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2999                 if (tp->led_ctrl == LED_CTRL_MODE_PHY_1)
3000                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
3001                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
3002                 else
3003                         tg3_writephy(tp, MII_TG3_EXT_CTRL, 0);
3004         }
3005
3006         current_link_up = 0;
3007         current_speed = SPEED_INVALID;
3008         current_duplex = DUPLEX_INVALID;
3009
3010         if (tp->tg3_flags2 & TG3_FLG2_CAPACITIVE_COUPLING) {
3011                 u32 val;
3012
3013                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4007);
3014                 tg3_readphy(tp, MII_TG3_AUX_CTRL, &val);
3015                 if (!(val & (1 << 10))) {
3016                         val |= (1 << 10);
3017                         tg3_writephy(tp, MII_TG3_AUX_CTRL, val);
3018                         goto relink;
3019                 }
3020         }
3021
3022         bmsr = 0;
3023         for (i = 0; i < 100; i++) {
3024                 tg3_readphy(tp, MII_BMSR, &bmsr);
3025                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
3026                     (bmsr & BMSR_LSTATUS))
3027                         break;
3028                 udelay(40);
3029         }
3030
3031         if (bmsr & BMSR_LSTATUS) {
3032                 u32 aux_stat, bmcr;
3033
3034                 tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
3035                 for (i = 0; i < 2000; i++) {
3036                         udelay(10);
3037                         if (!tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat) &&
3038                             aux_stat)
3039                                 break;
3040                 }
3041
3042                 tg3_aux_stat_to_speed_duplex(tp, aux_stat,
3043                                              &current_speed,
3044                                              &current_duplex);
3045
3046                 bmcr = 0;
3047                 for (i = 0; i < 200; i++) {
3048                         tg3_readphy(tp, MII_BMCR, &bmcr);
3049                         if (tg3_readphy(tp, MII_BMCR, &bmcr))
3050                                 continue;
3051                         if (bmcr && bmcr != 0x7fff)
3052                                 break;
3053                         udelay(10);
3054                 }
3055
3056                 lcl_adv = 0;
3057                 rmt_adv = 0;
3058
3059                 tp->link_config.active_speed = current_speed;
3060                 tp->link_config.active_duplex = current_duplex;
3061
3062                 if (tp->link_config.autoneg == AUTONEG_ENABLE) {
3063                         if ((bmcr & BMCR_ANENABLE) &&
3064                             tg3_copper_is_advertising_all(tp,
3065                                                 tp->link_config.advertising)) {
3066                                 if (tg3_adv_1000T_flowctrl_ok(tp, &lcl_adv,
3067                                                                   &rmt_adv))
3068                                         current_link_up = 1;
3069                         }
3070                 } else {
3071                         if (!(bmcr & BMCR_ANENABLE) &&
3072                             tp->link_config.speed == current_speed &&
3073                             tp->link_config.duplex == current_duplex &&
3074                             tp->link_config.flowctrl ==
3075                             tp->link_config.active_flowctrl) {
3076                                 current_link_up = 1;
3077                         }
3078                 }
3079
3080                 if (current_link_up == 1 &&
3081                     tp->link_config.active_duplex == DUPLEX_FULL)
3082                         tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
3083         }
3084
3085 relink:
3086         if (current_link_up == 0 || tp->link_config.phy_is_low_power) {
3087                 u32 tmp;
3088
3089                 tg3_phy_copper_begin(tp);
3090
3091                 tg3_readphy(tp, MII_BMSR, &tmp);
3092                 if (!tg3_readphy(tp, MII_BMSR, &tmp) &&
3093                     (tmp & BMSR_LSTATUS))
3094                         current_link_up = 1;
3095         }
3096
3097         tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
3098         if (current_link_up == 1) {
3099                 if (tp->link_config.active_speed == SPEED_100 ||
3100                     tp->link_config.active_speed == SPEED_10)
3101                         tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
3102                 else
3103                         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
3104         } else
3105                 tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
3106
3107         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
3108         if (tp->link_config.active_duplex == DUPLEX_HALF)
3109                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
3110
3111         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
3112                 if (current_link_up == 1 &&
3113                     tg3_5700_link_polarity(tp, tp->link_config.active_speed))
3114                         tp->mac_mode |= MAC_MODE_LINK_POLARITY;
3115                 else
3116                         tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
3117         }
3118
3119         /* ??? Without this setting Netgear GA302T PHY does not
3120          * ??? send/receive packets...
3121          */
3122         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411 &&
3123             tp->pci_chip_rev_id == CHIPREV_ID_5700_ALTIMA) {
3124                 tp->mi_mode |= MAC_MI_MODE_AUTO_POLL;
3125                 tw32_f(MAC_MI_MODE, tp->mi_mode);
3126                 udelay(80);
3127         }
3128
3129         tw32_f(MAC_MODE, tp->mac_mode);
3130         udelay(40);
3131
3132         if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
3133                 /* Polled via timer. */
3134                 tw32_f(MAC_EVENT, 0);
3135         } else {
3136                 tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
3137         }
3138         udelay(40);
3139
3140         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 &&
3141             current_link_up == 1 &&
3142             tp->link_config.active_speed == SPEED_1000 &&
3143             ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) ||
3144              (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED))) {
3145                 udelay(120);
3146                 tw32_f(MAC_STATUS,
3147                      (MAC_STATUS_SYNC_CHANGED |
3148                       MAC_STATUS_CFG_CHANGED));
3149                 udelay(40);
3150                 tg3_write_mem(tp,
3151                               NIC_SRAM_FIRMWARE_MBOX,
3152                               NIC_SRAM_FIRMWARE_MBOX_MAGIC2);
3153         }
3154
3155         /* Prevent send BD corruption. */
3156         if (tp->tg3_flags3 & TG3_FLG3_CLKREQ_BUG) {
3157                 u16 oldlnkctl, newlnkctl;
3158
3159                 pci_read_config_word(tp->pdev,
3160                                      tp->pcie_cap + PCI_EXP_LNKCTL,
3161                                      &oldlnkctl);
3162                 if (tp->link_config.active_speed == SPEED_100 ||
3163                     tp->link_config.active_speed == SPEED_10)
3164                         newlnkctl = oldlnkctl & ~PCI_EXP_LNKCTL_CLKREQ_EN;
3165                 else
3166                         newlnkctl = oldlnkctl | PCI_EXP_LNKCTL_CLKREQ_EN;
3167                 if (newlnkctl != oldlnkctl)
3168                         pci_write_config_word(tp->pdev,
3169                                               tp->pcie_cap + PCI_EXP_LNKCTL,
3170                                               newlnkctl);
3171         }
3172
3173         if (current_link_up != netif_carrier_ok(tp->dev)) {
3174                 if (current_link_up)
3175                         netif_carrier_on(tp->dev);
3176                 else
3177                         netif_carrier_off(tp->dev);
3178                 tg3_link_report(tp);
3179         }
3180
3181         return 0;
3182 }
3183
3184 struct tg3_fiber_aneginfo {
3185         int state;
3186 #define ANEG_STATE_UNKNOWN              0
3187 #define ANEG_STATE_AN_ENABLE            1
3188 #define ANEG_STATE_RESTART_INIT         2
3189 #define ANEG_STATE_RESTART              3
3190 #define ANEG_STATE_DISABLE_LINK_OK      4
3191 #define ANEG_STATE_ABILITY_DETECT_INIT  5
3192 #define ANEG_STATE_ABILITY_DETECT       6
3193 #define ANEG_STATE_ACK_DETECT_INIT      7
3194 #define ANEG_STATE_ACK_DETECT           8
3195 #define ANEG_STATE_COMPLETE_ACK_INIT    9
3196 #define ANEG_STATE_COMPLETE_ACK         10
3197 #define ANEG_STATE_IDLE_DETECT_INIT     11
3198 #define ANEG_STATE_IDLE_DETECT          12
3199 #define ANEG_STATE_LINK_OK              13
3200 #define ANEG_STATE_NEXT_PAGE_WAIT_INIT  14
3201 #define ANEG_STATE_NEXT_PAGE_WAIT       15
3202
3203         u32 flags;
3204 #define MR_AN_ENABLE            0x00000001
3205 #define MR_RESTART_AN           0x00000002
3206 #define MR_AN_COMPLETE          0x00000004
3207 #define MR_PAGE_RX              0x00000008
3208 #define MR_NP_LOADED            0x00000010
3209 #define MR_TOGGLE_TX            0x00000020
3210 #define MR_LP_ADV_FULL_DUPLEX   0x00000040
3211 #define MR_LP_ADV_HALF_DUPLEX   0x00000080
3212 #define MR_LP_ADV_SYM_PAUSE     0x00000100
3213 #define MR_LP_ADV_ASYM_PAUSE    0x00000200
3214 #define MR_LP_ADV_REMOTE_FAULT1 0x00000400
3215 #define MR_LP_ADV_REMOTE_FAULT2 0x00000800
3216 #define MR_LP_ADV_NEXT_PAGE     0x00001000
3217 #define MR_TOGGLE_RX            0x00002000
3218 #define MR_NP_RX                0x00004000
3219
3220 #define MR_LINK_OK              0x80000000
3221
3222         unsigned long link_time, cur_time;
3223
3224         u32 ability_match_cfg;
3225         int ability_match_count;
3226
3227         char ability_match, idle_match, ack_match;
3228
3229         u32 txconfig, rxconfig;
3230 #define ANEG_CFG_NP             0x00000080
3231 #define ANEG_CFG_ACK            0x00000040
3232 #define ANEG_CFG_RF2            0x00000020
3233 #define ANEG_CFG_RF1            0x00000010
3234 #define ANEG_CFG_PS2            0x00000001
3235 #define ANEG_CFG_PS1            0x00008000
3236 #define ANEG_CFG_HD             0x00004000
3237 #define ANEG_CFG_FD             0x00002000
3238 #define ANEG_CFG_INVAL          0x00001f06
3239
3240 };
3241 #define ANEG_OK         0
3242 #define ANEG_DONE       1
3243 #define ANEG_TIMER_ENAB 2
3244 #define ANEG_FAILED     -1
3245
3246 #define ANEG_STATE_SETTLE_TIME  10000
3247
3248 static int tg3_fiber_aneg_smachine(struct tg3 *tp,
3249                                    struct tg3_fiber_aneginfo *ap)
3250 {
3251         u16 flowctrl;
3252         unsigned long delta;
3253         u32 rx_cfg_reg;
3254         int ret;
3255
3256         if (ap->state == ANEG_STATE_UNKNOWN) {
3257                 ap->rxconfig = 0;
3258                 ap->link_time = 0;
3259                 ap->cur_time = 0;
3260                 ap->ability_match_cfg = 0;
3261                 ap->ability_match_count = 0;
3262                 ap->ability_match = 0;
3263                 ap->idle_match = 0;
3264                 ap->ack_match = 0;
3265         }
3266         ap->cur_time++;
3267
3268         if (tr32(MAC_STATUS) & MAC_STATUS_RCVD_CFG) {
3269                 rx_cfg_reg = tr32(MAC_RX_AUTO_NEG);
3270
3271                 if (rx_cfg_reg != ap->ability_match_cfg) {
3272                         ap->ability_match_cfg = rx_cfg_reg;
3273                         ap->ability_match = 0;
3274                         ap->ability_match_count = 0;
3275                 } else {
3276                         if (++ap->ability_match_count > 1) {
3277                                 ap->ability_match = 1;
3278                                 ap->ability_match_cfg = rx_cfg_reg;
3279                         }
3280                 }
3281                 if (rx_cfg_reg & ANEG_CFG_ACK)
3282                         ap->ack_match = 1;
3283                 else
3284                         ap->ack_match = 0;
3285
3286                 ap->idle_match = 0;
3287         } else {
3288                 ap->idle_match = 1;
3289                 ap->ability_match_cfg = 0;
3290                 ap->ability_match_count = 0;
3291                 ap->ability_match = 0;
3292                 ap->ack_match = 0;
3293
3294                 rx_cfg_reg = 0;
3295         }
3296
3297         ap->rxconfig = rx_cfg_reg;
3298         ret = ANEG_OK;
3299
3300         switch(ap->state) {
3301         case ANEG_STATE_UNKNOWN:
3302                 if (ap->flags & (MR_AN_ENABLE | MR_RESTART_AN))
3303                         ap->state = ANEG_STATE_AN_ENABLE;
3304
3305                 /* fallthru */
3306         case ANEG_STATE_AN_ENABLE:
3307                 ap->flags &= ~(MR_AN_COMPLETE | MR_PAGE_RX);
3308                 if (ap->flags & MR_AN_ENABLE) {
3309                         ap->link_time = 0;
3310                         ap->cur_time = 0;
3311                         ap->ability_match_cfg = 0;
3312                         ap->ability_match_count = 0;
3313                         ap->ability_match = 0;
3314                         ap->idle_match = 0;
3315                         ap->ack_match = 0;
3316
3317                         ap->state = ANEG_STATE_RESTART_INIT;
3318                 } else {
3319                         ap->state = ANEG_STATE_DISABLE_LINK_OK;
3320                 }
3321                 break;
3322
3323         case ANEG_STATE_RESTART_INIT:
3324                 ap->link_time = ap->cur_time;
3325                 ap->flags &= ~(MR_NP_LOADED);
3326                 ap->txconfig = 0;
3327                 tw32(MAC_TX_AUTO_NEG, 0);
3328                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
3329                 tw32_f(MAC_MODE, tp->mac_mode);
3330                 udelay(40);
3331
3332                 ret = ANEG_TIMER_ENAB;
3333                 ap->state = ANEG_STATE_RESTART;
3334
3335                 /* fallthru */
3336         case ANEG_STATE_RESTART:
3337                 delta = ap->cur_time - ap->link_time;
3338                 if (delta > ANEG_STATE_SETTLE_TIME) {
3339                         ap->state = ANEG_STATE_ABILITY_DETECT_INIT;
3340                 } else {
3341                         ret = ANEG_TIMER_ENAB;
3342                 }
3343                 break;
3344
3345         case ANEG_STATE_DISABLE_LINK_OK:
3346                 ret = ANEG_DONE;
3347                 break;
3348
3349         case ANEG_STATE_ABILITY_DETECT_INIT:
3350                 ap->flags &= ~(MR_TOGGLE_TX);
3351                 ap->txconfig = ANEG_CFG_FD;
3352                 flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
3353                 if (flowctrl & ADVERTISE_1000XPAUSE)
3354                         ap->txconfig |= ANEG_CFG_PS1;
3355                 if (flowctrl & ADVERTISE_1000XPSE_ASYM)
3356                         ap->txconfig |= ANEG_CFG_PS2;
3357                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
3358                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
3359                 tw32_f(MAC_MODE, tp->mac_mode);
3360                 udelay(40);
3361
3362                 ap->state = ANEG_STATE_ABILITY_DETECT;
3363                 break;
3364
3365         case ANEG_STATE_ABILITY_DETECT:
3366                 if (ap->ability_match != 0 && ap->rxconfig != 0) {
3367                         ap->state = ANEG_STATE_ACK_DETECT_INIT;
3368                 }
3369                 break;
3370
3371         case ANEG_STATE_ACK_DETECT_INIT:
3372                 ap->txconfig |= ANEG_CFG_ACK;
3373                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
3374                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
3375                 tw32_f(MAC_MODE, tp->mac_mode);
3376                 udelay(40);
3377
3378                 ap->state = ANEG_STATE_ACK_DETECT;
3379
3380                 /* fallthru */
3381         case ANEG_STATE_ACK_DETECT:
3382                 if (ap->ack_match != 0) {
3383                         if ((ap->rxconfig & ~ANEG_CFG_ACK) ==
3384                             (ap->ability_match_cfg & ~ANEG_CFG_ACK)) {
3385                                 ap->state = ANEG_STATE_COMPLETE_ACK_INIT;
3386                         } else {
3387                                 ap->state = ANEG_STATE_AN_ENABLE;
3388                         }
3389                 } else if (ap->ability_match != 0 &&
3390                            ap->rxconfig == 0) {
3391                         ap->state = ANEG_STATE_AN_ENABLE;
3392                 }
3393                 break;
3394
3395         case ANEG_STATE_COMPLETE_ACK_INIT:
3396                 if (ap->rxconfig & ANEG_CFG_INVAL) {
3397                         ret = ANEG_FAILED;
3398                         break;
3399                 }
3400                 ap->flags &= ~(MR_LP_ADV_FULL_DUPLEX |
3401                                MR_LP_ADV_HALF_DUPLEX |
3402                                MR_LP_ADV_SYM_PAUSE |
3403                                MR_LP_ADV_ASYM_PAUSE |
3404                                MR_LP_ADV_REMOTE_FAULT1 |
3405                                MR_LP_ADV_REMOTE_FAULT2 |
3406                                MR_LP_ADV_NEXT_PAGE |
3407                                MR_TOGGLE_RX |
3408                                MR_NP_RX);
3409                 if (ap->rxconfig & ANEG_CFG_FD)
3410                         ap->flags |= MR_LP_ADV_FULL_DUPLEX;
3411                 if (ap->rxconfig & ANEG_CFG_HD)
3412                         ap->flags |= MR_LP_ADV_HALF_DUPLEX;
3413                 if (ap->rxconfig & ANEG_CFG_PS1)
3414                         ap->flags |= MR_LP_ADV_SYM_PAUSE;
3415                 if (ap->rxconfig & ANEG_CFG_PS2)
3416                         ap->flags |= MR_LP_ADV_ASYM_PAUSE;
3417                 if (ap->rxconfig & ANEG_CFG_RF1)
3418                         ap->flags |= MR_LP_ADV_REMOTE_FAULT1;
3419                 if (ap->rxconfig & ANEG_CFG_RF2)
3420                         ap->flags |= MR_LP_ADV_REMOTE_FAULT2;
3421                 if (ap->rxconfig & ANEG_CFG_NP)
3422                         ap->flags |= MR_LP_ADV_NEXT_PAGE;
3423
3424                 ap->link_time = ap->cur_time;
3425
3426                 ap->flags ^= (MR_TOGGLE_TX);
3427                 if (ap->rxconfig & 0x0008)
3428                         ap->flags |= MR_TOGGLE_RX;
3429                 if (ap->rxconfig & ANEG_CFG_NP)
3430                         ap->flags |= MR_NP_RX;
3431                 ap->flags |= MR_PAGE_RX;
3432
3433                 ap->state = ANEG_STATE_COMPLETE_ACK;
3434                 ret = ANEG_TIMER_ENAB;
3435                 break;
3436
3437         case ANEG_STATE_COMPLETE_ACK:
3438                 if (ap->ability_match != 0 &&
3439                     ap->rxconfig == 0) {
3440                         ap->state = ANEG_STATE_AN_ENABLE;
3441                         break;
3442                 }
3443                 delta = ap->cur_time - ap->link_time;
3444                 if (delta > ANEG_STATE_SETTLE_TIME) {
3445                         if (!(ap->flags & (MR_LP_ADV_NEXT_PAGE))) {
3446                                 ap->state = ANEG_STATE_IDLE_DETECT_INIT;
3447                         } else {
3448                                 if ((ap->txconfig & ANEG_CFG_NP) == 0 &&
3449                                     !(ap->flags & MR_NP_RX)) {
3450                                         ap->state = ANEG_STATE_IDLE_DETECT_INIT;
3451                                 } else {
3452                                         ret = ANEG_FAILED;
3453                                 }
3454                         }
3455                 }
3456                 break;
3457
3458         case ANEG_STATE_IDLE_DETECT_INIT:
3459                 ap->link_time = ap->cur_time;
3460                 tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
3461                 tw32_f(MAC_MODE, tp->mac_mode);
3462                 udelay(40);
3463
3464                 ap->state = ANEG_STATE_IDLE_DETECT;
3465                 ret = ANEG_TIMER_ENAB;
3466                 break;
3467
3468         case ANEG_STATE_IDLE_DETECT:
3469                 if (ap->ability_match != 0 &&
3470                     ap->rxconfig == 0) {
3471                         ap->state = ANEG_STATE_AN_ENABLE;
3472                         break;
3473                 }
3474                 delta = ap->cur_time - ap->link_time;
3475                 if (delta > ANEG_STATE_SETTLE_TIME) {
3476                         /* XXX another gem from the Broadcom driver :( */
3477                         ap->state = ANEG_STATE_LINK_OK;
3478                 }
3479                 break;
3480
3481         case ANEG_STATE_LINK_OK:
3482                 ap->flags |= (MR_AN_COMPLETE | MR_LINK_OK);
3483                 ret = ANEG_DONE;
3484                 break;
3485
3486         case ANEG_STATE_NEXT_PAGE_WAIT_INIT:
3487                 /* ??? unimplemented */
3488                 break;
3489
3490         case ANEG_STATE_NEXT_PAGE_WAIT:
3491                 /* ??? unimplemented */
3492                 break;
3493
3494         default:
3495                 ret = ANEG_FAILED;
3496                 break;
3497         }
3498
3499         return ret;
3500 }
3501
3502 static int fiber_autoneg(struct tg3 *tp, u32 *txflags, u32 *rxflags)
3503 {
3504         int res = 0;
3505         struct tg3_fiber_aneginfo aninfo;
3506         int status = ANEG_FAILED;
3507         unsigned int tick;
3508         u32 tmp;
3509
3510         tw32_f(MAC_TX_AUTO_NEG, 0);
3511
3512         tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
3513         tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
3514         udelay(40);
3515
3516         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
3517         udelay(40);
3518
3519         memset(&aninfo, 0, sizeof(aninfo));
3520         aninfo.flags |= MR_AN_ENABLE;
3521         aninfo.state = ANEG_STATE_UNKNOWN;
3522         aninfo.cur_time = 0;
3523         tick = 0;
3524         while (++tick < 195000) {
3525                 status = tg3_fiber_aneg_smachine(tp, &aninfo);
3526                 if (status == ANEG_DONE || status == ANEG_FAILED)
3527                         break;
3528
3529                 udelay(1);
3530         }
3531
3532         tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
3533         tw32_f(MAC_MODE, tp->mac_mode);
3534         udelay(40);
3535
3536         *txflags = aninfo.txconfig;
3537         *rxflags = aninfo.flags;
3538
3539         if (status == ANEG_DONE &&
3540             (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
3541                              MR_LP_ADV_FULL_DUPLEX)))
3542                 res = 1;
3543
3544         return res;
3545 }
3546
3547 static void tg3_init_bcm8002(struct tg3 *tp)
3548 {
3549         u32 mac_status = tr32(MAC_STATUS);
3550         int i;
3551
3552         /* Reset when initting first time or we have a link. */
3553         if ((tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) &&
3554             !(mac_status & MAC_STATUS_PCS_SYNCED))
3555                 return;
3556
3557         /* Set PLL lock range. */
3558         tg3_writephy(tp, 0x16, 0x8007);
3559
3560         /* SW reset */
3561         tg3_writephy(tp, MII_BMCR, BMCR_RESET);
3562
3563         /* Wait for reset to complete. */
3564         /* XXX schedule_timeout() ... */
3565         for (i = 0; i < 500; i++)
3566                 udelay(10);
3567
3568         /* Config mode; select PMA/Ch 1 regs. */
3569         tg3_writephy(tp, 0x10, 0x8411);
3570
3571         /* Enable auto-lock and comdet, select txclk for tx. */
3572         tg3_writephy(tp, 0x11, 0x0a10);
3573
3574         tg3_writephy(tp, 0x18, 0x00a0);
3575         tg3_writephy(tp, 0x16, 0x41ff);
3576
3577         /* Assert and deassert POR. */
3578         tg3_writephy(tp, 0x13, 0x0400);
3579         udelay(40);
3580         tg3_writephy(tp, 0x13, 0x0000);
3581
3582         tg3_writephy(tp, 0x11, 0x0a50);
3583         udelay(40);
3584         tg3_writephy(tp, 0x11, 0x0a10);
3585
3586         /* Wait for signal to stabilize */
3587         /* XXX schedule_timeout() ... */
3588         for (i = 0; i < 15000; i++)
3589                 udelay(10);
3590
3591         /* Deselect the channel register so we can read the PHYID
3592          * later.
3593          */
3594         tg3_writephy(tp, 0x10, 0x8011);
3595 }
3596
3597 static int tg3_setup_fiber_hw_autoneg(struct tg3 *tp, u32 mac_status)
3598 {
3599         u16 flowctrl;
3600         u32 sg_dig_ctrl, sg_dig_status;
3601         u32 serdes_cfg, expected_sg_dig_ctrl;
3602         int workaround, port_a;
3603         int current_link_up;
3604
3605         serdes_cfg = 0;
3606         expected_sg_dig_ctrl = 0;
3607         workaround = 0;
3608         port_a = 1;
3609         current_link_up = 0;
3610
3611         if (tp->pci_chip_rev_id != CHIPREV_ID_5704_A0 &&
3612             tp->pci_chip_rev_id != CHIPREV_ID_5704_A1) {
3613                 workaround = 1;
3614                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
3615                         port_a = 0;
3616
3617                 /* preserve bits 0-11,13,14 for signal pre-emphasis */
3618                 /* preserve bits 20-23 for voltage regulator */
3619                 serdes_cfg = tr32(MAC_SERDES_CFG) & 0x00f06fff;
3620         }
3621
3622         sg_dig_ctrl = tr32(SG_DIG_CTRL);
3623
3624         if (tp->link_config.autoneg != AUTONEG_ENABLE) {
3625                 if (sg_dig_ctrl & SG_DIG_USING_HW_AUTONEG) {
3626                         if (workaround) {
3627                                 u32 val = serdes_cfg;
3628
3629                                 if (port_a)
3630                                         val |= 0xc010000;
3631                                 else
3632                                         val |= 0x4010000;
3633                                 tw32_f(MAC_SERDES_CFG, val);
3634                         }
3635
3636                         tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
3637                 }
3638                 if (mac_status & MAC_STATUS_PCS_SYNCED) {
3639                         tg3_setup_flow_control(tp, 0, 0);
3640                         current_link_up = 1;
3641                 }
3642                 goto out;
3643         }
3644
3645         /* Want auto-negotiation.  */
3646         expected_sg_dig_ctrl = SG_DIG_USING_HW_AUTONEG | SG_DIG_COMMON_SETUP;
3647
3648         flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
3649         if (flowctrl & ADVERTISE_1000XPAUSE)
3650                 expected_sg_dig_ctrl |= SG_DIG_PAUSE_CAP;
3651         if (flowctrl & ADVERTISE_1000XPSE_ASYM)
3652                 expected_sg_dig_ctrl |= SG_DIG_ASYM_PAUSE;
3653
3654         if (sg_dig_ctrl != expected_sg_dig_ctrl) {
3655                 if ((tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT) &&
3656                     tp->serdes_counter &&
3657                     ((mac_status & (MAC_STATUS_PCS_SYNCED |
3658                                     MAC_STATUS_RCVD_CFG)) ==
3659                      MAC_STATUS_PCS_SYNCED)) {
3660                         tp->serdes_counter--;
3661                         current_link_up = 1;
3662                         goto out;
3663                 }
3664 restart_autoneg:
3665                 if (workaround)
3666                         tw32_f(MAC_SERDES_CFG, serdes_cfg | 0xc011000);
3667                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl | SG_DIG_SOFT_RESET);
3668                 udelay(5);
3669                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl);
3670
3671                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
3672                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3673         } else if (mac_status & (MAC_STATUS_PCS_SYNCED |
3674                                  MAC_STATUS_SIGNAL_DET)) {
3675                 sg_dig_status = tr32(SG_DIG_STATUS);
3676                 mac_status = tr32(MAC_STATUS);
3677
3678                 if ((sg_dig_status & SG_DIG_AUTONEG_COMPLETE) &&
3679                     (mac_status & MAC_STATUS_PCS_SYNCED)) {
3680                         u32 local_adv = 0, remote_adv = 0;
3681
3682                         if (sg_dig_ctrl & SG_DIG_PAUSE_CAP)
3683                                 local_adv |= ADVERTISE_1000XPAUSE;
3684                         if (sg_dig_ctrl & SG_DIG_ASYM_PAUSE)
3685                                 local_adv |= ADVERTISE_1000XPSE_ASYM;
3686
3687                         if (sg_dig_status & SG_DIG_PARTNER_PAUSE_CAPABLE)
3688                                 remote_adv |= LPA_1000XPAUSE;
3689                         if (sg_dig_status & SG_DIG_PARTNER_ASYM_PAUSE)
3690                                 remote_adv |= LPA_1000XPAUSE_ASYM;
3691
3692                         tg3_setup_flow_control(tp, local_adv, remote_adv);
3693                         current_link_up = 1;
3694                         tp->serdes_counter = 0;
3695                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3696                 } else if (!(sg_dig_status & SG_DIG_AUTONEG_COMPLETE)) {
3697                         if (tp->serdes_counter)
3698                                 tp->serdes_counter--;
3699                         else {
3700                                 if (workaround) {
3701                                         u32 val = serdes_cfg;
3702
3703                                         if (port_a)
3704                                                 val |= 0xc010000;
3705                                         else
3706                                                 val |= 0x4010000;
3707
3708                                         tw32_f(MAC_SERDES_CFG, val);
3709                                 }
3710
3711                                 tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
3712                                 udelay(40);
3713
3714                                 /* Link parallel detection - link is up */
3715                                 /* only if we have PCS_SYNC and not */
3716                                 /* receiving config code words */
3717                                 mac_status = tr32(MAC_STATUS);
3718                                 if ((mac_status & MAC_STATUS_PCS_SYNCED) &&
3719                                     !(mac_status & MAC_STATUS_RCVD_CFG)) {
3720                                         tg3_setup_flow_control(tp, 0, 0);
3721                                         current_link_up = 1;
3722                                         tp->tg3_flags2 |=
3723                                                 TG3_FLG2_PARALLEL_DETECT;
3724                                         tp->serdes_counter =
3725                                                 SERDES_PARALLEL_DET_TIMEOUT;
3726                                 } else
3727                                         goto restart_autoneg;
3728                         }
3729                 }
3730         } else {
3731                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
3732                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3733         }
3734
3735 out:
3736         return current_link_up;
3737 }
3738
3739 static int tg3_setup_fiber_by_hand(struct tg3 *tp, u32 mac_status)
3740 {
3741         int current_link_up = 0;
3742
3743         if (!(mac_status & MAC_STATUS_PCS_SYNCED))
3744                 goto out;
3745
3746         if (tp->link_config.autoneg == AUTONEG_ENABLE) {
3747                 u32 txflags, rxflags;
3748                 int i;
3749
3750                 if (fiber_autoneg(tp, &txflags, &rxflags)) {
3751                         u32 local_adv = 0, remote_adv = 0;
3752
3753                         if (txflags & ANEG_CFG_PS1)
3754                                 local_adv |= ADVERTISE_1000XPAUSE;
3755                         if (txflags & ANEG_CFG_PS2)
3756                                 local_adv |= ADVERTISE_1000XPSE_ASYM;
3757
3758                         if (rxflags & MR_LP_ADV_SYM_PAUSE)
3759                                 remote_adv |= LPA_1000XPAUSE;
3760                         if (rxflags & MR_LP_ADV_ASYM_PAUSE)
3761                                 remote_adv |= LPA_1000XPAUSE_ASYM;
3762
3763                         tg3_setup_flow_control(tp, local_adv, remote_adv);
3764
3765                         current_link_up = 1;
3766                 }
3767                 for (i = 0; i < 30; i++) {
3768                         udelay(20);
3769                         tw32_f(MAC_STATUS,
3770                                (MAC_STATUS_SYNC_CHANGED |
3771                                 MAC_STATUS_CFG_CHANGED));
3772                         udelay(40);
3773                         if ((tr32(MAC_STATUS) &
3774                              (MAC_STATUS_SYNC_CHANGED |
3775                               MAC_STATUS_CFG_CHANGED)) == 0)
3776                                 break;
3777                 }
3778
3779                 mac_status = tr32(MAC_STATUS);
3780                 if (current_link_up == 0 &&
3781                     (mac_status & MAC_STATUS_PCS_SYNCED) &&
3782                     !(mac_status & MAC_STATUS_RCVD_CFG))
3783                         current_link_up = 1;
3784         } else {
3785                 tg3_setup_flow_control(tp, 0, 0);
3786
3787                 /* Forcing 1000FD link up. */
3788                 current_link_up = 1;
3789
3790                 tw32_f(MAC_MODE, (tp->mac_mode | MAC_MODE_SEND_CONFIGS));
3791                 udelay(40);
3792
3793                 tw32_f(MAC_MODE, tp->mac_mode);
3794                 udelay(40);
3795         }
3796
3797 out:
3798         return current_link_up;
3799 }
3800
3801 static int tg3_setup_fiber_phy(struct tg3 *tp, int force_reset)
3802 {
3803         u32 orig_pause_cfg;
3804         u16 orig_active_speed;
3805         u8 orig_active_duplex;
3806         u32 mac_status;
3807         int current_link_up;
3808         int i;
3809
3810         orig_pause_cfg = tp->link_config.active_flowctrl;
3811         orig_active_speed = tp->link_config.active_speed;
3812         orig_active_duplex = tp->link_config.active_duplex;
3813
3814         if (!(tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG) &&
3815             netif_carrier_ok(tp->dev) &&
3816             (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE)) {
3817                 mac_status = tr32(MAC_STATUS);
3818                 mac_status &= (MAC_STATUS_PCS_SYNCED |
3819                                MAC_STATUS_SIGNAL_DET |
3820                                MAC_STATUS_CFG_CHANGED |
3821                                MAC_STATUS_RCVD_CFG);
3822                 if (mac_status == (MAC_STATUS_PCS_SYNCED |
3823                                    MAC_STATUS_SIGNAL_DET)) {
3824                         tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
3825                                             MAC_STATUS_CFG_CHANGED));
3826                         return 0;
3827                 }
3828         }
3829
3830         tw32_f(MAC_TX_AUTO_NEG, 0);
3831
3832         tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
3833         tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
3834         tw32_f(MAC_MODE, tp->mac_mode);
3835         udelay(40);
3836
3837         if (tp->phy_id == PHY_ID_BCM8002)
3838                 tg3_init_bcm8002(tp);
3839
3840         /* Enable link change event even when serdes polling.  */
3841         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
3842         udelay(40);
3843
3844         current_link_up = 0;
3845         mac_status = tr32(MAC_STATUS);
3846
3847         if (tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG)
3848                 current_link_up = tg3_setup_fiber_hw_autoneg(tp, mac_status);
3849         else
3850                 current_link_up = tg3_setup_fiber_by_hand(tp, mac_status);
3851
3852         tp->hw_status->status =
3853                 (SD_STATUS_UPDATED |
3854                  (tp->hw_status->status & ~SD_STATUS_LINK_CHG));
3855
3856         for (i = 0; i < 100; i++) {
3857                 tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
3858                                     MAC_STATUS_CFG_CHANGED));
3859                 udelay(5);
3860                 if ((tr32(MAC_STATUS) & (MAC_STATUS_SYNC_CHANGED |
3861                                          MAC_STATUS_CFG_CHANGED |
3862                                          MAC_STATUS_LNKSTATE_CHANGED)) == 0)
3863                         break;
3864         }
3865
3866         mac_status = tr32(MAC_STATUS);
3867         if ((mac_status & MAC_STATUS_PCS_SYNCED) == 0) {
3868                 current_link_up = 0;
3869                 if (tp->link_config.autoneg == AUTONEG_ENABLE &&
3870                     tp->serdes_counter == 0) {
3871                         tw32_f(MAC_MODE, (tp->mac_mode |
3872                                           MAC_MODE_SEND_CONFIGS));
3873                         udelay(1);
3874                         tw32_f(MAC_MODE, tp->mac_mode);
3875                 }
3876         }
3877
3878         if (current_link_up == 1) {
3879                 tp->link_config.active_speed = SPEED_1000;
3880                 tp->link_config.active_duplex = DUPLEX_FULL;
3881                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
3882                                     LED_CTRL_LNKLED_OVERRIDE |
3883                                     LED_CTRL_1000MBPS_ON));
3884         } else {
3885                 tp->link_config.active_speed = SPEED_INVALID;
3886                 tp->link_config.active_duplex = DUPLEX_INVALID;
3887                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
3888                                     LED_CTRL_LNKLED_OVERRIDE |
3889                                     LED_CTRL_TRAFFIC_OVERRIDE));
3890         }
3891
3892         if (current_link_up != netif_carrier_ok(tp->dev)) {
3893                 if (current_link_up)
3894                         netif_carrier_on(tp->dev);
3895                 else
3896                         netif_carrier_off(tp->dev);
3897                 tg3_link_report(tp);
3898         } else {
3899                 u32 now_pause_cfg = tp->link_config.active_flowctrl;
3900                 if (orig_pause_cfg != now_pause_cfg ||
3901                     orig_active_speed != tp->link_config.active_speed ||
3902                     orig_active_duplex != tp->link_config.active_duplex)
3903                         tg3_link_report(tp);
3904         }
3905
3906         return 0;
3907 }
3908
3909 static int tg3_setup_fiber_mii_phy(struct tg3 *tp, int force_reset)
3910 {
3911         int current_link_up, err = 0;
3912         u32 bmsr, bmcr;
3913         u16 current_speed;
3914         u8 current_duplex;
3915         u32 local_adv, remote_adv;
3916
3917         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
3918         tw32_f(MAC_MODE, tp->mac_mode);
3919         udelay(40);
3920
3921         tw32(MAC_EVENT, 0);
3922
3923         tw32_f(MAC_STATUS,
3924              (MAC_STATUS_SYNC_CHANGED |
3925               MAC_STATUS_CFG_CHANGED |
3926               MAC_STATUS_MI_COMPLETION |
3927               MAC_STATUS_LNKSTATE_CHANGED));
3928         udelay(40);
3929
3930         if (force_reset)
3931                 tg3_phy_reset(tp);
3932
3933         current_link_up = 0;
3934         current_speed = SPEED_INVALID;
3935         current_duplex = DUPLEX_INVALID;
3936
3937         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
3938         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
3939         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
3940                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
3941                         bmsr |= BMSR_LSTATUS;
3942                 else
3943                         bmsr &= ~BMSR_LSTATUS;
3944         }
3945
3946         err |= tg3_readphy(tp, MII_BMCR, &bmcr);
3947
3948         if ((tp->link_config.autoneg == AUTONEG_ENABLE) && !force_reset &&
3949             (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
3950                 /* do nothing, just check for link up at the end */
3951         } else if (tp->link_config.autoneg == AUTONEG_ENABLE) {
3952                 u32 adv, new_adv;
3953
3954                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
3955                 new_adv = adv & ~(ADVERTISE_1000XFULL | ADVERTISE_1000XHALF |
3956                                   ADVERTISE_1000XPAUSE |
3957                                   ADVERTISE_1000XPSE_ASYM |
3958                                   ADVERTISE_SLCT);
3959
3960                 new_adv |= tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
3961
3962                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
3963                         new_adv |= ADVERTISE_1000XHALF;
3964                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
3965                         new_adv |= ADVERTISE_1000XFULL;
3966
3967                 if ((new_adv != adv) || !(bmcr & BMCR_ANENABLE)) {
3968                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
3969                         bmcr |= BMCR_ANENABLE | BMCR_ANRESTART;
3970                         tg3_writephy(tp, MII_BMCR, bmcr);
3971
3972                         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
3973                         tp->serdes_counter = SERDES_AN_TIMEOUT_5714S;
3974                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3975
3976                         return err;
3977                 }
3978         } else {
3979                 u32 new_bmcr;
3980
3981                 bmcr &= ~BMCR_SPEED1000;
3982                 new_bmcr = bmcr & ~(BMCR_ANENABLE | BMCR_FULLDPLX);
3983
3984                 if (tp->link_config.duplex == DUPLEX_FULL)
3985                         new_bmcr |= BMCR_FULLDPLX;
3986
3987                 if (new_bmcr != bmcr) {
3988                         /* BMCR_SPEED1000 is a reserved bit that needs
3989                          * to be set on write.
3990                          */
3991                         new_bmcr |= BMCR_SPEED1000;
3992
3993                         /* Force a linkdown */
3994                         if (netif_carrier_ok(tp->dev)) {
3995                                 u32 adv;
3996
3997                                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
3998                                 adv &= ~(ADVERTISE_1000XFULL |
3999                                          ADVERTISE_1000XHALF |
4000                                          ADVERTISE_SLCT);
4001                                 tg3_writephy(tp, MII_ADVERTISE, adv);
4002                                 tg3_writephy(tp, MII_BMCR, bmcr |
4003                                                            BMCR_ANRESTART |
4004                                                            BMCR_ANENABLE);
4005                                 udelay(10);
4006                                 netif_carrier_off(tp->dev);
4007                         }
4008                         tg3_writephy(tp, MII_BMCR, new_bmcr);
4009                         bmcr = new_bmcr;
4010                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4011                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4012                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
4013                             ASIC_REV_5714) {
4014                                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
4015                                         bmsr |= BMSR_LSTATUS;
4016                                 else
4017                                         bmsr &= ~BMSR_LSTATUS;
4018                         }
4019                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
4020                 }
4021         }
4022
4023         if (bmsr & BMSR_LSTATUS) {
4024                 current_speed = SPEED_1000;
4025                 current_link_up = 1;
4026                 if (bmcr & BMCR_FULLDPLX)
4027                         current_duplex = DUPLEX_FULL;
4028                 else
4029                         current_duplex = DUPLEX_HALF;
4030
4031                 local_adv = 0;
4032                 remote_adv = 0;
4033
4034                 if (bmcr & BMCR_ANENABLE) {
4035                         u32 common;
4036
4037                         err |= tg3_readphy(tp, MII_ADVERTISE, &local_adv);
4038                         err |= tg3_readphy(tp, MII_LPA, &remote_adv);
4039                         common = local_adv & remote_adv;
4040                         if (common & (ADVERTISE_1000XHALF |
4041                                       ADVERTISE_1000XFULL)) {
4042                                 if (common & ADVERTISE_1000XFULL)
4043                                         current_duplex = DUPLEX_FULL;
4044                                 else
4045                                         current_duplex = DUPLEX_HALF;
4046                         }
4047                         else
4048                                 current_link_up = 0;
4049                 }
4050         }
4051
4052         if (current_link_up == 1 && current_duplex == DUPLEX_FULL)
4053                 tg3_setup_flow_control(tp, local_adv, remote_adv);
4054
4055         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
4056         if (tp->link_config.active_duplex == DUPLEX_HALF)
4057                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
4058
4059         tw32_f(MAC_MODE, tp->mac_mode);
4060         udelay(40);
4061
4062         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
4063
4064         tp->link_config.active_speed = current_speed;
4065         tp->link_config.active_duplex = current_duplex;
4066
4067         if (current_link_up != netif_carrier_ok(tp->dev)) {
4068                 if (current_link_up)
4069                         netif_carrier_on(tp->dev);
4070                 else {
4071                         netif_carrier_off(tp->dev);
4072                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
4073                 }
4074                 tg3_link_report(tp);
4075         }
4076         return err;
4077 }
4078
4079 static void tg3_serdes_parallel_detect(struct tg3 *tp)
4080 {
4081         if (tp->serdes_counter) {
4082                 /* Give autoneg time to complete. */
4083                 tp->serdes_counter--;
4084                 return;
4085         }
4086         if (!netif_carrier_ok(tp->dev) &&
4087             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
4088                 u32 bmcr;
4089
4090                 tg3_readphy(tp, MII_BMCR, &bmcr);
4091                 if (bmcr & BMCR_ANENABLE) {
4092                         u32 phy1, phy2;
4093
4094                         /* Select shadow register 0x1f */
4095                         tg3_writephy(tp, 0x1c, 0x7c00);
4096                         tg3_readphy(tp, 0x1c, &phy1);
4097
4098                         /* Select expansion interrupt status register */
4099                         tg3_writephy(tp, 0x17, 0x0f01);
4100                         tg3_readphy(tp, 0x15, &phy2);
4101                         tg3_readphy(tp, 0x15, &phy2);
4102
4103                         if ((phy1 & 0x10) && !(phy2 & 0x20)) {
4104                                 /* We have signal detect and not receiving
4105                                  * config code words, link is up by parallel
4106                                  * detection.
4107                                  */
4108
4109                                 bmcr &= ~BMCR_ANENABLE;
4110                                 bmcr |= BMCR_SPEED1000 | BMCR_FULLDPLX;
4111                                 tg3_writephy(tp, MII_BMCR, bmcr);
4112                                 tp->tg3_flags2 |= TG3_FLG2_PARALLEL_DETECT;
4113                         }
4114                 }
4115         }
4116         else if (netif_carrier_ok(tp->dev) &&
4117                  (tp->link_config.autoneg == AUTONEG_ENABLE) &&
4118                  (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
4119                 u32 phy2;
4120
4121                 /* Select expansion interrupt status register */
4122                 tg3_writephy(tp, 0x17, 0x0f01);
4123                 tg3_readphy(tp, 0x15, &phy2);
4124                 if (phy2 & 0x20) {
4125                         u32 bmcr;
4126
4127                         /* Config code words received, turn on autoneg. */
4128                         tg3_readphy(tp, MII_BMCR, &bmcr);
4129                         tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANENABLE);
4130
4131                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
4132
4133                 }
4134         }
4135 }
4136
4137 static int tg3_setup_phy(struct tg3 *tp, int force_reset)
4138 {
4139         int err;
4140
4141         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
4142                 err = tg3_setup_fiber_phy(tp, force_reset);
4143         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
4144                 err = tg3_setup_fiber_mii_phy(tp, force_reset);
4145         } else {
4146                 err = tg3_setup_copper_phy(tp, force_reset);
4147         }
4148
4149         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX) {
4150                 u32 val, scale;
4151
4152                 val = tr32(TG3_CPMU_CLCK_STAT) & CPMU_CLCK_STAT_MAC_CLCK_MASK;
4153                 if (val == CPMU_CLCK_STAT_MAC_CLCK_62_5)
4154                         scale = 65;
4155                 else if (val == CPMU_CLCK_STAT_MAC_CLCK_6_25)
4156                         scale = 6;
4157                 else
4158                         scale = 12;
4159
4160                 val = tr32(GRC_MISC_CFG) & ~GRC_MISC_CFG_PRESCALAR_MASK;
4161                 val |= (scale << GRC_MISC_CFG_PRESCALAR_SHIFT);
4162                 tw32(GRC_MISC_CFG, val);
4163         }
4164
4165         if (tp->link_config.active_speed == SPEED_1000 &&
4166             tp->link_config.active_duplex == DUPLEX_HALF)
4167                 tw32(MAC_TX_LENGTHS,
4168                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
4169                       (6 << TX_LENGTHS_IPG_SHIFT) |
4170                       (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
4171         else
4172                 tw32(MAC_TX_LENGTHS,
4173                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
4174                       (6 << TX_LENGTHS_IPG_SHIFT) |
4175                       (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
4176
4177         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
4178                 if (netif_carrier_ok(tp->dev)) {
4179                         tw32(HOSTCC_STAT_COAL_TICKS,
4180                              tp->coal.stats_block_coalesce_usecs);
4181                 } else {
4182                         tw32(HOSTCC_STAT_COAL_TICKS, 0);
4183                 }
4184         }
4185
4186         if (tp->tg3_flags & TG3_FLAG_ASPM_WORKAROUND) {
4187                 u32 val = tr32(PCIE_PWR_MGMT_THRESH);
4188                 if (!netif_carrier_ok(tp->dev))
4189                         val = (val & ~PCIE_PWR_MGMT_L1_THRESH_MSK) |
4190                               tp->pwrmgmt_thresh;
4191                 else
4192                         val |= PCIE_PWR_MGMT_L1_THRESH_MSK;
4193                 tw32(PCIE_PWR_MGMT_THRESH, val);
4194         }
4195
4196         return err;
4197 }
4198
4199 /* This is called whenever we suspect that the system chipset is re-
4200  * ordering the sequence of MMIO to the tx send mailbox. The symptom
4201  * is bogus tx completions. We try to recover by setting the
4202  * TG3_FLAG_MBOX_WRITE_REORDER flag and resetting the chip later
4203  * in the workqueue.
4204  */
4205 static void tg3_tx_recover(struct tg3 *tp)
4206 {
4207         BUG_ON((tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) ||
4208                tp->write32_tx_mbox == tg3_write_indirect_mbox);
4209
4210         printk(KERN_WARNING PFX "%s: The system may be re-ordering memory-"
4211                "mapped I/O cycles to the network device, attempting to "
4212                "recover. Please report the problem to the driver maintainer "
4213                "and include system chipset information.\n", tp->dev->name);
4214
4215         spin_lock(&tp->lock);
4216         tp->tg3_flags |= TG3_FLAG_TX_RECOVERY_PENDING;
4217         spin_unlock(&tp->lock);
4218 }
4219
4220 static inline u32 tg3_tx_avail(struct tg3 *tp)
4221 {
4222         smp_mb();
4223         return (tp->tx_pending -
4224                 ((tp->tx_prod - tp->tx_cons) & (TG3_TX_RING_SIZE - 1)));
4225 }
4226
4227 /* Tigon3 never reports partial packet sends.  So we do not
4228  * need special logic to handle SKBs that have not had all
4229  * of their frags sent yet, like SunGEM does.
4230  */
4231 static void tg3_tx(struct tg3 *tp)
4232 {
4233         u32 hw_idx = tp->hw_status->idx[0].tx_consumer;
4234         u32 sw_idx = tp->tx_cons;
4235
4236         while (sw_idx != hw_idx) {
4237                 struct tx_ring_info *ri = &tp->tx_buffers[sw_idx];
4238                 struct sk_buff *skb = ri->skb;
4239                 int i, tx_bug = 0;
4240
4241                 if (unlikely(skb == NULL)) {
4242                         tg3_tx_recover(tp);
4243                         return;
4244                 }
4245
4246                 skb_dma_unmap(&tp->pdev->dev, skb, DMA_TO_DEVICE);
4247
4248                 ri->skb = NULL;
4249
4250                 sw_idx = NEXT_TX(sw_idx);
4251
4252                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4253                         ri = &tp->tx_buffers[sw_idx];
4254                         if (unlikely(ri->skb != NULL || sw_idx == hw_idx))
4255                                 tx_bug = 1;
4256                         sw_idx = NEXT_TX(sw_idx);
4257                 }
4258
4259                 dev_kfree_skb(skb);
4260
4261                 if (unlikely(tx_bug)) {
4262                         tg3_tx_recover(tp);
4263                         return;
4264                 }
4265         }
4266
4267         tp->tx_cons = sw_idx;
4268
4269         /* Need to make the tx_cons update visible to tg3_start_xmit()
4270          * before checking for netif_queue_stopped().  Without the
4271          * memory barrier, there is a small possibility that tg3_start_xmit()
4272          * will miss it and cause the queue to be stopped forever.
4273          */
4274         smp_mb();
4275
4276         if (unlikely(netif_queue_stopped(tp->dev) &&
4277                      (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp)))) {
4278                 netif_tx_lock(tp->dev);
4279                 if (netif_queue_stopped(tp->dev) &&
4280                     (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp)))
4281                         netif_wake_queue(tp->dev);
4282                 netif_tx_unlock(tp->dev);
4283         }
4284 }
4285
4286 /* Returns size of skb allocated or < 0 on error.
4287  *
4288  * We only need to fill in the address because the other members
4289  * of the RX descriptor are invariant, see tg3_init_rings.
4290  *
4291  * Note the purposeful assymetry of cpu vs. chip accesses.  For
4292  * posting buffers we only dirty the first cache line of the RX
4293  * descriptor (containing the address).  Whereas for the RX status
4294  * buffers the cpu only reads the last cacheline of the RX descriptor
4295  * (to fetch the error flags, vlan tag, checksum, and opaque cookie).
4296  */
4297 static int tg3_alloc_rx_skb(struct tg3 *tp, u32 opaque_key,
4298                             int src_idx, u32 dest_idx_unmasked)
4299 {
4300         struct tg3_rx_buffer_desc *desc;
4301         struct ring_info *map, *src_map;
4302         struct sk_buff *skb;
4303         dma_addr_t mapping;
4304         int skb_size, dest_idx;
4305
4306         src_map = NULL;
4307         switch (opaque_key) {
4308         case RXD_OPAQUE_RING_STD:
4309                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
4310                 desc = &tp->rx_std[dest_idx];
4311                 map = &tp->rx_std_buffers[dest_idx];
4312                 if (src_idx >= 0)
4313                         src_map = &tp->rx_std_buffers[src_idx];
4314                 skb_size = tp->rx_pkt_buf_sz;
4315                 break;
4316
4317         case RXD_OPAQUE_RING_JUMBO:
4318                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
4319                 desc = &tp->rx_jumbo[dest_idx];
4320                 map = &tp->rx_jumbo_buffers[dest_idx];
4321                 if (src_idx >= 0)
4322                         src_map = &tp->rx_jumbo_buffers[src_idx];
4323                 skb_size = RX_JUMBO_PKT_BUF_SZ;
4324                 break;
4325
4326         default:
4327                 return -EINVAL;
4328         }
4329
4330         /* Do not overwrite any of the map or rp information
4331          * until we are sure we can commit to a new buffer.
4332          *
4333          * Callers depend upon this behavior and assume that
4334          * we leave everything unchanged if we fail.
4335          */
4336         skb = netdev_alloc_skb(tp->dev, skb_size);
4337         if (skb == NULL)
4338                 return -ENOMEM;
4339
4340         skb_reserve(skb, tp->rx_offset);
4341
4342         mapping = pci_map_single(tp->pdev, skb->data,
4343                                  skb_size - tp->rx_offset,
4344                                  PCI_DMA_FROMDEVICE);
4345
4346         map->skb = skb;
4347         pci_unmap_addr_set(map, mapping, mapping);
4348
4349         if (src_map != NULL)
4350                 src_map->skb = NULL;
4351
4352         desc->addr_hi = ((u64)mapping >> 32);
4353         desc->addr_lo = ((u64)mapping & 0xffffffff);
4354
4355         return skb_size;
4356 }
4357
4358 /* We only need to move over in the address because the other
4359  * members of the RX descriptor are invariant.  See notes above
4360  * tg3_alloc_rx_skb for full details.
4361  */
4362 static void tg3_recycle_rx(struct tg3 *tp, u32 opaque_key,
4363                            int src_idx, u32 dest_idx_unmasked)
4364 {
4365         struct tg3_rx_buffer_desc *src_desc, *dest_desc;
4366         struct ring_info *src_map, *dest_map;
4367         int dest_idx;
4368
4369         switch (opaque_key) {
4370         case RXD_OPAQUE_RING_STD:
4371                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
4372                 dest_desc = &tp->rx_std[dest_idx];
4373                 dest_map = &tp->rx_std_buffers[dest_idx];
4374                 src_desc = &tp->rx_std[src_idx];
4375                 src_map = &tp->rx_std_buffers[src_idx];
4376                 break;
4377
4378         case RXD_OPAQUE_RING_JUMBO:
4379                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
4380                 dest_desc = &tp->rx_jumbo[dest_idx];
4381                 dest_map = &tp->rx_jumbo_buffers[dest_idx];
4382                 src_desc = &tp->rx_jumbo[src_idx];
4383                 src_map = &tp->rx_jumbo_buffers[src_idx];
4384                 break;
4385
4386         default:
4387                 return;
4388         }
4389
4390         dest_map->skb = src_map->skb;
4391         pci_unmap_addr_set(dest_map, mapping,
4392                            pci_unmap_addr(src_map, mapping));
4393         dest_desc->addr_hi = src_desc->addr_hi;
4394         dest_desc->addr_lo = src_desc->addr_lo;
4395
4396         src_map->skb = NULL;
4397 }
4398
4399 #if TG3_VLAN_TAG_USED
4400 static int tg3_vlan_rx(struct tg3 *tp, struct sk_buff *skb, u16 vlan_tag)
4401 {
4402         return vlan_gro_receive(&tp->napi, tp->vlgrp, vlan_tag, skb);
4403 }
4404 #endif
4405
4406 /* The RX ring scheme is composed of multiple rings which post fresh
4407  * buffers to the chip, and one special ring the chip uses to report
4408  * status back to the host.
4409  *
4410  * The special ring reports the status of received packets to the
4411  * host.  The chip does not write into the original descriptor the
4412  * RX buffer was obtained from.  The chip simply takes the original
4413  * descriptor as provided by the host, updates the status and length
4414  * field, then writes this into the next status ring entry.
4415  *
4416  * Each ring the host uses to post buffers to the chip is described
4417  * by a TG3_BDINFO entry in the chips SRAM area.  When a packet arrives,
4418  * it is first placed into the on-chip ram.  When the packet's length
4419  * is known, it walks down the TG3_BDINFO entries to select the ring.
4420  * Each TG3_BDINFO specifies a MAXLEN field and the first TG3_BDINFO
4421  * which is within the range of the new packet's length is chosen.
4422  *
4423  * The "separate ring for rx status" scheme may sound queer, but it makes
4424  * sense from a cache coherency perspective.  If only the host writes
4425  * to the buffer post rings, and only the chip writes to the rx status
4426  * rings, then cache lines never move beyond shared-modified state.
4427  * If both the host and chip were to write into the same ring, cache line
4428  * eviction could occur since both entities want it in an exclusive state.
4429  */
4430 static int tg3_rx(struct tg3 *tp, int budget)
4431 {
4432         u32 work_mask, rx_std_posted = 0;
4433         u32 sw_idx = tp->rx_rcb_ptr;
4434         u16 hw_idx;
4435         int received;
4436
4437         hw_idx = tp->hw_status->idx[0].rx_producer;
4438         /*
4439          * We need to order the read of hw_idx and the read of
4440          * the opaque cookie.
4441          */
4442         rmb();
4443         work_mask = 0;
4444         received = 0;
4445         while (sw_idx != hw_idx && budget > 0) {
4446                 struct tg3_rx_buffer_desc *desc = &tp->rx_rcb[sw_idx];
4447                 unsigned int len;
4448                 struct sk_buff *skb;
4449                 dma_addr_t dma_addr;
4450                 u32 opaque_key, desc_idx, *post_ptr;
4451
4452                 desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
4453                 opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
4454                 if (opaque_key == RXD_OPAQUE_RING_STD) {
4455                         dma_addr = pci_unmap_addr(&tp->rx_std_buffers[desc_idx],
4456                                                   mapping);
4457                         skb = tp->rx_std_buffers[desc_idx].skb;
4458                         post_ptr = &tp->rx_std_ptr;
4459                         rx_std_posted++;
4460                 } else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
4461                         dma_addr = pci_unmap_addr(&tp->rx_jumbo_buffers[desc_idx],
4462                                                   mapping);
4463                         skb = tp->rx_jumbo_buffers[desc_idx].skb;
4464                         post_ptr = &tp->rx_jumbo_ptr;
4465                 }
4466                 else {
4467                         goto next_pkt_nopost;
4468                 }
4469
4470                 work_mask |= opaque_key;
4471
4472                 if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
4473                     (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII)) {
4474                 drop_it:
4475                         tg3_recycle_rx(tp, opaque_key,
4476                                        desc_idx, *post_ptr);
4477                 drop_it_no_recycle:
4478                         /* Other statistics kept track of by card. */
4479                         tp->net_stats.rx_dropped++;
4480                         goto next_pkt;
4481                 }
4482
4483                 len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) -
4484                       ETH_FCS_LEN;
4485
4486                 if (len > RX_COPY_THRESHOLD
4487                         && tp->rx_offset == NET_IP_ALIGN
4488                         /* rx_offset will likely not equal NET_IP_ALIGN
4489                          * if this is a 5701 card running in PCI-X mode
4490                          * [see tg3_get_invariants()]
4491                          */
4492                 ) {
4493                         int skb_size;
4494
4495                         skb_size = tg3_alloc_rx_skb(tp, opaque_key,
4496                                                     desc_idx, *post_ptr);
4497                         if (skb_size < 0)
4498                                 goto drop_it;
4499
4500                         pci_unmap_single(tp->pdev, dma_addr,
4501                                          skb_size - tp->rx_offset,
4502                                          PCI_DMA_FROMDEVICE);
4503
4504                         skb_put(skb, len);
4505                 } else {
4506                         struct sk_buff *copy_skb;
4507
4508                         tg3_recycle_rx(tp, opaque_key,
4509                                        desc_idx, *post_ptr);
4510
4511                         copy_skb = netdev_alloc_skb(tp->dev,
4512                                                     len + TG3_RAW_IP_ALIGN);
4513                         if (copy_skb == NULL)
4514                                 goto drop_it_no_recycle;
4515
4516                         skb_reserve(copy_skb, TG3_RAW_IP_ALIGN);
4517                         skb_put(copy_skb, len);
4518                         pci_dma_sync_single_for_cpu(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
4519                         skb_copy_from_linear_data(skb, copy_skb->data, len);
4520                         pci_dma_sync_single_for_device(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
4521
4522                         /* We'll reuse the original ring buffer. */
4523                         skb = copy_skb;
4524                 }
4525
4526                 if ((tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) &&
4527                     (desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
4528                     (((desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
4529                       >> RXD_TCPCSUM_SHIFT) == 0xffff))
4530                         skb->ip_summed = CHECKSUM_UNNECESSARY;
4531                 else
4532                         skb->ip_summed = CHECKSUM_NONE;
4533
4534                 skb->protocol = eth_type_trans(skb, tp->dev);
4535
4536                 if (len > (tp->dev->mtu + ETH_HLEN) &&
4537                     skb->protocol != htons(ETH_P_8021Q)) {
4538                         dev_kfree_skb(skb);
4539                         goto next_pkt;
4540                 }
4541
4542 #if TG3_VLAN_TAG_USED
4543                 if (tp->vlgrp != NULL &&
4544                     desc->type_flags & RXD_FLAG_VLAN) {
4545                         tg3_vlan_rx(tp, skb,
4546                                     desc->err_vlan & RXD_VLAN_MASK);
4547                 } else
4548 #endif
4549                         napi_gro_receive(&tp->napi, skb);
4550
4551                 received++;
4552                 budget--;
4553
4554 next_pkt:
4555                 (*post_ptr)++;
4556
4557                 if (unlikely(rx_std_posted >= tp->rx_std_max_post)) {
4558                         u32 idx = *post_ptr % TG3_RX_RING_SIZE;
4559
4560                         tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX +
4561                                      TG3_64BIT_REG_LOW, idx);
4562                         work_mask &= ~RXD_OPAQUE_RING_STD;
4563                         rx_std_posted = 0;
4564                 }
4565 next_pkt_nopost:
4566                 sw_idx++;
4567                 sw_idx &= (TG3_RX_RCB_RING_SIZE(tp) - 1);
4568
4569                 /* Refresh hw_idx to see if there is new work */
4570                 if (sw_idx == hw_idx) {
4571                         hw_idx = tp->hw_status->idx[0].rx_producer;
4572                         rmb();
4573                 }
4574         }
4575
4576         /* ACK the status ring. */
4577         tp->rx_rcb_ptr = sw_idx;
4578         tw32_rx_mbox(MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW, sw_idx);
4579
4580         /* Refill RX ring(s). */
4581         if (work_mask & RXD_OPAQUE_RING_STD) {
4582                 sw_idx = tp->rx_std_ptr % TG3_RX_RING_SIZE;
4583                 tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW,
4584                              sw_idx);
4585         }
4586         if (work_mask & RXD_OPAQUE_RING_JUMBO) {
4587                 sw_idx = tp->rx_jumbo_ptr % TG3_RX_JUMBO_RING_SIZE;
4588                 tw32_rx_mbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW,
4589                              sw_idx);
4590         }
4591         mmiowb();
4592
4593         return received;
4594 }
4595
4596 static int tg3_poll_work(struct tg3 *tp, int work_done, int budget)
4597 {
4598         struct tg3_hw_status *sblk = tp->hw_status;
4599
4600         /* handle link change and other phy events */
4601         if (!(tp->tg3_flags &
4602               (TG3_FLAG_USE_LINKCHG_REG |
4603                TG3_FLAG_POLL_SERDES))) {
4604                 if (sblk->status & SD_STATUS_LINK_CHG) {
4605                         sblk->status = SD_STATUS_UPDATED |
4606                                 (sblk->status & ~SD_STATUS_LINK_CHG);
4607                         spin_lock(&tp->lock);
4608                         if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) {
4609                                 tw32_f(MAC_STATUS,
4610                                      (MAC_STATUS_SYNC_CHANGED |
4611                                       MAC_STATUS_CFG_CHANGED |
4612                                       MAC_STATUS_MI_COMPLETION |
4613                                       MAC_STATUS_LNKSTATE_CHANGED));
4614                                 udelay(40);
4615                         } else
4616                                 tg3_setup_phy(tp, 0);
4617                         spin_unlock(&tp->lock);
4618                 }
4619         }
4620
4621         /* run TX completion thread */
4622         if (sblk->idx[0].tx_consumer != tp->tx_cons) {
4623                 tg3_tx(tp);
4624                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING))
4625                         return work_done;
4626         }
4627
4628         /* run RX thread, within the bounds set by NAPI.
4629          * All RX "locking" is done by ensuring outside
4630          * code synchronizes with tg3->napi.poll()
4631          */
4632         if (sblk->idx[0].rx_producer != tp->rx_rcb_ptr)
4633                 work_done += tg3_rx(tp, budget - work_done);
4634
4635         return work_done;
4636 }
4637
4638 static int tg3_poll(struct napi_struct *napi, int budget)
4639 {
4640         struct tg3 *tp = container_of(napi, struct tg3, napi);
4641         int work_done = 0;
4642         struct tg3_hw_status *sblk = tp->hw_status;
4643
4644         while (1) {
4645                 work_done = tg3_poll_work(tp, work_done, budget);
4646
4647                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING))
4648                         goto tx_recovery;
4649
4650                 if (unlikely(work_done >= budget))
4651                         break;
4652
4653                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) {
4654                         /* tp->last_tag is used in tg3_restart_ints() below
4655                          * to tell the hw how much work has been processed,
4656                          * so we must read it before checking for more work.
4657                          */
4658                         tp->last_tag = sblk->status_tag;
4659                         rmb();
4660                 } else
4661                         sblk->status &= ~SD_STATUS_UPDATED;
4662
4663                 if (likely(!tg3_has_work(tp))) {
4664                         napi_complete(napi);
4665                         tg3_restart_ints(tp);
4666                         break;
4667                 }
4668         }
4669
4670         return work_done;
4671
4672 tx_recovery:
4673         /* work_done is guaranteed to be less than budget. */
4674         napi_complete(napi);
4675         schedule_work(&tp->reset_task);
4676         return work_done;
4677 }
4678
4679 static void tg3_irq_quiesce(struct tg3 *tp)
4680 {
4681         BUG_ON(tp->irq_sync);
4682
4683         tp->irq_sync = 1;
4684         smp_mb();
4685
4686         synchronize_irq(tp->pdev->irq);
4687 }
4688
4689 static inline int tg3_irq_sync(struct tg3 *tp)
4690 {
4691         return tp->irq_sync;
4692 }
4693
4694 /* Fully shutdown all tg3 driver activity elsewhere in the system.
4695  * If irq_sync is non-zero, then the IRQ handler must be synchronized
4696  * with as well.  Most of the time, this is not necessary except when
4697  * shutting down the device.
4698  */
4699 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync)
4700 {
4701         spin_lock_bh(&tp->lock);
4702         if (irq_sync)
4703                 tg3_irq_quiesce(tp);
4704 }
4705
4706 static inline void tg3_full_unlock(struct tg3 *tp)
4707 {
4708         spin_unlock_bh(&tp->lock);
4709 }
4710
4711 /* One-shot MSI handler - Chip automatically disables interrupt
4712  * after sending MSI so driver doesn't have to do it.
4713  */
4714 static irqreturn_t tg3_msi_1shot(int irq, void *dev_id)
4715 {
4716         struct net_device *dev = dev_id;
4717         struct tg3 *tp = netdev_priv(dev);
4718
4719         prefetch(tp->hw_status);
4720         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
4721
4722         if (likely(!tg3_irq_sync(tp)))
4723                 napi_schedule(&tp->napi);
4724
4725         return IRQ_HANDLED;
4726 }
4727
4728 /* MSI ISR - No need to check for interrupt sharing and no need to
4729  * flush status block and interrupt mailbox. PCI ordering rules
4730  * guarantee that MSI will arrive after the status block.
4731  */
4732 static irqreturn_t tg3_msi(int irq, void *dev_id)
4733 {
4734         struct net_device *dev = dev_id;
4735         struct tg3 *tp = netdev_priv(dev);
4736
4737         prefetch(tp->hw_status);
4738         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
4739         /*
4740          * Writing any value to intr-mbox-0 clears PCI INTA# and
4741          * chip-internal interrupt pending events.
4742          * Writing non-zero to intr-mbox-0 additional tells the
4743          * NIC to stop sending us irqs, engaging "in-intr-handler"
4744          * event coalescing.
4745          */
4746         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
4747         if (likely(!tg3_irq_sync(tp)))
4748                 napi_schedule(&tp->napi);
4749
4750         return IRQ_RETVAL(1);
4751 }
4752
4753 static irqreturn_t tg3_interrupt(int irq, void *dev_id)
4754 {
4755         struct net_device *dev = dev_id;
4756         struct tg3 *tp = netdev_priv(dev);
4757         struct tg3_hw_status *sblk = tp->hw_status;
4758         unsigned int handled = 1;
4759
4760         /* In INTx mode, it is possible for the interrupt to arrive at
4761          * the CPU before the status block posted prior to the interrupt.
4762          * Reading the PCI State register will confirm whether the
4763          * interrupt is ours and will flush the status block.
4764          */
4765         if (unlikely(!(sblk->status & SD_STATUS_UPDATED))) {
4766                 if ((tp->tg3_flags & TG3_FLAG_CHIP_RESETTING) ||
4767                     (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
4768                         handled = 0;
4769                         goto out;
4770                 }
4771         }
4772
4773         /*
4774          * Writing any value to intr-mbox-0 clears PCI INTA# and
4775          * chip-internal interrupt pending events.
4776          * Writing non-zero to intr-mbox-0 additional tells the
4777          * NIC to stop sending us irqs, engaging "in-intr-handler"
4778          * event coalescing.
4779          *
4780          * Flush the mailbox to de-assert the IRQ immediately to prevent
4781          * spurious interrupts.  The flush impacts performance but
4782          * excessive spurious interrupts can be worse in some cases.
4783          */
4784         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
4785         if (tg3_irq_sync(tp))
4786                 goto out;
4787         sblk->status &= ~SD_STATUS_UPDATED;
4788         if (likely(tg3_has_work(tp))) {
4789                 prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
4790                 napi_schedule(&tp->napi);
4791         } else {
4792                 /* No work, shared interrupt perhaps?  re-enable
4793                  * interrupts, and flush that PCI write
4794                  */
4795                 tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
4796                                0x00000000);
4797         }
4798 out:
4799         return IRQ_RETVAL(handled);
4800 }
4801
4802 static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id)
4803 {
4804         struct net_device *dev = dev_id;
4805         struct tg3 *tp = netdev_priv(dev);
4806         struct tg3_hw_status *sblk = tp->hw_status;
4807         unsigned int handled = 1;
4808
4809         /* In INTx mode, it is possible for the interrupt to arrive at
4810          * the CPU before the status block posted prior to the interrupt.
4811          * Reading the PCI State register will confirm whether the
4812          * interrupt is ours and will flush the status block.
4813          */
4814         if (unlikely(sblk->status_tag == tp->last_tag)) {
4815                 if ((tp->tg3_flags & TG3_FLAG_CHIP_RESETTING) ||
4816                     (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
4817                         handled = 0;
4818                         goto out;
4819                 }
4820         }
4821
4822         /*
4823          * writing any value to intr-mbox-0 clears PCI INTA# and
4824          * chip-internal interrupt pending events.
4825          * writing non-zero to intr-mbox-0 additional tells the
4826          * NIC to stop sending us irqs, engaging "in-intr-handler"
4827          * event coalescing.
4828          *
4829          * Flush the mailbox to de-assert the IRQ immediately to prevent
4830          * spurious interrupts.  The flush impacts performance but
4831          * excessive spurious interrupts can be worse in some cases.
4832          */
4833         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
4834         if (tg3_irq_sync(tp))
4835                 goto out;
4836         if (napi_schedule_prep(&tp->napi)) {
4837                 prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
4838                 /* Update last_tag to mark that this status has been
4839                  * seen. Because interrupt may be shared, we may be
4840                  * racing with tg3_poll(), so only update last_tag
4841                  * if tg3_poll() is not scheduled.
4842                  */
4843                 tp->last_tag = sblk->status_tag;
4844                 __napi_schedule(&tp->napi);
4845         }
4846 out:
4847         return IRQ_RETVAL(handled);
4848 }
4849
4850 /* ISR for interrupt test */
4851 static irqreturn_t tg3_test_isr(int irq, void *dev_id)
4852 {
4853         struct net_device *dev = dev_id;
4854         struct tg3 *tp = netdev_priv(dev);
4855         struct tg3_hw_status *sblk = tp->hw_status;
4856
4857         if ((sblk->status & SD_STATUS_UPDATED) ||
4858             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
4859                 tg3_disable_ints(tp);
4860                 return IRQ_RETVAL(1);
4861         }
4862         return IRQ_RETVAL(0);
4863 }
4864
4865 static int tg3_init_hw(struct tg3 *, int);
4866 static int tg3_halt(struct tg3 *, int, int);
4867
4868 /* Restart hardware after configuration changes, self-test, etc.
4869  * Invoked with tp->lock held.
4870  */
4871 static int tg3_restart_hw(struct tg3 *tp, int reset_phy)
4872         __releases(tp->lock)
4873         __acquires(tp->lock)
4874 {
4875         int err;
4876
4877         err = tg3_init_hw(tp, reset_phy);
4878         if (err) {
4879                 printk(KERN_ERR PFX "%s: Failed to re-initialize device, "
4880                        "aborting.\n", tp->dev->name);
4881                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
4882                 tg3_full_unlock(tp);
4883                 del_timer_sync(&tp->timer);
4884                 tp->irq_sync = 0;
4885                 napi_enable(&tp->napi);
4886                 dev_close(tp->dev);
4887                 tg3_full_lock(tp, 0);
4888         }
4889         return err;
4890 }
4891
4892 #ifdef CONFIG_NET_POLL_CONTROLLER
4893 static void tg3_poll_controller(struct net_device *dev)
4894 {
4895         struct tg3 *tp = netdev_priv(dev);
4896
4897         tg3_interrupt(tp->pdev->irq, dev);
4898 }
4899 #endif
4900
4901 static void tg3_reset_task(struct work_struct *work)
4902 {
4903         struct tg3 *tp = container_of(work, struct tg3, reset_task);
4904         int err;
4905         unsigned int restart_timer;
4906
4907         tg3_full_lock(tp, 0);
4908
4909         if (!netif_running(tp->dev)) {
4910                 tg3_full_unlock(tp);
4911                 return;
4912         }
4913
4914         tg3_full_unlock(tp);
4915
4916         tg3_phy_stop(tp);
4917
4918         tg3_netif_stop(tp);
4919
4920         tg3_full_lock(tp, 1);
4921
4922         restart_timer = tp->tg3_flags2 & TG3_FLG2_RESTART_TIMER;
4923         tp->tg3_flags2 &= ~TG3_FLG2_RESTART_TIMER;
4924
4925         if (tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING) {
4926                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
4927                 tp->write32_rx_mbox = tg3_write_flush_reg32;
4928                 tp->tg3_flags |= TG3_FLAG_MBOX_WRITE_REORDER;
4929                 tp->tg3_flags &= ~TG3_FLAG_TX_RECOVERY_PENDING;
4930         }
4931
4932         tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
4933         err = tg3_init_hw(tp, 1);
4934         if (err)
4935                 goto out;
4936
4937         tg3_netif_start(tp);
4938
4939         if (restart_timer)
4940                 mod_timer(&tp->timer, jiffies + 1);
4941
4942 out:
4943         tg3_full_unlock(tp);
4944
4945         if (!err)
4946                 tg3_phy_start(tp);
4947 }
4948
4949 static void tg3_dump_short_state(struct tg3 *tp)
4950 {
4951         printk(KERN_ERR PFX "DEBUG: MAC_TX_STATUS[%08x] MAC_RX_STATUS[%08x]\n",
4952                tr32(MAC_TX_STATUS), tr32(MAC_RX_STATUS));
4953         printk(KERN_ERR PFX "DEBUG: RDMAC_STATUS[%08x] WDMAC_STATUS[%08x]\n",
4954                tr32(RDMAC_STATUS), tr32(WDMAC_STATUS));
4955 }
4956
4957 static void tg3_tx_timeout(struct net_device *dev)
4958 {
4959         struct tg3 *tp = netdev_priv(dev);
4960
4961         if (netif_msg_tx_err(tp)) {
4962                 printk(KERN_ERR PFX "%s: transmit timed out, resetting\n",
4963                        dev->name);
4964                 tg3_dump_short_state(tp);
4965         }
4966
4967         schedule_work(&tp->reset_task);
4968 }
4969
4970 /* Test for DMA buffers crossing any 4GB boundaries: 4G, 8G, etc */
4971 static inline int tg3_4g_overflow_test(dma_addr_t mapping, int len)
4972 {
4973         u32 base = (u32) mapping & 0xffffffff;
4974
4975         return ((base > 0xffffdcc0) &&
4976                 (base + len + 8 < base));
4977 }
4978
4979 /* Test for DMA addresses > 40-bit */
4980 static inline int tg3_40bit_overflow_test(struct tg3 *tp, dma_addr_t mapping,
4981                                           int len)
4982 {
4983 #if defined(CONFIG_HIGHMEM) && (BITS_PER_LONG == 64)
4984         if (tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG)
4985                 return (((u64) mapping + len) > DMA_BIT_MASK(40));
4986         return 0;
4987 #else
4988         return 0;
4989 #endif
4990 }
4991
4992 static void tg3_set_txd(struct tg3 *, int, dma_addr_t, int, u32, u32);
4993
4994 /* Workaround 4GB and 40-bit hardware DMA bugs. */
4995 static int tigon3_dma_hwbug_workaround(struct tg3 *tp, struct sk_buff *skb,
4996                                        u32 last_plus_one, u32 *start,
4997                                        u32 base_flags, u32 mss)
4998 {
4999         struct sk_buff *new_skb;
5000         dma_addr_t new_addr = 0;
5001         u32 entry = *start;
5002         int i, ret = 0;
5003
5004         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701)
5005                 new_skb = skb_copy(skb, GFP_ATOMIC);
5006         else {
5007                 int more_headroom = 4 - ((unsigned long)skb->data & 3);
5008
5009                 new_skb = skb_copy_expand(skb,
5010                                           skb_headroom(skb) + more_headroom,
5011                                           skb_tailroom(skb), GFP_ATOMIC);
5012         }
5013
5014         if (!new_skb) {
5015                 ret = -1;
5016         } else {
5017                 /* New SKB is guaranteed to be linear. */
5018                 entry = *start;
5019                 ret = skb_dma_map(&tp->pdev->dev, new_skb, DMA_TO_DEVICE);
5020                 new_addr = skb_shinfo(new_skb)->dma_maps[0];
5021
5022                 /* Make sure new skb does not cross any 4G boundaries.
5023                  * Drop the packet if it does.
5024                  */
5025                 if (ret || tg3_4g_overflow_test(new_addr, new_skb->len)) {
5026                         if (!ret)
5027                                 skb_dma_unmap(&tp->pdev->dev, new_skb,
5028                                               DMA_TO_DEVICE);
5029                         ret = -1;
5030                         dev_kfree_skb(new_skb);
5031                         new_skb = NULL;
5032                 } else {
5033                         tg3_set_txd(tp, entry, new_addr, new_skb->len,
5034                                     base_flags, 1 | (mss << 1));
5035                         *start = NEXT_TX(entry);
5036                 }
5037         }
5038
5039         /* Now clean up the sw ring entries. */
5040         i = 0;
5041         while (entry != last_plus_one) {
5042                 if (i == 0) {
5043                         tp->tx_buffers[entry].skb = new_skb;
5044                 } else {
5045                         tp->tx_buffers[entry].skb = NULL;
5046                 }
5047                 entry = NEXT_TX(entry);
5048                 i++;
5049         }
5050
5051         skb_dma_unmap(&tp->pdev->dev, skb, DMA_TO_DEVICE);
5052         dev_kfree_skb(skb);
5053
5054         return ret;
5055 }
5056
5057 static void tg3_set_txd(struct tg3 *tp, int entry,
5058                         dma_addr_t mapping, int len, u32 flags,
5059                         u32 mss_and_is_end)
5060 {
5061         struct tg3_tx_buffer_desc *txd = &tp->tx_ring[entry];
5062         int is_end = (mss_and_is_end & 0x1);
5063         u32 mss = (mss_and_is_end >> 1);
5064         u32 vlan_tag = 0;
5065
5066         if (is_end)
5067                 flags |= TXD_FLAG_END;
5068         if (flags & TXD_FLAG_VLAN) {
5069                 vlan_tag = flags >> 16;
5070                 flags &= 0xffff;
5071         }
5072         vlan_tag |= (mss << TXD_MSS_SHIFT);
5073
5074         txd->addr_hi = ((u64) mapping >> 32);
5075         txd->addr_lo = ((u64) mapping & 0xffffffff);
5076         txd->len_flags = (len << TXD_LEN_SHIFT) | flags;
5077         txd->vlan_tag = vlan_tag << TXD_VLAN_TAG_SHIFT;
5078 }
5079
5080 /* hard_start_xmit for devices that don't have any bugs and
5081  * support TG3_FLG2_HW_TSO_2 only.
5082  */
5083 static int tg3_start_xmit(struct sk_buff *skb, struct net_device *dev)
5084 {
5085         struct tg3 *tp = netdev_priv(dev);
5086         u32 len, entry, base_flags, mss;
5087         struct skb_shared_info *sp;
5088         dma_addr_t mapping;
5089
5090         len = skb_headlen(skb);
5091
5092         /* We are running in BH disabled context with netif_tx_lock
5093          * and TX reclaim runs via tp->napi.poll inside of a software
5094          * interrupt.  Furthermore, IRQ processing runs lockless so we have
5095          * no IRQ context deadlocks to worry about either.  Rejoice!
5096          */
5097         if (unlikely(tg3_tx_avail(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
5098                 if (!netif_queue_stopped(dev)) {
5099                         netif_stop_queue(dev);
5100
5101                         /* This is a hard error, log it. */
5102                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
5103                                "queue awake!\n", dev->name);
5104                 }
5105                 return NETDEV_TX_BUSY;
5106         }
5107
5108         entry = tp->tx_prod;
5109         base_flags = 0;
5110         mss = 0;
5111         if ((mss = skb_shinfo(skb)->gso_size) != 0) {
5112                 int tcp_opt_len, ip_tcp_len;
5113
5114                 if (skb_header_cloned(skb) &&
5115                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
5116                         dev_kfree_skb(skb);
5117                         goto out_unlock;
5118                 }
5119
5120                 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
5121                         mss |= (skb_headlen(skb) - ETH_HLEN) << 9;
5122                 else {
5123                         struct iphdr *iph = ip_hdr(skb);
5124
5125                         tcp_opt_len = tcp_optlen(skb);
5126                         ip_tcp_len = ip_hdrlen(skb) + sizeof(struct tcphdr);
5127
5128                         iph->check = 0;
5129                         iph->tot_len = htons(mss + ip_tcp_len + tcp_opt_len);
5130                         mss |= (ip_tcp_len + tcp_opt_len) << 9;
5131                 }
5132
5133                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
5134                                TXD_FLAG_CPU_POST_DMA);
5135
5136                 tcp_hdr(skb)->check = 0;
5137
5138         }
5139         else if (skb->ip_summed == CHECKSUM_PARTIAL)
5140                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
5141 #if TG3_VLAN_TAG_USED
5142         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
5143                 base_flags |= (TXD_FLAG_VLAN |
5144                                (vlan_tx_tag_get(skb) << 16));
5145 #endif
5146
5147         if (skb_dma_map(&tp->pdev->dev, skb, DMA_TO_DEVICE)) {
5148                 dev_kfree_skb(skb);
5149                 goto out_unlock;
5150         }
5151
5152         sp = skb_shinfo(skb);
5153
5154         mapping = sp->dma_maps[0];
5155
5156         tp->tx_buffers[entry].skb = skb;
5157
5158         tg3_set_txd(tp, entry, mapping, len, base_flags,
5159                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
5160
5161         entry = NEXT_TX(entry);
5162
5163         /* Now loop through additional data fragments, and queue them. */
5164         if (skb_shinfo(skb)->nr_frags > 0) {
5165                 unsigned int i, last;
5166
5167                 last = skb_shinfo(skb)->nr_frags - 1;
5168                 for (i = 0; i <= last; i++) {
5169                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
5170
5171                         len = frag->size;
5172                         mapping = sp->dma_maps[i + 1];
5173                         tp->tx_buffers[entry].skb = NULL;
5174
5175                         tg3_set_txd(tp, entry, mapping, len,
5176                                     base_flags, (i == last) | (mss << 1));
5177
5178                         entry = NEXT_TX(entry);
5179                 }
5180         }
5181
5182         /* Packets are ready, update Tx producer idx local and on card. */
5183         tw32_tx_mbox((MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW), entry);
5184
5185         tp->tx_prod = entry;
5186         if (unlikely(tg3_tx_avail(tp) <= (MAX_SKB_FRAGS + 1))) {
5187                 netif_stop_queue(dev);
5188                 if (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp))
5189                         netif_wake_queue(tp->dev);
5190         }
5191
5192 out_unlock:
5193         mmiowb();
5194
5195         dev->trans_start = jiffies;
5196
5197         return NETDEV_TX_OK;
5198 }
5199
5200 static int tg3_start_xmit_dma_bug(struct sk_buff *, struct net_device *);
5201
5202 /* Use GSO to workaround a rare TSO bug that may be triggered when the
5203  * TSO header is greater than 80 bytes.
5204  */
5205 static int tg3_tso_bug(struct tg3 *tp, struct sk_buff *skb)
5206 {
5207         struct sk_buff *segs, *nskb;
5208
5209         /* Estimate the number of fragments in the worst case */
5210         if (unlikely(tg3_tx_avail(tp) <= (skb_shinfo(skb)->gso_segs * 3))) {
5211                 netif_stop_queue(tp->dev);
5212                 if (tg3_tx_avail(tp) <= (skb_shinfo(skb)->gso_segs * 3))
5213                         return NETDEV_TX_BUSY;
5214
5215                 netif_wake_queue(tp->dev);
5216         }
5217
5218         segs = skb_gso_segment(skb, tp->dev->features & ~NETIF_F_TSO);
5219         if (IS_ERR(segs))
5220                 goto tg3_tso_bug_end;
5221
5222         do {
5223                 nskb = segs;
5224                 segs = segs->next;
5225                 nskb->next = NULL;
5226                 tg3_start_xmit_dma_bug(nskb, tp->dev);
5227         } while (segs);
5228
5229 tg3_tso_bug_end:
5230         dev_kfree_skb(skb);
5231
5232         return NETDEV_TX_OK;
5233 }
5234
5235 /* hard_start_xmit for devices that have the 4G bug and/or 40-bit bug and
5236  * support TG3_FLG2_HW_TSO_1 or firmware TSO only.
5237  */
5238 static int tg3_start_xmit_dma_bug(struct sk_buff *skb, struct net_device *dev)
5239 {
5240         struct tg3 *tp = netdev_priv(dev);
5241         u32 len, entry, base_flags, mss;
5242         struct skb_shared_info *sp;
5243         int would_hit_hwbug;
5244         dma_addr_t mapping;
5245
5246         len = skb_headlen(skb);
5247
5248         /* We are running in BH disabled context with netif_tx_lock
5249          * and TX reclaim runs via tp->napi.poll inside of a software
5250          * interrupt.  Furthermore, IRQ processing runs lockless so we have
5251          * no IRQ context deadlocks to worry about either.  Rejoice!
5252          */
5253         if (unlikely(tg3_tx_avail(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
5254                 if (!netif_queue_stopped(dev)) {
5255                         netif_stop_queue(dev);
5256
5257                         /* This is a hard error, log it. */
5258                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
5259                                "queue awake!\n", dev->name);
5260                 }
5261                 return NETDEV_TX_BUSY;
5262         }
5263
5264         entry = tp->tx_prod;
5265         base_flags = 0;
5266         if (skb->ip_summed == CHECKSUM_PARTIAL)
5267                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
5268         mss = 0;
5269         if ((mss = skb_shinfo(skb)->gso_size) != 0) {
5270                 struct iphdr *iph;
5271                 int tcp_opt_len, ip_tcp_len, hdr_len;
5272
5273                 if (skb_header_cloned(skb) &&
5274                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
5275                         dev_kfree_skb(skb);
5276                         goto out_unlock;
5277                 }
5278
5279                 tcp_opt_len = tcp_optlen(skb);
5280                 ip_tcp_len = ip_hdrlen(skb) + sizeof(struct tcphdr);
5281
5282                 hdr_len = ip_tcp_len + tcp_opt_len;
5283                 if (unlikely((ETH_HLEN + hdr_len) > 80) &&
5284                              (tp->tg3_flags2 & TG3_FLG2_TSO_BUG))
5285                         return (tg3_tso_bug(tp, skb));
5286
5287                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
5288                                TXD_FLAG_CPU_POST_DMA);
5289
5290                 iph = ip_hdr(skb);
5291                 iph->check = 0;
5292                 iph->tot_len = htons(mss + hdr_len);
5293                 if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
5294                         tcp_hdr(skb)->check = 0;
5295                         base_flags &= ~TXD_FLAG_TCPUDP_CSUM;
5296                 } else
5297                         tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
5298                                                                  iph->daddr, 0,
5299                                                                  IPPROTO_TCP,
5300                                                                  0);
5301
5302                 if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO) ||
5303                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705)) {
5304                         if (tcp_opt_len || iph->ihl > 5) {
5305                                 int tsflags;
5306
5307                                 tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
5308                                 mss |= (tsflags << 11);
5309                         }
5310                 } else {
5311                         if (tcp_opt_len || iph->ihl > 5) {
5312                                 int tsflags;
5313
5314                                 tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
5315                                 base_flags |= tsflags << 12;
5316                         }
5317                 }
5318         }
5319 #if TG3_VLAN_TAG_USED
5320         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
5321                 base_flags |= (TXD_FLAG_VLAN |
5322                                (vlan_tx_tag_get(skb) << 16));
5323 #endif
5324
5325         if (skb_dma_map(&tp->pdev->dev, skb, DMA_TO_DEVICE)) {
5326                 dev_kfree_skb(skb);
5327                 goto out_unlock;
5328         }
5329
5330         sp = skb_shinfo(skb);
5331
5332         mapping = sp->dma_maps[0];
5333
5334         tp->tx_buffers[entry].skb = skb;
5335
5336         would_hit_hwbug = 0;
5337
5338         if (tp->tg3_flags3 & TG3_FLG3_5701_DMA_BUG)
5339                 would_hit_hwbug = 1;
5340         else if (tg3_4g_overflow_test(mapping, len))
5341                 would_hit_hwbug = 1;
5342
5343         tg3_set_txd(tp, entry, mapping, len, base_flags,
5344                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
5345
5346         entry = NEXT_TX(entry);
5347
5348         /* Now loop through additional data fragments, and queue them. */
5349         if (skb_shinfo(skb)->nr_frags > 0) {
5350                 unsigned int i, last;
5351
5352                 last = skb_shinfo(skb)->nr_frags - 1;
5353                 for (i = 0; i <= last; i++) {
5354                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
5355
5356                         len = frag->size;
5357                         mapping = sp->dma_maps[i + 1];
5358
5359                         tp->tx_buffers[entry].skb = NULL;
5360
5361                         if (tg3_4g_overflow_test(mapping, len))
5362                                 would_hit_hwbug = 1;
5363
5364                         if (tg3_40bit_overflow_test(tp, mapping, len))
5365                                 would_hit_hwbug = 1;
5366
5367                         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
5368                                 tg3_set_txd(tp, entry, mapping, len,
5369                                             base_flags, (i == last)|(mss << 1));
5370                         else
5371                                 tg3_set_txd(tp, entry, mapping, len,
5372                                             base_flags, (i == last));
5373
5374                         entry = NEXT_TX(entry);
5375                 }
5376         }
5377
5378         if (would_hit_hwbug) {
5379                 u32 last_plus_one = entry;
5380                 u32 start;
5381
5382                 start = entry - 1 - skb_shinfo(skb)->nr_frags;
5383                 start &= (TG3_TX_RING_SIZE - 1);
5384
5385                 /* If the workaround fails due to memory/mapping
5386                  * failure, silently drop this packet.
5387                  */
5388                 if (tigon3_dma_hwbug_workaround(tp, skb, last_plus_one,
5389                                                 &start, base_flags, mss))
5390                         goto out_unlock;
5391
5392                 entry = start;
5393         }
5394
5395         /* Packets are ready, update Tx producer idx local and on card. */
5396         tw32_tx_mbox((MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW), entry);
5397
5398         tp->tx_prod = entry;
5399         if (unlikely(tg3_tx_avail(tp) <= (MAX_SKB_FRAGS + 1))) {
5400                 netif_stop_queue(dev);
5401                 if (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp))
5402                         netif_wake_queue(tp->dev);
5403         }
5404
5405 out_unlock:
5406         mmiowb();
5407
5408         dev->trans_start = jiffies;
5409
5410         return NETDEV_TX_OK;
5411 }
5412
5413 static inline void tg3_set_mtu(struct net_device *dev, struct tg3 *tp,
5414                                int new_mtu)
5415 {
5416         dev->mtu = new_mtu;
5417
5418         if (new_mtu > ETH_DATA_LEN) {
5419                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
5420                         tp->tg3_flags2 &= ~TG3_FLG2_TSO_CAPABLE;
5421                         ethtool_op_set_tso(dev, 0);
5422                 }
5423                 else
5424                         tp->tg3_flags |= TG3_FLAG_JUMBO_RING_ENABLE;
5425         } else {
5426                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
5427                         tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
5428                 tp->tg3_flags &= ~TG3_FLAG_JUMBO_RING_ENABLE;
5429         }
5430 }
5431
5432 static int tg3_change_mtu(struct net_device *dev, int new_mtu)
5433 {
5434         struct tg3 *tp = netdev_priv(dev);
5435         int err;
5436
5437         if (new_mtu < TG3_MIN_MTU || new_mtu > TG3_MAX_MTU(tp))
5438                 return -EINVAL;
5439
5440         if (!netif_running(dev)) {
5441                 /* We'll just catch it later when the
5442                  * device is up'd.
5443                  */
5444                 tg3_set_mtu(dev, tp, new_mtu);
5445                 return 0;
5446         }
5447
5448         tg3_phy_stop(tp);
5449
5450         tg3_netif_stop(tp);
5451
5452         tg3_full_lock(tp, 1);
5453
5454         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
5455
5456         tg3_set_mtu(dev, tp, new_mtu);
5457
5458         err = tg3_restart_hw(tp, 0);
5459
5460         if (!err)
5461                 tg3_netif_start(tp);
5462
5463         tg3_full_unlock(tp);
5464
5465         if (!err)
5466                 tg3_phy_start(tp);
5467
5468         return err;
5469 }
5470
5471 /* Free up pending packets in all rx/tx rings.
5472  *
5473  * The chip has been shut down and the driver detached from
5474  * the networking, so no interrupts or new tx packets will
5475  * end up in the driver.  tp->{tx,}lock is not held and we are not
5476  * in an interrupt context and thus may sleep.
5477  */
5478 static void tg3_free_rings(struct tg3 *tp)
5479 {
5480         struct ring_info *rxp;
5481         int i;
5482
5483         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
5484                 rxp = &tp->rx_std_buffers[i];
5485
5486                 if (rxp->skb == NULL)
5487                         continue;
5488                 pci_unmap_single(tp->pdev,
5489                                  pci_unmap_addr(rxp, mapping),
5490                                  tp->rx_pkt_buf_sz - tp->rx_offset,
5491                                  PCI_DMA_FROMDEVICE);
5492                 dev_kfree_skb_any(rxp->skb);
5493                 rxp->skb = NULL;
5494         }
5495
5496         for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
5497                 rxp = &tp->rx_jumbo_buffers[i];
5498
5499                 if (rxp->skb == NULL)
5500                         continue;
5501                 pci_unmap_single(tp->pdev,
5502                                  pci_unmap_addr(rxp, mapping),
5503                                  RX_JUMBO_PKT_BUF_SZ - tp->rx_offset,
5504                                  PCI_DMA_FROMDEVICE);
5505                 dev_kfree_skb_any(rxp->skb);
5506                 rxp->skb = NULL;
5507         }
5508
5509         for (i = 0; i < TG3_TX_RING_SIZE; ) {
5510                 struct tx_ring_info *txp;
5511                 struct sk_buff *skb;
5512
5513                 txp = &tp->tx_buffers[i];
5514                 skb = txp->skb;
5515
5516                 if (skb == NULL) {
5517                         i++;
5518                         continue;
5519                 }
5520
5521                 skb_dma_unmap(&tp->pdev->dev, skb, DMA_TO_DEVICE);
5522
5523                 txp->skb = NULL;
5524
5525                 i += skb_shinfo(skb)->nr_frags + 1;
5526
5527                 dev_kfree_skb_any(skb);
5528         }
5529 }
5530
5531 /* Initialize tx/rx rings for packet processing.
5532  *
5533  * The chip has been shut down and the driver detached from
5534  * the networking, so no interrupts or new tx packets will
5535  * end up in the driver.  tp->{tx,}lock are held and thus
5536  * we may not sleep.
5537  */
5538 static int tg3_init_rings(struct tg3 *tp)
5539 {
5540         u32 i;
5541
5542         /* Free up all the SKBs. */
5543         tg3_free_rings(tp);
5544
5545         /* Zero out all descriptors. */
5546         memset(tp->rx_std, 0, TG3_RX_RING_BYTES);
5547         memset(tp->rx_jumbo, 0, TG3_RX_JUMBO_RING_BYTES);
5548         memset(tp->rx_rcb, 0, TG3_RX_RCB_RING_BYTES(tp));
5549         memset(tp->tx_ring, 0, TG3_TX_RING_BYTES);
5550
5551         tp->rx_pkt_buf_sz = RX_PKT_BUF_SZ;
5552         if ((tp->tg3_flags2 & TG3_FLG2_5780_CLASS) &&
5553             (tp->dev->mtu > ETH_DATA_LEN))
5554                 tp->rx_pkt_buf_sz = RX_JUMBO_PKT_BUF_SZ;
5555
5556         /* Initialize invariants of the rings, we only set this
5557          * stuff once.  This works because the card does not
5558          * write into the rx buffer posting rings.
5559          */
5560         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
5561                 struct tg3_rx_buffer_desc *rxd;
5562
5563                 rxd = &tp->rx_std[i];
5564                 rxd->idx_len = (tp->rx_pkt_buf_sz - tp->rx_offset - 64)
5565                         << RXD_LEN_SHIFT;
5566                 rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT);
5567                 rxd->opaque = (RXD_OPAQUE_RING_STD |
5568                                (i << RXD_OPAQUE_INDEX_SHIFT));
5569         }
5570
5571         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
5572                 for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
5573                         struct tg3_rx_buffer_desc *rxd;
5574
5575                         rxd = &tp->rx_jumbo[i];
5576                         rxd->idx_len = (RX_JUMBO_PKT_BUF_SZ - tp->rx_offset - 64)
5577                                 << RXD_LEN_SHIFT;
5578                         rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT) |
5579                                 RXD_FLAG_JUMBO;
5580                         rxd->opaque = (RXD_OPAQUE_RING_JUMBO |
5581                                (i << RXD_OPAQUE_INDEX_SHIFT));
5582                 }
5583         }
5584
5585         /* Now allocate fresh SKBs for each rx ring. */
5586         for (i = 0; i < tp->rx_pending; i++) {
5587                 if (tg3_alloc_rx_skb(tp, RXD_OPAQUE_RING_STD, -1, i) < 0) {
5588                         printk(KERN_WARNING PFX
5589                                "%s: Using a smaller RX standard ring, "
5590                                "only %d out of %d buffers were allocated "
5591                                "successfully.\n",
5592                                tp->dev->name, i, tp->rx_pending);
5593                         if (i == 0)
5594                                 return -ENOMEM;
5595                         tp->rx_pending = i;
5596                         break;
5597                 }
5598         }
5599
5600         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
5601                 for (i = 0; i < tp->rx_jumbo_pending; i++) {
5602                         if (tg3_alloc_rx_skb(tp, RXD_OPAQUE_RING_JUMBO,
5603                                              -1, i) < 0) {
5604                                 printk(KERN_WARNING PFX
5605                                        "%s: Using a smaller RX jumbo ring, "
5606                                        "only %d out of %d buffers were "
5607                                        "allocated successfully.\n",
5608                                        tp->dev->name, i, tp->rx_jumbo_pending);
5609                                 if (i == 0) {
5610                                         tg3_free_rings(tp);
5611                                         return -ENOMEM;
5612                                 }
5613                                 tp->rx_jumbo_pending = i;
5614                                 break;
5615                         }
5616                 }
5617         }
5618         return 0;
5619 }
5620
5621 /*
5622  * Must not be invoked with interrupt sources disabled and
5623  * the hardware shutdown down.
5624  */
5625 static void tg3_free_consistent(struct tg3 *tp)
5626 {
5627         kfree(tp->rx_std_buffers);
5628         tp->rx_std_buffers = NULL;
5629         if (tp->rx_std) {
5630                 pci_free_consistent(tp->pdev, TG3_RX_RING_BYTES,
5631                                     tp->rx_std, tp->rx_std_mapping);
5632                 tp->rx_std = NULL;
5633         }
5634         if (tp->rx_jumbo) {
5635                 pci_free_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
5636                                     tp->rx_jumbo, tp->rx_jumbo_mapping);
5637                 tp->rx_jumbo = NULL;
5638         }
5639         if (tp->rx_rcb) {
5640                 pci_free_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
5641                                     tp->rx_rcb, tp->rx_rcb_mapping);
5642                 tp->rx_rcb = NULL;
5643         }
5644         if (tp->tx_ring) {
5645                 pci_free_consistent(tp->pdev, TG3_TX_RING_BYTES,
5646                         tp->tx_ring, tp->tx_desc_mapping);
5647                 tp->tx_ring = NULL;
5648         }
5649         if (tp->hw_status) {
5650                 pci_free_consistent(tp->pdev, TG3_HW_STATUS_SIZE,
5651                                     tp->hw_status, tp->status_mapping);
5652                 tp->hw_status = NULL;
5653         }
5654         if (tp->hw_stats) {
5655                 pci_free_consistent(tp->pdev, sizeof(struct tg3_hw_stats),
5656                                     tp->hw_stats, tp->stats_mapping);
5657                 tp->hw_stats = NULL;
5658         }
5659 }
5660
5661 /*
5662  * Must not be invoked with interrupt sources disabled and
5663  * the hardware shutdown down.  Can sleep.
5664  */
5665 static int tg3_alloc_consistent(struct tg3 *tp)
5666 {
5667         tp->rx_std_buffers = kzalloc((sizeof(struct ring_info) *
5668                                       (TG3_RX_RING_SIZE +
5669                                        TG3_RX_JUMBO_RING_SIZE)) +
5670                                      (sizeof(struct tx_ring_info) *
5671                                       TG3_TX_RING_SIZE),
5672                                      GFP_KERNEL);
5673         if (!tp->rx_std_buffers)
5674                 return -ENOMEM;
5675
5676         tp->rx_jumbo_buffers = &tp->rx_std_buffers[TG3_RX_RING_SIZE];
5677         tp->tx_buffers = (struct tx_ring_info *)
5678                 &tp->rx_jumbo_buffers[TG3_RX_JUMBO_RING_SIZE];
5679
5680         tp->rx_std = pci_alloc_consistent(tp->pdev, TG3_RX_RING_BYTES,
5681                                           &tp->rx_std_mapping);
5682         if (!tp->rx_std)
5683                 goto err_out;
5684
5685         tp->rx_jumbo = pci_alloc_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
5686                                             &tp->rx_jumbo_mapping);
5687
5688         if (!tp->rx_jumbo)
5689                 goto err_out;
5690
5691         tp->rx_rcb = pci_alloc_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
5692                                           &tp->rx_rcb_mapping);
5693         if (!tp->rx_rcb)
5694                 goto err_out;
5695
5696         tp->tx_ring = pci_alloc_consistent(tp->pdev, TG3_TX_RING_BYTES,
5697                                            &tp->tx_desc_mapping);
5698         if (!tp->tx_ring)
5699                 goto err_out;
5700
5701         tp->hw_status = pci_alloc_consistent(tp->pdev,
5702                                              TG3_HW_STATUS_SIZE,
5703                                              &tp->status_mapping);
5704         if (!tp->hw_status)
5705                 goto err_out;
5706
5707         tp->hw_stats = pci_alloc_consistent(tp->pdev,
5708                                             sizeof(struct tg3_hw_stats),
5709                                             &tp->stats_mapping);
5710         if (!tp->hw_stats)
5711                 goto err_out;
5712
5713         memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
5714         memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
5715
5716         return 0;
5717
5718 err_out:
5719         tg3_free_consistent(tp);
5720         return -ENOMEM;
5721 }
5722
5723 #define MAX_WAIT_CNT 1000
5724
5725 /* To stop a block, clear the enable bit and poll till it
5726  * clears.  tp->lock is held.
5727  */
5728 static int tg3_stop_block(struct tg3 *tp, unsigned long ofs, u32 enable_bit, int silent)
5729 {
5730         unsigned int i;
5731         u32 val;
5732
5733         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
5734                 switch (ofs) {
5735                 case RCVLSC_MODE:
5736                 case DMAC_MODE:
5737                 case MBFREE_MODE:
5738                 case BUFMGR_MODE:
5739                 case MEMARB_MODE:
5740                         /* We can't enable/disable these bits of the
5741                          * 5705/5750, just say success.
5742                          */
5743                         return 0;
5744
5745                 default:
5746                         break;
5747                 }
5748         }
5749
5750         val = tr32(ofs);
5751         val &= ~enable_bit;
5752         tw32_f(ofs, val);
5753
5754         for (i = 0; i < MAX_WAIT_CNT; i++) {
5755                 udelay(100);
5756                 val = tr32(ofs);
5757                 if ((val & enable_bit) == 0)
5758                         break;
5759         }
5760
5761         if (i == MAX_WAIT_CNT && !silent) {
5762                 printk(KERN_ERR PFX "tg3_stop_block timed out, "
5763                        "ofs=%lx enable_bit=%x\n",
5764                        ofs, enable_bit);
5765                 return -ENODEV;
5766         }
5767
5768         return 0;
5769 }
5770
5771 /* tp->lock is held. */
5772 static int tg3_abort_hw(struct tg3 *tp, int silent)
5773 {
5774         int i, err;
5775
5776         tg3_disable_ints(tp);
5777
5778         tp->rx_mode &= ~RX_MODE_ENABLE;
5779         tw32_f(MAC_RX_MODE, tp->rx_mode);
5780         udelay(10);
5781
5782         err  = tg3_stop_block(tp, RCVBDI_MODE, RCVBDI_MODE_ENABLE, silent);
5783         err |= tg3_stop_block(tp, RCVLPC_MODE, RCVLPC_MODE_ENABLE, silent);
5784         err |= tg3_stop_block(tp, RCVLSC_MODE, RCVLSC_MODE_ENABLE, silent);
5785         err |= tg3_stop_block(tp, RCVDBDI_MODE, RCVDBDI_MODE_ENABLE, silent);