smsc95xx: Add module params to read MAC address
[linux-2.6.git] / drivers / net / tg3.c
1 /*
2  * tg3.c: Broadcom Tigon3 ethernet driver.
3  *
4  * Copyright (C) 2001, 2002, 2003, 2004 David S. Miller (davem@redhat.com)
5  * Copyright (C) 2001, 2002, 2003 Jeff Garzik (jgarzik@pobox.com)
6  * Copyright (C) 2004 Sun Microsystems Inc.
7  * Copyright (C) 2005-2011 Broadcom Corporation.
8  *
9  * Firmware is:
10  *      Derived from proprietary unpublished source code,
11  *      Copyright (C) 2000-2003 Broadcom Corporation.
12  *
13  *      Permission is hereby granted for the distribution of this firmware
14  *      data in hexadecimal or equivalent format, provided this copyright
15  *      notice is accompanying it.
16  */
17
18
19 #include <linux/module.h>
20 #include <linux/moduleparam.h>
21 #include <linux/stringify.h>
22 #include <linux/kernel.h>
23 #include <linux/types.h>
24 #include <linux/compiler.h>
25 #include <linux/slab.h>
26 #include <linux/delay.h>
27 #include <linux/in.h>
28 #include <linux/init.h>
29 #include <linux/interrupt.h>
30 #include <linux/ioport.h>
31 #include <linux/pci.h>
32 #include <linux/netdevice.h>
33 #include <linux/etherdevice.h>
34 #include <linux/skbuff.h>
35 #include <linux/ethtool.h>
36 #include <linux/mdio.h>
37 #include <linux/mii.h>
38 #include <linux/phy.h>
39 #include <linux/brcmphy.h>
40 #include <linux/if_vlan.h>
41 #include <linux/ip.h>
42 #include <linux/tcp.h>
43 #include <linux/workqueue.h>
44 #include <linux/prefetch.h>
45 #include <linux/dma-mapping.h>
46 #include <linux/firmware.h>
47
48 #include <net/checksum.h>
49 #include <net/ip.h>
50
51 #include <asm/system.h>
52 #include <linux/io.h>
53 #include <asm/byteorder.h>
54 #include <linux/uaccess.h>
55
56 #ifdef CONFIG_SPARC
57 #include <asm/idprom.h>
58 #include <asm/prom.h>
59 #endif
60
61 #define BAR_0   0
62 #define BAR_2   2
63
64 #include "tg3.h"
65
66 /* Functions & macros to verify TG3_FLAGS types */
67
68 static inline int _tg3_flag(enum TG3_FLAGS flag, unsigned long *bits)
69 {
70         return test_bit(flag, bits);
71 }
72
73 static inline void _tg3_flag_set(enum TG3_FLAGS flag, unsigned long *bits)
74 {
75         set_bit(flag, bits);
76 }
77
78 static inline void _tg3_flag_clear(enum TG3_FLAGS flag, unsigned long *bits)
79 {
80         clear_bit(flag, bits);
81 }
82
83 #define tg3_flag(tp, flag)                              \
84         _tg3_flag(TG3_FLAG_##flag, (tp)->tg3_flags)
85 #define tg3_flag_set(tp, flag)                          \
86         _tg3_flag_set(TG3_FLAG_##flag, (tp)->tg3_flags)
87 #define tg3_flag_clear(tp, flag)                        \
88         _tg3_flag_clear(TG3_FLAG_##flag, (tp)->tg3_flags)
89
90 #define DRV_MODULE_NAME         "tg3"
91 #define TG3_MAJ_NUM                     3
92 #define TG3_MIN_NUM                     119
93 #define DRV_MODULE_VERSION      \
94         __stringify(TG3_MAJ_NUM) "." __stringify(TG3_MIN_NUM)
95 #define DRV_MODULE_RELDATE      "May 18, 2011"
96
97 #define TG3_DEF_MAC_MODE        0
98 #define TG3_DEF_RX_MODE         0
99 #define TG3_DEF_TX_MODE         0
100 #define TG3_DEF_MSG_ENABLE        \
101         (NETIF_MSG_DRV          | \
102          NETIF_MSG_PROBE        | \
103          NETIF_MSG_LINK         | \
104          NETIF_MSG_TIMER        | \
105          NETIF_MSG_IFDOWN       | \
106          NETIF_MSG_IFUP         | \
107          NETIF_MSG_RX_ERR       | \
108          NETIF_MSG_TX_ERR)
109
110 #define TG3_GRC_LCLCTL_PWRSW_DELAY      100
111
112 /* length of time before we decide the hardware is borked,
113  * and dev->tx_timeout() should be called to fix the problem
114  */
115
116 #define TG3_TX_TIMEOUT                  (5 * HZ)
117
118 /* hardware minimum and maximum for a single frame's data payload */
119 #define TG3_MIN_MTU                     60
120 #define TG3_MAX_MTU(tp) \
121         (tg3_flag(tp, JUMBO_CAPABLE) ? 9000 : 1500)
122
123 /* These numbers seem to be hard coded in the NIC firmware somehow.
124  * You can't change the ring sizes, but you can change where you place
125  * them in the NIC onboard memory.
126  */
127 #define TG3_RX_STD_RING_SIZE(tp) \
128         (tg3_flag(tp, LRG_PROD_RING_CAP) ? \
129          TG3_RX_STD_MAX_SIZE_5717 : TG3_RX_STD_MAX_SIZE_5700)
130 #define TG3_DEF_RX_RING_PENDING         200
131 #define TG3_RX_JMB_RING_SIZE(tp) \
132         (tg3_flag(tp, LRG_PROD_RING_CAP) ? \
133          TG3_RX_JMB_MAX_SIZE_5717 : TG3_RX_JMB_MAX_SIZE_5700)
134 #define TG3_DEF_RX_JUMBO_RING_PENDING   100
135 #define TG3_RSS_INDIR_TBL_SIZE          128
136
137 /* Do not place this n-ring entries value into the tp struct itself,
138  * we really want to expose these constants to GCC so that modulo et
139  * al.  operations are done with shifts and masks instead of with
140  * hw multiply/modulo instructions.  Another solution would be to
141  * replace things like '% foo' with '& (foo - 1)'.
142  */
143
144 #define TG3_TX_RING_SIZE                512
145 #define TG3_DEF_TX_RING_PENDING         (TG3_TX_RING_SIZE - 1)
146
147 #define TG3_RX_STD_RING_BYTES(tp) \
148         (sizeof(struct tg3_rx_buffer_desc) * TG3_RX_STD_RING_SIZE(tp))
149 #define TG3_RX_JMB_RING_BYTES(tp) \
150         (sizeof(struct tg3_ext_rx_buffer_desc) * TG3_RX_JMB_RING_SIZE(tp))
151 #define TG3_RX_RCB_RING_BYTES(tp) \
152         (sizeof(struct tg3_rx_buffer_desc) * (tp->rx_ret_ring_mask + 1))
153 #define TG3_TX_RING_BYTES       (sizeof(struct tg3_tx_buffer_desc) * \
154                                  TG3_TX_RING_SIZE)
155 #define NEXT_TX(N)              (((N) + 1) & (TG3_TX_RING_SIZE - 1))
156
157 #define TG3_DMA_BYTE_ENAB               64
158
159 #define TG3_RX_STD_DMA_SZ               1536
160 #define TG3_RX_JMB_DMA_SZ               9046
161
162 #define TG3_RX_DMA_TO_MAP_SZ(x)         ((x) + TG3_DMA_BYTE_ENAB)
163
164 #define TG3_RX_STD_MAP_SZ               TG3_RX_DMA_TO_MAP_SZ(TG3_RX_STD_DMA_SZ)
165 #define TG3_RX_JMB_MAP_SZ               TG3_RX_DMA_TO_MAP_SZ(TG3_RX_JMB_DMA_SZ)
166
167 #define TG3_RX_STD_BUFF_RING_SIZE(tp) \
168         (sizeof(struct ring_info) * TG3_RX_STD_RING_SIZE(tp))
169
170 #define TG3_RX_JMB_BUFF_RING_SIZE(tp) \
171         (sizeof(struct ring_info) * TG3_RX_JMB_RING_SIZE(tp))
172
173 /* Due to a hardware bug, the 5701 can only DMA to memory addresses
174  * that are at least dword aligned when used in PCIX mode.  The driver
175  * works around this bug by double copying the packet.  This workaround
176  * is built into the normal double copy length check for efficiency.
177  *
178  * However, the double copy is only necessary on those architectures
179  * where unaligned memory accesses are inefficient.  For those architectures
180  * where unaligned memory accesses incur little penalty, we can reintegrate
181  * the 5701 in the normal rx path.  Doing so saves a device structure
182  * dereference by hardcoding the double copy threshold in place.
183  */
184 #define TG3_RX_COPY_THRESHOLD           256
185 #if NET_IP_ALIGN == 0 || defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
186         #define TG3_RX_COPY_THRESH(tp)  TG3_RX_COPY_THRESHOLD
187 #else
188         #define TG3_RX_COPY_THRESH(tp)  ((tp)->rx_copy_thresh)
189 #endif
190
191 /* minimum number of free TX descriptors required to wake up TX process */
192 #define TG3_TX_WAKEUP_THRESH(tnapi)             ((tnapi)->tx_pending / 4)
193 #define TG3_TX_BD_DMA_MAX               4096
194
195 #define TG3_RAW_IP_ALIGN 2
196
197 #define TG3_FW_UPDATE_TIMEOUT_SEC       5
198
199 #define FIRMWARE_TG3            "tigon/tg3.bin"
200 #define FIRMWARE_TG3TSO         "tigon/tg3_tso.bin"
201 #define FIRMWARE_TG3TSO5        "tigon/tg3_tso5.bin"
202
203 static char version[] __devinitdata =
204         DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")";
205
206 MODULE_AUTHOR("David S. Miller (davem@redhat.com) and Jeff Garzik (jgarzik@pobox.com)");
207 MODULE_DESCRIPTION("Broadcom Tigon3 ethernet driver");
208 MODULE_LICENSE("GPL");
209 MODULE_VERSION(DRV_MODULE_VERSION);
210 MODULE_FIRMWARE(FIRMWARE_TG3);
211 MODULE_FIRMWARE(FIRMWARE_TG3TSO);
212 MODULE_FIRMWARE(FIRMWARE_TG3TSO5);
213
214 static int tg3_debug = -1;      /* -1 == use TG3_DEF_MSG_ENABLE as value */
215 module_param(tg3_debug, int, 0);
216 MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");
217
218 static DEFINE_PCI_DEVICE_TABLE(tg3_pci_tbl) = {
219         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700)},
220         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701)},
221         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702)},
222         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703)},
223         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704)},
224         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE)},
225         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705)},
226         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2)},
227         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M)},
228         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2)},
229         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X)},
230         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X)},
231         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S)},
232         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3)},
233         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3)},
234         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782)},
235         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788)},
236         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789)},
237         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901)},
238         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2)},
239         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2)},
240         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F)},
241         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721)},
242         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5722)},
243         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751)},
244         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M)},
245         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F)},
246         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752)},
247         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M)},
248         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753)},
249         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M)},
250         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F)},
251         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754)},
252         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M)},
253         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755)},
254         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M)},
255         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5756)},
256         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786)},
257         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787)},
258         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M)},
259         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787F)},
260         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714)},
261         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S)},
262         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715)},
263         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S)},
264         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780)},
265         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S)},
266         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781)},
267         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906)},
268         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906M)},
269         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5784)},
270         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5764)},
271         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5723)},
272         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761)},
273         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761E)},
274         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5761S)},
275         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5761SE)},
276         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5785_G)},
277         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5785_F)},
278         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57780)},
279         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57760)},
280         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57790)},
281         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57788)},
282         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5717)},
283         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5718)},
284         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57781)},
285         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57785)},
286         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57761)},
287         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57765)},
288         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57791)},
289         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57795)},
290         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5719)},
291         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5720)},
292         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX)},
293         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX)},
294         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000)},
295         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001)},
296         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003)},
297         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100)},
298         {PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3)},
299         {PCI_DEVICE(0x10cf, 0x11a2)}, /* Fujitsu 1000base-SX with BCM5703SKHB */
300         {}
301 };
302
303 MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);
304
305 static const struct {
306         const char string[ETH_GSTRING_LEN];
307 } ethtool_stats_keys[] = {
308         { "rx_octets" },
309         { "rx_fragments" },
310         { "rx_ucast_packets" },
311         { "rx_mcast_packets" },
312         { "rx_bcast_packets" },
313         { "rx_fcs_errors" },
314         { "rx_align_errors" },
315         { "rx_xon_pause_rcvd" },
316         { "rx_xoff_pause_rcvd" },
317         { "rx_mac_ctrl_rcvd" },
318         { "rx_xoff_entered" },
319         { "rx_frame_too_long_errors" },
320         { "rx_jabbers" },
321         { "rx_undersize_packets" },
322         { "rx_in_length_errors" },
323         { "rx_out_length_errors" },
324         { "rx_64_or_less_octet_packets" },
325         { "rx_65_to_127_octet_packets" },
326         { "rx_128_to_255_octet_packets" },
327         { "rx_256_to_511_octet_packets" },
328         { "rx_512_to_1023_octet_packets" },
329         { "rx_1024_to_1522_octet_packets" },
330         { "rx_1523_to_2047_octet_packets" },
331         { "rx_2048_to_4095_octet_packets" },
332         { "rx_4096_to_8191_octet_packets" },
333         { "rx_8192_to_9022_octet_packets" },
334
335         { "tx_octets" },
336         { "tx_collisions" },
337
338         { "tx_xon_sent" },
339         { "tx_xoff_sent" },
340         { "tx_flow_control" },
341         { "tx_mac_errors" },
342         { "tx_single_collisions" },
343         { "tx_mult_collisions" },
344         { "tx_deferred" },
345         { "tx_excessive_collisions" },
346         { "tx_late_collisions" },
347         { "tx_collide_2times" },
348         { "tx_collide_3times" },
349         { "tx_collide_4times" },
350         { "tx_collide_5times" },
351         { "tx_collide_6times" },
352         { "tx_collide_7times" },
353         { "tx_collide_8times" },
354         { "tx_collide_9times" },
355         { "tx_collide_10times" },
356         { "tx_collide_11times" },
357         { "tx_collide_12times" },
358         { "tx_collide_13times" },
359         { "tx_collide_14times" },
360         { "tx_collide_15times" },
361         { "tx_ucast_packets" },
362         { "tx_mcast_packets" },
363         { "tx_bcast_packets" },
364         { "tx_carrier_sense_errors" },
365         { "tx_discards" },
366         { "tx_errors" },
367
368         { "dma_writeq_full" },
369         { "dma_write_prioq_full" },
370         { "rxbds_empty" },
371         { "rx_discards" },
372         { "rx_errors" },
373         { "rx_threshold_hit" },
374
375         { "dma_readq_full" },
376         { "dma_read_prioq_full" },
377         { "tx_comp_queue_full" },
378
379         { "ring_set_send_prod_index" },
380         { "ring_status_update" },
381         { "nic_irqs" },
382         { "nic_avoided_irqs" },
383         { "nic_tx_threshold_hit" },
384
385         { "mbuf_lwm_thresh_hit" },
386 };
387
388 #define TG3_NUM_STATS   ARRAY_SIZE(ethtool_stats_keys)
389
390
391 static const struct {
392         const char string[ETH_GSTRING_LEN];
393 } ethtool_test_keys[] = {
394         { "nvram test     (online) " },
395         { "link test      (online) " },
396         { "register test  (offline)" },
397         { "memory test    (offline)" },
398         { "loopback test  (offline)" },
399         { "interrupt test (offline)" },
400 };
401
402 #define TG3_NUM_TEST    ARRAY_SIZE(ethtool_test_keys)
403
404
405 static void tg3_write32(struct tg3 *tp, u32 off, u32 val)
406 {
407         writel(val, tp->regs + off);
408 }
409
410 static u32 tg3_read32(struct tg3 *tp, u32 off)
411 {
412         return readl(tp->regs + off);
413 }
414
415 static void tg3_ape_write32(struct tg3 *tp, u32 off, u32 val)
416 {
417         writel(val, tp->aperegs + off);
418 }
419
420 static u32 tg3_ape_read32(struct tg3 *tp, u32 off)
421 {
422         return readl(tp->aperegs + off);
423 }
424
425 static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val)
426 {
427         unsigned long flags;
428
429         spin_lock_irqsave(&tp->indirect_lock, flags);
430         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
431         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
432         spin_unlock_irqrestore(&tp->indirect_lock, flags);
433 }
434
435 static void tg3_write_flush_reg32(struct tg3 *tp, u32 off, u32 val)
436 {
437         writel(val, tp->regs + off);
438         readl(tp->regs + off);
439 }
440
441 static u32 tg3_read_indirect_reg32(struct tg3 *tp, u32 off)
442 {
443         unsigned long flags;
444         u32 val;
445
446         spin_lock_irqsave(&tp->indirect_lock, flags);
447         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
448         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
449         spin_unlock_irqrestore(&tp->indirect_lock, flags);
450         return val;
451 }
452
453 static void tg3_write_indirect_mbox(struct tg3 *tp, u32 off, u32 val)
454 {
455         unsigned long flags;
456
457         if (off == (MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW)) {
458                 pci_write_config_dword(tp->pdev, TG3PCI_RCV_RET_RING_CON_IDX +
459                                        TG3_64BIT_REG_LOW, val);
460                 return;
461         }
462         if (off == TG3_RX_STD_PROD_IDX_REG) {
463                 pci_write_config_dword(tp->pdev, TG3PCI_STD_RING_PROD_IDX +
464                                        TG3_64BIT_REG_LOW, val);
465                 return;
466         }
467
468         spin_lock_irqsave(&tp->indirect_lock, flags);
469         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
470         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
471         spin_unlock_irqrestore(&tp->indirect_lock, flags);
472
473         /* In indirect mode when disabling interrupts, we also need
474          * to clear the interrupt bit in the GRC local ctrl register.
475          */
476         if ((off == (MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW)) &&
477             (val == 0x1)) {
478                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_LOCAL_CTRL,
479                                        tp->grc_local_ctrl|GRC_LCLCTRL_CLEARINT);
480         }
481 }
482
483 static u32 tg3_read_indirect_mbox(struct tg3 *tp, u32 off)
484 {
485         unsigned long flags;
486         u32 val;
487
488         spin_lock_irqsave(&tp->indirect_lock, flags);
489         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
490         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
491         spin_unlock_irqrestore(&tp->indirect_lock, flags);
492         return val;
493 }
494
495 /* usec_wait specifies the wait time in usec when writing to certain registers
496  * where it is unsafe to read back the register without some delay.
497  * GRC_LOCAL_CTRL is one example if the GPIOs are toggled to switch power.
498  * TG3PCI_CLOCK_CTRL is another example if the clock frequencies are changed.
499  */
500 static void _tw32_flush(struct tg3 *tp, u32 off, u32 val, u32 usec_wait)
501 {
502         if (tg3_flag(tp, PCIX_TARGET_HWBUG) || tg3_flag(tp, ICH_WORKAROUND))
503                 /* Non-posted methods */
504                 tp->write32(tp, off, val);
505         else {
506                 /* Posted method */
507                 tg3_write32(tp, off, val);
508                 if (usec_wait)
509                         udelay(usec_wait);
510                 tp->read32(tp, off);
511         }
512         /* Wait again after the read for the posted method to guarantee that
513          * the wait time is met.
514          */
515         if (usec_wait)
516                 udelay(usec_wait);
517 }
518
519 static inline void tw32_mailbox_flush(struct tg3 *tp, u32 off, u32 val)
520 {
521         tp->write32_mbox(tp, off, val);
522         if (!tg3_flag(tp, MBOX_WRITE_REORDER) && !tg3_flag(tp, ICH_WORKAROUND))
523                 tp->read32_mbox(tp, off);
524 }
525
526 static void tg3_write32_tx_mbox(struct tg3 *tp, u32 off, u32 val)
527 {
528         void __iomem *mbox = tp->regs + off;
529         writel(val, mbox);
530         if (tg3_flag(tp, TXD_MBOX_HWBUG))
531                 writel(val, mbox);
532         if (tg3_flag(tp, MBOX_WRITE_REORDER))
533                 readl(mbox);
534 }
535
536 static u32 tg3_read32_mbox_5906(struct tg3 *tp, u32 off)
537 {
538         return readl(tp->regs + off + GRCMBOX_BASE);
539 }
540
541 static void tg3_write32_mbox_5906(struct tg3 *tp, u32 off, u32 val)
542 {
543         writel(val, tp->regs + off + GRCMBOX_BASE);
544 }
545
546 #define tw32_mailbox(reg, val)          tp->write32_mbox(tp, reg, val)
547 #define tw32_mailbox_f(reg, val)        tw32_mailbox_flush(tp, (reg), (val))
548 #define tw32_rx_mbox(reg, val)          tp->write32_rx_mbox(tp, reg, val)
549 #define tw32_tx_mbox(reg, val)          tp->write32_tx_mbox(tp, reg, val)
550 #define tr32_mailbox(reg)               tp->read32_mbox(tp, reg)
551
552 #define tw32(reg, val)                  tp->write32(tp, reg, val)
553 #define tw32_f(reg, val)                _tw32_flush(tp, (reg), (val), 0)
554 #define tw32_wait_f(reg, val, us)       _tw32_flush(tp, (reg), (val), (us))
555 #define tr32(reg)                       tp->read32(tp, reg)
556
557 static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val)
558 {
559         unsigned long flags;
560
561         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906 &&
562             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC))
563                 return;
564
565         spin_lock_irqsave(&tp->indirect_lock, flags);
566         if (tg3_flag(tp, SRAM_USE_CONFIG)) {
567                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
568                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
569
570                 /* Always leave this as zero. */
571                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
572         } else {
573                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
574                 tw32_f(TG3PCI_MEM_WIN_DATA, val);
575
576                 /* Always leave this as zero. */
577                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
578         }
579         spin_unlock_irqrestore(&tp->indirect_lock, flags);
580 }
581
582 static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val)
583 {
584         unsigned long flags;
585
586         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906 &&
587             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC)) {
588                 *val = 0;
589                 return;
590         }
591
592         spin_lock_irqsave(&tp->indirect_lock, flags);
593         if (tg3_flag(tp, SRAM_USE_CONFIG)) {
594                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
595                 pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
596
597                 /* Always leave this as zero. */
598                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
599         } else {
600                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
601                 *val = tr32(TG3PCI_MEM_WIN_DATA);
602
603                 /* Always leave this as zero. */
604                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
605         }
606         spin_unlock_irqrestore(&tp->indirect_lock, flags);
607 }
608
609 static void tg3_ape_lock_init(struct tg3 *tp)
610 {
611         int i;
612         u32 regbase, bit;
613
614         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
615                 regbase = TG3_APE_LOCK_GRANT;
616         else
617                 regbase = TG3_APE_PER_LOCK_GRANT;
618
619         /* Make sure the driver hasn't any stale locks. */
620         for (i = 0; i < 8; i++) {
621                 if (i == TG3_APE_LOCK_GPIO)
622                         continue;
623                 tg3_ape_write32(tp, regbase + 4 * i, APE_LOCK_GRANT_DRIVER);
624         }
625
626         /* Clear the correct bit of the GPIO lock too. */
627         if (!tp->pci_fn)
628                 bit = APE_LOCK_GRANT_DRIVER;
629         else
630                 bit = 1 << tp->pci_fn;
631
632         tg3_ape_write32(tp, regbase + 4 * TG3_APE_LOCK_GPIO, bit);
633 }
634
635 static int tg3_ape_lock(struct tg3 *tp, int locknum)
636 {
637         int i, off;
638         int ret = 0;
639         u32 status, req, gnt, bit;
640
641         if (!tg3_flag(tp, ENABLE_APE))
642                 return 0;
643
644         switch (locknum) {
645         case TG3_APE_LOCK_GPIO:
646                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
647                         return 0;
648         case TG3_APE_LOCK_GRC:
649         case TG3_APE_LOCK_MEM:
650                 break;
651         default:
652                 return -EINVAL;
653         }
654
655         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761) {
656                 req = TG3_APE_LOCK_REQ;
657                 gnt = TG3_APE_LOCK_GRANT;
658         } else {
659                 req = TG3_APE_PER_LOCK_REQ;
660                 gnt = TG3_APE_PER_LOCK_GRANT;
661         }
662
663         off = 4 * locknum;
664
665         if (locknum != TG3_APE_LOCK_GPIO || !tp->pci_fn)
666                 bit = APE_LOCK_REQ_DRIVER;
667         else
668                 bit = 1 << tp->pci_fn;
669
670         tg3_ape_write32(tp, req + off, bit);
671
672         /* Wait for up to 1 millisecond to acquire lock. */
673         for (i = 0; i < 100; i++) {
674                 status = tg3_ape_read32(tp, gnt + off);
675                 if (status == bit)
676                         break;
677                 udelay(10);
678         }
679
680         if (status != bit) {
681                 /* Revoke the lock request. */
682                 tg3_ape_write32(tp, gnt + off, bit);
683                 ret = -EBUSY;
684         }
685
686         return ret;
687 }
688
689 static void tg3_ape_unlock(struct tg3 *tp, int locknum)
690 {
691         u32 gnt, bit;
692
693         if (!tg3_flag(tp, ENABLE_APE))
694                 return;
695
696         switch (locknum) {
697         case TG3_APE_LOCK_GPIO:
698                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
699                         return;
700         case TG3_APE_LOCK_GRC:
701         case TG3_APE_LOCK_MEM:
702                 break;
703         default:
704                 return;
705         }
706
707         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
708                 gnt = TG3_APE_LOCK_GRANT;
709         else
710                 gnt = TG3_APE_PER_LOCK_GRANT;
711
712         if (locknum != TG3_APE_LOCK_GPIO || !tp->pci_fn)
713                 bit = APE_LOCK_GRANT_DRIVER;
714         else
715                 bit = 1 << tp->pci_fn;
716
717         tg3_ape_write32(tp, gnt + 4 * locknum, bit);
718 }
719
720 static void tg3_disable_ints(struct tg3 *tp)
721 {
722         int i;
723
724         tw32(TG3PCI_MISC_HOST_CTRL,
725              (tp->misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT));
726         for (i = 0; i < tp->irq_max; i++)
727                 tw32_mailbox_f(tp->napi[i].int_mbox, 0x00000001);
728 }
729
730 static void tg3_enable_ints(struct tg3 *tp)
731 {
732         int i;
733
734         tp->irq_sync = 0;
735         wmb();
736
737         tw32(TG3PCI_MISC_HOST_CTRL,
738              (tp->misc_host_ctrl & ~MISC_HOST_CTRL_MASK_PCI_INT));
739
740         tp->coal_now = tp->coalesce_mode | HOSTCC_MODE_ENABLE;
741         for (i = 0; i < tp->irq_cnt; i++) {
742                 struct tg3_napi *tnapi = &tp->napi[i];
743
744                 tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
745                 if (tg3_flag(tp, 1SHOT_MSI))
746                         tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
747
748                 tp->coal_now |= tnapi->coal_now;
749         }
750
751         /* Force an initial interrupt */
752         if (!tg3_flag(tp, TAGGED_STATUS) &&
753             (tp->napi[0].hw_status->status & SD_STATUS_UPDATED))
754                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
755         else
756                 tw32(HOSTCC_MODE, tp->coal_now);
757
758         tp->coal_now &= ~(tp->napi[0].coal_now | tp->napi[1].coal_now);
759 }
760
761 static inline unsigned int tg3_has_work(struct tg3_napi *tnapi)
762 {
763         struct tg3 *tp = tnapi->tp;
764         struct tg3_hw_status *sblk = tnapi->hw_status;
765         unsigned int work_exists = 0;
766
767         /* check for phy events */
768         if (!(tg3_flag(tp, USE_LINKCHG_REG) || tg3_flag(tp, POLL_SERDES))) {
769                 if (sblk->status & SD_STATUS_LINK_CHG)
770                         work_exists = 1;
771         }
772         /* check for RX/TX work to do */
773         if (sblk->idx[0].tx_consumer != tnapi->tx_cons ||
774             *(tnapi->rx_rcb_prod_idx) != tnapi->rx_rcb_ptr)
775                 work_exists = 1;
776
777         return work_exists;
778 }
779
780 /* tg3_int_reenable
781  *  similar to tg3_enable_ints, but it accurately determines whether there
782  *  is new work pending and can return without flushing the PIO write
783  *  which reenables interrupts
784  */
785 static void tg3_int_reenable(struct tg3_napi *tnapi)
786 {
787         struct tg3 *tp = tnapi->tp;
788
789         tw32_mailbox(tnapi->int_mbox, tnapi->last_tag << 24);
790         mmiowb();
791
792         /* When doing tagged status, this work check is unnecessary.
793          * The last_tag we write above tells the chip which piece of
794          * work we've completed.
795          */
796         if (!tg3_flag(tp, TAGGED_STATUS) && tg3_has_work(tnapi))
797                 tw32(HOSTCC_MODE, tp->coalesce_mode |
798                      HOSTCC_MODE_ENABLE | tnapi->coal_now);
799 }
800
801 static void tg3_switch_clocks(struct tg3 *tp)
802 {
803         u32 clock_ctrl;
804         u32 orig_clock_ctrl;
805
806         if (tg3_flag(tp, CPMU_PRESENT) || tg3_flag(tp, 5780_CLASS))
807                 return;
808
809         clock_ctrl = tr32(TG3PCI_CLOCK_CTRL);
810
811         orig_clock_ctrl = clock_ctrl;
812         clock_ctrl &= (CLOCK_CTRL_FORCE_CLKRUN |
813                        CLOCK_CTRL_CLKRUN_OENABLE |
814                        0x1f);
815         tp->pci_clock_ctrl = clock_ctrl;
816
817         if (tg3_flag(tp, 5705_PLUS)) {
818                 if (orig_clock_ctrl & CLOCK_CTRL_625_CORE) {
819                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
820                                     clock_ctrl | CLOCK_CTRL_625_CORE, 40);
821                 }
822         } else if ((orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
823                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
824                             clock_ctrl |
825                             (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK),
826                             40);
827                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
828                             clock_ctrl | (CLOCK_CTRL_ALTCLK),
829                             40);
830         }
831         tw32_wait_f(TG3PCI_CLOCK_CTRL, clock_ctrl, 40);
832 }
833
834 #define PHY_BUSY_LOOPS  5000
835
836 static int tg3_readphy(struct tg3 *tp, int reg, u32 *val)
837 {
838         u32 frame_val;
839         unsigned int loops;
840         int ret;
841
842         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
843                 tw32_f(MAC_MI_MODE,
844                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
845                 udelay(80);
846         }
847
848         *val = 0x0;
849
850         frame_val  = ((tp->phy_addr << MI_COM_PHY_ADDR_SHIFT) &
851                       MI_COM_PHY_ADDR_MASK);
852         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
853                       MI_COM_REG_ADDR_MASK);
854         frame_val |= (MI_COM_CMD_READ | MI_COM_START);
855
856         tw32_f(MAC_MI_COM, frame_val);
857
858         loops = PHY_BUSY_LOOPS;
859         while (loops != 0) {
860                 udelay(10);
861                 frame_val = tr32(MAC_MI_COM);
862
863                 if ((frame_val & MI_COM_BUSY) == 0) {
864                         udelay(5);
865                         frame_val = tr32(MAC_MI_COM);
866                         break;
867                 }
868                 loops -= 1;
869         }
870
871         ret = -EBUSY;
872         if (loops != 0) {
873                 *val = frame_val & MI_COM_DATA_MASK;
874                 ret = 0;
875         }
876
877         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
878                 tw32_f(MAC_MI_MODE, tp->mi_mode);
879                 udelay(80);
880         }
881
882         return ret;
883 }
884
885 static int tg3_writephy(struct tg3 *tp, int reg, u32 val)
886 {
887         u32 frame_val;
888         unsigned int loops;
889         int ret;
890
891         if ((tp->phy_flags & TG3_PHYFLG_IS_FET) &&
892             (reg == MII_CTRL1000 || reg == MII_TG3_AUX_CTRL))
893                 return 0;
894
895         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
896                 tw32_f(MAC_MI_MODE,
897                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
898                 udelay(80);
899         }
900
901         frame_val  = ((tp->phy_addr << MI_COM_PHY_ADDR_SHIFT) &
902                       MI_COM_PHY_ADDR_MASK);
903         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
904                       MI_COM_REG_ADDR_MASK);
905         frame_val |= (val & MI_COM_DATA_MASK);
906         frame_val |= (MI_COM_CMD_WRITE | MI_COM_START);
907
908         tw32_f(MAC_MI_COM, frame_val);
909
910         loops = PHY_BUSY_LOOPS;
911         while (loops != 0) {
912                 udelay(10);
913                 frame_val = tr32(MAC_MI_COM);
914                 if ((frame_val & MI_COM_BUSY) == 0) {
915                         udelay(5);
916                         frame_val = tr32(MAC_MI_COM);
917                         break;
918                 }
919                 loops -= 1;
920         }
921
922         ret = -EBUSY;
923         if (loops != 0)
924                 ret = 0;
925
926         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
927                 tw32_f(MAC_MI_MODE, tp->mi_mode);
928                 udelay(80);
929         }
930
931         return ret;
932 }
933
934 static int tg3_phy_cl45_write(struct tg3 *tp, u32 devad, u32 addr, u32 val)
935 {
936         int err;
937
938         err = tg3_writephy(tp, MII_TG3_MMD_CTRL, devad);
939         if (err)
940                 goto done;
941
942         err = tg3_writephy(tp, MII_TG3_MMD_ADDRESS, addr);
943         if (err)
944                 goto done;
945
946         err = tg3_writephy(tp, MII_TG3_MMD_CTRL,
947                            MII_TG3_MMD_CTRL_DATA_NOINC | devad);
948         if (err)
949                 goto done;
950
951         err = tg3_writephy(tp, MII_TG3_MMD_ADDRESS, val);
952
953 done:
954         return err;
955 }
956
957 static int tg3_phy_cl45_read(struct tg3 *tp, u32 devad, u32 addr, u32 *val)
958 {
959         int err;
960
961         err = tg3_writephy(tp, MII_TG3_MMD_CTRL, devad);
962         if (err)
963                 goto done;
964
965         err = tg3_writephy(tp, MII_TG3_MMD_ADDRESS, addr);
966         if (err)
967                 goto done;
968
969         err = tg3_writephy(tp, MII_TG3_MMD_CTRL,
970                            MII_TG3_MMD_CTRL_DATA_NOINC | devad);
971         if (err)
972                 goto done;
973
974         err = tg3_readphy(tp, MII_TG3_MMD_ADDRESS, val);
975
976 done:
977         return err;
978 }
979
980 static int tg3_phydsp_read(struct tg3 *tp, u32 reg, u32 *val)
981 {
982         int err;
983
984         err = tg3_writephy(tp, MII_TG3_DSP_ADDRESS, reg);
985         if (!err)
986                 err = tg3_readphy(tp, MII_TG3_DSP_RW_PORT, val);
987
988         return err;
989 }
990
991 static int tg3_phydsp_write(struct tg3 *tp, u32 reg, u32 val)
992 {
993         int err;
994
995         err = tg3_writephy(tp, MII_TG3_DSP_ADDRESS, reg);
996         if (!err)
997                 err = tg3_writephy(tp, MII_TG3_DSP_RW_PORT, val);
998
999         return err;
1000 }
1001
1002 static int tg3_phy_auxctl_read(struct tg3 *tp, int reg, u32 *val)
1003 {
1004         int err;
1005
1006         err = tg3_writephy(tp, MII_TG3_AUX_CTRL,
1007                            (reg << MII_TG3_AUXCTL_MISC_RDSEL_SHIFT) |
1008                            MII_TG3_AUXCTL_SHDWSEL_MISC);
1009         if (!err)
1010                 err = tg3_readphy(tp, MII_TG3_AUX_CTRL, val);
1011
1012         return err;
1013 }
1014
1015 static int tg3_phy_auxctl_write(struct tg3 *tp, int reg, u32 set)
1016 {
1017         if (reg == MII_TG3_AUXCTL_SHDWSEL_MISC)
1018                 set |= MII_TG3_AUXCTL_MISC_WREN;
1019
1020         return tg3_writephy(tp, MII_TG3_AUX_CTRL, set | reg);
1021 }
1022
1023 #define TG3_PHY_AUXCTL_SMDSP_ENABLE(tp) \
1024         tg3_phy_auxctl_write((tp), MII_TG3_AUXCTL_SHDWSEL_AUXCTL, \
1025                              MII_TG3_AUXCTL_ACTL_SMDSP_ENA | \
1026                              MII_TG3_AUXCTL_ACTL_TX_6DB)
1027
1028 #define TG3_PHY_AUXCTL_SMDSP_DISABLE(tp) \
1029         tg3_phy_auxctl_write((tp), MII_TG3_AUXCTL_SHDWSEL_AUXCTL, \
1030                              MII_TG3_AUXCTL_ACTL_TX_6DB);
1031
1032 static int tg3_bmcr_reset(struct tg3 *tp)
1033 {
1034         u32 phy_control;
1035         int limit, err;
1036
1037         /* OK, reset it, and poll the BMCR_RESET bit until it
1038          * clears or we time out.
1039          */
1040         phy_control = BMCR_RESET;
1041         err = tg3_writephy(tp, MII_BMCR, phy_control);
1042         if (err != 0)
1043                 return -EBUSY;
1044
1045         limit = 5000;
1046         while (limit--) {
1047                 err = tg3_readphy(tp, MII_BMCR, &phy_control);
1048                 if (err != 0)
1049                         return -EBUSY;
1050
1051                 if ((phy_control & BMCR_RESET) == 0) {
1052                         udelay(40);
1053                         break;
1054                 }
1055                 udelay(10);
1056         }
1057         if (limit < 0)
1058                 return -EBUSY;
1059
1060         return 0;
1061 }
1062
1063 static int tg3_mdio_read(struct mii_bus *bp, int mii_id, int reg)
1064 {
1065         struct tg3 *tp = bp->priv;
1066         u32 val;
1067
1068         spin_lock_bh(&tp->lock);
1069
1070         if (tg3_readphy(tp, reg, &val))
1071                 val = -EIO;
1072
1073         spin_unlock_bh(&tp->lock);
1074
1075         return val;
1076 }
1077
1078 static int tg3_mdio_write(struct mii_bus *bp, int mii_id, int reg, u16 val)
1079 {
1080         struct tg3 *tp = bp->priv;
1081         u32 ret = 0;
1082
1083         spin_lock_bh(&tp->lock);
1084
1085         if (tg3_writephy(tp, reg, val))
1086                 ret = -EIO;
1087
1088         spin_unlock_bh(&tp->lock);
1089
1090         return ret;
1091 }
1092
1093 static int tg3_mdio_reset(struct mii_bus *bp)
1094 {
1095         return 0;
1096 }
1097
1098 static void tg3_mdio_config_5785(struct tg3 *tp)
1099 {
1100         u32 val;
1101         struct phy_device *phydev;
1102
1103         phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
1104         switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) {
1105         case PHY_ID_BCM50610:
1106         case PHY_ID_BCM50610M:
1107                 val = MAC_PHYCFG2_50610_LED_MODES;
1108                 break;
1109         case PHY_ID_BCMAC131:
1110                 val = MAC_PHYCFG2_AC131_LED_MODES;
1111                 break;
1112         case PHY_ID_RTL8211C:
1113                 val = MAC_PHYCFG2_RTL8211C_LED_MODES;
1114                 break;
1115         case PHY_ID_RTL8201E:
1116                 val = MAC_PHYCFG2_RTL8201E_LED_MODES;
1117                 break;
1118         default:
1119                 return;
1120         }
1121
1122         if (phydev->interface != PHY_INTERFACE_MODE_RGMII) {
1123                 tw32(MAC_PHYCFG2, val);
1124
1125                 val = tr32(MAC_PHYCFG1);
1126                 val &= ~(MAC_PHYCFG1_RGMII_INT |
1127                          MAC_PHYCFG1_RXCLK_TO_MASK | MAC_PHYCFG1_TXCLK_TO_MASK);
1128                 val |= MAC_PHYCFG1_RXCLK_TIMEOUT | MAC_PHYCFG1_TXCLK_TIMEOUT;
1129                 tw32(MAC_PHYCFG1, val);
1130
1131                 return;
1132         }
1133
1134         if (!tg3_flag(tp, RGMII_INBAND_DISABLE))
1135                 val |= MAC_PHYCFG2_EMODE_MASK_MASK |
1136                        MAC_PHYCFG2_FMODE_MASK_MASK |
1137                        MAC_PHYCFG2_GMODE_MASK_MASK |
1138                        MAC_PHYCFG2_ACT_MASK_MASK   |
1139                        MAC_PHYCFG2_QUAL_MASK_MASK |
1140                        MAC_PHYCFG2_INBAND_ENABLE;
1141
1142         tw32(MAC_PHYCFG2, val);
1143
1144         val = tr32(MAC_PHYCFG1);
1145         val &= ~(MAC_PHYCFG1_RXCLK_TO_MASK | MAC_PHYCFG1_TXCLK_TO_MASK |
1146                  MAC_PHYCFG1_RGMII_EXT_RX_DEC | MAC_PHYCFG1_RGMII_SND_STAT_EN);
1147         if (!tg3_flag(tp, RGMII_INBAND_DISABLE)) {
1148                 if (tg3_flag(tp, RGMII_EXT_IBND_RX_EN))
1149                         val |= MAC_PHYCFG1_RGMII_EXT_RX_DEC;
1150                 if (tg3_flag(tp, RGMII_EXT_IBND_TX_EN))
1151                         val |= MAC_PHYCFG1_RGMII_SND_STAT_EN;
1152         }
1153         val |= MAC_PHYCFG1_RXCLK_TIMEOUT | MAC_PHYCFG1_TXCLK_TIMEOUT |
1154                MAC_PHYCFG1_RGMII_INT | MAC_PHYCFG1_TXC_DRV;
1155         tw32(MAC_PHYCFG1, val);
1156
1157         val = tr32(MAC_EXT_RGMII_MODE);
1158         val &= ~(MAC_RGMII_MODE_RX_INT_B |
1159                  MAC_RGMII_MODE_RX_QUALITY |
1160                  MAC_RGMII_MODE_RX_ACTIVITY |
1161                  MAC_RGMII_MODE_RX_ENG_DET |
1162                  MAC_RGMII_MODE_TX_ENABLE |
1163                  MAC_RGMII_MODE_TX_LOWPWR |
1164                  MAC_RGMII_MODE_TX_RESET);
1165         if (!tg3_flag(tp, RGMII_INBAND_DISABLE)) {
1166                 if (tg3_flag(tp, RGMII_EXT_IBND_RX_EN))
1167                         val |= MAC_RGMII_MODE_RX_INT_B |
1168                                MAC_RGMII_MODE_RX_QUALITY |
1169                                MAC_RGMII_MODE_RX_ACTIVITY |
1170                                MAC_RGMII_MODE_RX_ENG_DET;
1171                 if (tg3_flag(tp, RGMII_EXT_IBND_TX_EN))
1172                         val |= MAC_RGMII_MODE_TX_ENABLE |
1173                                MAC_RGMII_MODE_TX_LOWPWR |
1174                                MAC_RGMII_MODE_TX_RESET;
1175         }
1176         tw32(MAC_EXT_RGMII_MODE, val);
1177 }
1178
1179 static void tg3_mdio_start(struct tg3 *tp)
1180 {
1181         tp->mi_mode &= ~MAC_MI_MODE_AUTO_POLL;
1182         tw32_f(MAC_MI_MODE, tp->mi_mode);
1183         udelay(80);
1184
1185         if (tg3_flag(tp, MDIOBUS_INITED) &&
1186             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785)
1187                 tg3_mdio_config_5785(tp);
1188 }
1189
1190 static int tg3_mdio_init(struct tg3 *tp)
1191 {
1192         int i;
1193         u32 reg;
1194         struct phy_device *phydev;
1195
1196         if (tg3_flag(tp, 5717_PLUS)) {
1197                 u32 is_serdes;
1198
1199                 tp->phy_addr = tp->pci_fn + 1;
1200
1201                 if (tp->pci_chip_rev_id != CHIPREV_ID_5717_A0)
1202                         is_serdes = tr32(SG_DIG_STATUS) & SG_DIG_IS_SERDES;
1203                 else
1204                         is_serdes = tr32(TG3_CPMU_PHY_STRAP) &
1205                                     TG3_CPMU_PHY_STRAP_IS_SERDES;
1206                 if (is_serdes)
1207                         tp->phy_addr += 7;
1208         } else
1209                 tp->phy_addr = TG3_PHY_MII_ADDR;
1210
1211         tg3_mdio_start(tp);
1212
1213         if (!tg3_flag(tp, USE_PHYLIB) || tg3_flag(tp, MDIOBUS_INITED))
1214                 return 0;
1215
1216         tp->mdio_bus = mdiobus_alloc();
1217         if (tp->mdio_bus == NULL)
1218                 return -ENOMEM;
1219
1220         tp->mdio_bus->name     = "tg3 mdio bus";
1221         snprintf(tp->mdio_bus->id, MII_BUS_ID_SIZE, "%x",
1222                  (tp->pdev->bus->number << 8) | tp->pdev->devfn);
1223         tp->mdio_bus->priv     = tp;
1224         tp->mdio_bus->parent   = &tp->pdev->dev;
1225         tp->mdio_bus->read     = &tg3_mdio_read;
1226         tp->mdio_bus->write    = &tg3_mdio_write;
1227         tp->mdio_bus->reset    = &tg3_mdio_reset;
1228         tp->mdio_bus->phy_mask = ~(1 << TG3_PHY_MII_ADDR);
1229         tp->mdio_bus->irq      = &tp->mdio_irq[0];
1230
1231         for (i = 0; i < PHY_MAX_ADDR; i++)
1232                 tp->mdio_bus->irq[i] = PHY_POLL;
1233
1234         /* The bus registration will look for all the PHYs on the mdio bus.
1235          * Unfortunately, it does not ensure the PHY is powered up before
1236          * accessing the PHY ID registers.  A chip reset is the
1237          * quickest way to bring the device back to an operational state..
1238          */
1239         if (tg3_readphy(tp, MII_BMCR, &reg) || (reg & BMCR_PDOWN))
1240                 tg3_bmcr_reset(tp);
1241
1242         i = mdiobus_register(tp->mdio_bus);
1243         if (i) {
1244                 dev_warn(&tp->pdev->dev, "mdiobus_reg failed (0x%x)\n", i);
1245                 mdiobus_free(tp->mdio_bus);
1246                 return i;
1247         }
1248
1249         phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
1250
1251         if (!phydev || !phydev->drv) {
1252                 dev_warn(&tp->pdev->dev, "No PHY devices\n");
1253                 mdiobus_unregister(tp->mdio_bus);
1254                 mdiobus_free(tp->mdio_bus);
1255                 return -ENODEV;
1256         }
1257
1258         switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) {
1259         case PHY_ID_BCM57780:
1260                 phydev->interface = PHY_INTERFACE_MODE_GMII;
1261                 phydev->dev_flags |= PHY_BRCM_AUTO_PWRDWN_ENABLE;
1262                 break;
1263         case PHY_ID_BCM50610:
1264         case PHY_ID_BCM50610M:
1265                 phydev->dev_flags |= PHY_BRCM_CLEAR_RGMII_MODE |
1266                                      PHY_BRCM_RX_REFCLK_UNUSED |
1267                                      PHY_BRCM_DIS_TXCRXC_NOENRGY |
1268                                      PHY_BRCM_AUTO_PWRDWN_ENABLE;
1269                 if (tg3_flag(tp, RGMII_INBAND_DISABLE))
1270                         phydev->dev_flags |= PHY_BRCM_STD_IBND_DISABLE;
1271                 if (tg3_flag(tp, RGMII_EXT_IBND_RX_EN))
1272                         phydev->dev_flags |= PHY_BRCM_EXT_IBND_RX_ENABLE;
1273                 if (tg3_flag(tp, RGMII_EXT_IBND_TX_EN))
1274                         phydev->dev_flags |= PHY_BRCM_EXT_IBND_TX_ENABLE;
1275                 /* fallthru */
1276         case PHY_ID_RTL8211C:
1277                 phydev->interface = PHY_INTERFACE_MODE_RGMII;
1278                 break;
1279         case PHY_ID_RTL8201E:
1280         case PHY_ID_BCMAC131:
1281                 phydev->interface = PHY_INTERFACE_MODE_MII;
1282                 phydev->dev_flags |= PHY_BRCM_AUTO_PWRDWN_ENABLE;
1283                 tp->phy_flags |= TG3_PHYFLG_IS_FET;
1284                 break;
1285         }
1286
1287         tg3_flag_set(tp, MDIOBUS_INITED);
1288
1289         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785)
1290                 tg3_mdio_config_5785(tp);
1291
1292         return 0;
1293 }
1294
1295 static void tg3_mdio_fini(struct tg3 *tp)
1296 {
1297         if (tg3_flag(tp, MDIOBUS_INITED)) {
1298                 tg3_flag_clear(tp, MDIOBUS_INITED);
1299                 mdiobus_unregister(tp->mdio_bus);
1300                 mdiobus_free(tp->mdio_bus);
1301         }
1302 }
1303
1304 /* tp->lock is held. */
1305 static inline void tg3_generate_fw_event(struct tg3 *tp)
1306 {
1307         u32 val;
1308
1309         val = tr32(GRC_RX_CPU_EVENT);
1310         val |= GRC_RX_CPU_DRIVER_EVENT;
1311         tw32_f(GRC_RX_CPU_EVENT, val);
1312
1313         tp->last_event_jiffies = jiffies;
1314 }
1315
1316 #define TG3_FW_EVENT_TIMEOUT_USEC 2500
1317
1318 /* tp->lock is held. */
1319 static void tg3_wait_for_event_ack(struct tg3 *tp)
1320 {
1321         int i;
1322         unsigned int delay_cnt;
1323         long time_remain;
1324
1325         /* If enough time has passed, no wait is necessary. */
1326         time_remain = (long)(tp->last_event_jiffies + 1 +
1327                       usecs_to_jiffies(TG3_FW_EVENT_TIMEOUT_USEC)) -
1328                       (long)jiffies;
1329         if (time_remain < 0)
1330                 return;
1331
1332         /* Check if we can shorten the wait time. */
1333         delay_cnt = jiffies_to_usecs(time_remain);
1334         if (delay_cnt > TG3_FW_EVENT_TIMEOUT_USEC)
1335                 delay_cnt = TG3_FW_EVENT_TIMEOUT_USEC;
1336         delay_cnt = (delay_cnt >> 3) + 1;
1337
1338         for (i = 0; i < delay_cnt; i++) {
1339                 if (!(tr32(GRC_RX_CPU_EVENT) & GRC_RX_CPU_DRIVER_EVENT))
1340                         break;
1341                 udelay(8);
1342         }
1343 }
1344
1345 /* tp->lock is held. */
1346 static void tg3_ump_link_report(struct tg3 *tp)
1347 {
1348         u32 reg;
1349         u32 val;
1350
1351         if (!tg3_flag(tp, 5780_CLASS) || !tg3_flag(tp, ENABLE_ASF))
1352                 return;
1353
1354         tg3_wait_for_event_ack(tp);
1355
1356         tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_LINK_UPDATE);
1357
1358         tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 14);
1359
1360         val = 0;
1361         if (!tg3_readphy(tp, MII_BMCR, &reg))
1362                 val = reg << 16;
1363         if (!tg3_readphy(tp, MII_BMSR, &reg))
1364                 val |= (reg & 0xffff);
1365         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX, val);
1366
1367         val = 0;
1368         if (!tg3_readphy(tp, MII_ADVERTISE, &reg))
1369                 val = reg << 16;
1370         if (!tg3_readphy(tp, MII_LPA, &reg))
1371                 val |= (reg & 0xffff);
1372         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 4, val);
1373
1374         val = 0;
1375         if (!(tp->phy_flags & TG3_PHYFLG_MII_SERDES)) {
1376                 if (!tg3_readphy(tp, MII_CTRL1000, &reg))
1377                         val = reg << 16;
1378                 if (!tg3_readphy(tp, MII_STAT1000, &reg))
1379                         val |= (reg & 0xffff);
1380         }
1381         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 8, val);
1382
1383         if (!tg3_readphy(tp, MII_PHYADDR, &reg))
1384                 val = reg << 16;
1385         else
1386                 val = 0;
1387         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 12, val);
1388
1389         tg3_generate_fw_event(tp);
1390 }
1391
1392 static void tg3_link_report(struct tg3 *tp)
1393 {
1394         if (!netif_carrier_ok(tp->dev)) {
1395                 netif_info(tp, link, tp->dev, "Link is down\n");
1396                 tg3_ump_link_report(tp);
1397         } else if (netif_msg_link(tp)) {
1398                 netdev_info(tp->dev, "Link is up at %d Mbps, %s duplex\n",
1399                             (tp->link_config.active_speed == SPEED_1000 ?
1400                              1000 :
1401                              (tp->link_config.active_speed == SPEED_100 ?
1402                               100 : 10)),
1403                             (tp->link_config.active_duplex == DUPLEX_FULL ?
1404                              "full" : "half"));
1405
1406                 netdev_info(tp->dev, "Flow control is %s for TX and %s for RX\n",
1407                             (tp->link_config.active_flowctrl & FLOW_CTRL_TX) ?
1408                             "on" : "off",
1409                             (tp->link_config.active_flowctrl & FLOW_CTRL_RX) ?
1410                             "on" : "off");
1411
1412                 if (tp->phy_flags & TG3_PHYFLG_EEE_CAP)
1413                         netdev_info(tp->dev, "EEE is %s\n",
1414                                     tp->setlpicnt ? "enabled" : "disabled");
1415
1416                 tg3_ump_link_report(tp);
1417         }
1418 }
1419
1420 static u16 tg3_advert_flowctrl_1000T(u8 flow_ctrl)
1421 {
1422         u16 miireg;
1423
1424         if ((flow_ctrl & FLOW_CTRL_TX) && (flow_ctrl & FLOW_CTRL_RX))
1425                 miireg = ADVERTISE_PAUSE_CAP;
1426         else if (flow_ctrl & FLOW_CTRL_TX)
1427                 miireg = ADVERTISE_PAUSE_ASYM;
1428         else if (flow_ctrl & FLOW_CTRL_RX)
1429                 miireg = ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM;
1430         else
1431                 miireg = 0;
1432
1433         return miireg;
1434 }
1435
1436 static u16 tg3_advert_flowctrl_1000X(u8 flow_ctrl)
1437 {
1438         u16 miireg;
1439
1440         if ((flow_ctrl & FLOW_CTRL_TX) && (flow_ctrl & FLOW_CTRL_RX))
1441                 miireg = ADVERTISE_1000XPAUSE;
1442         else if (flow_ctrl & FLOW_CTRL_TX)
1443                 miireg = ADVERTISE_1000XPSE_ASYM;
1444         else if (flow_ctrl & FLOW_CTRL_RX)
1445                 miireg = ADVERTISE_1000XPAUSE | ADVERTISE_1000XPSE_ASYM;
1446         else
1447                 miireg = 0;
1448
1449         return miireg;
1450 }
1451
1452 static u8 tg3_resolve_flowctrl_1000X(u16 lcladv, u16 rmtadv)
1453 {
1454         u8 cap = 0;
1455
1456         if (lcladv & ADVERTISE_1000XPAUSE) {
1457                 if (lcladv & ADVERTISE_1000XPSE_ASYM) {
1458                         if (rmtadv & LPA_1000XPAUSE)
1459                                 cap = FLOW_CTRL_TX | FLOW_CTRL_RX;
1460                         else if (rmtadv & LPA_1000XPAUSE_ASYM)
1461                                 cap = FLOW_CTRL_RX;
1462                 } else {
1463                         if (rmtadv & LPA_1000XPAUSE)
1464                                 cap = FLOW_CTRL_TX | FLOW_CTRL_RX;
1465                 }
1466         } else if (lcladv & ADVERTISE_1000XPSE_ASYM) {
1467                 if ((rmtadv & LPA_1000XPAUSE) && (rmtadv & LPA_1000XPAUSE_ASYM))
1468                         cap = FLOW_CTRL_TX;
1469         }
1470
1471         return cap;
1472 }
1473
1474 static void tg3_setup_flow_control(struct tg3 *tp, u32 lcladv, u32 rmtadv)
1475 {
1476         u8 autoneg;
1477         u8 flowctrl = 0;
1478         u32 old_rx_mode = tp->rx_mode;
1479         u32 old_tx_mode = tp->tx_mode;
1480
1481         if (tg3_flag(tp, USE_PHYLIB))
1482                 autoneg = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]->autoneg;
1483         else
1484                 autoneg = tp->link_config.autoneg;
1485
1486         if (autoneg == AUTONEG_ENABLE && tg3_flag(tp, PAUSE_AUTONEG)) {
1487                 if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
1488                         flowctrl = tg3_resolve_flowctrl_1000X(lcladv, rmtadv);
1489                 else
1490                         flowctrl = mii_resolve_flowctrl_fdx(lcladv, rmtadv);
1491         } else
1492                 flowctrl = tp->link_config.flowctrl;
1493
1494         tp->link_config.active_flowctrl = flowctrl;
1495
1496         if (flowctrl & FLOW_CTRL_RX)
1497                 tp->rx_mode |= RX_MODE_FLOW_CTRL_ENABLE;
1498         else
1499                 tp->rx_mode &= ~RX_MODE_FLOW_CTRL_ENABLE;
1500
1501         if (old_rx_mode != tp->rx_mode)
1502                 tw32_f(MAC_RX_MODE, tp->rx_mode);
1503
1504         if (flowctrl & FLOW_CTRL_TX)
1505                 tp->tx_mode |= TX_MODE_FLOW_CTRL_ENABLE;
1506         else
1507                 tp->tx_mode &= ~TX_MODE_FLOW_CTRL_ENABLE;
1508
1509         if (old_tx_mode != tp->tx_mode)
1510                 tw32_f(MAC_TX_MODE, tp->tx_mode);
1511 }
1512
1513 static void tg3_adjust_link(struct net_device *dev)
1514 {
1515         u8 oldflowctrl, linkmesg = 0;
1516         u32 mac_mode, lcl_adv, rmt_adv;
1517         struct tg3 *tp = netdev_priv(dev);
1518         struct phy_device *phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
1519
1520         spin_lock_bh(&tp->lock);
1521
1522         mac_mode = tp->mac_mode & ~(MAC_MODE_PORT_MODE_MASK |
1523                                     MAC_MODE_HALF_DUPLEX);
1524
1525         oldflowctrl = tp->link_config.active_flowctrl;
1526
1527         if (phydev->link) {
1528                 lcl_adv = 0;
1529                 rmt_adv = 0;
1530
1531                 if (phydev->speed == SPEED_100 || phydev->speed == SPEED_10)
1532                         mac_mode |= MAC_MODE_PORT_MODE_MII;
1533                 else if (phydev->speed == SPEED_1000 ||
1534                          GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5785)
1535                         mac_mode |= MAC_MODE_PORT_MODE_GMII;
1536                 else
1537                         mac_mode |= MAC_MODE_PORT_MODE_MII;
1538
1539                 if (phydev->duplex == DUPLEX_HALF)
1540                         mac_mode |= MAC_MODE_HALF_DUPLEX;
1541                 else {
1542                         lcl_adv = tg3_advert_flowctrl_1000T(
1543                                   tp->link_config.flowctrl);
1544
1545                         if (phydev->pause)
1546                                 rmt_adv = LPA_PAUSE_CAP;
1547                         if (phydev->asym_pause)
1548                                 rmt_adv |= LPA_PAUSE_ASYM;
1549                 }
1550
1551                 tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
1552         } else
1553                 mac_mode |= MAC_MODE_PORT_MODE_GMII;
1554
1555         if (mac_mode != tp->mac_mode) {
1556                 tp->mac_mode = mac_mode;
1557                 tw32_f(MAC_MODE, tp->mac_mode);
1558                 udelay(40);
1559         }
1560
1561         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785) {
1562                 if (phydev->speed == SPEED_10)
1563                         tw32(MAC_MI_STAT,
1564                              MAC_MI_STAT_10MBPS_MODE |
1565                              MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
1566                 else
1567                         tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
1568         }
1569
1570         if (phydev->speed == SPEED_1000 && phydev->duplex == DUPLEX_HALF)
1571                 tw32(MAC_TX_LENGTHS,
1572                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
1573                       (6 << TX_LENGTHS_IPG_SHIFT) |
1574                       (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
1575         else
1576                 tw32(MAC_TX_LENGTHS,
1577                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
1578                       (6 << TX_LENGTHS_IPG_SHIFT) |
1579                       (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
1580
1581         if ((phydev->link && tp->link_config.active_speed == SPEED_INVALID) ||
1582             (!phydev->link && tp->link_config.active_speed != SPEED_INVALID) ||
1583             phydev->speed != tp->link_config.active_speed ||
1584             phydev->duplex != tp->link_config.active_duplex ||
1585             oldflowctrl != tp->link_config.active_flowctrl)
1586                 linkmesg = 1;
1587
1588         tp->link_config.active_speed = phydev->speed;
1589         tp->link_config.active_duplex = phydev->duplex;
1590
1591         spin_unlock_bh(&tp->lock);
1592
1593         if (linkmesg)
1594                 tg3_link_report(tp);
1595 }
1596
1597 static int tg3_phy_init(struct tg3 *tp)
1598 {
1599         struct phy_device *phydev;
1600
1601         if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED)
1602                 return 0;
1603
1604         /* Bring the PHY back to a known state. */
1605         tg3_bmcr_reset(tp);
1606
1607         phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
1608
1609         /* Attach the MAC to the PHY. */
1610         phydev = phy_connect(tp->dev, dev_name(&phydev->dev), tg3_adjust_link,
1611                              phydev->dev_flags, phydev->interface);
1612         if (IS_ERR(phydev)) {
1613                 dev_err(&tp->pdev->dev, "Could not attach to PHY\n");
1614                 return PTR_ERR(phydev);
1615         }
1616
1617         /* Mask with MAC supported features. */
1618         switch (phydev->interface) {
1619         case PHY_INTERFACE_MODE_GMII:
1620         case PHY_INTERFACE_MODE_RGMII:
1621                 if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
1622                         phydev->supported &= (PHY_GBIT_FEATURES |
1623                                               SUPPORTED_Pause |
1624                                               SUPPORTED_Asym_Pause);
1625                         break;
1626                 }
1627                 /* fallthru */
1628         case PHY_INTERFACE_MODE_MII:
1629                 phydev->supported &= (PHY_BASIC_FEATURES |
1630                                       SUPPORTED_Pause |
1631                                       SUPPORTED_Asym_Pause);
1632                 break;
1633         default:
1634                 phy_disconnect(tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]);
1635                 return -EINVAL;
1636         }
1637
1638         tp->phy_flags |= TG3_PHYFLG_IS_CONNECTED;
1639
1640         phydev->advertising = phydev->supported;
1641
1642         return 0;
1643 }
1644
1645 static void tg3_phy_start(struct tg3 *tp)
1646 {
1647         struct phy_device *phydev;
1648
1649         if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
1650                 return;
1651
1652         phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
1653
1654         if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) {
1655                 tp->phy_flags &= ~TG3_PHYFLG_IS_LOW_POWER;
1656                 phydev->speed = tp->link_config.orig_speed;
1657                 phydev->duplex = tp->link_config.orig_duplex;
1658                 phydev->autoneg = tp->link_config.orig_autoneg;
1659                 phydev->advertising = tp->link_config.orig_advertising;
1660         }
1661
1662         phy_start(phydev);
1663
1664         phy_start_aneg(phydev);
1665 }
1666
1667 static void tg3_phy_stop(struct tg3 *tp)
1668 {
1669         if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
1670                 return;
1671
1672         phy_stop(tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]);
1673 }
1674
1675 static void tg3_phy_fini(struct tg3 *tp)
1676 {
1677         if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) {
1678                 phy_disconnect(tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]);
1679                 tp->phy_flags &= ~TG3_PHYFLG_IS_CONNECTED;
1680         }
1681 }
1682
1683 static void tg3_phy_fet_toggle_apd(struct tg3 *tp, bool enable)
1684 {
1685         u32 phytest;
1686
1687         if (!tg3_readphy(tp, MII_TG3_FET_TEST, &phytest)) {
1688                 u32 phy;
1689
1690                 tg3_writephy(tp, MII_TG3_FET_TEST,
1691                              phytest | MII_TG3_FET_SHADOW_EN);
1692                 if (!tg3_readphy(tp, MII_TG3_FET_SHDW_AUXSTAT2, &phy)) {
1693                         if (enable)
1694                                 phy |= MII_TG3_FET_SHDW_AUXSTAT2_APD;
1695                         else
1696                                 phy &= ~MII_TG3_FET_SHDW_AUXSTAT2_APD;
1697                         tg3_writephy(tp, MII_TG3_FET_SHDW_AUXSTAT2, phy);
1698                 }
1699                 tg3_writephy(tp, MII_TG3_FET_TEST, phytest);
1700         }
1701 }
1702
1703 static void tg3_phy_toggle_apd(struct tg3 *tp, bool enable)
1704 {
1705         u32 reg;
1706
1707         if (!tg3_flag(tp, 5705_PLUS) ||
1708             (tg3_flag(tp, 5717_PLUS) &&
1709              (tp->phy_flags & TG3_PHYFLG_MII_SERDES)))
1710                 return;
1711
1712         if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
1713                 tg3_phy_fet_toggle_apd(tp, enable);
1714                 return;
1715         }
1716
1717         reg = MII_TG3_MISC_SHDW_WREN |
1718               MII_TG3_MISC_SHDW_SCR5_SEL |
1719               MII_TG3_MISC_SHDW_SCR5_LPED |
1720               MII_TG3_MISC_SHDW_SCR5_DLPTLM |
1721               MII_TG3_MISC_SHDW_SCR5_SDTL |
1722               MII_TG3_MISC_SHDW_SCR5_C125OE;
1723         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5784 || !enable)
1724                 reg |= MII_TG3_MISC_SHDW_SCR5_DLLAPD;
1725
1726         tg3_writephy(tp, MII_TG3_MISC_SHDW, reg);
1727
1728
1729         reg = MII_TG3_MISC_SHDW_WREN |
1730               MII_TG3_MISC_SHDW_APD_SEL |
1731               MII_TG3_MISC_SHDW_APD_WKTM_84MS;
1732         if (enable)
1733                 reg |= MII_TG3_MISC_SHDW_APD_ENABLE;
1734
1735         tg3_writephy(tp, MII_TG3_MISC_SHDW, reg);
1736 }
1737
1738 static void tg3_phy_toggle_automdix(struct tg3 *tp, int enable)
1739 {
1740         u32 phy;
1741
1742         if (!tg3_flag(tp, 5705_PLUS) ||
1743             (tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
1744                 return;
1745
1746         if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
1747                 u32 ephy;
1748
1749                 if (!tg3_readphy(tp, MII_TG3_FET_TEST, &ephy)) {
1750                         u32 reg = MII_TG3_FET_SHDW_MISCCTRL;
1751
1752                         tg3_writephy(tp, MII_TG3_FET_TEST,
1753                                      ephy | MII_TG3_FET_SHADOW_EN);
1754                         if (!tg3_readphy(tp, reg, &phy)) {
1755                                 if (enable)
1756                                         phy |= MII_TG3_FET_SHDW_MISCCTRL_MDIX;
1757                                 else
1758                                         phy &= ~MII_TG3_FET_SHDW_MISCCTRL_MDIX;
1759                                 tg3_writephy(tp, reg, phy);
1760                         }
1761                         tg3_writephy(tp, MII_TG3_FET_TEST, ephy);
1762                 }
1763         } else {
1764                 int ret;
1765
1766                 ret = tg3_phy_auxctl_read(tp,
1767                                           MII_TG3_AUXCTL_SHDWSEL_MISC, &phy);
1768                 if (!ret) {
1769                         if (enable)
1770                                 phy |= MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
1771                         else
1772                                 phy &= ~MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
1773                         tg3_phy_auxctl_write(tp,
1774                                              MII_TG3_AUXCTL_SHDWSEL_MISC, phy);
1775                 }
1776         }
1777 }
1778
1779 static void tg3_phy_set_wirespeed(struct tg3 *tp)
1780 {
1781         int ret;
1782         u32 val;
1783
1784         if (tp->phy_flags & TG3_PHYFLG_NO_ETH_WIRE_SPEED)
1785                 return;
1786
1787         ret = tg3_phy_auxctl_read(tp, MII_TG3_AUXCTL_SHDWSEL_MISC, &val);
1788         if (!ret)
1789                 tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_MISC,
1790                                      val | MII_TG3_AUXCTL_MISC_WIRESPD_EN);
1791 }
1792
1793 static void tg3_phy_apply_otp(struct tg3 *tp)
1794 {
1795         u32 otp, phy;
1796
1797         if (!tp->phy_otp)
1798                 return;
1799
1800         otp = tp->phy_otp;
1801
1802         if (TG3_PHY_AUXCTL_SMDSP_ENABLE(tp))
1803                 return;
1804
1805         phy = ((otp & TG3_OTP_AGCTGT_MASK) >> TG3_OTP_AGCTGT_SHIFT);
1806         phy |= MII_TG3_DSP_TAP1_AGCTGT_DFLT;
1807         tg3_phydsp_write(tp, MII_TG3_DSP_TAP1, phy);
1808
1809         phy = ((otp & TG3_OTP_HPFFLTR_MASK) >> TG3_OTP_HPFFLTR_SHIFT) |
1810               ((otp & TG3_OTP_HPFOVER_MASK) >> TG3_OTP_HPFOVER_SHIFT);
1811         tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH0, phy);
1812
1813         phy = ((otp & TG3_OTP_LPFDIS_MASK) >> TG3_OTP_LPFDIS_SHIFT);
1814         phy |= MII_TG3_DSP_AADJ1CH3_ADCCKADJ;
1815         tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH3, phy);
1816
1817         phy = ((otp & TG3_OTP_VDAC_MASK) >> TG3_OTP_VDAC_SHIFT);
1818         tg3_phydsp_write(tp, MII_TG3_DSP_EXP75, phy);
1819
1820         phy = ((otp & TG3_OTP_10BTAMP_MASK) >> TG3_OTP_10BTAMP_SHIFT);
1821         tg3_phydsp_write(tp, MII_TG3_DSP_EXP96, phy);
1822
1823         phy = ((otp & TG3_OTP_ROFF_MASK) >> TG3_OTP_ROFF_SHIFT) |
1824               ((otp & TG3_OTP_RCOFF_MASK) >> TG3_OTP_RCOFF_SHIFT);
1825         tg3_phydsp_write(tp, MII_TG3_DSP_EXP97, phy);
1826
1827         TG3_PHY_AUXCTL_SMDSP_DISABLE(tp);
1828 }
1829
1830 static void tg3_phy_eee_adjust(struct tg3 *tp, u32 current_link_up)
1831 {
1832         u32 val;
1833
1834         if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP))
1835                 return;
1836
1837         tp->setlpicnt = 0;
1838
1839         if (tp->link_config.autoneg == AUTONEG_ENABLE &&
1840             current_link_up == 1 &&
1841             tp->link_config.active_duplex == DUPLEX_FULL &&
1842             (tp->link_config.active_speed == SPEED_100 ||
1843              tp->link_config.active_speed == SPEED_1000)) {
1844                 u32 eeectl;
1845
1846                 if (tp->link_config.active_speed == SPEED_1000)
1847                         eeectl = TG3_CPMU_EEE_CTRL_EXIT_16_5_US;
1848                 else
1849                         eeectl = TG3_CPMU_EEE_CTRL_EXIT_36_US;
1850
1851                 tw32(TG3_CPMU_EEE_CTRL, eeectl);
1852
1853                 tg3_phy_cl45_read(tp, MDIO_MMD_AN,
1854                                   TG3_CL45_D7_EEERES_STAT, &val);
1855
1856                 if (val == TG3_CL45_D7_EEERES_STAT_LP_1000T ||
1857                     val == TG3_CL45_D7_EEERES_STAT_LP_100TX)
1858                         tp->setlpicnt = 2;
1859         }
1860
1861         if (!tp->setlpicnt) {
1862                 if (current_link_up == 1 &&
1863                    !TG3_PHY_AUXCTL_SMDSP_ENABLE(tp)) {
1864                         tg3_phydsp_write(tp, MII_TG3_DSP_TAP26, 0x0000);
1865                         TG3_PHY_AUXCTL_SMDSP_DISABLE(tp);
1866                 }
1867
1868                 val = tr32(TG3_CPMU_EEE_MODE);
1869                 tw32(TG3_CPMU_EEE_MODE, val & ~TG3_CPMU_EEEMD_LPI_ENABLE);
1870         }
1871 }
1872
1873 static void tg3_phy_eee_enable(struct tg3 *tp)
1874 {
1875         u32 val;
1876
1877         if (tp->link_config.active_speed == SPEED_1000 &&
1878             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 ||
1879              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5719 ||
1880              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57765) &&
1881             !TG3_PHY_AUXCTL_SMDSP_ENABLE(tp)) {
1882                 val = MII_TG3_DSP_TAP26_ALNOKO |
1883                       MII_TG3_DSP_TAP26_RMRXSTO;
1884                 tg3_phydsp_write(tp, MII_TG3_DSP_TAP26, val);
1885                 TG3_PHY_AUXCTL_SMDSP_DISABLE(tp);
1886         }
1887
1888         val = tr32(TG3_CPMU_EEE_MODE);
1889         tw32(TG3_CPMU_EEE_MODE, val | TG3_CPMU_EEEMD_LPI_ENABLE);
1890 }
1891
1892 static int tg3_wait_macro_done(struct tg3 *tp)
1893 {
1894         int limit = 100;
1895
1896         while (limit--) {
1897                 u32 tmp32;
1898
1899                 if (!tg3_readphy(tp, MII_TG3_DSP_CONTROL, &tmp32)) {
1900                         if ((tmp32 & 0x1000) == 0)
1901                                 break;
1902                 }
1903         }
1904         if (limit < 0)
1905                 return -EBUSY;
1906
1907         return 0;
1908 }
1909
1910 static int tg3_phy_write_and_check_testpat(struct tg3 *tp, int *resetp)
1911 {
1912         static const u32 test_pat[4][6] = {
1913         { 0x00005555, 0x00000005, 0x00002aaa, 0x0000000a, 0x00003456, 0x00000003 },
1914         { 0x00002aaa, 0x0000000a, 0x00003333, 0x00000003, 0x0000789a, 0x00000005 },
1915         { 0x00005a5a, 0x00000005, 0x00002a6a, 0x0000000a, 0x00001bcd, 0x00000003 },
1916         { 0x00002a5a, 0x0000000a, 0x000033c3, 0x00000003, 0x00002ef1, 0x00000005 }
1917         };
1918         int chan;
1919
1920         for (chan = 0; chan < 4; chan++) {
1921                 int i;
1922
1923                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
1924                              (chan * 0x2000) | 0x0200);
1925                 tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0002);
1926
1927                 for (i = 0; i < 6; i++)
1928                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT,
1929                                      test_pat[chan][i]);
1930
1931                 tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0202);
1932                 if (tg3_wait_macro_done(tp)) {
1933                         *resetp = 1;
1934                         return -EBUSY;
1935                 }
1936
1937                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
1938                              (chan * 0x2000) | 0x0200);
1939                 tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0082);
1940                 if (tg3_wait_macro_done(tp)) {
1941                         *resetp = 1;
1942                         return -EBUSY;
1943                 }
1944
1945                 tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0802);
1946                 if (tg3_wait_macro_done(tp)) {
1947                         *resetp = 1;
1948                         return -EBUSY;
1949                 }
1950
1951                 for (i = 0; i < 6; i += 2) {
1952                         u32 low, high;
1953
1954                         if (tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &low) ||
1955                             tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &high) ||
1956                             tg3_wait_macro_done(tp)) {
1957                                 *resetp = 1;
1958                                 return -EBUSY;
1959                         }
1960                         low &= 0x7fff;
1961                         high &= 0x000f;
1962                         if (low != test_pat[chan][i] ||
1963                             high != test_pat[chan][i+1]) {
1964                                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000b);
1965                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4001);
1966                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4005);
1967
1968                                 return -EBUSY;
1969                         }
1970                 }
1971         }
1972
1973         return 0;
1974 }
1975
1976 static int tg3_phy_reset_chanpat(struct tg3 *tp)
1977 {
1978         int chan;
1979
1980         for (chan = 0; chan < 4; chan++) {
1981                 int i;
1982
1983                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
1984                              (chan * 0x2000) | 0x0200);
1985                 tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0002);
1986                 for (i = 0; i < 6; i++)
1987                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x000);
1988                 tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0202);
1989                 if (tg3_wait_macro_done(tp))
1990                         return -EBUSY;
1991         }
1992
1993         return 0;
1994 }
1995
1996 static int tg3_phy_reset_5703_4_5(struct tg3 *tp)
1997 {
1998         u32 reg32, phy9_orig;
1999         int retries, do_phy_reset, err;
2000
2001         retries = 10;
2002         do_phy_reset = 1;
2003         do {
2004                 if (do_phy_reset) {
2005                         err = tg3_bmcr_reset(tp);
2006                         if (err)
2007                                 return err;
2008                         do_phy_reset = 0;
2009                 }
2010
2011                 /* Disable transmitter and interrupt.  */
2012                 if (tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32))
2013                         continue;
2014
2015                 reg32 |= 0x3000;
2016                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
2017
2018                 /* Set full-duplex, 1000 mbps.  */
2019                 tg3_writephy(tp, MII_BMCR,
2020                              BMCR_FULLDPLX | BMCR_SPEED1000);
2021
2022                 /* Set to master mode.  */
2023                 if (tg3_readphy(tp, MII_CTRL1000, &phy9_orig))
2024                         continue;
2025
2026                 tg3_writephy(tp, MII_CTRL1000,
2027                              CTL1000_AS_MASTER | CTL1000_ENABLE_MASTER);
2028
2029                 err = TG3_PHY_AUXCTL_SMDSP_ENABLE(tp);
2030                 if (err)
2031                         return err;
2032
2033                 /* Block the PHY control access.  */
2034                 tg3_phydsp_write(tp, 0x8005, 0x0800);
2035
2036                 err = tg3_phy_write_and_check_testpat(tp, &do_phy_reset);
2037                 if (!err)
2038                         break;
2039         } while (--retries);
2040
2041         err = tg3_phy_reset_chanpat(tp);
2042         if (err)
2043                 return err;
2044
2045         tg3_phydsp_write(tp, 0x8005, 0x0000);
2046
2047         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8200);
2048         tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0000);
2049
2050         TG3_PHY_AUXCTL_SMDSP_DISABLE(tp);
2051
2052         tg3_writephy(tp, MII_CTRL1000, phy9_orig);
2053
2054         if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32)) {
2055                 reg32 &= ~0x3000;
2056                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
2057         } else if (!err)
2058                 err = -EBUSY;
2059
2060         return err;
2061 }
2062
2063 /* This will reset the tigon3 PHY if there is no valid
2064  * link unless the FORCE argument is non-zero.
2065  */
2066 static int tg3_phy_reset(struct tg3 *tp)
2067 {
2068         u32 val, cpmuctrl;
2069         int err;
2070
2071         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2072                 val = tr32(GRC_MISC_CFG);
2073                 tw32_f(GRC_MISC_CFG, val & ~GRC_MISC_CFG_EPHY_IDDQ);
2074                 udelay(40);
2075         }
2076         err  = tg3_readphy(tp, MII_BMSR, &val);
2077         err |= tg3_readphy(tp, MII_BMSR, &val);
2078         if (err != 0)
2079                 return -EBUSY;
2080
2081         if (netif_running(tp->dev) && netif_carrier_ok(tp->dev)) {
2082                 netif_carrier_off(tp->dev);
2083                 tg3_link_report(tp);
2084         }
2085
2086         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
2087             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
2088             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
2089                 err = tg3_phy_reset_5703_4_5(tp);
2090                 if (err)
2091                         return err;
2092                 goto out;
2093         }
2094
2095         cpmuctrl = 0;
2096         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 &&
2097             GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5784_AX) {
2098                 cpmuctrl = tr32(TG3_CPMU_CTRL);
2099                 if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY)
2100                         tw32(TG3_CPMU_CTRL,
2101                              cpmuctrl & ~CPMU_CTRL_GPHY_10MB_RXONLY);
2102         }
2103
2104         err = tg3_bmcr_reset(tp);
2105         if (err)
2106                 return err;
2107
2108         if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY) {
2109                 val = MII_TG3_DSP_EXP8_AEDW | MII_TG3_DSP_EXP8_REJ2MHz;
2110                 tg3_phydsp_write(tp, MII_TG3_DSP_EXP8, val);
2111
2112                 tw32(TG3_CPMU_CTRL, cpmuctrl);
2113         }
2114
2115         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX ||
2116             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5761_AX) {
2117                 val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
2118                 if ((val & CPMU_LSPD_1000MB_MACCLK_MASK) ==
2119                     CPMU_LSPD_1000MB_MACCLK_12_5) {
2120                         val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
2121                         udelay(40);
2122                         tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
2123                 }
2124         }
2125
2126         if (tg3_flag(tp, 5717_PLUS) &&
2127             (tp->phy_flags & TG3_PHYFLG_MII_SERDES))
2128                 return 0;
2129
2130         tg3_phy_apply_otp(tp);
2131
2132         if (tp->phy_flags & TG3_PHYFLG_ENABLE_APD)
2133                 tg3_phy_toggle_apd(tp, true);
2134         else
2135                 tg3_phy_toggle_apd(tp, false);
2136
2137 out:
2138         if ((tp->phy_flags & TG3_PHYFLG_ADC_BUG) &&
2139             !TG3_PHY_AUXCTL_SMDSP_ENABLE(tp)) {
2140                 tg3_phydsp_write(tp, 0x201f, 0x2aaa);
2141                 tg3_phydsp_write(tp, 0x000a, 0x0323);
2142                 TG3_PHY_AUXCTL_SMDSP_DISABLE(tp);
2143         }
2144
2145         if (tp->phy_flags & TG3_PHYFLG_5704_A0_BUG) {
2146                 tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8d68);
2147                 tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8d68);
2148         }
2149
2150         if (tp->phy_flags & TG3_PHYFLG_BER_BUG) {
2151                 if (!TG3_PHY_AUXCTL_SMDSP_ENABLE(tp)) {
2152                         tg3_phydsp_write(tp, 0x000a, 0x310b);
2153                         tg3_phydsp_write(tp, 0x201f, 0x9506);
2154                         tg3_phydsp_write(tp, 0x401f, 0x14e2);
2155                         TG3_PHY_AUXCTL_SMDSP_DISABLE(tp);
2156                 }
2157         } else if (tp->phy_flags & TG3_PHYFLG_JITTER_BUG) {
2158                 if (!TG3_PHY_AUXCTL_SMDSP_ENABLE(tp)) {
2159                         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
2160                         if (tp->phy_flags & TG3_PHYFLG_ADJUST_TRIM) {
2161                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x110b);
2162                                 tg3_writephy(tp, MII_TG3_TEST1,
2163                                              MII_TG3_TEST1_TRIM_EN | 0x4);
2164                         } else
2165                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x010b);
2166
2167                         TG3_PHY_AUXCTL_SMDSP_DISABLE(tp);
2168                 }
2169         }
2170
2171         /* Set Extended packet length bit (bit 14) on all chips that */
2172         /* support jumbo frames */
2173         if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
2174                 /* Cannot do read-modify-write on 5401 */
2175                 tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL, 0x4c20);
2176         } else if (tg3_flag(tp, JUMBO_CAPABLE)) {
2177                 /* Set bit 14 with read-modify-write to preserve other bits */
2178                 err = tg3_phy_auxctl_read(tp,
2179                                           MII_TG3_AUXCTL_SHDWSEL_AUXCTL, &val);
2180                 if (!err)
2181                         tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL,
2182                                            val | MII_TG3_AUXCTL_ACTL_EXTPKTLEN);
2183         }
2184
2185         /* Set phy register 0x10 bit 0 to high fifo elasticity to support
2186          * jumbo frames transmission.
2187          */
2188         if (tg3_flag(tp, JUMBO_CAPABLE)) {
2189                 if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &val))
2190                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
2191                                      val | MII_TG3_EXT_CTRL_FIFO_ELASTIC);
2192         }
2193
2194         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2195                 /* adjust output voltage */
2196                 tg3_writephy(tp, MII_TG3_FET_PTEST, 0x12);
2197         }
2198
2199         tg3_phy_toggle_automdix(tp, 1);
2200         tg3_phy_set_wirespeed(tp);
2201         return 0;
2202 }
2203
2204 #define TG3_GPIO_MSG_DRVR_PRES           0x00000001
2205 #define TG3_GPIO_MSG_NEED_VAUX           0x00000002
2206 #define TG3_GPIO_MSG_MASK                (TG3_GPIO_MSG_DRVR_PRES | \
2207                                           TG3_GPIO_MSG_NEED_VAUX)
2208 #define TG3_GPIO_MSG_ALL_DRVR_PRES_MASK \
2209         ((TG3_GPIO_MSG_DRVR_PRES << 0) | \
2210          (TG3_GPIO_MSG_DRVR_PRES << 4) | \
2211          (TG3_GPIO_MSG_DRVR_PRES << 8) | \
2212          (TG3_GPIO_MSG_DRVR_PRES << 12))
2213
2214 #define TG3_GPIO_MSG_ALL_NEED_VAUX_MASK \
2215         ((TG3_GPIO_MSG_NEED_VAUX << 0) | \
2216          (TG3_GPIO_MSG_NEED_VAUX << 4) | \
2217          (TG3_GPIO_MSG_NEED_VAUX << 8) | \
2218          (TG3_GPIO_MSG_NEED_VAUX << 12))
2219
2220 static inline u32 tg3_set_function_status(struct tg3 *tp, u32 newstat)
2221 {
2222         u32 status, shift;
2223
2224         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 ||
2225             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5719)
2226                 status = tg3_ape_read32(tp, TG3_APE_GPIO_MSG);
2227         else
2228                 status = tr32(TG3_CPMU_DRV_STATUS);
2229
2230         shift = TG3_APE_GPIO_MSG_SHIFT + 4 * tp->pci_fn;
2231         status &= ~(TG3_GPIO_MSG_MASK << shift);
2232         status |= (newstat << shift);
2233
2234         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 ||
2235             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5719)
2236                 tg3_ape_write32(tp, TG3_APE_GPIO_MSG, status);
2237         else
2238                 tw32(TG3_CPMU_DRV_STATUS, status);
2239
2240         return status >> TG3_APE_GPIO_MSG_SHIFT;
2241 }
2242
2243 static inline int tg3_pwrsrc_switch_to_vmain(struct tg3 *tp)
2244 {
2245         if (!tg3_flag(tp, IS_NIC))
2246                 return 0;
2247
2248         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 ||
2249             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5719 ||
2250             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720) {
2251                 if (tg3_ape_lock(tp, TG3_APE_LOCK_GPIO))
2252                         return -EIO;
2253
2254                 tg3_set_function_status(tp, TG3_GPIO_MSG_DRVR_PRES);
2255
2256                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl,
2257                             TG3_GRC_LCLCTL_PWRSW_DELAY);
2258
2259                 tg3_ape_unlock(tp, TG3_APE_LOCK_GPIO);
2260         } else {
2261                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl,
2262                             TG3_GRC_LCLCTL_PWRSW_DELAY);
2263         }
2264
2265         return 0;
2266 }
2267
2268 static void tg3_pwrsrc_die_with_vmain(struct tg3 *tp)
2269 {
2270         u32 grc_local_ctrl;
2271
2272         if (!tg3_flag(tp, IS_NIC) ||
2273             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2274             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)
2275                 return;
2276
2277         grc_local_ctrl = tp->grc_local_ctrl | GRC_LCLCTRL_GPIO_OE1;
2278
2279         tw32_wait_f(GRC_LOCAL_CTRL,
2280                     grc_local_ctrl | GRC_LCLCTRL_GPIO_OUTPUT1,
2281                     TG3_GRC_LCLCTL_PWRSW_DELAY);
2282
2283         tw32_wait_f(GRC_LOCAL_CTRL,
2284                     grc_local_ctrl,
2285                     TG3_GRC_LCLCTL_PWRSW_DELAY);
2286
2287         tw32_wait_f(GRC_LOCAL_CTRL,
2288                     grc_local_ctrl | GRC_LCLCTRL_GPIO_OUTPUT1,
2289                     TG3_GRC_LCLCTL_PWRSW_DELAY);
2290 }
2291
2292 static void tg3_pwrsrc_switch_to_vaux(struct tg3 *tp)
2293 {
2294         if (!tg3_flag(tp, IS_NIC))
2295                 return;
2296
2297         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2298             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2299                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2300                             (GRC_LCLCTRL_GPIO_OE0 |
2301                              GRC_LCLCTRL_GPIO_OE1 |
2302                              GRC_LCLCTRL_GPIO_OE2 |
2303                              GRC_LCLCTRL_GPIO_OUTPUT0 |
2304                              GRC_LCLCTRL_GPIO_OUTPUT1),
2305                             TG3_GRC_LCLCTL_PWRSW_DELAY);
2306         } else if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761 ||
2307                    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761S) {
2308                 /* The 5761 non-e device swaps GPIO 0 and GPIO 2. */
2309                 u32 grc_local_ctrl = GRC_LCLCTRL_GPIO_OE0 |
2310                                      GRC_LCLCTRL_GPIO_OE1 |
2311                                      GRC_LCLCTRL_GPIO_OE2 |
2312                                      GRC_LCLCTRL_GPIO_OUTPUT0 |
2313                                      GRC_LCLCTRL_GPIO_OUTPUT1 |
2314                                      tp->grc_local_ctrl;
2315                 tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl,
2316                             TG3_GRC_LCLCTL_PWRSW_DELAY);
2317
2318                 grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT2;
2319                 tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl,
2320                             TG3_GRC_LCLCTL_PWRSW_DELAY);
2321
2322                 grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT0;
2323                 tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl,
2324                             TG3_GRC_LCLCTL_PWRSW_DELAY);
2325         } else {
2326                 u32 no_gpio2;
2327                 u32 grc_local_ctrl = 0;
2328
2329                 /* Workaround to prevent overdrawing Amps. */
2330                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
2331                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
2332                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2333                                     grc_local_ctrl,
2334                                     TG3_GRC_LCLCTL_PWRSW_DELAY);
2335                 }
2336
2337                 /* On 5753 and variants, GPIO2 cannot be used. */
2338                 no_gpio2 = tp->nic_sram_data_cfg &
2339                            NIC_SRAM_DATA_CFG_NO_GPIO2;
2340
2341                 grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
2342                                   GRC_LCLCTRL_GPIO_OE1 |
2343                                   GRC_LCLCTRL_GPIO_OE2 |
2344                                   GRC_LCLCTRL_GPIO_OUTPUT1 |
2345                                   GRC_LCLCTRL_GPIO_OUTPUT2;
2346                 if (no_gpio2) {
2347                         grc_local_ctrl &= ~(GRC_LCLCTRL_GPIO_OE2 |
2348                                             GRC_LCLCTRL_GPIO_OUTPUT2);
2349                 }
2350                 tw32_wait_f(GRC_LOCAL_CTRL,
2351                             tp->grc_local_ctrl | grc_local_ctrl,
2352                             TG3_GRC_LCLCTL_PWRSW_DELAY);
2353
2354                 grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT0;
2355
2356                 tw32_wait_f(GRC_LOCAL_CTRL,
2357                             tp->grc_local_ctrl | grc_local_ctrl,
2358                             TG3_GRC_LCLCTL_PWRSW_DELAY);
2359
2360                 if (!no_gpio2) {
2361                         grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT2;
2362                         tw32_wait_f(GRC_LOCAL_CTRL,
2363                                     tp->grc_local_ctrl | grc_local_ctrl,
2364                                     TG3_GRC_LCLCTL_PWRSW_DELAY);
2365                 }
2366         }
2367 }
2368
2369 static void tg3_frob_aux_power_5717(struct tg3 *tp, bool wol_enable)
2370 {
2371         u32 msg = 0;
2372
2373         /* Serialize power state transitions */
2374         if (tg3_ape_lock(tp, TG3_APE_LOCK_GPIO))
2375                 return;
2376
2377         if (tg3_flag(tp, ENABLE_ASF) || tg3_flag(tp, ENABLE_APE) || wol_enable)
2378                 msg = TG3_GPIO_MSG_NEED_VAUX;
2379
2380         msg = tg3_set_function_status(tp, msg);
2381
2382         if (msg & TG3_GPIO_MSG_ALL_DRVR_PRES_MASK)
2383                 goto done;
2384
2385         if (msg & TG3_GPIO_MSG_ALL_NEED_VAUX_MASK)
2386                 tg3_pwrsrc_switch_to_vaux(tp);
2387         else
2388                 tg3_pwrsrc_die_with_vmain(tp);
2389
2390 done:
2391         tg3_ape_unlock(tp, TG3_APE_LOCK_GPIO);
2392 }
2393
2394 static void tg3_frob_aux_power(struct tg3 *tp, bool include_wol)
2395 {
2396         bool need_vaux = false;
2397
2398         /* The GPIOs do something completely different on 57765. */
2399         if (!tg3_flag(tp, IS_NIC) ||
2400             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57765)
2401                 return;
2402
2403         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 ||
2404             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5719 ||
2405             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720) {
2406                 tg3_frob_aux_power_5717(tp, include_wol ?
2407                                         tg3_flag(tp, WOL_ENABLE) != 0 : 0);
2408                 return;
2409         }
2410
2411         if (tp->pdev_peer && tp->pdev_peer != tp->pdev) {
2412                 struct net_device *dev_peer;
2413
2414                 dev_peer = pci_get_drvdata(tp->pdev_peer);
2415
2416                 /* remove_one() may have been run on the peer. */
2417                 if (dev_peer) {
2418                         struct tg3 *tp_peer = netdev_priv(dev_peer);
2419
2420                         if (tg3_flag(tp_peer, INIT_COMPLETE))
2421                                 return;
2422
2423                         if ((include_wol && tg3_flag(tp_peer, WOL_ENABLE)) ||
2424                             tg3_flag(tp_peer, ENABLE_ASF))
2425                                 need_vaux = true;
2426                 }
2427         }
2428
2429         if ((include_wol && tg3_flag(tp, WOL_ENABLE)) ||
2430             tg3_flag(tp, ENABLE_ASF))
2431                 need_vaux = true;
2432
2433         if (need_vaux)
2434                 tg3_pwrsrc_switch_to_vaux(tp);
2435         else
2436                 tg3_pwrsrc_die_with_vmain(tp);
2437 }
2438
2439 static int tg3_5700_link_polarity(struct tg3 *tp, u32 speed)
2440 {
2441         if (tp->led_ctrl == LED_CTRL_MODE_PHY_2)
2442                 return 1;
2443         else if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5411) {
2444                 if (speed != SPEED_10)
2445                         return 1;
2446         } else if (speed == SPEED_10)
2447                 return 1;
2448
2449         return 0;
2450 }
2451
2452 static int tg3_setup_phy(struct tg3 *, int);
2453
2454 #define RESET_KIND_SHUTDOWN     0
2455 #define RESET_KIND_INIT         1
2456 #define RESET_KIND_SUSPEND      2
2457
2458 static void tg3_write_sig_post_reset(struct tg3 *, int);
2459 static int tg3_halt_cpu(struct tg3 *, u32);
2460
2461 static void tg3_power_down_phy(struct tg3 *tp, bool do_low_power)
2462 {
2463         u32 val;
2464
2465         if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
2466                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
2467                         u32 sg_dig_ctrl = tr32(SG_DIG_CTRL);
2468                         u32 serdes_cfg = tr32(MAC_SERDES_CFG);
2469
2470                         sg_dig_ctrl |=
2471                                 SG_DIG_USING_HW_AUTONEG | SG_DIG_SOFT_RESET;
2472                         tw32(SG_DIG_CTRL, sg_dig_ctrl);
2473                         tw32(MAC_SERDES_CFG, serdes_cfg | (1 << 15));
2474                 }
2475                 return;
2476         }
2477
2478         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2479                 tg3_bmcr_reset(tp);
2480                 val = tr32(GRC_MISC_CFG);
2481                 tw32_f(GRC_MISC_CFG, val | GRC_MISC_CFG_EPHY_IDDQ);
2482                 udelay(40);
2483                 return;
2484         } else if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
2485                 u32 phytest;
2486                 if (!tg3_readphy(tp, MII_TG3_FET_TEST, &phytest)) {
2487                         u32 phy;
2488
2489                         tg3_writephy(tp, MII_ADVERTISE, 0);
2490                         tg3_writephy(tp, MII_BMCR,
2491                                      BMCR_ANENABLE | BMCR_ANRESTART);
2492
2493                         tg3_writephy(tp, MII_TG3_FET_TEST,
2494                                      phytest | MII_TG3_FET_SHADOW_EN);
2495                         if (!tg3_readphy(tp, MII_TG3_FET_SHDW_AUXMODE4, &phy)) {
2496                                 phy |= MII_TG3_FET_SHDW_AUXMODE4_SBPD;
2497                                 tg3_writephy(tp,
2498                                              MII_TG3_FET_SHDW_AUXMODE4,
2499                                              phy);
2500                         }
2501                         tg3_writephy(tp, MII_TG3_FET_TEST, phytest);
2502                 }
2503                 return;
2504         } else if (do_low_power) {
2505                 tg3_writephy(tp, MII_TG3_EXT_CTRL,
2506                              MII_TG3_EXT_CTRL_FORCE_LED_OFF);
2507
2508                 val = MII_TG3_AUXCTL_PCTL_100TX_LPWR |
2509                       MII_TG3_AUXCTL_PCTL_SPR_ISOLATE |
2510                       MII_TG3_AUXCTL_PCTL_VREG_11V;
2511                 tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_PWRCTL, val);
2512         }
2513
2514         /* The PHY should not be powered down on some chips because
2515          * of bugs.
2516          */
2517         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2518             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
2519             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 &&
2520              (tp->phy_flags & TG3_PHYFLG_MII_SERDES)))
2521                 return;
2522
2523         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX ||
2524             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5761_AX) {
2525                 val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
2526                 val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
2527                 val |= CPMU_LSPD_1000MB_MACCLK_12_5;
2528                 tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
2529         }
2530
2531         tg3_writephy(tp, MII_BMCR, BMCR_PDOWN);
2532 }
2533
2534 /* tp->lock is held. */
2535 static int tg3_nvram_lock(struct tg3 *tp)
2536 {
2537         if (tg3_flag(tp, NVRAM)) {
2538                 int i;
2539
2540                 if (tp->nvram_lock_cnt == 0) {
2541                         tw32(NVRAM_SWARB, SWARB_REQ_SET1);
2542                         for (i = 0; i < 8000; i++) {
2543                                 if (tr32(NVRAM_SWARB) & SWARB_GNT1)
2544                                         break;
2545                                 udelay(20);
2546                         }
2547                         if (i == 8000) {
2548                                 tw32(NVRAM_SWARB, SWARB_REQ_CLR1);
2549                                 return -ENODEV;
2550                         }
2551                 }
2552                 tp->nvram_lock_cnt++;
2553         }
2554         return 0;
2555 }
2556
2557 /* tp->lock is held. */
2558 static void tg3_nvram_unlock(struct tg3 *tp)
2559 {
2560         if (tg3_flag(tp, NVRAM)) {
2561                 if (tp->nvram_lock_cnt > 0)
2562                         tp->nvram_lock_cnt--;
2563                 if (tp->nvram_lock_cnt == 0)
2564                         tw32_f(NVRAM_SWARB, SWARB_REQ_CLR1);
2565         }
2566 }
2567
2568 /* tp->lock is held. */
2569 static void tg3_enable_nvram_access(struct tg3 *tp)
2570 {
2571         if (tg3_flag(tp, 5750_PLUS) && !tg3_flag(tp, PROTECTED_NVRAM)) {
2572                 u32 nvaccess = tr32(NVRAM_ACCESS);
2573
2574                 tw32(NVRAM_ACCESS, nvaccess | ACCESS_ENABLE);
2575         }
2576 }
2577
2578 /* tp->lock is held. */
2579 static void tg3_disable_nvram_access(struct tg3 *tp)
2580 {
2581         if (tg3_flag(tp, 5750_PLUS) && !tg3_flag(tp, PROTECTED_NVRAM)) {
2582                 u32 nvaccess = tr32(NVRAM_ACCESS);
2583
2584                 tw32(NVRAM_ACCESS, nvaccess & ~ACCESS_ENABLE);
2585         }
2586 }
2587
2588 static int tg3_nvram_read_using_eeprom(struct tg3 *tp,
2589                                         u32 offset, u32 *val)
2590 {
2591         u32 tmp;
2592         int i;
2593
2594         if (offset > EEPROM_ADDR_ADDR_MASK || (offset % 4) != 0)
2595                 return -EINVAL;
2596
2597         tmp = tr32(GRC_EEPROM_ADDR) & ~(EEPROM_ADDR_ADDR_MASK |
2598                                         EEPROM_ADDR_DEVID_MASK |
2599                                         EEPROM_ADDR_READ);
2600         tw32(GRC_EEPROM_ADDR,
2601              tmp |
2602              (0 << EEPROM_ADDR_DEVID_SHIFT) |
2603              ((offset << EEPROM_ADDR_ADDR_SHIFT) &
2604               EEPROM_ADDR_ADDR_MASK) |
2605              EEPROM_ADDR_READ | EEPROM_ADDR_START);
2606
2607         for (i = 0; i < 1000; i++) {
2608                 tmp = tr32(GRC_EEPROM_ADDR);
2609
2610                 if (tmp & EEPROM_ADDR_COMPLETE)
2611                         break;
2612                 msleep(1);
2613         }
2614         if (!(tmp & EEPROM_ADDR_COMPLETE))
2615                 return -EBUSY;
2616
2617         tmp = tr32(GRC_EEPROM_DATA);
2618
2619         /*
2620          * The data will always be opposite the native endian
2621          * format.  Perform a blind byteswap to compensate.
2622          */
2623         *val = swab32(tmp);
2624
2625         return 0;
2626 }
2627
2628 #define NVRAM_CMD_TIMEOUT 10000
2629
2630 static int tg3_nvram_exec_cmd(struct tg3 *tp, u32 nvram_cmd)
2631 {
2632         int i;
2633
2634         tw32(NVRAM_CMD, nvram_cmd);
2635         for (i = 0; i < NVRAM_CMD_TIMEOUT; i++) {
2636                 udelay(10);
2637                 if (tr32(NVRAM_CMD) & NVRAM_CMD_DONE) {
2638                         udelay(10);
2639                         break;
2640                 }
2641         }
2642
2643         if (i == NVRAM_CMD_TIMEOUT)
2644                 return -EBUSY;
2645
2646         return 0;
2647 }
2648
2649 static u32 tg3_nvram_phys_addr(struct tg3 *tp, u32 addr)
2650 {
2651         if (tg3_flag(tp, NVRAM) &&
2652             tg3_flag(tp, NVRAM_BUFFERED) &&
2653             tg3_flag(tp, FLASH) &&
2654             !tg3_flag(tp, NO_NVRAM_ADDR_TRANS) &&
2655             (tp->nvram_jedecnum == JEDEC_ATMEL))
2656
2657                 addr = ((addr / tp->nvram_pagesize) <<
2658                         ATMEL_AT45DB0X1B_PAGE_POS) +
2659                        (addr % tp->nvram_pagesize);
2660
2661         return addr;
2662 }
2663
2664 static u32 tg3_nvram_logical_addr(struct tg3 *tp, u32 addr)
2665 {
2666         if (tg3_flag(tp, NVRAM) &&
2667             tg3_flag(tp, NVRAM_BUFFERED) &&
2668             tg3_flag(tp, FLASH) &&
2669             !tg3_flag(tp, NO_NVRAM_ADDR_TRANS) &&
2670             (tp->nvram_jedecnum == JEDEC_ATMEL))
2671
2672                 addr = ((addr >> ATMEL_AT45DB0X1B_PAGE_POS) *
2673                         tp->nvram_pagesize) +
2674                        (addr & ((1 << ATMEL_AT45DB0X1B_PAGE_POS) - 1));
2675
2676         return addr;
2677 }
2678
2679 /* NOTE: Data read in from NVRAM is byteswapped according to
2680  * the byteswapping settings for all other register accesses.
2681  * tg3 devices are BE devices, so on a BE machine, the data
2682  * returned will be exactly as it is seen in NVRAM.  On a LE
2683  * machine, the 32-bit value will be byteswapped.
2684  */
2685 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val)
2686 {
2687         int ret;
2688
2689         if (!tg3_flag(tp, NVRAM))
2690                 return tg3_nvram_read_using_eeprom(tp, offset, val);
2691
2692         offset = tg3_nvram_phys_addr(tp, offset);
2693
2694         if (offset > NVRAM_ADDR_MSK)
2695                 return -EINVAL;
2696
2697         ret = tg3_nvram_lock(tp);
2698         if (ret)
2699                 return ret;
2700
2701         tg3_enable_nvram_access(tp);
2702
2703         tw32(NVRAM_ADDR, offset);
2704         ret = tg3_nvram_exec_cmd(tp, NVRAM_CMD_RD | NVRAM_CMD_GO |
2705                 NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_DONE);
2706
2707         if (ret == 0)
2708                 *val = tr32(NVRAM_RDDATA);
2709
2710         tg3_disable_nvram_access(tp);
2711
2712         tg3_nvram_unlock(tp);
2713
2714         return ret;
2715 }
2716
2717 /* Ensures NVRAM data is in bytestream format. */
2718 static int tg3_nvram_read_be32(struct tg3 *tp, u32 offset, __be32 *val)
2719 {
2720         u32 v;
2721         int res = tg3_nvram_read(tp, offset, &v);
2722         if (!res)
2723                 *val = cpu_to_be32(v);
2724         return res;
2725 }
2726
2727 /* tp->lock is held. */
2728 static void __tg3_set_mac_addr(struct tg3 *tp, int skip_mac_1)
2729 {
2730         u32 addr_high, addr_low;
2731         int i;
2732
2733         addr_high = ((tp->dev->dev_addr[0] << 8) |
2734                      tp->dev->dev_addr[1]);
2735         addr_low = ((tp->dev->dev_addr[2] << 24) |
2736                     (tp->dev->dev_addr[3] << 16) |
2737                     (tp->dev->dev_addr[4] <<  8) |
2738                     (tp->dev->dev_addr[5] <<  0));
2739         for (i = 0; i < 4; i++) {
2740                 if (i == 1 && skip_mac_1)
2741                         continue;
2742                 tw32(MAC_ADDR_0_HIGH + (i * 8), addr_high);
2743                 tw32(MAC_ADDR_0_LOW + (i * 8), addr_low);
2744         }
2745
2746         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
2747             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
2748                 for (i = 0; i < 12; i++) {
2749                         tw32(MAC_EXTADDR_0_HIGH + (i * 8), addr_high);
2750                         tw32(MAC_EXTADDR_0_LOW + (i * 8), addr_low);
2751                 }
2752         }
2753
2754         addr_high = (tp->dev->dev_addr[0] +
2755                      tp->dev->dev_addr[1] +
2756                      tp->dev->dev_addr[2] +
2757                      tp->dev->dev_addr[3] +
2758                      tp->dev->dev_addr[4] +
2759                      tp->dev->dev_addr[5]) &
2760                 TX_BACKOFF_SEED_MASK;
2761         tw32(MAC_TX_BACKOFF_SEED, addr_high);
2762 }
2763
2764 static void tg3_enable_register_access(struct tg3 *tp)
2765 {
2766         /*
2767          * Make sure register accesses (indirect or otherwise) will function
2768          * correctly.
2769          */
2770         pci_write_config_dword(tp->pdev,
2771                                TG3PCI_MISC_HOST_CTRL, tp->misc_host_ctrl);
2772 }
2773
2774 static int tg3_power_up(struct tg3 *tp)
2775 {
2776         int err;
2777
2778         tg3_enable_register_access(tp);
2779
2780         err = pci_set_power_state(tp->pdev, PCI_D0);
2781         if (!err) {
2782                 /* Switch out of Vaux if it is a NIC */
2783                 tg3_pwrsrc_switch_to_vmain(tp);
2784         } else {
2785                 netdev_err(tp->dev, "Transition to D0 failed\n");
2786         }
2787
2788         return err;
2789 }
2790
2791 static int tg3_power_down_prepare(struct tg3 *tp)
2792 {
2793         u32 misc_host_ctrl;
2794         bool device_should_wake, do_low_power;
2795
2796         tg3_enable_register_access(tp);
2797
2798         /* Restore the CLKREQ setting. */
2799         if (tg3_flag(tp, CLKREQ_BUG)) {
2800                 u16 lnkctl;
2801
2802                 pci_read_config_word(tp->pdev,
2803                                      pci_pcie_cap(tp->pdev) + PCI_EXP_LNKCTL,
2804                                      &lnkctl);
2805                 lnkctl |= PCI_EXP_LNKCTL_CLKREQ_EN;
2806                 pci_write_config_word(tp->pdev,
2807                                       pci_pcie_cap(tp->pdev) + PCI_EXP_LNKCTL,
2808                                       lnkctl);
2809         }
2810
2811         misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
2812         tw32(TG3PCI_MISC_HOST_CTRL,
2813              misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT);
2814
2815         device_should_wake = device_may_wakeup(&tp->pdev->dev) &&
2816                              tg3_flag(tp, WOL_ENABLE);
2817
2818         if (tg3_flag(tp, USE_PHYLIB)) {
2819                 do_low_power = false;
2820                 if ((tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) &&
2821                     !(tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)) {
2822                         struct phy_device *phydev;
2823                         u32 phyid, advertising;
2824
2825                         phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
2826
2827                         tp->phy_flags |= TG3_PHYFLG_IS_LOW_POWER;
2828
2829                         tp->link_config.orig_speed = phydev->speed;
2830                         tp->link_config.orig_duplex = phydev->duplex;
2831                         tp->link_config.orig_autoneg = phydev->autoneg;
2832                         tp->link_config.orig_advertising = phydev->advertising;
2833
2834                         advertising = ADVERTISED_TP |
2835                                       ADVERTISED_Pause |
2836                                       ADVERTISED_Autoneg |
2837                                       ADVERTISED_10baseT_Half;
2838
2839                         if (tg3_flag(tp, ENABLE_ASF) || device_should_wake) {
2840                                 if (tg3_flag(tp, WOL_SPEED_100MB))
2841                                         advertising |=
2842                                                 ADVERTISED_100baseT_Half |
2843                                                 ADVERTISED_100baseT_Full |
2844                                                 ADVERTISED_10baseT_Full;
2845                                 else
2846                                         advertising |= ADVERTISED_10baseT_Full;
2847                         }
2848
2849                         phydev->advertising = advertising;
2850
2851                         phy_start_aneg(phydev);
2852
2853                         phyid = phydev->drv->phy_id & phydev->drv->phy_id_mask;
2854                         if (phyid != PHY_ID_BCMAC131) {
2855                                 phyid &= PHY_BCM_OUI_MASK;
2856                                 if (phyid == PHY_BCM_OUI_1 ||
2857                                     phyid == PHY_BCM_OUI_2 ||
2858                                     phyid == PHY_BCM_OUI_3)
2859                                         do_low_power = true;
2860                         }
2861                 }
2862         } else {
2863                 do_low_power = true;
2864
2865                 if (!(tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)) {
2866                         tp->phy_flags |= TG3_PHYFLG_IS_LOW_POWER;
2867                         tp->link_config.orig_speed = tp->link_config.speed;
2868                         tp->link_config.orig_duplex = tp->link_config.duplex;
2869                         tp->link_config.orig_autoneg = tp->link_config.autoneg;
2870                 }
2871
2872                 if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
2873                         tp->link_config.speed = SPEED_10;
2874                         tp->link_config.duplex = DUPLEX_HALF;
2875                         tp->link_config.autoneg = AUTONEG_ENABLE;
2876                         tg3_setup_phy(tp, 0);
2877                 }
2878         }
2879
2880         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2881                 u32 val;
2882
2883                 val = tr32(GRC_VCPU_EXT_CTRL);
2884                 tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_DISABLE_WOL);
2885         } else if (!tg3_flag(tp, ENABLE_ASF)) {
2886                 int i;
2887                 u32 val;
2888
2889                 for (i = 0; i < 200; i++) {
2890                         tg3_read_mem(tp, NIC_SRAM_FW_ASF_STATUS_MBOX, &val);
2891                         if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
2892                                 break;
2893                         msleep(1);
2894                 }
2895         }
2896         if (tg3_flag(tp, WOL_CAP))
2897                 tg3_write_mem(tp, NIC_SRAM_WOL_MBOX, WOL_SIGNATURE |
2898                                                      WOL_DRV_STATE_SHUTDOWN |
2899                                                      WOL_DRV_WOL |
2900                                                      WOL_SET_MAGIC_PKT);
2901
2902         if (device_should_wake) {
2903                 u32 mac_mode;
2904
2905                 if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES)) {
2906                         if (do_low_power &&
2907                             !(tp->phy_flags & TG3_PHYFLG_IS_FET)) {
2908                                 tg3_phy_auxctl_write(tp,
2909                                                MII_TG3_AUXCTL_SHDWSEL_PWRCTL,
2910                                                MII_TG3_AUXCTL_PCTL_WOL_EN |
2911                                                MII_TG3_AUXCTL_PCTL_100TX_LPWR |
2912                                                MII_TG3_AUXCTL_PCTL_CL_AB_TXDAC);
2913                                 udelay(40);
2914                         }
2915
2916                         if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
2917                                 mac_mode = MAC_MODE_PORT_MODE_GMII;
2918                         else
2919                                 mac_mode = MAC_MODE_PORT_MODE_MII;
2920
2921                         mac_mode |= tp->mac_mode & MAC_MODE_LINK_POLARITY;
2922                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
2923                             ASIC_REV_5700) {
2924                                 u32 speed = tg3_flag(tp, WOL_SPEED_100MB) ?
2925                                              SPEED_100 : SPEED_10;
2926                                 if (tg3_5700_link_polarity(tp, speed))
2927                                         mac_mode |= MAC_MODE_LINK_POLARITY;
2928                                 else
2929                                         mac_mode &= ~MAC_MODE_LINK_POLARITY;
2930                         }
2931                 } else {
2932                         mac_mode = MAC_MODE_PORT_MODE_TBI;
2933                 }
2934
2935                 if (!tg3_flag(tp, 5750_PLUS))
2936                         tw32(MAC_LED_CTRL, tp->led_ctrl);
2937
2938                 mac_mode |= MAC_MODE_MAGIC_PKT_ENABLE;
2939                 if ((tg3_flag(tp, 5705_PLUS) && !tg3_flag(tp, 5780_CLASS)) &&
2940                     (tg3_flag(tp, ENABLE_ASF) || tg3_flag(tp, ENABLE_APE)))
2941                         mac_mode |= MAC_MODE_KEEP_FRAME_IN_WOL;
2942
2943                 if (tg3_flag(tp, ENABLE_APE))
2944                         mac_mode |= MAC_MODE_APE_TX_EN |
2945                                     MAC_MODE_APE_RX_EN |
2946                                     MAC_MODE_TDE_ENABLE;
2947
2948                 tw32_f(MAC_MODE, mac_mode);
2949                 udelay(100);
2950
2951                 tw32_f(MAC_RX_MODE, RX_MODE_ENABLE);
2952                 udelay(10);
2953         }
2954
2955         if (!tg3_flag(tp, WOL_SPEED_100MB) &&
2956             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2957              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
2958                 u32 base_val;
2959
2960                 base_val = tp->pci_clock_ctrl;
2961                 base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
2962                              CLOCK_CTRL_TXCLK_DISABLE);
2963
2964                 tw32_wait_f(TG3PCI_CLOCK_CTRL, base_val | CLOCK_CTRL_ALTCLK |
2965                             CLOCK_CTRL_PWRDOWN_PLL133, 40);
2966         } else if (tg3_flag(tp, 5780_CLASS) ||
2967                    tg3_flag(tp, CPMU_PRESENT) ||
2968                    GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2969                 /* do nothing */
2970         } else if (!(tg3_flag(tp, 5750_PLUS) && tg3_flag(tp, ENABLE_ASF))) {
2971                 u32 newbits1, newbits2;
2972
2973                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2974                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2975                         newbits1 = (CLOCK_CTRL_RXCLK_DISABLE |
2976                                     CLOCK_CTRL_TXCLK_DISABLE |
2977                                     CLOCK_CTRL_ALTCLK);
2978                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
2979                 } else if (tg3_flag(tp, 5705_PLUS)) {
2980                         newbits1 = CLOCK_CTRL_625_CORE;
2981                         newbits2 = newbits1 | CLOCK_CTRL_ALTCLK;
2982                 } else {
2983                         newbits1 = CLOCK_CTRL_ALTCLK;
2984                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
2985                 }
2986
2987                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits1,
2988                             40);
2989
2990                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits2,
2991                             40);
2992
2993                 if (!tg3_flag(tp, 5705_PLUS)) {
2994                         u32 newbits3;
2995
2996                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2997                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2998                                 newbits3 = (CLOCK_CTRL_RXCLK_DISABLE |
2999                                             CLOCK_CTRL_TXCLK_DISABLE |
3000                                             CLOCK_CTRL_44MHZ_CORE);
3001                         } else {
3002                                 newbits3 = CLOCK_CTRL_44MHZ_CORE;
3003                         }
3004
3005                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
3006                                     tp->pci_clock_ctrl | newbits3, 40);
3007                 }
3008         }
3009
3010         if (!(device_should_wake) && !tg3_flag(tp, ENABLE_ASF))
3011                 tg3_power_down_phy(tp, do_low_power);
3012
3013         tg3_frob_aux_power(tp, true);
3014
3015         /* Workaround for unstable PLL clock */
3016         if ((GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX) ||
3017             (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX)) {
3018                 u32 val = tr32(0x7d00);
3019
3020                 val &= ~((1 << 16) | (1 << 4) | (1 << 2) | (1 << 1) | 1);
3021                 tw32(0x7d00, val);
3022                 if (!tg3_flag(tp, ENABLE_ASF)) {
3023                         int err;
3024
3025                         err = tg3_nvram_lock(tp);
3026                         tg3_halt_cpu(tp, RX_CPU_BASE);
3027                         if (!err)
3028                                 tg3_nvram_unlock(tp);
3029                 }
3030         }
3031
3032         tg3_write_sig_post_reset(tp, RESET_KIND_SHUTDOWN);
3033
3034         return 0;
3035 }
3036
3037 static void tg3_power_down(struct tg3 *tp)
3038 {
3039         tg3_power_down_prepare(tp);
3040
3041         pci_wake_from_d3(tp->pdev, tg3_flag(tp, WOL_ENABLE));
3042         pci_set_power_state(tp->pdev, PCI_D3hot);
3043 }
3044
3045 static void tg3_aux_stat_to_speed_duplex(struct tg3 *tp, u32 val, u16 *speed, u8 *duplex)
3046 {
3047         switch (val & MII_TG3_AUX_STAT_SPDMASK) {
3048         case MII_TG3_AUX_STAT_10HALF:
3049                 *speed = SPEED_10;
3050                 *duplex = DUPLEX_HALF;
3051                 break;
3052
3053         case MII_TG3_AUX_STAT_10FULL:
3054                 *speed = SPEED_10;
3055                 *duplex = DUPLEX_FULL;
3056                 break;
3057
3058         case MII_TG3_AUX_STAT_100HALF:
3059                 *speed = SPEED_100;
3060                 *duplex = DUPLEX_HALF;
3061                 break;
3062
3063         case MII_TG3_AUX_STAT_100FULL:
3064                 *speed = SPEED_100;
3065                 *duplex = DUPLEX_FULL;
3066                 break;
3067
3068         case MII_TG3_AUX_STAT_1000HALF:
3069                 *speed = SPEED_1000;
3070                 *duplex = DUPLEX_HALF;
3071                 break;
3072
3073         case MII_TG3_AUX_STAT_1000FULL:
3074                 *speed = SPEED_1000;
3075                 *duplex = DUPLEX_FULL;
3076                 break;
3077
3078         default:
3079                 if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
3080                         *speed = (val & MII_TG3_AUX_STAT_100) ? SPEED_100 :
3081                                  SPEED_10;
3082                         *duplex = (val & MII_TG3_AUX_STAT_FULL) ? DUPLEX_FULL :
3083                                   DUPLEX_HALF;
3084                         break;
3085                 }
3086                 *speed = SPEED_INVALID;
3087                 *duplex = DUPLEX_INVALID;
3088                 break;
3089         }
3090 }
3091
3092 static int tg3_phy_autoneg_cfg(struct tg3 *tp, u32 advertise, u32 flowctrl)
3093 {
3094         int err = 0;
3095         u32 val, new_adv;
3096
3097         new_adv = ADVERTISE_CSMA;
3098         if (advertise & ADVERTISED_10baseT_Half)
3099                 new_adv |= ADVERTISE_10HALF;
3100         if (advertise & ADVERTISED_10baseT_Full)
3101                 new_adv |= ADVERTISE_10FULL;
3102         if (advertise & ADVERTISED_100baseT_Half)
3103                 new_adv |= ADVERTISE_100HALF;
3104         if (advertise & ADVERTISED_100baseT_Full)
3105                 new_adv |= ADVERTISE_100FULL;
3106
3107         new_adv |= tg3_advert_flowctrl_1000T(flowctrl);
3108
3109         err = tg3_writephy(tp, MII_ADVERTISE, new_adv);
3110         if (err)
3111                 goto done;
3112
3113         if (tp->phy_flags & TG3_PHYFLG_10_100_ONLY)
3114                 goto done;
3115
3116         new_adv = 0;
3117         if (advertise & ADVERTISED_1000baseT_Half)
3118                 new_adv |= ADVERTISE_1000HALF;
3119         if (advertise & ADVERTISED_1000baseT_Full)
3120                 new_adv |= ADVERTISE_1000FULL;
3121
3122         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
3123             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
3124                 new_adv |= CTL1000_AS_MASTER | CTL1000_ENABLE_MASTER;
3125
3126         err = tg3_writephy(tp, MII_CTRL1000, new_adv);
3127         if (err)
3128                 goto done;
3129
3130         if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP))
3131                 goto done;
3132
3133         tw32(TG3_CPMU_EEE_MODE,
3134              tr32(TG3_CPMU_EEE_MODE) & ~TG3_CPMU_EEEMD_LPI_ENABLE);
3135
3136         err = TG3_PHY_AUXCTL_SMDSP_ENABLE(tp);
3137         if (!err) {
3138                 u32 err2;
3139
3140                 val = 0;
3141                 /* Advertise 100-BaseTX EEE ability */
3142                 if (advertise & ADVERTISED_100baseT_Full)
3143                         val |= MDIO_AN_EEE_ADV_100TX;
3144                 /* Advertise 1000-BaseT EEE ability */
3145                 if (advertise & ADVERTISED_1000baseT_Full)
3146                         val |= MDIO_AN_EEE_ADV_1000T;
3147                 err = tg3_phy_cl45_write(tp, MDIO_MMD_AN, MDIO_AN_EEE_ADV, val);
3148                 if (err)
3149                         val = 0;
3150
3151                 switch (GET_ASIC_REV(tp->pci_chip_rev_id)) {
3152                 case ASIC_REV_5717:
3153                 case ASIC_REV_57765:
3154                 case ASIC_REV_5719:
3155                         /* If we advertised any eee advertisements above... */
3156                         if (val)
3157                                 val = MII_TG3_DSP_TAP26_ALNOKO |
3158                                       MII_TG3_DSP_TAP26_RMRXSTO |
3159                                       MII_TG3_DSP_TAP26_OPCSINPT;
3160                         tg3_phydsp_write(tp, MII_TG3_DSP_TAP26, val);
3161                         /* Fall through */
3162                 case ASIC_REV_5720:
3163                         if (!tg3_phydsp_read(tp, MII_TG3_DSP_CH34TP2, &val))
3164                                 tg3_phydsp_write(tp, MII_TG3_DSP_CH34TP2, val |
3165                                                  MII_TG3_DSP_CH34TP2_HIBW01);
3166                 }
3167
3168                 err2 = TG3_PHY_AUXCTL_SMDSP_DISABLE(tp);
3169                 if (!err)
3170                         err = err2;
3171         }
3172
3173 done:
3174         return err;
3175 }
3176
3177 static void tg3_phy_copper_begin(struct tg3 *tp)
3178 {
3179         u32 new_adv;
3180         int i;
3181
3182         if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) {
3183                 new_adv = ADVERTISED_10baseT_Half |
3184                           ADVERTISED_10baseT_Full;
3185                 if (tg3_flag(tp, WOL_SPEED_100MB))
3186                         new_adv |= ADVERTISED_100baseT_Half |
3187                                    ADVERTISED_100baseT_Full;
3188
3189                 tg3_phy_autoneg_cfg(tp, new_adv,
3190                                     FLOW_CTRL_TX | FLOW_CTRL_RX);
3191         } else if (tp->link_config.speed == SPEED_INVALID) {
3192                 if (tp->phy_flags & TG3_PHYFLG_10_100_ONLY)
3193                         tp->link_config.advertising &=
3194                                 ~(ADVERTISED_1000baseT_Half |
3195                                   ADVERTISED_1000baseT_Full);
3196
3197                 tg3_phy_autoneg_cfg(tp, tp->link_config.advertising,
3198                                     tp->link_config.flowctrl);
3199         } else {
3200                 /* Asking for a specific link mode. */
3201                 if (tp->link_config.speed == SPEED_1000) {
3202                         if (tp->link_config.duplex == DUPLEX_FULL)
3203                                 new_adv = ADVERTISED_1000baseT_Full;
3204                         else
3205                                 new_adv = ADVERTISED_1000baseT_Half;
3206                 } else if (tp->link_config.speed == SPEED_100) {
3207                         if (tp->link_config.duplex == DUPLEX_FULL)
3208                                 new_adv = ADVERTISED_100baseT_Full;
3209                         else
3210                                 new_adv = ADVERTISED_100baseT_Half;
3211                 } else {
3212                         if (tp->link_config.duplex == DUPLEX_FULL)
3213                                 new_adv = ADVERTISED_10baseT_Full;
3214                         else
3215                                 new_adv = ADVERTISED_10baseT_Half;
3216                 }
3217
3218                 tg3_phy_autoneg_cfg(tp, new_adv,
3219                                     tp->link_config.flowctrl);
3220         }
3221
3222         if (tp->link_config.autoneg == AUTONEG_DISABLE &&
3223             tp->link_config.speed != SPEED_INVALID) {
3224                 u32 bmcr, orig_bmcr;
3225
3226                 tp->link_config.active_speed = tp->link_config.speed;
3227                 tp->link_config.active_duplex = tp->link_config.duplex;
3228
3229                 bmcr = 0;
3230                 switch (tp->link_config.speed) {
3231                 default:
3232                 case SPEED_10:
3233                         break;
3234
3235                 case SPEED_100:
3236                         bmcr |= BMCR_SPEED100;
3237                         break;
3238
3239                 case SPEED_1000:
3240                         bmcr |= BMCR_SPEED1000;
3241                         break;
3242                 }
3243
3244                 if (tp->link_config.duplex == DUPLEX_FULL)
3245                         bmcr |= BMCR_FULLDPLX;
3246
3247                 if (!tg3_readphy(tp, MII_BMCR, &orig_bmcr) &&
3248                     (bmcr != orig_bmcr)) {
3249                         tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK);
3250                         for (i = 0; i < 1500; i++) {
3251                                 u32 tmp;
3252
3253                                 udelay(10);
3254                                 if (tg3_readphy(tp, MII_BMSR, &tmp) ||
3255                                     tg3_readphy(tp, MII_BMSR, &tmp))
3256                                         continue;
3257                                 if (!(tmp & BMSR_LSTATUS)) {
3258                                         udelay(40);
3259                                         break;
3260                                 }
3261                         }
3262                         tg3_writephy(tp, MII_BMCR, bmcr);
3263                         udelay(40);
3264                 }
3265         } else {
3266                 tg3_writephy(tp, MII_BMCR,
3267                              BMCR_ANENABLE | BMCR_ANRESTART);
3268         }
3269 }
3270
3271 static int tg3_init_5401phy_dsp(struct tg3 *tp)
3272 {
3273         int err;
3274
3275         /* Turn off tap power management. */
3276         /* Set Extended packet length bit */
3277         err = tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL, 0x4c20);
3278
3279         err |= tg3_phydsp_write(tp, 0x0012, 0x1804);
3280         err |= tg3_phydsp_write(tp, 0x0013, 0x1204);
3281         err |= tg3_phydsp_write(tp, 0x8006, 0x0132);
3282         err |= tg3_phydsp_write(tp, 0x8006, 0x0232);
3283         err |= tg3_phydsp_write(tp, 0x201f, 0x0a20);
3284
3285         udelay(40);
3286
3287         return err;
3288 }
3289
3290 static int tg3_copper_is_advertising_all(struct tg3 *tp, u32 mask)
3291 {
3292         u32 adv_reg, all_mask = 0;
3293
3294         if (mask & ADVERTISED_10baseT_Half)
3295                 all_mask |= ADVERTISE_10HALF;
3296         if (mask & ADVERTISED_10baseT_Full)
3297                 all_mask |= ADVERTISE_10FULL;
3298         if (mask & ADVERTISED_100baseT_Half)
3299                 all_mask |= ADVERTISE_100HALF;
3300         if (mask & ADVERTISED_100baseT_Full)
3301                 all_mask |= ADVERTISE_100FULL;
3302
3303         if (tg3_readphy(tp, MII_ADVERTISE, &adv_reg))
3304                 return 0;
3305
3306         if ((adv_reg & all_mask) != all_mask)
3307                 return 0;
3308         if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
3309                 u32 tg3_ctrl;
3310
3311                 all_mask = 0;
3312                 if (mask & ADVERTISED_1000baseT_Half)
3313                         all_mask |= ADVERTISE_1000HALF;
3314                 if (mask & ADVERTISED_1000baseT_Full)
3315                         all_mask |= ADVERTISE_1000FULL;
3316
3317                 if (tg3_readphy(tp, MII_CTRL1000, &tg3_ctrl))
3318                         return 0;
3319
3320                 if ((tg3_ctrl & all_mask) != all_mask)
3321                         return 0;
3322         }
3323         return 1;
3324 }
3325
3326 static int tg3_adv_1000T_flowctrl_ok(struct tg3 *tp, u32 *lcladv, u32 *rmtadv)
3327 {
3328         u32 curadv, reqadv;
3329
3330         if (tg3_readphy(tp, MII_ADVERTISE, lcladv))
3331                 return 1;
3332
3333         curadv = *lcladv & (ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM);
3334         reqadv = tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
3335
3336         if (tp->link_config.active_duplex == DUPLEX_FULL) {
3337                 if (curadv != reqadv)
3338                         return 0;
3339
3340                 if (tg3_flag(tp, PAUSE_AUTONEG))
3341                         tg3_readphy(tp, MII_LPA, rmtadv);
3342         } else {
3343                 /* Reprogram the advertisement register, even if it
3344                  * does not affect the current link.  If the link
3345                  * gets renegotiated in the future, we can save an
3346                  * additional renegotiation cycle by advertising
3347                  * it correctly in the first place.
3348                  */
3349                 if (curadv != reqadv) {
3350                         *lcladv &= ~(ADVERTISE_PAUSE_CAP |
3351                                      ADVERTISE_PAUSE_ASYM);
3352                         tg3_writephy(tp, MII_ADVERTISE, *lcladv | reqadv);
3353                 }
3354         }
3355
3356         return 1;
3357 }
3358
3359 static int tg3_setup_copper_phy(struct tg3 *tp, int force_reset)
3360 {
3361         int current_link_up;
3362         u32 bmsr, val;
3363         u32 lcl_adv, rmt_adv;
3364         u16 current_speed;
3365         u8 current_duplex;
3366         int i, err;
3367
3368         tw32(MAC_EVENT, 0);
3369
3370         tw32_f(MAC_STATUS,
3371              (MAC_STATUS_SYNC_CHANGED |
3372               MAC_STATUS_CFG_CHANGED |
3373               MAC_STATUS_MI_COMPLETION |
3374               MAC_STATUS_LNKSTATE_CHANGED));
3375         udelay(40);
3376
3377         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
3378                 tw32_f(MAC_MI_MODE,
3379                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
3380                 udelay(80);
3381         }
3382
3383         tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_PWRCTL, 0);
3384
3385         /* Some third-party PHYs need to be reset on link going
3386          * down.
3387          */
3388         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
3389              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
3390              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
3391             netif_carrier_ok(tp->dev)) {
3392                 tg3_readphy(tp, MII_BMSR, &bmsr);
3393                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
3394                     !(bmsr & BMSR_LSTATUS))
3395                         force_reset = 1;
3396         }
3397         if (force_reset)
3398                 tg3_phy_reset(tp);
3399
3400         if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
3401                 tg3_readphy(tp, MII_BMSR, &bmsr);
3402                 if (tg3_readphy(tp, MII_BMSR, &bmsr) ||
3403                     !tg3_flag(tp, INIT_COMPLETE))
3404                         bmsr = 0;
3405
3406                 if (!(bmsr & BMSR_LSTATUS)) {
3407                         err = tg3_init_5401phy_dsp(tp);
3408                         if (err)
3409                                 return err;
3410
3411                         tg3_readphy(tp, MII_BMSR, &bmsr);
3412                         for (i = 0; i < 1000; i++) {
3413                                 udelay(10);
3414                                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
3415                                     (bmsr & BMSR_LSTATUS)) {
3416                                         udelay(40);
3417                                         break;
3418                                 }
3419                         }
3420
3421                         if ((tp->phy_id & TG3_PHY_ID_REV_MASK) ==
3422                             TG3_PHY_REV_BCM5401_B0 &&
3423                             !(bmsr & BMSR_LSTATUS) &&
3424                             tp->link_config.active_speed == SPEED_1000) {
3425                                 err = tg3_phy_reset(tp);
3426                                 if (!err)
3427                                         err = tg3_init_5401phy_dsp(tp);
3428                                 if (err)
3429                                         return err;
3430                         }
3431                 }
3432         } else if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
3433                    tp->pci_chip_rev_id == CHIPREV_ID_5701_B0) {
3434                 /* 5701 {A0,B0} CRC bug workaround */
3435                 tg3_writephy(tp, 0x15, 0x0a75);
3436                 tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8c68);
3437                 tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8d68);
3438                 tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8c68);
3439         }
3440
3441         /* Clear pending interrupts... */
3442         tg3_readphy(tp, MII_TG3_ISTAT, &val);
3443         tg3_readphy(tp, MII_TG3_ISTAT, &val);
3444
3445         if (tp->phy_flags & TG3_PHYFLG_USE_MI_INTERRUPT)
3446                 tg3_writephy(tp, MII_TG3_IMASK, ~MII_TG3_INT_LINKCHG);
3447         else if (!(tp->phy_flags & TG3_PHYFLG_IS_FET))
3448                 tg3_writephy(tp, MII_TG3_IMASK, ~0);
3449
3450         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
3451             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
3452                 if (tp->led_ctrl == LED_CTRL_MODE_PHY_1)
3453                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
3454                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
3455                 else
3456                         tg3_writephy(tp, MII_TG3_EXT_CTRL, 0);
3457         }
3458
3459         current_link_up = 0;
3460         current_speed = SPEED_INVALID;
3461         current_duplex = DUPLEX_INVALID;
3462
3463         if (tp->phy_flags & TG3_PHYFLG_CAPACITIVE_COUPLING) {
3464                 err = tg3_phy_auxctl_read(tp,
3465                                           MII_TG3_AUXCTL_SHDWSEL_MISCTEST,
3466                                           &val);
3467                 if (!err && !(val & (1 << 10))) {
3468                         tg3_phy_auxctl_write(tp,
3469                                              MII_TG3_AUXCTL_SHDWSEL_MISCTEST,
3470                                              val | (1 << 10));
3471                         goto relink;
3472                 }
3473         }
3474
3475         bmsr = 0;
3476         for (i = 0; i < 100; i++) {
3477                 tg3_readphy(tp, MII_BMSR, &bmsr);
3478                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
3479                     (bmsr & BMSR_LSTATUS))
3480                         break;
3481                 udelay(40);
3482         }
3483
3484         if (bmsr & BMSR_LSTATUS) {
3485                 u32 aux_stat, bmcr;
3486
3487                 tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
3488                 for (i = 0; i < 2000; i++) {
3489                         udelay(10);
3490                         if (!tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat) &&
3491                             aux_stat)
3492                                 break;
3493                 }
3494
3495                 tg3_aux_stat_to_speed_duplex(tp, aux_stat,
3496                                              &current_speed,
3497                                              &current_duplex);
3498
3499                 bmcr = 0;
3500                 for (i = 0; i < 200; i++) {
3501                         tg3_readphy(tp, MII_BMCR, &bmcr);
3502                         if (tg3_readphy(tp, MII_BMCR, &bmcr))
3503                                 continue;
3504                         if (bmcr && bmcr != 0x7fff)
3505                                 break;
3506                         udelay(10);
3507                 }
3508
3509                 lcl_adv = 0;
3510                 rmt_adv = 0;
3511
3512                 tp->link_config.active_speed = current_speed;
3513                 tp->link_config.active_duplex = current_duplex;
3514
3515                 if (tp->link_config.autoneg == AUTONEG_ENABLE) {
3516                         if ((bmcr & BMCR_ANENABLE) &&
3517                             tg3_copper_is_advertising_all(tp,
3518                                                 tp->link_config.advertising)) {
3519                                 if (tg3_adv_1000T_flowctrl_ok(tp, &lcl_adv,
3520                                                                   &rmt_adv))
3521                                         current_link_up = 1;
3522                         }
3523                 } else {
3524                         if (!(bmcr & BMCR_ANENABLE) &&
3525                             tp->link_config.speed == current_speed &&
3526                             tp->link_config.duplex == current_duplex &&
3527                             tp->link_config.flowctrl ==
3528                             tp->link_config.active_flowctrl) {
3529                                 current_link_up = 1;
3530                         }
3531                 }
3532
3533                 if (current_link_up == 1 &&
3534                     tp->link_config.active_duplex == DUPLEX_FULL)
3535                         tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
3536         }
3537
3538 relink:
3539         if (current_link_up == 0 || (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)) {
3540                 tg3_phy_copper_begin(tp);
3541
3542                 tg3_readphy(tp, MII_BMSR, &bmsr);
3543                 if ((!tg3_readphy(tp, MII_BMSR, &bmsr) && (bmsr & BMSR_LSTATUS)) ||
3544                     (tp->mac_mode & MAC_MODE_PORT_INT_LPBACK))
3545                         current_link_up = 1;
3546         }
3547
3548         tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
3549         if (current_link_up == 1) {
3550                 if (tp->link_config.active_speed == SPEED_100 ||
3551                     tp->link_config.active_speed == SPEED_10)
3552                         tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
3553                 else
3554                         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
3555         } else if (tp->phy_flags & TG3_PHYFLG_IS_FET)
3556                 tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
3557         else
3558                 tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
3559
3560         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
3561         if (tp->link_config.active_duplex == DUPLEX_HALF)
3562                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
3563
3564         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
3565                 if (current_link_up == 1 &&
3566                     tg3_5700_link_polarity(tp, tp->link_config.active_speed))
3567                         tp->mac_mode |= MAC_MODE_LINK_POLARITY;
3568                 else
3569                         tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
3570         }
3571
3572         /* ??? Without this setting Netgear GA302T PHY does not
3573          * ??? send/receive packets...
3574          */
3575         if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5411 &&
3576             tp->pci_chip_rev_id == CHIPREV_ID_5700_ALTIMA) {
3577                 tp->mi_mode |= MAC_MI_MODE_AUTO_POLL;
3578                 tw32_f(MAC_MI_MODE, tp->mi_mode);
3579                 udelay(80);
3580         }
3581
3582         tw32_f(MAC_MODE, tp->mac_mode);
3583         udelay(40);
3584
3585         tg3_phy_eee_adjust(tp, current_link_up);
3586
3587         if (tg3_flag(tp, USE_LINKCHG_REG)) {
3588                 /* Polled via timer. */
3589                 tw32_f(MAC_EVENT, 0);
3590         } else {
3591                 tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
3592         }
3593         udelay(40);
3594
3595         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 &&
3596             current_link_up == 1 &&
3597             tp->link_config.active_speed == SPEED_1000 &&
3598             (tg3_flag(tp, PCIX_MODE) || tg3_flag(tp, PCI_HIGH_SPEED))) {
3599                 udelay(120);
3600                 tw32_f(MAC_STATUS,
3601                      (MAC_STATUS_SYNC_CHANGED |
3602                       MAC_STATUS_CFG_CHANGED));
3603                 udelay(40);
3604                 tg3_write_mem(tp,
3605                               NIC_SRAM_FIRMWARE_MBOX,
3606                               NIC_SRAM_FIRMWARE_MBOX_MAGIC2);
3607         }
3608
3609         /* Prevent send BD corruption. */
3610         if (tg3_flag(tp, CLKREQ_BUG)) {
3611                 u16 oldlnkctl, newlnkctl;
3612
3613                 pci_read_config_word(tp->pdev,
3614                                      pci_pcie_cap(tp->pdev) + PCI_EXP_LNKCTL,
3615                                      &oldlnkctl);
3616                 if (tp->link_config.active_speed == SPEED_100 ||
3617                     tp->link_config.active_speed == SPEED_10)
3618                         newlnkctl = oldlnkctl & ~PCI_EXP_LNKCTL_CLKREQ_EN;
3619                 else
3620                         newlnkctl = oldlnkctl | PCI_EXP_LNKCTL_CLKREQ_EN;
3621                 if (newlnkctl != oldlnkctl)
3622                         pci_write_config_word(tp->pdev,
3623                                               pci_pcie_cap(tp->pdev) + PCI_EXP_LNKCTL,
3624                                               newlnkctl);
3625         }
3626
3627         if (current_link_up != netif_carrier_ok(tp->dev)) {
3628                 if (current_link_up)
3629                         netif_carrier_on(tp->dev);
3630                 else
3631                         netif_carrier_off(tp->dev);
3632                 tg3_link_report(tp);
3633         }
3634
3635         return 0;
3636 }
3637
3638 struct tg3_fiber_aneginfo {
3639         int state;
3640 #define ANEG_STATE_UNKNOWN              0
3641 #define ANEG_STATE_AN_ENABLE            1
3642 #define ANEG_STATE_RESTART_INIT         2
3643 #define ANEG_STATE_RESTART              3
3644 #define ANEG_STATE_DISABLE_LINK_OK      4
3645 #define ANEG_STATE_ABILITY_DETECT_INIT  5
3646 #define ANEG_STATE_ABILITY_DETECT       6
3647 #define ANEG_STATE_ACK_DETECT_INIT      7
3648 #define ANEG_STATE_ACK_DETECT           8
3649 #define ANEG_STATE_COMPLETE_ACK_INIT    9
3650 #define ANEG_STATE_COMPLETE_ACK         10
3651 #define ANEG_STATE_IDLE_DETECT_INIT     11
3652 #define ANEG_STATE_IDLE_DETECT          12
3653 #define ANEG_STATE_LINK_OK              13
3654 #define ANEG_STATE_NEXT_PAGE_WAIT_INIT  14
3655 #define ANEG_STATE_NEXT_PAGE_WAIT       15
3656
3657         u32 flags;
3658 #define MR_AN_ENABLE            0x00000001
3659 #define MR_RESTART_AN           0x00000002
3660 #define MR_AN_COMPLETE          0x00000004
3661 #define MR_PAGE_RX              0x00000008
3662 #define MR_NP_LOADED            0x00000010
3663 #define MR_TOGGLE_TX            0x00000020
3664 #define MR_LP_ADV_FULL_DUPLEX   0x00000040
3665 #define MR_LP_ADV_HALF_DUPLEX   0x00000080
3666 #define MR_LP_ADV_SYM_PAUSE     0x00000100
3667 #define MR_LP_ADV_ASYM_PAUSE    0x00000200
3668 #define MR_LP_ADV_REMOTE_FAULT1 0x00000400
3669 #define MR_LP_ADV_REMOTE_FAULT2 0x00000800
3670 #define MR_LP_ADV_NEXT_PAGE     0x00001000
3671 #define MR_TOGGLE_RX            0x00002000
3672 #define MR_NP_RX                0x00004000
3673
3674 #define MR_LINK_OK              0x80000000
3675
3676         unsigned long link_time, cur_time;
3677
3678         u32 ability_match_cfg;
3679         int ability_match_count;
3680
3681         char ability_match, idle_match, ack_match;
3682
3683         u32 txconfig, rxconfig;
3684 #define ANEG_CFG_NP             0x00000080
3685 #define ANEG_CFG_ACK            0x00000040
3686 #define ANEG_CFG_RF2            0x00000020
3687 #define ANEG_CFG_RF1            0x00000010
3688 #define ANEG_CFG_PS2            0x00000001
3689 #define ANEG_CFG_PS1            0x00008000
3690 #define ANEG_CFG_HD             0x00004000
3691 #define ANEG_CFG_FD             0x00002000
3692 #define ANEG_CFG_INVAL          0x00001f06
3693
3694 };
3695 #define ANEG_OK         0
3696 #define ANEG_DONE       1
3697 #define ANEG_TIMER_ENAB 2
3698 #define ANEG_FAILED     -1
3699
3700 #define ANEG_STATE_SETTLE_TIME  10000
3701
3702 static int tg3_fiber_aneg_smachine(struct tg3 *tp,
3703                                    struct tg3_fiber_aneginfo *ap)
3704 {
3705         u16 flowctrl;
3706         unsigned long delta;
3707         u32 rx_cfg_reg;
3708         int ret;
3709
3710         if (ap->state == ANEG_STATE_UNKNOWN) {
3711                 ap->rxconfig = 0;
3712                 ap->link_time = 0;
3713                 ap->cur_time = 0;
3714                 ap->ability_match_cfg = 0;
3715                 ap->ability_match_count = 0;
3716                 ap->ability_match = 0;
3717                 ap->idle_match = 0;
3718                 ap->ack_match = 0;
3719         }
3720         ap->cur_time++;
3721
3722         if (tr32(MAC_STATUS) & MAC_STATUS_RCVD_CFG) {
3723                 rx_cfg_reg = tr32(MAC_RX_AUTO_NEG);
3724
3725                 if (rx_cfg_reg != ap->ability_match_cfg) {
3726                         ap->ability_match_cfg = rx_cfg_reg;
3727                         ap->ability_match = 0;
3728                         ap->ability_match_count = 0;
3729                 } else {
3730                         if (++ap->ability_match_count > 1) {
3731                                 ap->ability_match = 1;
3732                                 ap->ability_match_cfg = rx_cfg_reg;
3733                         }
3734                 }
3735                 if (rx_cfg_reg & ANEG_CFG_ACK)
3736                         ap->ack_match = 1;
3737                 else
3738                         ap->ack_match = 0;
3739
3740                 ap->idle_match = 0;
3741         } else {
3742                 ap->idle_match = 1;
3743                 ap->ability_match_cfg = 0;
3744                 ap->ability_match_count = 0;
3745                 ap->ability_match = 0;
3746                 ap->ack_match = 0;
3747
3748                 rx_cfg_reg = 0;
3749         }
3750
3751         ap->rxconfig = rx_cfg_reg;
3752         ret = ANEG_OK;
3753
3754         switch (ap->state) {
3755         case ANEG_STATE_UNKNOWN:
3756                 if (ap->flags & (MR_AN_ENABLE | MR_RESTART_AN))
3757                         ap->state = ANEG_STATE_AN_ENABLE;
3758
3759                 /* fallthru */
3760         case ANEG_STATE_AN_ENABLE:
3761                 ap->flags &= ~(MR_AN_COMPLETE | MR_PAGE_RX);
3762                 if (ap->flags & MR_AN_ENABLE) {
3763                         ap->link_time = 0;
3764                         ap->cur_time = 0;
3765                         ap->ability_match_cfg = 0;
3766                         ap->ability_match_count = 0;
3767                         ap->ability_match = 0;
3768                         ap->idle_match = 0;
3769                         ap->ack_match = 0;
3770
3771                         ap->state = ANEG_STATE_RESTART_INIT;
3772                 } else {
3773                         ap->state = ANEG_STATE_DISABLE_LINK_OK;
3774                 }
3775                 break;
3776
3777         case ANEG_STATE_RESTART_INIT:
3778                 ap->link_time = ap->cur_time;
3779                 ap->flags &= ~(MR_NP_LOADED);
3780                 ap->txconfig = 0;
3781                 tw32(MAC_TX_AUTO_NEG, 0);
3782                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
3783                 tw32_f(MAC_MODE, tp->mac_mode);
3784                 udelay(40);
3785
3786                 ret = ANEG_TIMER_ENAB;
3787                 ap->state = ANEG_STATE_RESTART;
3788
3789                 /* fallthru */
3790         case ANEG_STATE_RESTART:
3791                 delta = ap->cur_time - ap->link_time;
3792                 if (delta > ANEG_STATE_SETTLE_TIME)
3793                         ap->state = ANEG_STATE_ABILITY_DETECT_INIT;
3794                 else
3795                         ret = ANEG_TIMER_ENAB;
3796                 break;
3797
3798         case ANEG_STATE_DISABLE_LINK_OK:
3799                 ret = ANEG_DONE;
3800                 break;
3801
3802         case ANEG_STATE_ABILITY_DETECT_INIT:
3803                 ap->flags &= ~(MR_TOGGLE_TX);
3804                 ap->txconfig = ANEG_CFG_FD;
3805                 flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
3806                 if (flowctrl & ADVERTISE_1000XPAUSE)
3807                         ap->txconfig |= ANEG_CFG_PS1;
3808                 if (flowctrl & ADVERTISE_1000XPSE_ASYM)
3809                         ap->txconfig |= ANEG_CFG_PS2;
3810                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
3811                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
3812                 tw32_f(MAC_MODE, tp->mac_mode);
3813                 udelay(40);
3814
3815                 ap->state = ANEG_STATE_ABILITY_DETECT;
3816                 break;
3817
3818         case ANEG_STATE_ABILITY_DETECT:
3819                 if (ap->ability_match != 0 && ap->rxconfig != 0)
3820                         ap->state = ANEG_STATE_ACK_DETECT_INIT;
3821                 break;
3822
3823         case ANEG_STATE_ACK_DETECT_INIT:
3824                 ap->txconfig |= ANEG_CFG_ACK;
3825                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
3826                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
3827                 tw32_f(MAC_MODE, tp->mac_mode);
3828                 udelay(40);
3829
3830                 ap->state = ANEG_STATE_ACK_DETECT;
3831
3832                 /* fallthru */
3833         case ANEG_STATE_ACK_DETECT:
3834                 if (ap->ack_match != 0) {
3835                         if ((ap->rxconfig & ~ANEG_CFG_ACK) ==
3836                             (ap->ability_match_cfg & ~ANEG_CFG_ACK)) {
3837                                 ap->state = ANEG_STATE_COMPLETE_ACK_INIT;
3838                         } else {
3839                                 ap->state = ANEG_STATE_AN_ENABLE;
3840                         }
3841                 } else if (ap->ability_match != 0 &&
3842                            ap->rxconfig == 0) {
3843                         ap->state = ANEG_STATE_AN_ENABLE;
3844                 }
3845                 break;
3846
3847         case ANEG_STATE_COMPLETE_ACK_INIT:
3848                 if (ap->rxconfig & ANEG_CFG_INVAL) {
3849                         ret = ANEG_FAILED;
3850                         break;
3851                 }
3852                 ap->flags &= ~(MR_LP_ADV_FULL_DUPLEX |
3853                                MR_LP_ADV_HALF_DUPLEX |
3854                                MR_LP_ADV_SYM_PAUSE |
3855                                MR_LP_ADV_ASYM_PAUSE |
3856                                MR_LP_ADV_REMOTE_FAULT1 |
3857                                MR_LP_ADV_REMOTE_FAULT2 |
3858                                MR_LP_ADV_NEXT_PAGE |
3859                                MR_TOGGLE_RX |
3860                                MR_NP_RX);
3861                 if (ap->rxconfig & ANEG_CFG_FD)
3862                         ap->flags |= MR_LP_ADV_FULL_DUPLEX;
3863                 if (ap->rxconfig & ANEG_CFG_HD)
3864                         ap->flags |= MR_LP_ADV_HALF_DUPLEX;
3865                 if (ap->rxconfig & ANEG_CFG_PS1)
3866                         ap->flags |= MR_LP_ADV_SYM_PAUSE;
3867                 if (ap->rxconfig & ANEG_CFG_PS2)
3868                         ap->flags |= MR_LP_ADV_ASYM_PAUSE;
3869                 if (ap->rxconfig & ANEG_CFG_RF1)
3870                         ap->flags |= MR_LP_ADV_REMOTE_FAULT1;
3871                 if (ap->rxconfig & ANEG_CFG_RF2)
3872                         ap->flags |= MR_LP_ADV_REMOTE_FAULT2;
3873                 if (ap->rxconfig & ANEG_CFG_NP)
3874                         ap->flags |= MR_LP_ADV_NEXT_PAGE;
3875
3876                 ap->link_time = ap->cur_time;
3877
3878                 ap->flags ^= (MR_TOGGLE_TX);
3879                 if (ap->rxconfig & 0x0008)
3880                         ap->flags |= MR_TOGGLE_RX;
3881                 if (ap->rxconfig & ANEG_CFG_NP)
3882                         ap->flags |= MR_NP_RX;
3883                 ap->flags |= MR_PAGE_RX;
3884
3885                 ap->state = ANEG_STATE_COMPLETE_ACK;
3886                 ret = ANEG_TIMER_ENAB;
3887                 break;
3888
3889         case ANEG_STATE_COMPLETE_ACK:
3890                 if (ap->ability_match != 0 &&
3891                     ap->rxconfig == 0) {
3892                         ap->state = ANEG_STATE_AN_ENABLE;
3893                         break;
3894                 }
3895                 delta = ap->cur_time - ap->link_time;
3896                 if (delta > ANEG_STATE_SETTLE_TIME) {
3897                         if (!(ap->flags & (MR_LP_ADV_NEXT_PAGE))) {
3898                                 ap->state = ANEG_STATE_IDLE_DETECT_INIT;
3899                         } else {
3900                                 if ((ap->txconfig & ANEG_CFG_NP) == 0 &&
3901                                     !(ap->flags & MR_NP_RX)) {
3902                                         ap->state = ANEG_STATE_IDLE_DETECT_INIT;
3903                                 } else {
3904                                         ret = ANEG_FAILED;
3905                                 }
3906                         }
3907                 }
3908                 break;
3909
3910         case ANEG_STATE_IDLE_DETECT_INIT:
3911                 ap->link_time = ap->cur_time;
3912                 tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
3913                 tw32_f(MAC_MODE, tp->mac_mode);
3914                 udelay(40);
3915
3916                 ap->state = ANEG_STATE_IDLE_DETECT;
3917                 ret = ANEG_TIMER_ENAB;
3918                 break;
3919
3920         case ANEG_STATE_IDLE_DETECT:
3921                 if (ap->ability_match != 0 &&
3922                     ap->rxconfig == 0) {
3923                         ap->state = ANEG_STATE_AN_ENABLE;
3924                         break;
3925                 }
3926                 delta = ap->cur_time - ap->link_time;
3927                 if (delta > ANEG_STATE_SETTLE_TIME) {
3928                         /* XXX another gem from the Broadcom driver :( */
3929                         ap->state = ANEG_STATE_LINK_OK;
3930                 }
3931                 break;
3932
3933         case ANEG_STATE_LINK_OK:
3934                 ap->flags |= (MR_AN_COMPLETE | MR_LINK_OK);
3935                 ret = ANEG_DONE;
3936                 break;
3937
3938         case ANEG_STATE_NEXT_PAGE_WAIT_INIT:
3939                 /* ??? unimplemented */
3940                 break;
3941
3942         case ANEG_STATE_NEXT_PAGE_WAIT:
3943                 /* ??? unimplemented */
3944                 break;
3945
3946         default:
3947                 ret = ANEG_FAILED;
3948                 break;
3949         }
3950
3951         return ret;
3952 }
3953
3954 static int fiber_autoneg(struct tg3 *tp, u32 *txflags, u32 *rxflags)
3955 {
3956         int res = 0;
3957         struct tg3_fiber_aneginfo aninfo;
3958         int status = ANEG_FAILED;
3959         unsigned int tick;
3960         u32 tmp;
3961
3962         tw32_f(MAC_TX_AUTO_NEG, 0);
3963
3964         tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
3965         tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
3966         udelay(40);
3967
3968         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
3969         udelay(40);
3970
3971         memset(&aninfo, 0, sizeof(aninfo));
3972         aninfo.flags |= MR_AN_ENABLE;
3973         aninfo.state = ANEG_STATE_UNKNOWN;
3974         aninfo.cur_time = 0;
3975         tick = 0;
3976         while (++tick < 195000) {
3977                 status = tg3_fiber_aneg_smachine(tp, &aninfo);
3978                 if (status == ANEG_DONE || status == ANEG_FAILED)
3979                         break;
3980
3981                 udelay(1);
3982         }
3983
3984         tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
3985         tw32_f(MAC_MODE, tp->mac_mode);
3986         udelay(40);
3987
3988         *txflags = aninfo.txconfig;
3989         *rxflags = aninfo.flags;
3990
3991         if (status == ANEG_DONE &&
3992             (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
3993                              MR_LP_ADV_FULL_DUPLEX)))
3994                 res = 1;
3995
3996         return res;
3997 }
3998
3999 static void tg3_init_bcm8002(struct tg3 *tp)
4000 {
4001         u32 mac_status = tr32(MAC_STATUS);
4002         int i;
4003
4004         /* Reset when initting first time or we have a link. */
4005         if (tg3_flag(tp, INIT_COMPLETE) &&
4006             !(mac_status & MAC_STATUS_PCS_SYNCED))
4007                 return;
4008
4009         /* Set PLL lock range. */
4010         tg3_writephy(tp, 0x16, 0x8007);
4011
4012         /* SW reset */
4013         tg3_writephy(tp, MII_BMCR, BMCR_RESET);
4014
4015         /* Wait for reset to complete. */
4016         /* XXX schedule_timeout() ... */
4017         for (i = 0; i < 500; i++)
4018                 udelay(10);
4019
4020         /* Config mode; select PMA/Ch 1 regs. */
4021         tg3_writephy(tp, 0x10, 0x8411);
4022
4023         /* Enable auto-lock and comdet, select txclk for tx. */
4024         tg3_writephy(tp, 0x11, 0x0a10);
4025
4026         tg3_writephy(tp, 0x18, 0x00a0);
4027         tg3_writephy(tp, 0x16, 0x41ff);
4028
4029         /* Assert and deassert POR. */
4030         tg3_writephy(tp, 0x13, 0x0400);
4031         udelay(40);
4032         tg3_writephy(tp, 0x13, 0x0000);
4033
4034         tg3_writephy(tp, 0x11, 0x0a50);
4035         udelay(40);
4036         tg3_writephy(tp, 0x11, 0x0a10);
4037
4038         /* Wait for signal to stabilize */
4039         /* XXX schedule_timeout() ... */
4040         for (i = 0; i < 15000; i++)
4041                 udelay(10);
4042
4043         /* Deselect the channel register so we can read the PHYID
4044          * later.
4045          */
4046         tg3_writephy(tp, 0x10, 0x8011);
4047 }
4048
4049 static int tg3_setup_fiber_hw_autoneg(struct tg3 *tp, u32 mac_status)
4050 {
4051         u16 flowctrl;
4052         u32 sg_dig_ctrl, sg_dig_status;
4053         u32 serdes_cfg, expected_sg_dig_ctrl;
4054         int workaround, port_a;
4055         int current_link_up;
4056
4057         serdes_cfg = 0;
4058         expected_sg_dig_ctrl = 0;
4059         workaround = 0;
4060         port_a = 1;
4061         current_link_up = 0;
4062
4063         if (tp->pci_chip_rev_id != CHIPREV_ID_5704_A0 &&
4064             tp->pci_chip_rev_id != CHIPREV_ID_5704_A1) {
4065                 workaround = 1;
4066                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
4067                         port_a = 0;
4068
4069                 /* preserve bits 0-11,13,14 for signal pre-emphasis */
4070                 /* preserve bits 20-23 for voltage regulator */
4071                 serdes_cfg = tr32(MAC_SERDES_CFG) & 0x00f06fff;
4072         }
4073
4074         sg_dig_ctrl = tr32(SG_DIG_CTRL);
4075
4076         if (tp->link_config.autoneg != AUTONEG_ENABLE) {
4077                 if (sg_dig_ctrl & SG_DIG_USING_HW_AUTONEG) {
4078                         if (workaround) {
4079                                 u32 val = serdes_cfg;
4080
4081                                 if (port_a)
4082                                         val |= 0xc010000;
4083                                 else
4084                                         val |= 0x4010000;
4085                                 tw32_f(MAC_SERDES_CFG, val);
4086                         }
4087
4088                         tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
4089                 }
4090                 if (mac_status & MAC_STATUS_PCS_SYNCED) {
4091                         tg3_setup_flow_control(tp, 0, 0);
4092                         current_link_up = 1;
4093                 }
4094                 goto out;
4095         }
4096
4097         /* Want auto-negotiation.  */
4098         expected_sg_dig_ctrl = SG_DIG_USING_HW_AUTONEG | SG_DIG_COMMON_SETUP;
4099
4100         flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
4101         if (flowctrl & ADVERTISE_1000XPAUSE)
4102                 expected_sg_dig_ctrl |= SG_DIG_PAUSE_CAP;
4103         if (flowctrl & ADVERTISE_1000XPSE_ASYM)
4104                 expected_sg_dig_ctrl |= SG_DIG_ASYM_PAUSE;
4105
4106         if (sg_dig_ctrl != expected_sg_dig_ctrl) {
4107                 if ((tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT) &&
4108                     tp->serdes_counter &&
4109                     ((mac_status & (MAC_STATUS_PCS_SYNCED |
4110                                     MAC_STATUS_RCVD_CFG)) ==
4111                      MAC_STATUS_PCS_SYNCED)) {
4112                         tp->serdes_counter--;
4113                         current_link_up = 1;
4114                         goto out;
4115                 }
4116 restart_autoneg:
4117                 if (workaround)
4118                         tw32_f(MAC_SERDES_CFG, serdes_cfg | 0xc011000);
4119                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl | SG_DIG_SOFT_RESET);
4120                 udelay(5);
4121                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl);
4122
4123                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
4124                 tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
4125         } else if (mac_status & (MAC_STATUS_PCS_SYNCED |
4126                                  MAC_STATUS_SIGNAL_DET)) {
4127                 sg_dig_status = tr32(SG_DIG_STATUS);
4128                 mac_status = tr32(MAC_STATUS);
4129
4130                 if ((sg_dig_status & SG_DIG_AUTONEG_COMPLETE) &&
4131                     (mac_status & MAC_STATUS_PCS_SYNCED)) {
4132                         u32 local_adv = 0, remote_adv = 0;
4133
4134                         if (sg_dig_ctrl & SG_DIG_PAUSE_CAP)
4135                                 local_adv |= ADVERTISE_1000XPAUSE;
4136                         if (sg_dig_ctrl & SG_DIG_ASYM_PAUSE)
4137                                 local_adv |= ADVERTISE_1000XPSE_ASYM;
4138
4139                         if (sg_dig_status & SG_DIG_PARTNER_PAUSE_CAPABLE)
4140                                 remote_adv |= LPA_1000XPAUSE;
4141                         if (sg_dig_status & SG_DIG_PARTNER_ASYM_PAUSE)
4142                                 remote_adv |= LPA_1000XPAUSE_ASYM;
4143
4144                         tg3_setup_flow_control(tp, local_adv, remote_adv);
4145                         current_link_up = 1;
4146                         tp->serdes_counter = 0;
4147                         tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
4148                 } else if (!(sg_dig_status & SG_DIG_AUTONEG_COMPLETE)) {
4149                         if (tp->serdes_counter)
4150                                 tp->serdes_counter--;
4151                         else {
4152                                 if (workaround) {
4153                                         u32 val = serdes_cfg;
4154
4155                                         if (port_a)
4156                                                 val |= 0xc010000;
4157                                         else
4158                                                 val |= 0x4010000;
4159
4160                                         tw32_f(MAC_SERDES_CFG, val);
4161                                 }
4162
4163                                 tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
4164                                 udelay(40);
4165
4166                                 /* Link parallel detection - link is up */
4167                                 /* only if we have PCS_SYNC and not */
4168                                 /* receiving config code words */
4169                                 mac_status = tr32(MAC_STATUS);
4170                                 if ((mac_status & MAC_STATUS_PCS_SYNCED) &&
4171                                     !(mac_status & MAC_STATUS_RCVD_CFG)) {
4172                                         tg3_setup_flow_control(tp, 0, 0);
4173                                         current_link_up = 1;
4174                                         tp->phy_flags |=
4175                                                 TG3_PHYFLG_PARALLEL_DETECT;
4176                                         tp->serdes_counter =
4177                                                 SERDES_PARALLEL_DET_TIMEOUT;
4178                                 } else
4179                                         goto restart_autoneg;
4180                         }
4181                 }
4182         } else {
4183                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
4184                 tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
4185         }
4186
4187 out:
4188         return current_link_up;
4189 }
4190
4191 static int tg3_setup_fiber_by_hand(struct tg3 *tp, u32 mac_status)
4192 {
4193         int current_link_up = 0;
4194
4195         if (!(mac_status & MAC_STATUS_PCS_SYNCED))
4196                 goto out;
4197
4198         if (tp->link_config.autoneg == AUTONEG_ENABLE) {
4199                 u32 txflags, rxflags;
4200                 int i;
4201
4202                 if (fiber_autoneg(tp, &txflags, &rxflags)) {
4203                         u32 local_adv = 0, remote_adv = 0;
4204
4205                         if (txflags & ANEG_CFG_PS1)
4206                                 local_adv |= ADVERTISE_1000XPAUSE;
4207                         if (txflags & ANEG_CFG_PS2)
4208                                 local_adv |= ADVERTISE_1000XPSE_ASYM;
4209
4210                         if (rxflags & MR_LP_ADV_SYM_PAUSE)
4211                                 remote_adv |= LPA_1000XPAUSE;
4212                         if (rxflags & MR_LP_ADV_ASYM_PAUSE)
4213                                 remote_adv |= LPA_1000XPAUSE_ASYM;
4214
4215                         tg3_setup_flow_control(tp, local_adv, remote_adv);
4216
4217                         current_link_up = 1;
4218                 }
4219                 for (i = 0; i < 30; i++) {
4220                         udelay(20);
4221                         tw32_f(MAC_STATUS,
4222                                (MAC_STATUS_SYNC_CHANGED |
4223                                 MAC_STATUS_CFG_CHANGED));
4224                         udelay(40);
4225                         if ((tr32(MAC_STATUS) &
4226                              (MAC_STATUS_SYNC_CHANGED |
4227                               MAC_STATUS_CFG_CHANGED)) == 0)
4228                                 break;
4229                 }
4230
4231                 mac_status = tr32(MAC_STATUS);
4232                 if (current_link_up == 0 &&
4233                     (mac_status & MAC_STATUS_PCS_SYNCED) &&
4234                     !(mac_status & MAC_STATUS_RCVD_CFG))
4235                         current_link_up = 1;
4236         } else {
4237                 tg3_setup_flow_control(tp, 0, 0);
4238
4239                 /* Forcing 1000FD link up. */
4240                 current_link_up = 1;
4241
4242                 tw32_f(MAC_MODE, (tp->mac_mode | MAC_MODE_SEND_CONFIGS));
4243                 udelay(40);
4244
4245                 tw32_f(MAC_MODE, tp->mac_mode);
4246                 udelay(40);
4247         }
4248
4249 out:
4250         return current_link_up;
4251 }
4252
4253 static int tg3_setup_fiber_phy(struct tg3 *tp, int force_reset)
4254 {
4255         u32 orig_pause_cfg;
4256         u16 orig_active_speed;
4257         u8 orig_active_duplex;
4258         u32 mac_status;
4259         int current_link_up;
4260         int i;
4261
4262         orig_pause_cfg = tp->link_config.active_flowctrl;
4263         orig_active_speed = tp->link_config.active_speed;
4264         orig_active_duplex = tp->link_config.active_duplex;
4265
4266         if (!tg3_flag(tp, HW_AUTONEG) &&
4267             netif_carrier_ok(tp->dev) &&
4268             tg3_flag(tp, INIT_COMPLETE)) {
4269                 mac_status = tr32(MAC_STATUS);
4270                 mac_status &= (MAC_STATUS_PCS_SYNCED |
4271                                MAC_STATUS_SIGNAL_DET |
4272                                MAC_STATUS_CFG_CHANGED |
4273                                MAC_STATUS_RCVD_CFG);
4274                 if (mac_status == (MAC_STATUS_PCS_SYNCED |
4275                                    MAC_STATUS_SIGNAL_DET)) {
4276                         tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
4277                                             MAC_STATUS_CFG_CHANGED));
4278                         return 0;
4279                 }
4280         }
4281
4282         tw32_f(MAC_TX_AUTO_NEG, 0);
4283
4284         tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
4285         tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
4286         tw32_f(MAC_MODE, tp->mac_mode);
4287         udelay(40);
4288
4289         if (tp->phy_id == TG3_PHY_ID_BCM8002)
4290                 tg3_init_bcm8002(tp);
4291
4292         /* Enable link change event even when serdes polling.  */
4293         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
4294         udelay(40);
4295
4296         current_link_up = 0;
4297         mac_status = tr32(MAC_STATUS);
4298
4299         if (tg3_flag(tp, HW_AUTONEG))
4300                 current_link_up = tg3_setup_fiber_hw_autoneg(tp, mac_status);
4301         else
4302                 current_link_up = tg3_setup_fiber_by_hand(tp, mac_status);
4303
4304         tp->napi[0].hw_status->status =
4305                 (SD_STATUS_UPDATED |
4306                  (tp->napi[0].hw_status->status & ~SD_STATUS_LINK_CHG));
4307
4308         for (i = 0; i < 100; i++) {
4309                 tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
4310                                     MAC_STATUS_CFG_CHANGED));
4311                 udelay(5);
4312                 if ((tr32(MAC_STATUS) & (MAC_STATUS_SYNC_CHANGED |
4313                                          MAC_STATUS_CFG_CHANGED |
4314                                          MAC_STATUS_LNKSTATE_CHANGED)) == 0)
4315                         break;
4316         }
4317
4318         mac_status = tr32(MAC_STATUS);
4319         if ((mac_status & MAC_STATUS_PCS_SYNCED) == 0) {
4320                 current_link_up = 0;
4321                 if (tp->link_config.autoneg == AUTONEG_ENABLE &&
4322                     tp->serdes_counter == 0) {
4323                         tw32_f(MAC_MODE, (tp->mac_mode |
4324                                           MAC_MODE_SEND_CONFIGS));
4325                         udelay(1);
4326                         tw32_f(MAC_MODE, tp->mac_mode);
4327                 }
4328         }
4329
4330         if (current_link_up == 1) {
4331                 tp->link_config.active_speed = SPEED_1000;
4332                 tp->link_config.active_duplex = DUPLEX_FULL;
4333                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
4334                                     LED_CTRL_LNKLED_OVERRIDE |
4335                                     LED_CTRL_1000MBPS_ON));
4336         } else {
4337                 tp->link_config.active_speed = SPEED_INVALID;
4338                 tp->link_config.active_duplex = DUPLEX_INVALID;
4339                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
4340                                     LED_CTRL_LNKLED_OVERRIDE |
4341                                     LED_CTRL_TRAFFIC_OVERRIDE));
4342         }
4343
4344         if (current_link_up != netif_carrier_ok(tp->dev)) {
4345                 if (current_link_up)
4346                         netif_carrier_on(tp->dev);
4347                 else
4348                         netif_carrier_off(tp->dev);
4349                 tg3_link_report(tp);
4350         } else {
4351                 u32 now_pause_cfg = tp->link_config.active_flowctrl;
4352                 if (orig_pause_cfg != now_pause_cfg ||
4353                     orig_active_speed != tp->link_config.active_speed ||
4354                     orig_active_duplex != tp->link_config.active_duplex)
4355                         tg3_link_report(tp);
4356         }
4357
4358         return 0;
4359 }
4360
4361 static int tg3_setup_fiber_mii_phy(struct tg3 *tp, int force_reset)
4362 {
4363         int current_link_up, err = 0;
4364         u32 bmsr, bmcr;
4365         u16 current_speed;
4366         u8 current_duplex;
4367         u32 local_adv, remote_adv;
4368
4369         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
4370         tw32_f(MAC_MODE, tp->mac_mode);
4371         udelay(40);
4372
4373         tw32(MAC_EVENT, 0);
4374
4375         tw32_f(MAC_STATUS,
4376              (MAC_STATUS_SYNC_CHANGED |
4377               MAC_STATUS_CFG_CHANGED |
4378               MAC_STATUS_MI_COMPLETION |
4379               MAC_STATUS_LNKSTATE_CHANGED));
4380         udelay(40);
4381
4382         if (force_reset)
4383                 tg3_phy_reset(tp);
4384
4385         current_link_up = 0;
4386         current_speed = SPEED_INVALID;
4387         current_duplex = DUPLEX_INVALID;
4388
4389         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4390         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4391         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
4392                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
4393                         bmsr |= BMSR_LSTATUS;
4394                 else
4395                         bmsr &= ~BMSR_LSTATUS;
4396         }
4397
4398         err |= tg3_readphy(tp, MII_BMCR, &bmcr);
4399
4400         if ((tp->link_config.autoneg == AUTONEG_ENABLE) && !force_reset &&
4401             (tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT)) {
4402                 /* do nothing, just check for link up at the end */
4403         } else if (tp->link_config.autoneg == AUTONEG_ENABLE) {
4404                 u32 adv, new_adv;
4405
4406                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
4407                 new_adv = adv & ~(ADVERTISE_1000XFULL | ADVERTISE_1000XHALF |
4408                                   ADVERTISE_1000XPAUSE |
4409                                   ADVERTISE_1000XPSE_ASYM |
4410                                   ADVERTISE_SLCT);
4411
4412                 new_adv |= tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
4413
4414                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
4415                         new_adv |= ADVERTISE_1000XHALF;
4416                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
4417                         new_adv |= ADVERTISE_1000XFULL;
4418
4419                 if ((new_adv != adv) || !(bmcr & BMCR_ANENABLE)) {
4420                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
4421                         bmcr |= BMCR_ANENABLE | BMCR_ANRESTART;
4422                         tg3_writephy(tp, MII_BMCR, bmcr);
4423
4424                         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
4425                         tp->serdes_counter = SERDES_AN_TIMEOUT_5714S;
4426                         tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
4427
4428                         return err;
4429                 }
4430         } else {
4431                 u32 new_bmcr;
4432
4433                 bmcr &= ~BMCR_SPEED1000;
4434                 new_bmcr = bmcr & ~(BMCR_ANENABLE | BMCR_FULLDPLX);
4435
4436                 if (tp->link_config.duplex == DUPLEX_FULL)
4437                         new_bmcr |= BMCR_FULLDPLX;
4438
4439                 if (new_bmcr != bmcr) {
4440                         /* BMCR_SPEED1000 is a reserved bit that needs
4441                          * to be set on write.
4442                          */
4443                         new_bmcr |= BMCR_SPEED1000;
4444
4445                         /* Force a linkdown */
4446                         if (netif_carrier_ok(tp->dev)) {
4447                                 u32 adv;
4448
4449                                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
4450                                 adv &= ~(ADVERTISE_1000XFULL |
4451                                          ADVERTISE_1000XHALF |
4452                                          ADVERTISE_SLCT);
4453                                 tg3_writephy(tp, MII_ADVERTISE, adv);
4454                                 tg3_writephy(tp, MII_BMCR, bmcr |
4455                                                            BMCR_ANRESTART |
4456                                                            BMCR_ANENABLE);
4457                                 udelay(10);
4458                                 netif_carrier_off(tp->dev);
4459                         }
4460                         tg3_writephy(tp, MII_BMCR, new_bmcr);
4461                         bmcr = new_bmcr;
4462                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4463                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4464                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
4465                             ASIC_REV_5714) {
4466                                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
4467                                         bmsr |= BMSR_LSTATUS;
4468                                 else
4469                                         bmsr &= ~BMSR_LSTATUS;
4470                         }
4471                         tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
4472                 }
4473         }
4474
4475         if (bmsr & BMSR_LSTATUS) {
4476                 current_speed = SPEED_1000;
4477                 current_link_up = 1;
4478                 if (bmcr & BMCR_FULLDPLX)
4479                         current_duplex = DUPLEX_FULL;
4480                 else
4481                         current_duplex = DUPLEX_HALF;
4482
4483                 local_adv = 0;
4484                 remote_adv = 0;
4485
4486                 if (bmcr & BMCR_ANENABLE) {
4487                         u32 common;
4488
4489                         err |= tg3_readphy(tp, MII_ADVERTISE, &local_adv);
4490                         err |= tg3_readphy(tp, MII_LPA, &remote_adv);
4491                         common = local_adv & remote_adv;
4492                         if (common & (ADVERTISE_1000XHALF |
4493                                       ADVERTISE_1000XFULL)) {
4494                                 if (common & ADVERTISE_1000XFULL)
4495                                         current_duplex = DUPLEX_FULL;
4496                                 else
4497                                         current_duplex = DUPLEX_HALF;
4498                         } else if (!tg3_flag(tp, 5780_CLASS)) {
4499                                 /* Link is up via parallel detect */
4500                         } else {
4501                                 current_link_up = 0;
4502                         }
4503                 }
4504         }
4505
4506         if (current_link_up == 1 && current_duplex == DUPLEX_FULL)
4507                 tg3_setup_flow_control(tp, local_adv, remote_adv);
4508
4509         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
4510         if (tp->link_config.active_duplex == DUPLEX_HALF)
4511                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
4512
4513         tw32_f(MAC_MODE, tp->mac_mode);
4514         udelay(40);
4515
4516         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
4517
4518         tp->link_config.active_speed = current_speed;
4519         tp->link_config.active_duplex = current_duplex;
4520
4521         if (current_link_up != netif_carrier_ok(tp->dev)) {
4522                 if (current_link_up)
4523                         netif_carrier_on(tp->dev);
4524                 else {
4525                         netif_carrier_off(tp->dev);
4526                         tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
4527                 }
4528                 tg3_link_report(tp);
4529         }
4530         return err;
4531 }
4532
4533 static void tg3_serdes_parallel_detect(struct tg3 *tp)
4534 {
4535         if (tp->serdes_counter) {
4536                 /* Give autoneg time to complete. */
4537                 tp->serdes_counter--;
4538                 return;
4539         }
4540
4541         if (!netif_carrier_ok(tp->dev) &&
4542             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
4543                 u32 bmcr;
4544
4545                 tg3_readphy(tp, MII_BMCR, &bmcr);
4546                 if (bmcr & BMCR_ANENABLE) {
4547                         u32 phy1, phy2;
4548
4549                         /* Select shadow register 0x1f */
4550                         tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x7c00);
4551                         tg3_readphy(tp, MII_TG3_MISC_SHDW, &phy1);
4552
4553                         /* Select expansion interrupt status register */
4554                         tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
4555                                          MII_TG3_DSP_EXP1_INT_STAT);
4556                         tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &phy2);
4557                         tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &phy2);
4558
4559                         if ((phy1 & 0x10) && !(phy2 & 0x20)) {
4560                                 /* We have signal detect and not receiving
4561                                  * config code words, link is up by parallel
4562                                  * detection.
4563                                  */
4564
4565                                 bmcr &= ~BMCR_ANENABLE;
4566                                 bmcr |= BMCR_SPEED1000 | BMCR_FULLDPLX;
4567                                 tg3_writephy(tp, MII_BMCR, bmcr);
4568                                 tp->phy_flags |= TG3_PHYFLG_PARALLEL_DETECT;
4569                         }
4570                 }
4571         } else if (netif_carrier_ok(tp->dev) &&
4572                    (tp->link_config.autoneg == AUTONEG_ENABLE) &&
4573                    (tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT)) {
4574                 u32 phy2;
4575
4576                 /* Select expansion interrupt status register */
4577                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
4578                                  MII_TG3_DSP_EXP1_INT_STAT);
4579                 tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &phy2);
4580                 if (phy2 & 0x20) {
4581                         u32 bmcr;
4582
4583                         /* Config code words received, turn on autoneg. */
4584                         tg3_readphy(tp, MII_BMCR, &bmcr);
4585                         tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANENABLE);
4586
4587                         tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
4588
4589                 }
4590         }
4591 }
4592
4593 static int tg3_setup_phy(struct tg3 *tp, int force_reset)
4594 {
4595         u32 val;
4596         int err;
4597
4598         if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
4599                 err = tg3_setup_fiber_phy(tp, force_reset);
4600         else if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
4601                 err = tg3_setup_fiber_mii_phy(tp, force_reset);
4602         else
4603                 err = tg3_setup_copper_phy(tp, force_reset);
4604
4605         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX) {
4606                 u32 scale;
4607
4608                 val = tr32(TG3_CPMU_CLCK_STAT) & CPMU_CLCK_STAT_MAC_CLCK_MASK;
4609                 if (val == CPMU_CLCK_STAT_MAC_CLCK_62_5)
4610                         scale = 65;
4611                 else if (val == CPMU_CLCK_STAT_MAC_CLCK_6_25)
4612                         scale = 6;
4613                 else
4614                         scale = 12;
4615
4616                 val = tr32(GRC_MISC_CFG) & ~GRC_MISC_CFG_PRESCALAR_MASK;
4617                 val |= (scale << GRC_MISC_CFG_PRESCALAR_SHIFT);
4618                 tw32(GRC_MISC_CFG, val);
4619         }
4620
4621         val = (2 << TX_LENGTHS_IPG_CRS_SHIFT) |
4622               (6 << TX_LENGTHS_IPG_SHIFT);
4623         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720)
4624                 val |= tr32(MAC_TX_LENGTHS) &
4625                        (TX_LENGTHS_JMB_FRM_LEN_MSK |
4626                         TX_LENGTHS_CNT_DWN_VAL_MSK);
4627
4628         if (tp->link_config.active_speed == SPEED_1000 &&
4629             tp->link_config.active_duplex == DUPLEX_HALF)
4630                 tw32(MAC_TX_LENGTHS, val |
4631                      (0xff << TX_LENGTHS_SLOT_TIME_SHIFT));
4632         else
4633                 tw32(MAC_TX_LENGTHS, val |
4634                      (32 << TX_LENGTHS_SLOT_TIME_SHIFT));
4635
4636         if (!tg3_flag(tp, 5705_PLUS)) {
4637                 if (netif_carrier_ok(tp->dev)) {
4638                         tw32(HOSTCC_STAT_COAL_TICKS,
4639                              tp->coal.stats_block_coalesce_usecs);
4640                 } else {
4641                         tw32(HOSTCC_STAT_COAL_TICKS, 0);
4642                 }
4643         }
4644
4645         if (tg3_flag(tp, ASPM_WORKAROUND)) {
4646                 val = tr32(PCIE_PWR_MGMT_THRESH);
4647                 if (!netif_carrier_ok(tp->dev))
4648                         val = (val & ~PCIE_PWR_MGMT_L1_THRESH_MSK) |
4649                               tp->pwrmgmt_thresh;
4650                 else
4651                         val |= PCIE_PWR_MGMT_L1_THRESH_MSK;
4652                 tw32(PCIE_PWR_MGMT_THRESH, val);
4653         }
4654
4655         return err;
4656 }
4657
4658 static inline int tg3_irq_sync(struct tg3 *tp)
4659 {
4660         return tp->irq_sync;
4661 }
4662
4663 static inline void tg3_rd32_loop(struct tg3 *tp, u32 *dst, u32 off, u32 len)
4664 {
4665         int i;
4666
4667         dst = (u32 *)((u8 *)dst + off);
4668         for (i = 0; i < len; i += sizeof(u32))
4669                 *dst++ = tr32(off + i);
4670 }
4671
4672 static void tg3_dump_legacy_regs(struct tg3 *tp, u32 *regs)
4673 {
4674         tg3_rd32_loop(tp, regs, TG3PCI_VENDOR, 0xb0);
4675         tg3_rd32_loop(tp, regs, MAILBOX_INTERRUPT_0, 0x200);
4676         tg3_rd32_loop(tp, regs, MAC_MODE, 0x4f0);
4677         tg3_rd32_loop(tp, regs, SNDDATAI_MODE, 0xe0);
4678         tg3_rd32_loop(tp, regs, SNDDATAC_MODE, 0x04);
4679         tg3_rd32_loop(tp, regs, SNDBDS_MODE, 0x80);
4680         tg3_rd32_loop(tp, regs, SNDBDI_MODE, 0x48);
4681         tg3_rd32_loop(tp, regs, SNDBDC_MODE, 0x04);
4682         tg3_rd32_loop(tp, regs, RCVLPC_MODE, 0x20);
4683         tg3_rd32_loop(tp, regs, RCVLPC_SELLST_BASE, 0x15c);
4684         tg3_rd32_loop(tp, regs, RCVDBDI_MODE, 0x0c);
4685         tg3_rd32_loop(tp, regs, RCVDBDI_JUMBO_BD, 0x3c);
4686         tg3_rd32_loop(tp, regs, RCVDBDI_BD_PROD_IDX_0, 0x44);
4687         tg3_rd32_loop(tp, regs, RCVDCC_MODE, 0x04);
4688         tg3_rd32_loop(tp, regs, RCVBDI_MODE, 0x20);
4689         tg3_rd32_loop(tp, regs, RCVCC_MODE, 0x14);
4690         tg3_rd32_loop(tp, regs, RCVLSC_MODE, 0x08);
4691         tg3_rd32_loop(tp, regs, MBFREE_MODE, 0x08);
4692         tg3_rd32_loop(tp, regs, HOSTCC_MODE, 0x100);
4693
4694         if (tg3_flag(tp, SUPPORT_MSIX))
4695                 tg3_rd32_loop(tp, regs, HOSTCC_RXCOL_TICKS_VEC1, 0x180);
4696
4697         tg3_rd32_loop(tp, regs, MEMARB_MODE, 0x10);
4698         tg3_rd32_loop(tp, regs, BUFMGR_MODE, 0x58);
4699         tg3_rd32_loop(tp, regs, RDMAC_MODE, 0x08);
4700         tg3_rd32_loop(tp, regs, WDMAC_MODE, 0x08);
4701         tg3_rd32_loop(tp, regs, RX_CPU_MODE, 0x04);
4702         tg3_rd32_loop(tp, regs, RX_CPU_STATE, 0x04);
4703         tg3_rd32_loop(tp, regs, RX_CPU_PGMCTR, 0x04);
4704         tg3_rd32_loop(tp, regs, RX_CPU_HWBKPT, 0x04);
4705
4706         if (!tg3_flag(tp, 5705_PLUS)) {
4707                 tg3_rd32_loop(tp, regs, TX_CPU_MODE, 0x04);
4708                 tg3_rd32_loop(tp, regs, TX_CPU_STATE, 0x04);
4709                 tg3_rd32_loop(tp, regs, TX_CPU_PGMCTR, 0x04);
4710         }
4711
4712         tg3_rd32_loop(tp, regs, GRCMBOX_INTERRUPT_0, 0x110);
4713         tg3_rd32_loop(tp, regs, FTQ_RESET, 0x120);
4714         tg3_rd32_loop(tp, regs, MSGINT_MODE, 0x0c);
4715         tg3_rd32_loop(tp, regs, DMAC_MODE, 0x04);
4716         tg3_rd32_loop(tp, regs, GRC_MODE, 0x4c);
4717
4718         if (tg3_flag(tp, NVRAM))
4719                 tg3_rd32_loop(tp, regs, NVRAM_CMD, 0x24);
4720 }
4721
4722 static void tg3_dump_state(struct tg3 *tp)
4723 {
4724         int i;
4725         u32 *regs;
4726
4727         regs = kzalloc(TG3_REG_BLK_SIZE, GFP_ATOMIC);
4728         if (!regs) {
4729                 netdev_err(tp->dev, "Failed allocating register dump buffer\n");
4730                 return;
4731         }
4732
4733         if (tg3_flag(tp, PCI_EXPRESS)) {
4734                 /* Read up to but not including private PCI registers */
4735                 for (i = 0; i < TG3_PCIE_TLDLPL_PORT; i += sizeof(u32))
4736                         regs[i / sizeof(u32)] = tr32(i);
4737         } else
4738                 tg3_dump_legacy_regs(tp, regs);
4739
4740         for (i = 0; i < TG3_REG_BLK_SIZE / sizeof(u32); i += 4) {
4741                 if (!regs[i + 0] && !regs[i + 1] &&
4742                     !regs[i + 2] && !regs[i + 3])
4743                         continue;
4744
4745                 netdev_err(tp->dev, "0x%08x: 0x%08x, 0x%08x, 0x%08x, 0x%08x\n",
4746                            i * 4,
4747                            regs[i + 0], regs[i + 1], regs[i + 2], regs[i + 3]);
4748         }
4749
4750         kfree(regs);
4751
4752         for (i = 0; i < tp->irq_cnt; i++) {
4753                 struct tg3_napi *tnapi = &tp->napi[i];
4754
4755                 /* SW status block */
4756                 netdev_err(tp->dev,
4757                          "%d: Host status block [%08x:%08x:(%04x:%04x:%04x):(%04x:%04x)]\n",
4758                            i,
4759                            tnapi->hw_status->status,
4760                            tnapi->hw_status->status_tag,
4761                            tnapi->hw_status->rx_jumbo_consumer,
4762                            tnapi->hw_status->rx_consumer,
4763                            tnapi->hw_status->rx_mini_consumer,
4764                            tnapi->hw_status->idx[0].rx_producer,
4765                            tnapi->hw_status->idx[0].tx_consumer);
4766
4767                 netdev_err(tp->dev,
4768                 "%d: NAPI info [%08x:%08x:(%04x:%04x:%04x):%04x:(%04x:%04x:%04x:%04x)]\n",
4769                            i,
4770                            tnapi->last_tag, tnapi->last_irq_tag,
4771                            tnapi->tx_prod, tnapi->tx_cons, tnapi->tx_pending,
4772                            tnapi->rx_rcb_ptr,
4773                            tnapi->prodring.rx_std_prod_idx,
4774                            tnapi->prodring.rx_std_cons_idx,
4775                            tnapi->prodring.rx_jmb_prod_idx,
4776                            tnapi->prodring.rx_jmb_cons_idx);
4777         }
4778 }
4779
4780 /* This is called whenever we suspect that the system chipset is re-
4781  * ordering the sequence of MMIO to the tx send mailbox. The symptom
4782  * is bogus tx completions. We try to recover by setting the
4783  * TG3_FLAG_MBOX_WRITE_REORDER flag and resetting the chip later
4784  * in the workqueue.
4785  */
4786 static void tg3_tx_recover(struct tg3 *tp)
4787 {
4788         BUG_ON(tg3_flag(tp, MBOX_WRITE_REORDER) ||
4789                tp->write32_tx_mbox == tg3_write_indirect_mbox);
4790
4791         netdev_warn(tp->dev,
4792                     "The system may be re-ordering memory-mapped I/O "
4793                     "cycles to the network device, attempting to recover. "
4794                     "Please report the problem to the driver maintainer "
4795                     "and include system chipset information.\n");
4796
4797         spin_lock(&tp->lock);
4798         tg3_flag_set(tp, TX_RECOVERY_PENDING);
4799         spin_unlock(&tp->lock);
4800 }
4801
4802 static inline u32 tg3_tx_avail(struct tg3_napi *tnapi)
4803 {
4804         /* Tell compiler to fetch tx indices from memory. */
4805         barrier();
4806         return tnapi->tx_pending -
4807                ((tnapi->tx_prod - tnapi->tx_cons) & (TG3_TX_RING_SIZE - 1));
4808 }
4809
4810 /* Tigon3 never reports partial packet sends.  So we do not
4811  * need special logic to handle SKBs that have not had all
4812  * of their frags sent yet, like SunGEM does.
4813  */
4814 static void tg3_tx(struct tg3_napi *tnapi)
4815 {
4816         struct tg3 *tp = tnapi->tp;
4817         u32 hw_idx = tnapi->hw_status->idx[0].tx_consumer;
4818         u32 sw_idx = tnapi->tx_cons;
4819         struct netdev_queue *txq;
4820         int index = tnapi - tp->napi;
4821
4822         if (tg3_flag(tp, ENABLE_TSS))
4823                 index--;
4824
4825         txq = netdev_get_tx_queue(tp->dev, index);
4826
4827         while (sw_idx != hw_idx) {
4828                 struct tg3_tx_ring_info *ri = &tnapi->tx_buffers[sw_idx];
4829                 struct sk_buff *skb = ri->skb;
4830                 int i, tx_bug = 0;
4831
4832                 if (unlikely(skb == NULL)) {
4833                         tg3_tx_recover(tp);
4834                         return;
4835                 }
4836
4837                 pci_unmap_single(tp->pdev,
4838                                  dma_unmap_addr(ri, mapping),
4839                                  skb_headlen(skb),
4840                                  PCI_DMA_TODEVICE);
4841
4842                 ri->skb = NULL;
4843
4844                 while (ri->fragmented) {
4845                         ri->fragmented = false;
4846                         sw_idx = NEXT_TX(sw_idx);
4847                         ri = &tnapi->tx_buffers[sw_idx];
4848                 }
4849
4850                 sw_idx = NEXT_TX(sw_idx);
4851
4852                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4853                         ri = &tnapi->tx_buffers[sw_idx];
4854                         if (unlikely(ri->skb != NULL || sw_idx == hw_idx))
4855                                 tx_bug = 1;
4856
4857                         pci_unmap_page(tp->pdev,
4858                                        dma_unmap_addr(ri, mapping),
4859                                        skb_shinfo(skb)->frags[i].size,
4860                                        PCI_DMA_TODEVICE);
4861
4862                         while (ri->fragmented) {
4863                                 ri->fragmented = false;
4864                                 sw_idx = NEXT_TX(sw_idx);
4865                                 ri = &tnapi->tx_buffers[sw_idx];
4866                         }
4867
4868                         sw_idx = NEXT_TX(sw_idx);
4869                 }
4870
4871                 dev_kfree_skb(skb);
4872
4873                 if (unlikely(tx_bug)) {
4874                         tg3_tx_recover(tp);
4875                         return;
4876                 }
4877         }
4878
4879         tnapi->tx_cons = sw_idx;
4880
4881         /* Need to make the tx_cons update visible to tg3_start_xmit()
4882          * before checking for netif_queue_stopped().  Without the
4883          * memory barrier, there is a small possibility that tg3_start_xmit()
4884          * will miss it and cause the queue to be stopped forever.
4885          */
4886         smp_mb();
4887
4888         if (unlikely(netif_tx_queue_stopped(txq) &&
4889                      (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi)))) {
4890                 __netif_tx_lock(txq, smp_processor_id());
4891                 if (netif_tx_queue_stopped(txq) &&
4892                     (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi)))
4893                         netif_tx_wake_queue(txq);
4894                 __netif_tx_unlock(txq);
4895         }
4896 }
4897
4898 static void tg3_rx_skb_free(struct tg3 *tp, struct ring_info *ri, u32 map_sz)
4899 {
4900         if (!ri->skb)
4901                 return;
4902
4903         pci_unmap_single(tp->pdev, dma_unmap_addr(ri, mapping),
4904                          map_sz, PCI_DMA_FROMDEVICE);
4905         dev_kfree_skb_any(ri->skb);
4906         ri->skb = NULL;
4907 }
4908
4909 /* Returns size of skb allocated or < 0 on error.
4910  *
4911  * We only need to fill in the address because the other members
4912  * of the RX descriptor are invariant, see tg3_init_rings.
4913  *
4914  * Note the purposeful assymetry of cpu vs. chip accesses.  For
4915  * posting buffers we only dirty the first cache line of the RX
4916  * descriptor (containing the address).  Whereas for the RX status
4917  * buffers the cpu only reads the last cacheline of the RX descriptor
4918  * (to fetch the error flags, vlan tag, checksum, and opaque cookie).
4919  */
4920 static int tg3_alloc_rx_skb(struct tg3 *tp, struct tg3_rx_prodring_set *tpr,
4921                             u32 opaque_key, u32 dest_idx_unmasked)
4922 {
4923         struct tg3_rx_buffer_desc *desc;
4924         struct ring_info *map;
4925         struct sk_buff *skb;
4926         dma_addr_t mapping;
4927         int skb_size, dest_idx;
4928
4929         switch (opaque_key) {
4930         case RXD_OPAQUE_RING_STD:
4931                 dest_idx = dest_idx_unmasked & tp->rx_std_ring_mask;
4932                 desc = &tpr->rx_std[dest_idx];
4933                 map = &tpr->rx_std_buffers[dest_idx];
4934                 skb_size = tp->rx_pkt_map_sz;
4935                 break;
4936
4937         case RXD_OPAQUE_RING_JUMBO:
4938                 dest_idx = dest_idx_unmasked & tp->rx_jmb_ring_mask;
4939                 desc = &tpr->rx_jmb[dest_idx].std;
4940                 map = &tpr->rx_jmb_buffers[dest_idx];
4941                 skb_size = TG3_RX_JMB_MAP_SZ;
4942                 break;
4943
4944         default:
4945                 return -EINVAL;
4946         }
4947
4948         /* Do not overwrite any of the map or rp information
4949          * until we are sure we can commit to a new buffer.
4950          *
4951          * Callers depend upon this behavior and assume that
4952          * we leave everything unchanged if we fail.
4953          */
4954         skb = netdev_alloc_skb(tp->dev, skb_size + tp->rx_offset);
4955         if (skb == NULL)
4956                 return -ENOMEM;
4957
4958         skb_reserve(skb, tp->rx_offset);
4959
4960         mapping = pci_map_single(tp->pdev, skb->data, skb_size,
4961                                  PCI_DMA_FROMDEVICE);
4962         if (pci_dma_mapping_error(tp->pdev, mapping)) {
4963                 dev_kfree_skb(skb);
4964                 return -EIO;
4965         }
4966
4967         map->skb = skb;
4968         dma_unmap_addr_set(map, mapping, mapping);
4969
4970         desc->addr_hi = ((u64)mapping >> 32);
4971         desc->addr_lo = ((u64)mapping & 0xffffffff);
4972
4973         return skb_size;
4974 }
4975
4976 /* We only need to move over in the address because the other
4977  * members of the RX descriptor are invariant.  See notes above
4978  * tg3_alloc_rx_skb for full details.
4979  */
4980 static void tg3_recycle_rx(struct tg3_napi *tnapi,
4981                            struct tg3_rx_prodring_set *dpr,
4982                            u32 opaque_key, int src_idx,
4983                            u32 dest_idx_unmasked)
4984 {
4985         struct tg3 *tp = tnapi->tp;
4986         struct tg3_rx_buffer_desc *src_desc, *dest_desc;
4987         struct ring_info *src_map, *dest_map;
4988         struct tg3_rx_prodring_set *spr = &tp->napi[0].prodring;
4989         int dest_idx;
4990
4991         switch (opaque_key) {
4992         case RXD_OPAQUE_RING_STD:
4993                 dest_idx = dest_idx_unmasked & tp->rx_std_ring_mask;
4994                 dest_desc = &dpr->rx_std[dest_idx];
4995                 dest_map = &dpr->rx_std_buffers[dest_idx];
4996                 src_desc = &spr->rx_std[src_idx];
4997                 src_map = &spr->rx_std_buffers[src_idx];
4998                 break;
4999
5000         case RXD_OPAQUE_RING_JUMBO:
5001                 dest_idx = dest_idx_unmasked & tp->rx_jmb_ring_mask;
5002                 dest_desc = &dpr->rx_jmb[dest_idx].std;
5003                 dest_map = &dpr->rx_jmb_buffers[dest_idx];
5004                 src_desc = &spr->rx_jmb[src_idx].std;
5005                 src_map = &spr->rx_jmb_buffers[src_idx];
5006                 break;
5007
5008         default:
5009                 return;
5010         }
5011
5012         dest_map->skb = src_map->skb;
5013         dma_unmap_addr_set(dest_map, mapping,
5014                            dma_unmap_addr(src_map, mapping));
5015         dest_desc->addr_hi = src_desc->addr_hi;
5016         dest_desc->addr_lo = src_desc->addr_lo;
5017
5018         /* Ensure that the update to the skb happens after the physical
5019          * addresses have been transferred to the new BD location.
5020          */
5021         smp_wmb();
5022
5023         src_map->skb = NULL;
5024 }
5025
5026 /* The RX ring scheme is composed of multiple rings which post fresh
5027  * buffers to the chip, and one special ring the chip uses to report
5028  * status back to the host.
5029  *
5030  * The special ring reports the status of received packets to the
5031  * host.  The chip does not write into the original descriptor the
5032  * RX buffer was obtained from.  The chip simply takes the original
5033  * descriptor as provided by the host, updates the status and length
5034  * field, then writes this into the next status ring entry.
5035  *
5036  * Each ring the host uses to post buffers to the chip is described
5037  * by a TG3_BDINFO entry in the chips SRAM area.  When a packet arrives,
5038  * it is first placed into the on-chip ram.  When the packet's length
5039  * is known, it walks down the TG3_BDINFO entries to select the ring.
5040  * Each TG3_BDINFO specifies a MAXLEN field and the first TG3_BDINFO
5041  * which is within the range of the new packet's length is chosen.
5042  *
5043  * The "separate ring for rx status" scheme may sound queer, but it makes
5044  * sense from a cache coherency perspective.  If only the host writes
5045  * to the buffer post rings, and only the chip writes to the rx status
5046  * rings, then cache lines never move beyond shared-modified state.
5047  * If both the host and chip were to write into the same ring, cache line
5048  * eviction could occur since both entities want it in an exclusive state.
5049  */
5050 static int tg3_rx(struct tg3_napi *tnapi, int budget)
5051 {
5052         struct tg3 *tp = tnapi->tp;
5053         u32 work_mask, rx_std_posted = 0;
5054         u32 std_prod_idx, jmb_prod_idx;
5055         u32 sw_idx = tnapi->rx_rcb_ptr;
5056         u16 hw_idx;
5057         int received;
5058         struct tg3_rx_prodring_set *tpr = &tnapi->prodring;
5059
5060         hw_idx = *(tnapi->rx_rcb_prod_idx);
5061         /*
5062          * We need to order the read of hw_idx and the read of
5063          * the opaque cookie.
5064          */
5065         rmb();
5066         work_mask = 0;
5067         received = 0;
5068         std_prod_idx = tpr->rx_std_prod_idx;
5069         jmb_prod_idx = tpr->rx_jmb_prod_idx;
5070         while (sw_idx != hw_idx && budget > 0) {
5071                 struct ring_info *ri;
5072                 struct tg3_rx_buffer_desc *desc = &tnapi->rx_rcb[sw_idx];
5073                 unsigned int len;
5074                 struct sk_buff *skb;
5075                 dma_addr_t dma_addr;
5076                 u32 opaque_key, desc_idx, *post_ptr;
5077
5078                 desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
5079                 opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
5080                 if (opaque_key == RXD_OPAQUE_RING_STD) {
5081                         ri = &tp->napi[0].prodring.rx_std_buffers[desc_idx];
5082                         dma_addr = dma_unmap_addr(ri, mapping);
5083                         skb = ri->skb;
5084                         post_ptr = &std_prod_idx;
5085                         rx_std_posted++;
5086                 } else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
5087                         ri = &tp->napi[0].prodring.rx_jmb_buffers[desc_idx];
5088                         dma_addr = dma_unmap_addr(ri, mapping);
5089                         skb = ri->skb;
5090                         post_ptr = &jmb_prod_idx;
5091                 } else
5092                         goto next_pkt_nopost;
5093
5094                 work_mask |= opaque_key;
5095
5096                 if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
5097                     (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII)) {
5098                 drop_it:
5099                         tg3_recycle_rx(tnapi, tpr, opaque_key,
5100                                        desc_idx, *post_ptr);
5101                 drop_it_no_recycle:
5102                         /* Other statistics kept track of by card. */
5103                         tp->rx_dropped++;
5104                         goto next_pkt;
5105                 }
5106
5107                 len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) -
5108                       ETH_FCS_LEN;
5109
5110                 if (len > TG3_RX_COPY_THRESH(tp)) {
5111                         int skb_size;
5112
5113                         skb_size = tg3_alloc_rx_skb(tp, tpr, opaque_key,
5114                                                     *post_ptr);
5115                         if (skb_size < 0)
5116                                 goto drop_it;
5117
5118                         pci_unmap_single(tp->pdev, dma_addr, skb_size,
5119                                          PCI_DMA_FROMDEVICE);
5120
5121                         /* Ensure that the update to the skb happens
5122                          * after the usage of the old DMA mapping.
5123                          */
5124                         smp_wmb();
5125
5126                         ri->skb = NULL;
5127
5128                         skb_put(skb, len);
5129                 } else {
5130                         struct sk_buff *copy_skb;
5131
5132                         tg3_recycle_rx(tnapi, tpr, opaque_key,
5133                                        desc_idx, *post_ptr);
5134
5135                         copy_skb = netdev_alloc_skb(tp->dev, len +
5136                                                     TG3_RAW_IP_ALIGN);
5137                         if (copy_skb == NULL)
5138                                 goto drop_it_no_recycle;
5139
5140                         skb_reserve(copy_skb, TG3_RAW_IP_ALIGN);
5141                         skb_put(copy_skb, len);
5142                         pci_dma_sync_single_for_cpu(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
5143                         skb_copy_from_linear_data(skb, copy_skb->data, len);
5144                         pci_dma_sync_single_for_device(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
5145
5146                         /* We'll reuse the original ring buffer. */
5147                         skb = copy_skb;
5148                 }
5149
5150                 if ((tp->dev->features & NETIF_F_RXCSUM) &&
5151                     (desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
5152                     (((desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
5153                       >> RXD_TCPCSUM_SHIFT) == 0xffff))
5154                         skb->ip_summed = CHECKSUM_UNNECESSARY;
5155                 else
5156                         skb_checksum_none_assert(skb);
5157
5158                 skb->protocol = eth_type_trans(skb, tp->dev);
5159
5160                 if (len > (tp->dev->mtu + ETH_HLEN) &&
5161                     skb->protocol != htons(ETH_P_8021Q)) {
5162                         dev_kfree_skb(skb);
5163                         goto drop_it_no_recycle;
5164                 }
5165
5166                 if (desc->type_flags & RXD_FLAG_VLAN &&
5167                     !(tp->rx_mode & RX_MODE_KEEP_VLAN_TAG))
5168                         __vlan_hwaccel_put_tag(skb,
5169                                                desc->err_vlan & RXD_VLAN_MASK);
5170
5171                 napi_gro_receive(&tnapi->napi, skb);
5172
5173                 received++;
5174                 budget--;
5175
5176 next_pkt:
5177                 (*post_ptr)++;
5178
5179                 if (unlikely(rx_std_posted >= tp->rx_std_max_post)) {
5180                         tpr->rx_std_prod_idx = std_prod_idx &
5181                                                tp->rx_std_ring_mask;
5182                         tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG,
5183                                      tpr->rx_std_prod_idx);
5184                         work_mask &= ~RXD_OPAQUE_RING_STD;
5185                         rx_std_posted = 0;
5186                 }
5187 next_pkt_nopost:
5188                 sw_idx++;
5189                 sw_idx &= tp->rx_ret_ring_mask;
5190
5191                 /* Refresh hw_idx to see if there is new work */
5192                 if (sw_idx == hw_idx) {
5193                         hw_idx = *(tnapi->rx_rcb_prod_idx);
5194                         rmb();
5195                 }
5196         }
5197
5198         /* ACK the status ring. */
5199         tnapi->rx_rcb_ptr = sw_idx;
5200         tw32_rx_mbox(tnapi->consmbox, sw_idx);
5201
5202         /* Refill RX ring(s). */
5203         if (!tg3_flag(tp, ENABLE_RSS)) {
5204                 if (work_mask & RXD_OPAQUE_RING_STD) {
5205                         tpr->rx_std_prod_idx = std_prod_idx &
5206                                                tp->rx_std_ring_mask;
5207                         tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG,
5208                                      tpr->rx_std_prod_idx);
5209                 }
5210                 if (work_mask & RXD_OPAQUE_RING_JUMBO) {
5211                         tpr->rx_jmb_prod_idx = jmb_prod_idx &
5212                                                tp->rx_jmb_ring_mask;
5213                         tw32_rx_mbox(TG3_RX_JMB_PROD_IDX_REG,
5214                                      tpr->rx_jmb_prod_idx);
5215                 }
5216                 mmiowb();
5217         } else if (work_mask) {
5218                 /* rx_std_buffers[] and rx_jmb_buffers[] entries must be
5219                  * updated before the producer indices can be updated.
5220                  */
5221                 smp_wmb();
5222
5223                 tpr->rx_std_prod_idx = std_prod_idx & tp->rx_std_ring_mask;
5224                 tpr->rx_jmb_prod_idx = jmb_prod_idx & tp->rx_jmb_ring_mask;
5225
5226                 if (tnapi != &tp->napi[1])
5227                         napi_schedule(&tp->napi[1].napi);
5228         }
5229
5230         return received;
5231 }
5232
5233 static void tg3_poll_link(struct tg3 *tp)
5234 {
5235         /* handle link change and other phy events */
5236         if (!(tg3_flag(tp, USE_LINKCHG_REG) || tg3_flag(tp, POLL_SERDES))) {
5237                 struct tg3_hw_status *sblk = tp->napi[0].hw_status;
5238
5239                 if (sblk->status & SD_STATUS_LINK_CHG) {
5240                         sblk->status = SD_STATUS_UPDATED |
5241                                        (sblk->status & ~SD_STATUS_LINK_CHG);
5242                         spin_lock(&tp->lock);
5243                         if (tg3_flag(tp, USE_PHYLIB)) {
5244                                 tw32_f(MAC_STATUS,
5245                                      (MAC_STATUS_SYNC_CHANGED |
5246                                       MAC_STATUS_CFG_CHANGED |
5247                                       MAC_STATUS_MI_COMPLETION |
5248                                       MAC_STATUS_LNKSTATE_CHANGED));
5249                                 udelay(40);
5250                         } else
5251                                 tg3_setup_phy(tp, 0);
5252                         spin_unlock(&tp->lock);
5253                 }
5254         }
5255 }
5256
5257 static int tg3_rx_prodring_xfer(struct tg3 *tp,
5258                                 struct tg3_rx_prodring_set *dpr,
5259                                 struct tg3_rx_prodring_set *spr)
5260 {
5261         u32 si, di, cpycnt, src_prod_idx;
5262         int i, err = 0;
5263
5264         while (1) {
5265                 src_prod_idx = spr->rx_std_prod_idx;
5266
5267                 /* Make sure updates to the rx_std_buffers[] entries and the
5268                  * standard producer index are seen in the correct order.
5269                  */
5270                 smp_rmb();
5271
5272                 if (spr->rx_std_cons_idx == src_prod_idx)
5273                         break;
5274
5275                 if (spr->rx_std_cons_idx < src_prod_idx)
5276                         cpycnt = src_prod_idx - spr->rx_std_cons_idx;
5277                 else
5278                         cpycnt = tp->rx_std_ring_mask + 1 -
5279                                  spr->rx_std_cons_idx;
5280
5281                 cpycnt = min(cpycnt,
5282                              tp->rx_std_ring_mask + 1 - dpr->rx_std_prod_idx);
5283
5284                 si = spr->rx_std_cons_idx;
5285                 di = dpr->rx_std_prod_idx;
5286
5287                 for (i = di; i < di + cpycnt; i++) {
5288                         if (dpr->rx_std_buffers[i].skb) {
5289                                 cpycnt = i - di;
5290                                 err = -ENOSPC;
5291                                 break;
5292                         }
5293                 }
5294
5295                 if (!cpycnt)
5296                         break;
5297
5298                 /* Ensure that updates to the rx_std_buffers ring and the
5299                  * shadowed hardware producer ring from tg3_recycle_skb() are
5300                  * ordered correctly WRT the skb check above.
5301                  */
5302                 smp_rmb();
5303
5304                 memcpy(&dpr->rx_std_buffers[di],
5305                        &spr->rx_std_buffers[si],
5306                        cpycnt * sizeof(struct ring_info));
5307
5308                 for (i = 0; i < cpycnt; i++, di++, si++) {
5309                         struct tg3_rx_buffer_desc *sbd, *dbd;
5310                         sbd = &spr->rx_std[si];
5311                         dbd = &dpr->rx_std[di];
5312                         dbd->addr_hi = sbd->addr_hi;
5313                         dbd->addr_lo = sbd->addr_lo;
5314                 }
5315
5316                 spr->rx_std_cons_idx = (spr->rx_std_cons_idx + cpycnt) &
5317                                        tp->rx_std_ring_mask;
5318                 dpr->rx_std_prod_idx = (dpr->rx_std_prod_idx + cpycnt) &
5319                                        tp->rx_std_ring_mask;
5320         }
5321
5322         while (1) {
5323                 src_prod_idx = spr->rx_jmb_prod_idx;
5324
5325                 /* Make sure updates to the rx_jmb_buffers[] entries and
5326                  * the jumbo producer index are seen in the correct order.
5327                  */
5328                 smp_rmb();
5329
5330                 if (spr->rx_jmb_cons_idx == src_prod_idx)
5331                         break;
5332
5333                 if (spr->rx_jmb_cons_idx < src_prod_idx)
5334                         cpycnt = src_prod_idx - spr->rx_jmb_cons_idx;
5335                 else
5336                         cpycnt = tp->rx_jmb_ring_mask + 1 -
5337                                  spr->rx_jmb_cons_idx;
5338
5339                 cpycnt = min(cpycnt,
5340                              tp->rx_jmb_ring_mask + 1 - dpr->rx_jmb_prod_idx);
5341
5342                 si = spr->rx_jmb_cons_idx;
5343                 di = dpr->rx_jmb_prod_idx;
5344
5345                 for (i = di; i < di + cpycnt; i++) {
5346                         if (dpr->rx_jmb_buffers[i].skb) {
5347                                 cpycnt = i - di;
5348                                 err = -ENOSPC;
5349                                 break;
5350                         }
5351                 }
5352
5353                 if (!cpycnt)
5354                         break;
5355
5356                 /* Ensure that updates to the rx_jmb_buffers ring and the
5357                  * shadowed hardware producer ring from tg3_recycle_skb() are
5358                  * ordered correctly WRT the skb check above.
5359                  */
5360                 smp_rmb();
5361
5362                 memcpy(&dpr->rx_jmb_buffers[di],
5363                        &spr->rx_jmb_buffers[si],
5364                        cpycnt * sizeof(struct ring_info));
5365
5366                 for (i = 0; i < cpycnt; i++, di++, si++) {
5367                         struct tg3_rx_buffer_desc *sbd, *dbd;
5368                         sbd = &spr->rx_jmb[si].std;
5369                         dbd = &dpr->rx_jmb[di].std;
5370                         dbd->addr_hi = sbd->addr_hi;
5371                         dbd->addr_lo = sbd->addr_lo;
5372                 }
5373
5374                 spr->rx_jmb_cons_idx = (spr->rx_jmb_cons_idx + cpycnt) &
5375                                        tp->rx_jmb_ring_mask;
5376                 dpr->rx_jmb_prod_idx = (dpr->rx_jmb_prod_idx + cpycnt) &
5377                                        tp->rx_jmb_ring_mask;
5378         }
5379
5380         return err;
5381 }
5382
5383 static int tg3_poll_work(struct tg3_napi *tnapi, int work_done, int budget)
5384 {
5385         struct tg3 *tp = tnapi->tp;
5386
5387         /* run TX completion thread */
5388         if (tnapi->hw_status->idx[0].tx_consumer != tnapi->tx_cons) {
5389                 tg3_tx(tnapi);
5390                 if (unlikely(tg3_flag(tp, TX_RECOVERY_PENDING)))
5391                         return work_done;
5392         }
5393
5394         /* run RX thread, within the bounds set by NAPI.
5395          * All RX "locking" is done by ensuring outside
5396          * code synchronizes with tg3->napi.poll()
5397          */
5398         if (*(tnapi->rx_rcb_prod_idx) != tnapi->rx_rcb_ptr)
5399                 work_done += tg3_rx(tnapi, budget - work_done);
5400
5401         if (tg3_flag(tp, ENABLE_RSS) && tnapi == &tp->napi[1]) {
5402                 struct tg3_rx_prodring_set *dpr = &tp->napi[0].prodring;
5403                 int i, err = 0;
5404                 u32 std_prod_idx = dpr->rx_std_prod_idx;
5405                 u32 jmb_prod_idx = dpr->rx_jmb_prod_idx;
5406
5407                 for (i = 1; i < tp->irq_cnt; i++)
5408                         err |= tg3_rx_prodring_xfer(tp, dpr,
5409                                                     &tp->napi[i].prodring);
5410
5411                 wmb();
5412
5413                 if (std_prod_idx != dpr->rx_std_prod_idx)
5414                         tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG,
5415                                      dpr->rx_std_prod_idx);
5416
5417                 if (jmb_prod_idx != dpr->rx_jmb_prod_idx)
5418                         tw32_rx_mbox(TG3_RX_JMB_PROD_IDX_REG,
5419                                      dpr->rx_jmb_prod_idx);
5420
5421                 mmiowb();
5422
5423                 if (err)
5424                         tw32_f(HOSTCC_MODE, tp->coal_now);
5425         }
5426
5427         return work_done;
5428 }
5429
5430 static int tg3_poll_msix(struct napi_struct *napi, int budget)
5431 {
5432         struct tg3_napi *tnapi = container_of(napi, struct tg3_napi, napi);
5433         struct tg3 *tp = tnapi->tp;
5434         int work_done = 0;
5435         struct tg3_hw_status *sblk = tnapi->hw_status;
5436
5437         while (1) {
5438                 work_done = tg3_poll_work(tnapi, work_done, budget);
5439
5440                 if (unlikely(tg3_flag(tp, TX_RECOVERY_PENDING)))
5441                         goto tx_recovery;
5442
5443                 if (unlikely(work_done >= budget))
5444                         break;
5445
5446                 /* tp->last_tag is used in tg3_int_reenable() below
5447                  * to tell the hw how much work has been processed,
5448                  * so we must read it before checking for more work.
5449                  */
5450                 tnapi->last_tag = sblk->status_tag;
5451                 tnapi->last_irq_tag = tnapi->last_tag;
5452                 rmb();
5453
5454                 /* check for RX/TX work to do */
5455                 if (likely(sblk->idx[0].tx_consumer == tnapi->tx_cons &&
5456                            *(tnapi->rx_rcb_prod_idx) == tnapi->rx_rcb_ptr)) {
5457                         napi_complete(napi);
5458                         /* Reenable interrupts. */
5459                         tw32_mailbox(tnapi->int_mbox, tnapi->last_tag << 24);
5460                         mmiowb();
5461                         break;
5462                 }
5463         }
5464
5465         return work_done;
5466
5467 tx_recovery:
5468         /* work_done is guaranteed to be less than budget. */
5469         napi_complete(napi);
5470         schedule_work(&tp->reset_task);
5471         return work_done;
5472 }
5473
5474 static void tg3_process_error(struct tg3 *tp)
5475 {
5476         u32 val;
5477         bool real_error = false;
5478
5479         if (tg3_flag(tp, ERROR_PROCESSED))
5480                 return;
5481
5482         /* Check Flow Attention register */
5483         val = tr32(HOSTCC_FLOW_ATTN);
5484         if (val & ~HOSTCC_FLOW_ATTN_MBUF_LWM) {
5485                 netdev_err(tp->dev, "FLOW Attention error.  Resetting chip.\n");
5486                 real_error = true;
5487         }
5488
5489         if (tr32(MSGINT_STATUS) & ~MSGINT_STATUS_MSI_REQ) {
5490                 netdev_err(tp->dev, "MSI Status error.  Resetting chip.\n");
5491                 real_error = true;
5492         }
5493
5494         if (tr32(RDMAC_STATUS) || tr32(WDMAC_STATUS)) {
5495                 netdev_err(tp->dev, "DMA Status error.  Resetting chip.\n");
5496                 real_error = true;
5497         }
5498
5499         if (!real_error)
5500                 return;
5501
5502         tg3_dump_state(tp);
5503
5504         tg3_flag_set(tp, ERROR_PROCESSED);
5505         schedule_work(&tp->reset_task);
5506 }
5507
5508 static int tg3_poll(struct napi_struct *napi, int budget)
5509 {
5510         struct tg3_napi *tnapi = container_of(napi, struct tg3_napi, napi);
5511         struct tg3 *tp = tnapi->tp;
5512         int work_done = 0;
5513         struct tg3_hw_status *sblk = tnapi->hw_status;
5514
5515         while (1) {
5516                 if (sblk->status & SD_STATUS_ERROR)
5517                         tg3_process_error(tp);
5518
5519                 tg3_poll_link(tp);
5520
5521                 work_done = tg3_poll_work(tnapi, work_done, budget);
5522
5523                 if (unlikely(tg3_flag(tp, TX_RECOVERY_PENDING)))
5524                         goto tx_recovery;
5525
5526                 if (unlikely(work_done >= budget))
5527                         break;
5528
5529                 if (tg3_flag(tp, TAGGED_STATUS)) {
5530                         /* tp->last_tag is used in tg3_int_reenable() below
5531                          * to tell the hw how much work has been processed,
5532                          * so we must read it before checking for more work.
5533                          */
5534                         tnapi->last_tag = sblk->status_tag;
5535                         tnapi->last_irq_tag = tnapi->last_tag;
5536                         rmb();
5537                 } else
5538                         sblk->status &= ~SD_STATUS_UPDATED;
5539
5540                 if (likely(!tg3_has_work(tnapi))) {
5541                         napi_complete(napi);
5542                         tg3_int_reenable(tnapi);
5543                         break;
5544                 }
5545         }
5546
5547         return work_done;
5548
5549 tx_recovery:
5550         /* work_done is guaranteed to be less than budget. */
5551         napi_complete(napi);
5552         schedule_work(&tp->reset_task);
5553         return work_done;
5554 }
5555
5556 static void tg3_napi_disable(struct tg3 *tp)
5557 {
5558         int i;
5559
5560         for (i = tp->irq_cnt - 1; i >= 0; i--)
5561                 napi_disable(&tp->napi[i].napi);
5562 }
5563
5564 static void tg3_napi_enable(struct tg3 *tp)
5565 {
5566         int i;
5567
5568         for (i = 0; i < tp->irq_cnt; i++)
5569                 napi_enable(&tp->napi[i].napi);
5570 }
5571
5572 static void tg3_napi_init(struct tg3 *tp)
5573 {
5574         int i;
5575
5576         netif_napi_add(tp->dev, &tp->napi[0].napi, tg3_poll, 64);
5577         for (i = 1; i < tp->irq_cnt; i++)
5578                 netif_napi_add(tp->dev, &tp->napi[i].napi, tg3_poll_msix, 64);
5579 }
5580
5581 static void tg3_napi_fini(struct tg3 *tp)
5582 {
5583         int i;
5584
5585         for (i = 0; i < tp->irq_cnt; i++)
5586                 netif_napi_del(&tp->napi[i].napi);
5587 }
5588
5589 static inline void tg3_netif_stop(struct tg3 *tp)
5590 {
5591         tp->dev->trans_start = jiffies; /* prevent tx timeout */
5592         tg3_napi_disable(tp);
5593         netif_tx_disable(tp->dev);
5594 }
5595
5596 static inline void tg3_netif_start(struct tg3 *tp)
5597 {
5598         /* NOTE: unconditional netif_tx_wake_all_queues is only
5599          * appropriate so long as all callers are assured to
5600          * have free tx slots (such as after tg3_init_hw)
5601          */
5602         netif_tx_wake_all_queues(tp->dev);
5603
5604         tg3_napi_enable(tp);
5605         tp->napi[0].hw_status->status |= SD_STATUS_UPDATED;
5606         tg3_enable_ints(tp);
5607 }
5608
5609 static void tg3_irq_quiesce(struct tg3 *tp)
5610 {
5611         int i;
5612
5613         BUG_ON(tp->irq_sync);
5614
5615         tp->irq_sync = 1;
5616         smp_mb();
5617
5618         for (i = 0; i < tp->irq_cnt; i++)
5619                 synchronize_irq(tp->napi[i].irq_vec);
5620 }
5621
5622 /* Fully shutdown all tg3 driver activity elsewhere in the system.
5623  * If irq_sync is non-zero, then the IRQ handler must be synchronized
5624  * with as well.  Most of the time, this is not necessary except when
5625  * shutting down the device.
5626  */
5627 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync)
5628 {
5629         spin_lock_bh(&tp->lock);
5630         if (irq_sync)
5631                 tg3_irq_quiesce(tp);
5632 }
5633
5634 static inline void tg3_full_unlock(struct tg3 *tp)
5635 {
5636         spin_unlock_bh(&tp->lock);
5637 }
5638
5639 /* One-shot MSI handler - Chip automatically disables interrupt
5640  * after sending MSI so driver doesn't have to do it.
5641  */
5642 static irqreturn_t tg3_msi_1shot(int irq, void *dev_id)
5643 {
5644         struct tg3_napi *tnapi = dev_id;
5645         struct tg3 *tp = tnapi->tp;
5646
5647         prefetch(tnapi->hw_status);
5648         if (tnapi->rx_rcb)
5649                 prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
5650
5651         if (likely(!tg3_irq_sync(tp)))
5652                 napi_schedule(&tnapi->napi);
5653
5654         return IRQ_HANDLED;
5655 }
5656
5657 /* MSI ISR - No need to check for interrupt sharing and no need to
5658  * flush status block and interrupt mailbox. PCI ordering rules
5659  * guarantee that MSI will arrive after the status block.
5660  */
5661 static irqreturn_t tg3_msi(int irq, void *dev_id)
5662 {
5663         struct tg3_napi *tnapi = dev_id;
5664         struct tg3 *tp = tnapi->tp;
5665
5666         prefetch(tnapi->hw_status);
5667         if (tnapi->rx_rcb)
5668                 prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
5669         /*
5670          * Writing any value to intr-mbox-0 clears PCI INTA# and
5671          * chip-internal interrupt pending events.
5672          * Writing non-zero to intr-mbox-0 additional tells the
5673          * NIC to stop sending us irqs, engaging "in-intr-handler"
5674          * event coalescing.
5675          */
5676         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
5677         if (likely(!tg3_irq_sync(tp)))
5678                 napi_schedule(&tnapi->napi);
5679
5680         return IRQ_RETVAL(1);
5681 }
5682
5683 static irqreturn_t tg3_interrupt(int irq, void *dev_id)
5684 {
5685         struct tg3_napi *tnapi = dev_id;
5686         struct tg3 *tp = tnapi->tp;
5687         struct tg3_hw_status *sblk = tnapi->hw_status;
5688         unsigned int handled = 1;
5689
5690         /* In INTx mode, it is possible for the interrupt to arrive at
5691          * the CPU before the status block posted prior to the interrupt.
5692          * Reading the PCI State register will confirm whether the
5693          * interrupt is ours and will flush the status block.
5694          */
5695         if (unlikely(!(sblk->status & SD_STATUS_UPDATED))) {
5696                 if (tg3_flag(tp, CHIP_RESETTING) ||
5697                     (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
5698                         handled = 0;
5699                         goto out;
5700                 }
5701         }
5702
5703         /*
5704          * Writing any value to intr-mbox-0 clears PCI INTA# and
5705          * chip-internal interrupt pending events.
5706          * Writing non-zero to intr-mbox-0 additional tells the
5707          * NIC to stop sending us irqs, engaging "in-intr-handler"
5708          * event coalescing.
5709          *
5710          * Flush the mailbox to de-assert the IRQ immediately to prevent
5711          * spurious interrupts.  The flush impacts performance but
5712          * excessive spurious interrupts can be worse in some cases.
5713          */
5714         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
5715         if (tg3_irq_sync(tp))
5716                 goto out;
5717         sblk->status &= ~SD_STATUS_UPDATED;
5718         if (likely(tg3_has_work(tnapi))) {
5719                 prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
5720                 napi_schedule(&tnapi->napi);
5721         } else {
5722                 /* No work, shared interrupt perhaps?  re-enable
5723                  * interrupts, and flush that PCI write
5724                  */
5725                 tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
5726                                0x00000000);
5727         }
5728 out:
5729         return IRQ_RETVAL(handled);
5730 }
5731
5732 static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id)
5733 {
5734         struct tg3_napi *tnapi = dev_id;
5735         struct tg3 *tp = tnapi->tp;
5736         struct tg3_hw_status *sblk = tnapi->hw_status;
5737         unsigned int handled = 1;
5738
5739         /* In INTx mode, it is possible for the interrupt to arrive at
5740          * the CPU before the status block posted prior to the interrupt.
5741          * Reading the PCI State register will confirm whether the
5742          * interrupt is ours and will flush the status block.
5743          */
5744         if (unlikely(sblk->status_tag == tnapi->last_irq_tag)) {
5745                 if (tg3_flag(tp, CHIP_RESETTING) ||
5746                     (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
5747                         handled = 0;
5748                         goto out;
5749                 }
5750         }
5751
5752         /*
5753          * writing any value to intr-mbox-0 clears PCI INTA# and
5754          * chip-internal interrupt pending events.
5755          * writing non-zero to intr-mbox-0 additional tells the
5756          * NIC to stop sending us irqs, engaging "in-intr-handler"
5757          * event coalescing.
5758          *
5759          * Flush the mailbox to de-assert the IRQ immediately to prevent
5760          * spurious interrupts.  The flush impacts performance but
5761          * excessive spurious interrupts can be worse in some cases.
5762          */
5763         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
5764
5765         /*
5766          * In a shared interrupt configuration, sometimes other devices'
5767          * interrupts will scream.  We record the current status tag here
5768          * so that the above check can report that the screaming interrupts
5769          * are unhandled.  Eventually they will be silenced.
5770          */
5771         tnapi->last_irq_tag = sblk->status_tag;
5772
5773         if (tg3_irq_sync(tp))
5774                 goto out;
5775
5776         prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
5777
5778         napi_schedule(&tnapi->napi);
5779
5780 out:
5781         return IRQ_RETVAL(handled);
5782 }
5783
5784 /* ISR for interrupt test */
5785 static irqreturn_t tg3_test_isr(int irq, void *dev_id)
5786 {
5787         struct tg3_napi *tnapi = dev_id;
5788         struct tg3 *tp = tnapi->tp;
5789         struct tg3_hw_status *sblk = tnapi->hw_status;
5790
5791         if ((sblk->status & SD_STATUS_UPDATED) ||
5792             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
5793                 tg3_disable_ints(tp);
5794                 return IRQ_RETVAL(1);
5795         }
5796         return IRQ_RETVAL(0);
5797 }
5798
5799 static int tg3_init_hw(struct tg3 *, int);
5800 static int tg3_halt(struct tg3 *, int, int);
5801
5802 /* Restart hardware after configuration changes, self-test, etc.
5803  * Invoked with tp->lock held.
5804  */
5805 static int tg3_restart_hw(struct tg3 *tp, int reset_phy)
5806         __releases(tp->lock)
5807         __acquires(tp->lock)
5808 {
5809         int err;
5810
5811         err = tg3_init_hw(tp, reset_phy);
5812         if (err) {
5813                 netdev_err(tp->dev,
5814                            "Failed to re-initialize device, aborting\n");
5815                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
5816                 tg3_full_unlock(tp);
5817                 del_timer_sync(&tp->timer);
5818                 tp->irq_sync = 0;
5819                 tg3_napi_enable(tp);
5820                 dev_close(tp->dev);
5821                 tg3_full_lock(tp, 0);
5822         }
5823         return err;
5824 }
5825
5826 #ifdef CONFIG_NET_POLL_CONTROLLER
5827 static void tg3_poll_controller(struct net_device *dev)
5828 {
5829         int i;
5830         struct tg3 *tp = netdev_priv(dev);
5831
5832         for (i = 0; i < tp->irq_cnt; i++)
5833                 tg3_interrupt(tp->napi[i].irq_vec, &tp->napi[i]);
5834 }
5835 #endif
5836
5837 static void tg3_reset_task(struct work_struct *work)
5838 {
5839         struct tg3 *tp = container_of(work, struct tg3, reset_task);
5840         int err;
5841         unsigned int restart_timer;
5842
5843         tg3_full_lock(tp, 0);
5844
5845         if (!netif_running(tp->dev)) {
5846                 tg3_full_unlock(tp);
5847                 return;
5848         }
5849
5850         tg3_full_unlock(tp);
5851 <