tg3: Add 57765 phy ID and enable devices.
[linux-2.6.git] / drivers / net / tg3.c
1 /*
2  * tg3.c: Broadcom Tigon3 ethernet driver.
3  *
4  * Copyright (C) 2001, 2002, 2003, 2004 David S. Miller (davem@redhat.com)
5  * Copyright (C) 2001, 2002, 2003 Jeff Garzik (jgarzik@pobox.com)
6  * Copyright (C) 2004 Sun Microsystems Inc.
7  * Copyright (C) 2005-2009 Broadcom Corporation.
8  *
9  * Firmware is:
10  *      Derived from proprietary unpublished source code,
11  *      Copyright (C) 2000-2003 Broadcom Corporation.
12  *
13  *      Permission is hereby granted for the distribution of this firmware
14  *      data in hexadecimal or equivalent format, provided this copyright
15  *      notice is accompanying it.
16  */
17
18
19 #include <linux/module.h>
20 #include <linux/moduleparam.h>
21 #include <linux/kernel.h>
22 #include <linux/types.h>
23 #include <linux/compiler.h>
24 #include <linux/slab.h>
25 #include <linux/delay.h>
26 #include <linux/in.h>
27 #include <linux/init.h>
28 #include <linux/ioport.h>
29 #include <linux/pci.h>
30 #include <linux/netdevice.h>
31 #include <linux/etherdevice.h>
32 #include <linux/skbuff.h>
33 #include <linux/ethtool.h>
34 #include <linux/mii.h>
35 #include <linux/phy.h>
36 #include <linux/brcmphy.h>
37 #include <linux/if_vlan.h>
38 #include <linux/ip.h>
39 #include <linux/tcp.h>
40 #include <linux/workqueue.h>
41 #include <linux/prefetch.h>
42 #include <linux/dma-mapping.h>
43 #include <linux/firmware.h>
44
45 #include <net/checksum.h>
46 #include <net/ip.h>
47
48 #include <asm/system.h>
49 #include <asm/io.h>
50 #include <asm/byteorder.h>
51 #include <asm/uaccess.h>
52
53 #ifdef CONFIG_SPARC
54 #include <asm/idprom.h>
55 #include <asm/prom.h>
56 #endif
57
58 #define BAR_0   0
59 #define BAR_2   2
60
61 #if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
62 #define TG3_VLAN_TAG_USED 1
63 #else
64 #define TG3_VLAN_TAG_USED 0
65 #endif
66
67 #include "tg3.h"
68
69 #define DRV_MODULE_NAME         "tg3"
70 #define PFX DRV_MODULE_NAME     ": "
71 #define DRV_MODULE_VERSION      "3.105"
72 #define DRV_MODULE_RELDATE      "December 2, 2009"
73
74 #define TG3_DEF_MAC_MODE        0
75 #define TG3_DEF_RX_MODE         0
76 #define TG3_DEF_TX_MODE         0
77 #define TG3_DEF_MSG_ENABLE        \
78         (NETIF_MSG_DRV          | \
79          NETIF_MSG_PROBE        | \
80          NETIF_MSG_LINK         | \
81          NETIF_MSG_TIMER        | \
82          NETIF_MSG_IFDOWN       | \
83          NETIF_MSG_IFUP         | \
84          NETIF_MSG_RX_ERR       | \
85          NETIF_MSG_TX_ERR)
86
87 /* length of time before we decide the hardware is borked,
88  * and dev->tx_timeout() should be called to fix the problem
89  */
90 #define TG3_TX_TIMEOUT                  (5 * HZ)
91
92 /* hardware minimum and maximum for a single frame's data payload */
93 #define TG3_MIN_MTU                     60
94 #define TG3_MAX_MTU(tp) \
95         ((tp->tg3_flags & TG3_FLAG_JUMBO_CAPABLE) ? 9000 : 1500)
96
97 /* These numbers seem to be hard coded in the NIC firmware somehow.
98  * You can't change the ring sizes, but you can change where you place
99  * them in the NIC onboard memory.
100  */
101 #define TG3_RX_RING_SIZE                512
102 #define TG3_DEF_RX_RING_PENDING         200
103 #define TG3_RX_JUMBO_RING_SIZE          256
104 #define TG3_DEF_RX_JUMBO_RING_PENDING   100
105 #define TG3_RSS_INDIR_TBL_SIZE 128
106
107 /* Do not place this n-ring entries value into the tp struct itself,
108  * we really want to expose these constants to GCC so that modulo et
109  * al.  operations are done with shifts and masks instead of with
110  * hw multiply/modulo instructions.  Another solution would be to
111  * replace things like '% foo' with '& (foo - 1)'.
112  */
113 #define TG3_RX_RCB_RING_SIZE(tp)        \
114         (((tp->tg3_flags & TG3_FLAG_JUMBO_CAPABLE) && \
115           !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) ? 1024 : 512)
116
117 #define TG3_TX_RING_SIZE                512
118 #define TG3_DEF_TX_RING_PENDING         (TG3_TX_RING_SIZE - 1)
119
120 #define TG3_RX_RING_BYTES       (sizeof(struct tg3_rx_buffer_desc) * \
121                                  TG3_RX_RING_SIZE)
122 #define TG3_RX_JUMBO_RING_BYTES (sizeof(struct tg3_ext_rx_buffer_desc) * \
123                                  TG3_RX_JUMBO_RING_SIZE)
124 #define TG3_RX_RCB_RING_BYTES(tp) (sizeof(struct tg3_rx_buffer_desc) * \
125                                  TG3_RX_RCB_RING_SIZE(tp))
126 #define TG3_TX_RING_BYTES       (sizeof(struct tg3_tx_buffer_desc) * \
127                                  TG3_TX_RING_SIZE)
128 #define NEXT_TX(N)              (((N) + 1) & (TG3_TX_RING_SIZE - 1))
129
130 #define TG3_DMA_BYTE_ENAB               64
131
132 #define TG3_RX_STD_DMA_SZ               1536
133 #define TG3_RX_JMB_DMA_SZ               9046
134
135 #define TG3_RX_DMA_TO_MAP_SZ(x)         ((x) + TG3_DMA_BYTE_ENAB)
136
137 #define TG3_RX_STD_MAP_SZ               TG3_RX_DMA_TO_MAP_SZ(TG3_RX_STD_DMA_SZ)
138 #define TG3_RX_JMB_MAP_SZ               TG3_RX_DMA_TO_MAP_SZ(TG3_RX_JMB_DMA_SZ)
139
140 #define TG3_RX_STD_BUFF_RING_SIZE \
141         (sizeof(struct ring_info) * TG3_RX_RING_SIZE)
142
143 #define TG3_RX_JMB_BUFF_RING_SIZE \
144         (sizeof(struct ring_info) * TG3_RX_JUMBO_RING_SIZE)
145
146 /* minimum number of free TX descriptors required to wake up TX process */
147 #define TG3_TX_WAKEUP_THRESH(tnapi)             ((tnapi)->tx_pending / 4)
148
149 #define TG3_RAW_IP_ALIGN 2
150
151 /* number of ETHTOOL_GSTATS u64's */
152 #define TG3_NUM_STATS           (sizeof(struct tg3_ethtool_stats)/sizeof(u64))
153
154 #define TG3_NUM_TEST            6
155
156 #define FIRMWARE_TG3            "tigon/tg3.bin"
157 #define FIRMWARE_TG3TSO         "tigon/tg3_tso.bin"
158 #define FIRMWARE_TG3TSO5        "tigon/tg3_tso5.bin"
159
160 static char version[] __devinitdata =
161         DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
162
163 MODULE_AUTHOR("David S. Miller (davem@redhat.com) and Jeff Garzik (jgarzik@pobox.com)");
164 MODULE_DESCRIPTION("Broadcom Tigon3 ethernet driver");
165 MODULE_LICENSE("GPL");
166 MODULE_VERSION(DRV_MODULE_VERSION);
167 MODULE_FIRMWARE(FIRMWARE_TG3);
168 MODULE_FIRMWARE(FIRMWARE_TG3TSO);
169 MODULE_FIRMWARE(FIRMWARE_TG3TSO5);
170
171 #define TG3_RSS_MIN_NUM_MSIX_VECS       2
172
173 static int tg3_debug = -1;      /* -1 == use TG3_DEF_MSG_ENABLE as value */
174 module_param(tg3_debug, int, 0);
175 MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");
176
177 static DEFINE_PCI_DEVICE_TABLE(tg3_pci_tbl) = {
178         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700)},
179         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701)},
180         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702)},
181         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703)},
182         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704)},
183         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE)},
184         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705)},
185         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2)},
186         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M)},
187         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2)},
188         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X)},
189         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X)},
190         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S)},
191         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3)},
192         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3)},
193         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782)},
194         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788)},
195         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789)},
196         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901)},
197         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2)},
198         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2)},
199         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F)},
200         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5720)},
201         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721)},
202         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5722)},
203         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750)},
204         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751)},
205         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750M)},
206         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M)},
207         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F)},
208         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752)},
209         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M)},
210         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753)},
211         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M)},
212         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F)},
213         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754)},
214         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M)},
215         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755)},
216         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M)},
217         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5756)},
218         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786)},
219         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787)},
220         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M)},
221         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787F)},
222         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714)},
223         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S)},
224         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715)},
225         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S)},
226         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780)},
227         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S)},
228         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781)},
229         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906)},
230         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906M)},
231         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5784)},
232         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5764)},
233         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5723)},
234         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761)},
235         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761E)},
236         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5761S)},
237         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5761SE)},
238         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5785_G)},
239         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5785_F)},
240         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57780)},
241         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57760)},
242         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57790)},
243         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57788)},
244         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5717)},
245         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5718)},
246         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5724)},
247         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57781)},
248         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57785)},
249         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57761)},
250         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57765)},
251         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57791)},
252         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57795)},
253         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX)},
254         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX)},
255         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000)},
256         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001)},
257         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003)},
258         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100)},
259         {PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3)},
260         {}
261 };
262
263 MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);
264
265 static const struct {
266         const char string[ETH_GSTRING_LEN];
267 } ethtool_stats_keys[TG3_NUM_STATS] = {
268         { "rx_octets" },
269         { "rx_fragments" },
270         { "rx_ucast_packets" },
271         { "rx_mcast_packets" },
272         { "rx_bcast_packets" },
273         { "rx_fcs_errors" },
274         { "rx_align_errors" },
275         { "rx_xon_pause_rcvd" },
276         { "rx_xoff_pause_rcvd" },
277         { "rx_mac_ctrl_rcvd" },
278         { "rx_xoff_entered" },
279         { "rx_frame_too_long_errors" },
280         { "rx_jabbers" },
281         { "rx_undersize_packets" },
282         { "rx_in_length_errors" },
283         { "rx_out_length_errors" },
284         { "rx_64_or_less_octet_packets" },
285         { "rx_65_to_127_octet_packets" },
286         { "rx_128_to_255_octet_packets" },
287         { "rx_256_to_511_octet_packets" },
288         { "rx_512_to_1023_octet_packets" },
289         { "rx_1024_to_1522_octet_packets" },
290         { "rx_1523_to_2047_octet_packets" },
291         { "rx_2048_to_4095_octet_packets" },
292         { "rx_4096_to_8191_octet_packets" },
293         { "rx_8192_to_9022_octet_packets" },
294
295         { "tx_octets" },
296         { "tx_collisions" },
297
298         { "tx_xon_sent" },
299         { "tx_xoff_sent" },
300         { "tx_flow_control" },
301         { "tx_mac_errors" },
302         { "tx_single_collisions" },
303         { "tx_mult_collisions" },
304         { "tx_deferred" },
305         { "tx_excessive_collisions" },
306         { "tx_late_collisions" },
307         { "tx_collide_2times" },
308         { "tx_collide_3times" },
309         { "tx_collide_4times" },
310         { "tx_collide_5times" },
311         { "tx_collide_6times" },
312         { "tx_collide_7times" },
313         { "tx_collide_8times" },
314         { "tx_collide_9times" },
315         { "tx_collide_10times" },
316         { "tx_collide_11times" },
317         { "tx_collide_12times" },
318         { "tx_collide_13times" },
319         { "tx_collide_14times" },
320         { "tx_collide_15times" },
321         { "tx_ucast_packets" },
322         { "tx_mcast_packets" },
323         { "tx_bcast_packets" },
324         { "tx_carrier_sense_errors" },
325         { "tx_discards" },
326         { "tx_errors" },
327
328         { "dma_writeq_full" },
329         { "dma_write_prioq_full" },
330         { "rxbds_empty" },
331         { "rx_discards" },
332         { "rx_errors" },
333         { "rx_threshold_hit" },
334
335         { "dma_readq_full" },
336         { "dma_read_prioq_full" },
337         { "tx_comp_queue_full" },
338
339         { "ring_set_send_prod_index" },
340         { "ring_status_update" },
341         { "nic_irqs" },
342         { "nic_avoided_irqs" },
343         { "nic_tx_threshold_hit" }
344 };
345
346 static const struct {
347         const char string[ETH_GSTRING_LEN];
348 } ethtool_test_keys[TG3_NUM_TEST] = {
349         { "nvram test     (online) " },
350         { "link test      (online) " },
351         { "register test  (offline)" },
352         { "memory test    (offline)" },
353         { "loopback test  (offline)" },
354         { "interrupt test (offline)" },
355 };
356
357 static void tg3_write32(struct tg3 *tp, u32 off, u32 val)
358 {
359         writel(val, tp->regs + off);
360 }
361
362 static u32 tg3_read32(struct tg3 *tp, u32 off)
363 {
364         return (readl(tp->regs + off));
365 }
366
367 static void tg3_ape_write32(struct tg3 *tp, u32 off, u32 val)
368 {
369         writel(val, tp->aperegs + off);
370 }
371
372 static u32 tg3_ape_read32(struct tg3 *tp, u32 off)
373 {
374         return (readl(tp->aperegs + off));
375 }
376
377 static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val)
378 {
379         unsigned long flags;
380
381         spin_lock_irqsave(&tp->indirect_lock, flags);
382         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
383         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
384         spin_unlock_irqrestore(&tp->indirect_lock, flags);
385 }
386
387 static void tg3_write_flush_reg32(struct tg3 *tp, u32 off, u32 val)
388 {
389         writel(val, tp->regs + off);
390         readl(tp->regs + off);
391 }
392
393 static u32 tg3_read_indirect_reg32(struct tg3 *tp, u32 off)
394 {
395         unsigned long flags;
396         u32 val;
397
398         spin_lock_irqsave(&tp->indirect_lock, flags);
399         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
400         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
401         spin_unlock_irqrestore(&tp->indirect_lock, flags);
402         return val;
403 }
404
405 static void tg3_write_indirect_mbox(struct tg3 *tp, u32 off, u32 val)
406 {
407         unsigned long flags;
408
409         if (off == (MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW)) {
410                 pci_write_config_dword(tp->pdev, TG3PCI_RCV_RET_RING_CON_IDX +
411                                        TG3_64BIT_REG_LOW, val);
412                 return;
413         }
414         if (off == TG3_RX_STD_PROD_IDX_REG) {
415                 pci_write_config_dword(tp->pdev, TG3PCI_STD_RING_PROD_IDX +
416                                        TG3_64BIT_REG_LOW, val);
417                 return;
418         }
419
420         spin_lock_irqsave(&tp->indirect_lock, flags);
421         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
422         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
423         spin_unlock_irqrestore(&tp->indirect_lock, flags);
424
425         /* In indirect mode when disabling interrupts, we also need
426          * to clear the interrupt bit in the GRC local ctrl register.
427          */
428         if ((off == (MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW)) &&
429             (val == 0x1)) {
430                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_LOCAL_CTRL,
431                                        tp->grc_local_ctrl|GRC_LCLCTRL_CLEARINT);
432         }
433 }
434
435 static u32 tg3_read_indirect_mbox(struct tg3 *tp, u32 off)
436 {
437         unsigned long flags;
438         u32 val;
439
440         spin_lock_irqsave(&tp->indirect_lock, flags);
441         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
442         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
443         spin_unlock_irqrestore(&tp->indirect_lock, flags);
444         return val;
445 }
446
447 /* usec_wait specifies the wait time in usec when writing to certain registers
448  * where it is unsafe to read back the register without some delay.
449  * GRC_LOCAL_CTRL is one example if the GPIOs are toggled to switch power.
450  * TG3PCI_CLOCK_CTRL is another example if the clock frequencies are changed.
451  */
452 static void _tw32_flush(struct tg3 *tp, u32 off, u32 val, u32 usec_wait)
453 {
454         if ((tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) ||
455             (tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
456                 /* Non-posted methods */
457                 tp->write32(tp, off, val);
458         else {
459                 /* Posted method */
460                 tg3_write32(tp, off, val);
461                 if (usec_wait)
462                         udelay(usec_wait);
463                 tp->read32(tp, off);
464         }
465         /* Wait again after the read for the posted method to guarantee that
466          * the wait time is met.
467          */
468         if (usec_wait)
469                 udelay(usec_wait);
470 }
471
472 static inline void tw32_mailbox_flush(struct tg3 *tp, u32 off, u32 val)
473 {
474         tp->write32_mbox(tp, off, val);
475         if (!(tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) &&
476             !(tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
477                 tp->read32_mbox(tp, off);
478 }
479
480 static void tg3_write32_tx_mbox(struct tg3 *tp, u32 off, u32 val)
481 {
482         void __iomem *mbox = tp->regs + off;
483         writel(val, mbox);
484         if (tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG)
485                 writel(val, mbox);
486         if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
487                 readl(mbox);
488 }
489
490 static u32 tg3_read32_mbox_5906(struct tg3 *tp, u32 off)
491 {
492         return (readl(tp->regs + off + GRCMBOX_BASE));
493 }
494
495 static void tg3_write32_mbox_5906(struct tg3 *tp, u32 off, u32 val)
496 {
497         writel(val, tp->regs + off + GRCMBOX_BASE);
498 }
499
500 #define tw32_mailbox(reg, val)  tp->write32_mbox(tp, reg, val)
501 #define tw32_mailbox_f(reg, val)        tw32_mailbox_flush(tp, (reg), (val))
502 #define tw32_rx_mbox(reg, val)  tp->write32_rx_mbox(tp, reg, val)
503 #define tw32_tx_mbox(reg, val)  tp->write32_tx_mbox(tp, reg, val)
504 #define tr32_mailbox(reg)       tp->read32_mbox(tp, reg)
505
506 #define tw32(reg,val)           tp->write32(tp, reg, val)
507 #define tw32_f(reg,val)         _tw32_flush(tp,(reg),(val), 0)
508 #define tw32_wait_f(reg,val,us) _tw32_flush(tp,(reg),(val), (us))
509 #define tr32(reg)               tp->read32(tp, reg)
510
511 static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val)
512 {
513         unsigned long flags;
514
515         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) &&
516             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC))
517                 return;
518
519         spin_lock_irqsave(&tp->indirect_lock, flags);
520         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
521                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
522                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
523
524                 /* Always leave this as zero. */
525                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
526         } else {
527                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
528                 tw32_f(TG3PCI_MEM_WIN_DATA, val);
529
530                 /* Always leave this as zero. */
531                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
532         }
533         spin_unlock_irqrestore(&tp->indirect_lock, flags);
534 }
535
536 static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val)
537 {
538         unsigned long flags;
539
540         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) &&
541             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC)) {
542                 *val = 0;
543                 return;
544         }
545
546         spin_lock_irqsave(&tp->indirect_lock, flags);
547         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
548                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
549                 pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
550
551                 /* Always leave this as zero. */
552                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
553         } else {
554                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
555                 *val = tr32(TG3PCI_MEM_WIN_DATA);
556
557                 /* Always leave this as zero. */
558                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
559         }
560         spin_unlock_irqrestore(&tp->indirect_lock, flags);
561 }
562
563 static void tg3_ape_lock_init(struct tg3 *tp)
564 {
565         int i;
566
567         /* Make sure the driver hasn't any stale locks. */
568         for (i = 0; i < 8; i++)
569                 tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + 4 * i,
570                                 APE_LOCK_GRANT_DRIVER);
571 }
572
573 static int tg3_ape_lock(struct tg3 *tp, int locknum)
574 {
575         int i, off;
576         int ret = 0;
577         u32 status;
578
579         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
580                 return 0;
581
582         switch (locknum) {
583                 case TG3_APE_LOCK_GRC:
584                 case TG3_APE_LOCK_MEM:
585                         break;
586                 default:
587                         return -EINVAL;
588         }
589
590         off = 4 * locknum;
591
592         tg3_ape_write32(tp, TG3_APE_LOCK_REQ + off, APE_LOCK_REQ_DRIVER);
593
594         /* Wait for up to 1 millisecond to acquire lock. */
595         for (i = 0; i < 100; i++) {
596                 status = tg3_ape_read32(tp, TG3_APE_LOCK_GRANT + off);
597                 if (status == APE_LOCK_GRANT_DRIVER)
598                         break;
599                 udelay(10);
600         }
601
602         if (status != APE_LOCK_GRANT_DRIVER) {
603                 /* Revoke the lock request. */
604                 tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + off,
605                                 APE_LOCK_GRANT_DRIVER);
606
607                 ret = -EBUSY;
608         }
609
610         return ret;
611 }
612
613 static void tg3_ape_unlock(struct tg3 *tp, int locknum)
614 {
615         int off;
616
617         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
618                 return;
619
620         switch (locknum) {
621                 case TG3_APE_LOCK_GRC:
622                 case TG3_APE_LOCK_MEM:
623                         break;
624                 default:
625                         return;
626         }
627
628         off = 4 * locknum;
629         tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + off, APE_LOCK_GRANT_DRIVER);
630 }
631
632 static void tg3_disable_ints(struct tg3 *tp)
633 {
634         int i;
635
636         tw32(TG3PCI_MISC_HOST_CTRL,
637              (tp->misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT));
638         for (i = 0; i < tp->irq_max; i++)
639                 tw32_mailbox_f(tp->napi[i].int_mbox, 0x00000001);
640 }
641
642 static void tg3_enable_ints(struct tg3 *tp)
643 {
644         int i;
645         u32 coal_now = 0;
646
647         tp->irq_sync = 0;
648         wmb();
649
650         tw32(TG3PCI_MISC_HOST_CTRL,
651              (tp->misc_host_ctrl & ~MISC_HOST_CTRL_MASK_PCI_INT));
652
653         for (i = 0; i < tp->irq_cnt; i++) {
654                 struct tg3_napi *tnapi = &tp->napi[i];
655                 tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
656                 if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)
657                         tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
658
659                 coal_now |= tnapi->coal_now;
660         }
661
662         /* Force an initial interrupt */
663         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
664             (tp->napi[0].hw_status->status & SD_STATUS_UPDATED))
665                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
666         else
667                 tw32(HOSTCC_MODE, tp->coalesce_mode |
668                      HOSTCC_MODE_ENABLE | coal_now);
669 }
670
671 static inline unsigned int tg3_has_work(struct tg3_napi *tnapi)
672 {
673         struct tg3 *tp = tnapi->tp;
674         struct tg3_hw_status *sblk = tnapi->hw_status;
675         unsigned int work_exists = 0;
676
677         /* check for phy events */
678         if (!(tp->tg3_flags &
679               (TG3_FLAG_USE_LINKCHG_REG |
680                TG3_FLAG_POLL_SERDES))) {
681                 if (sblk->status & SD_STATUS_LINK_CHG)
682                         work_exists = 1;
683         }
684         /* check for RX/TX work to do */
685         if (sblk->idx[0].tx_consumer != tnapi->tx_cons ||
686             *(tnapi->rx_rcb_prod_idx) != tnapi->rx_rcb_ptr)
687                 work_exists = 1;
688
689         return work_exists;
690 }
691
692 /* tg3_int_reenable
693  *  similar to tg3_enable_ints, but it accurately determines whether there
694  *  is new work pending and can return without flushing the PIO write
695  *  which reenables interrupts
696  */
697 static void tg3_int_reenable(struct tg3_napi *tnapi)
698 {
699         struct tg3 *tp = tnapi->tp;
700
701         tw32_mailbox(tnapi->int_mbox, tnapi->last_tag << 24);
702         mmiowb();
703
704         /* When doing tagged status, this work check is unnecessary.
705          * The last_tag we write above tells the chip which piece of
706          * work we've completed.
707          */
708         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
709             tg3_has_work(tnapi))
710                 tw32(HOSTCC_MODE, tp->coalesce_mode |
711                      HOSTCC_MODE_ENABLE | tnapi->coal_now);
712 }
713
714 static void tg3_napi_disable(struct tg3 *tp)
715 {
716         int i;
717
718         for (i = tp->irq_cnt - 1; i >= 0; i--)
719                 napi_disable(&tp->napi[i].napi);
720 }
721
722 static void tg3_napi_enable(struct tg3 *tp)
723 {
724         int i;
725
726         for (i = 0; i < tp->irq_cnt; i++)
727                 napi_enable(&tp->napi[i].napi);
728 }
729
730 static inline void tg3_netif_stop(struct tg3 *tp)
731 {
732         tp->dev->trans_start = jiffies; /* prevent tx timeout */
733         tg3_napi_disable(tp);
734         netif_tx_disable(tp->dev);
735 }
736
737 static inline void tg3_netif_start(struct tg3 *tp)
738 {
739         /* NOTE: unconditional netif_tx_wake_all_queues is only
740          * appropriate so long as all callers are assured to
741          * have free tx slots (such as after tg3_init_hw)
742          */
743         netif_tx_wake_all_queues(tp->dev);
744
745         tg3_napi_enable(tp);
746         tp->napi[0].hw_status->status |= SD_STATUS_UPDATED;
747         tg3_enable_ints(tp);
748 }
749
750 static void tg3_switch_clocks(struct tg3 *tp)
751 {
752         u32 clock_ctrl;
753         u32 orig_clock_ctrl;
754
755         if ((tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) ||
756             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
757                 return;
758
759         clock_ctrl = tr32(TG3PCI_CLOCK_CTRL);
760
761         orig_clock_ctrl = clock_ctrl;
762         clock_ctrl &= (CLOCK_CTRL_FORCE_CLKRUN |
763                        CLOCK_CTRL_CLKRUN_OENABLE |
764                        0x1f);
765         tp->pci_clock_ctrl = clock_ctrl;
766
767         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
768                 if (orig_clock_ctrl & CLOCK_CTRL_625_CORE) {
769                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
770                                     clock_ctrl | CLOCK_CTRL_625_CORE, 40);
771                 }
772         } else if ((orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
773                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
774                             clock_ctrl |
775                             (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK),
776                             40);
777                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
778                             clock_ctrl | (CLOCK_CTRL_ALTCLK),
779                             40);
780         }
781         tw32_wait_f(TG3PCI_CLOCK_CTRL, clock_ctrl, 40);
782 }
783
784 #define PHY_BUSY_LOOPS  5000
785
786 static int tg3_readphy(struct tg3 *tp, int reg, u32 *val)
787 {
788         u32 frame_val;
789         unsigned int loops;
790         int ret;
791
792         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
793                 tw32_f(MAC_MI_MODE,
794                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
795                 udelay(80);
796         }
797
798         *val = 0x0;
799
800         frame_val  = ((tp->phy_addr << MI_COM_PHY_ADDR_SHIFT) &
801                       MI_COM_PHY_ADDR_MASK);
802         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
803                       MI_COM_REG_ADDR_MASK);
804         frame_val |= (MI_COM_CMD_READ | MI_COM_START);
805
806         tw32_f(MAC_MI_COM, frame_val);
807
808         loops = PHY_BUSY_LOOPS;
809         while (loops != 0) {
810                 udelay(10);
811                 frame_val = tr32(MAC_MI_COM);
812
813                 if ((frame_val & MI_COM_BUSY) == 0) {
814                         udelay(5);
815                         frame_val = tr32(MAC_MI_COM);
816                         break;
817                 }
818                 loops -= 1;
819         }
820
821         ret = -EBUSY;
822         if (loops != 0) {
823                 *val = frame_val & MI_COM_DATA_MASK;
824                 ret = 0;
825         }
826
827         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
828                 tw32_f(MAC_MI_MODE, tp->mi_mode);
829                 udelay(80);
830         }
831
832         return ret;
833 }
834
835 static int tg3_writephy(struct tg3 *tp, int reg, u32 val)
836 {
837         u32 frame_val;
838         unsigned int loops;
839         int ret;
840
841         if ((tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET) &&
842             (reg == MII_TG3_CTRL || reg == MII_TG3_AUX_CTRL))
843                 return 0;
844
845         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
846                 tw32_f(MAC_MI_MODE,
847                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
848                 udelay(80);
849         }
850
851         frame_val  = ((tp->phy_addr << MI_COM_PHY_ADDR_SHIFT) &
852                       MI_COM_PHY_ADDR_MASK);
853         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
854                       MI_COM_REG_ADDR_MASK);
855         frame_val |= (val & MI_COM_DATA_MASK);
856         frame_val |= (MI_COM_CMD_WRITE | MI_COM_START);
857
858         tw32_f(MAC_MI_COM, frame_val);
859
860         loops = PHY_BUSY_LOOPS;
861         while (loops != 0) {
862                 udelay(10);
863                 frame_val = tr32(MAC_MI_COM);
864                 if ((frame_val & MI_COM_BUSY) == 0) {
865                         udelay(5);
866                         frame_val = tr32(MAC_MI_COM);
867                         break;
868                 }
869                 loops -= 1;
870         }
871
872         ret = -EBUSY;
873         if (loops != 0)
874                 ret = 0;
875
876         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
877                 tw32_f(MAC_MI_MODE, tp->mi_mode);
878                 udelay(80);
879         }
880
881         return ret;
882 }
883
884 static int tg3_bmcr_reset(struct tg3 *tp)
885 {
886         u32 phy_control;
887         int limit, err;
888
889         /* OK, reset it, and poll the BMCR_RESET bit until it
890          * clears or we time out.
891          */
892         phy_control = BMCR_RESET;
893         err = tg3_writephy(tp, MII_BMCR, phy_control);
894         if (err != 0)
895                 return -EBUSY;
896
897         limit = 5000;
898         while (limit--) {
899                 err = tg3_readphy(tp, MII_BMCR, &phy_control);
900                 if (err != 0)
901                         return -EBUSY;
902
903                 if ((phy_control & BMCR_RESET) == 0) {
904                         udelay(40);
905                         break;
906                 }
907                 udelay(10);
908         }
909         if (limit < 0)
910                 return -EBUSY;
911
912         return 0;
913 }
914
915 static int tg3_mdio_read(struct mii_bus *bp, int mii_id, int reg)
916 {
917         struct tg3 *tp = bp->priv;
918         u32 val;
919
920         spin_lock_bh(&tp->lock);
921
922         if (tg3_readphy(tp, reg, &val))
923                 val = -EIO;
924
925         spin_unlock_bh(&tp->lock);
926
927         return val;
928 }
929
930 static int tg3_mdio_write(struct mii_bus *bp, int mii_id, int reg, u16 val)
931 {
932         struct tg3 *tp = bp->priv;
933         u32 ret = 0;
934
935         spin_lock_bh(&tp->lock);
936
937         if (tg3_writephy(tp, reg, val))
938                 ret = -EIO;
939
940         spin_unlock_bh(&tp->lock);
941
942         return ret;
943 }
944
945 static int tg3_mdio_reset(struct mii_bus *bp)
946 {
947         return 0;
948 }
949
950 static void tg3_mdio_config_5785(struct tg3 *tp)
951 {
952         u32 val;
953         struct phy_device *phydev;
954
955         phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
956         switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) {
957         case TG3_PHY_ID_BCM50610:
958         case TG3_PHY_ID_BCM50610M:
959                 val = MAC_PHYCFG2_50610_LED_MODES;
960                 break;
961         case TG3_PHY_ID_BCMAC131:
962                 val = MAC_PHYCFG2_AC131_LED_MODES;
963                 break;
964         case TG3_PHY_ID_RTL8211C:
965                 val = MAC_PHYCFG2_RTL8211C_LED_MODES;
966                 break;
967         case TG3_PHY_ID_RTL8201E:
968                 val = MAC_PHYCFG2_RTL8201E_LED_MODES;
969                 break;
970         default:
971                 return;
972         }
973
974         if (phydev->interface != PHY_INTERFACE_MODE_RGMII) {
975                 tw32(MAC_PHYCFG2, val);
976
977                 val = tr32(MAC_PHYCFG1);
978                 val &= ~(MAC_PHYCFG1_RGMII_INT |
979                          MAC_PHYCFG1_RXCLK_TO_MASK | MAC_PHYCFG1_TXCLK_TO_MASK);
980                 val |= MAC_PHYCFG1_RXCLK_TIMEOUT | MAC_PHYCFG1_TXCLK_TIMEOUT;
981                 tw32(MAC_PHYCFG1, val);
982
983                 return;
984         }
985
986         if (!(tp->tg3_flags3 & TG3_FLG3_RGMII_STD_IBND_DISABLE))
987                 val |= MAC_PHYCFG2_EMODE_MASK_MASK |
988                        MAC_PHYCFG2_FMODE_MASK_MASK |
989                        MAC_PHYCFG2_GMODE_MASK_MASK |
990                        MAC_PHYCFG2_ACT_MASK_MASK   |
991                        MAC_PHYCFG2_QUAL_MASK_MASK |
992                        MAC_PHYCFG2_INBAND_ENABLE;
993
994         tw32(MAC_PHYCFG2, val);
995
996         val = tr32(MAC_PHYCFG1);
997         val &= ~(MAC_PHYCFG1_RXCLK_TO_MASK | MAC_PHYCFG1_TXCLK_TO_MASK |
998                  MAC_PHYCFG1_RGMII_EXT_RX_DEC | MAC_PHYCFG1_RGMII_SND_STAT_EN);
999         if (!(tp->tg3_flags3 & TG3_FLG3_RGMII_STD_IBND_DISABLE)) {
1000                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_RX_EN)
1001                         val |= MAC_PHYCFG1_RGMII_EXT_RX_DEC;
1002                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_TX_EN)
1003                         val |= MAC_PHYCFG1_RGMII_SND_STAT_EN;
1004         }
1005         val |= MAC_PHYCFG1_RXCLK_TIMEOUT | MAC_PHYCFG1_TXCLK_TIMEOUT |
1006                MAC_PHYCFG1_RGMII_INT | MAC_PHYCFG1_TXC_DRV;
1007         tw32(MAC_PHYCFG1, val);
1008
1009         val = tr32(MAC_EXT_RGMII_MODE);
1010         val &= ~(MAC_RGMII_MODE_RX_INT_B |
1011                  MAC_RGMII_MODE_RX_QUALITY |
1012                  MAC_RGMII_MODE_RX_ACTIVITY |
1013                  MAC_RGMII_MODE_RX_ENG_DET |
1014                  MAC_RGMII_MODE_TX_ENABLE |
1015                  MAC_RGMII_MODE_TX_LOWPWR |
1016                  MAC_RGMII_MODE_TX_RESET);
1017         if (!(tp->tg3_flags3 & TG3_FLG3_RGMII_STD_IBND_DISABLE)) {
1018                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_RX_EN)
1019                         val |= MAC_RGMII_MODE_RX_INT_B |
1020                                MAC_RGMII_MODE_RX_QUALITY |
1021                                MAC_RGMII_MODE_RX_ACTIVITY |
1022                                MAC_RGMII_MODE_RX_ENG_DET;
1023                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_TX_EN)
1024                         val |= MAC_RGMII_MODE_TX_ENABLE |
1025                                MAC_RGMII_MODE_TX_LOWPWR |
1026                                MAC_RGMII_MODE_TX_RESET;
1027         }
1028         tw32(MAC_EXT_RGMII_MODE, val);
1029 }
1030
1031 static void tg3_mdio_start(struct tg3 *tp)
1032 {
1033         tp->mi_mode &= ~MAC_MI_MODE_AUTO_POLL;
1034         tw32_f(MAC_MI_MODE, tp->mi_mode);
1035         udelay(80);
1036
1037         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717) {
1038                 u32 funcnum, is_serdes;
1039
1040                 funcnum = tr32(TG3_CPMU_STATUS) & TG3_CPMU_STATUS_PCIE_FUNC;
1041                 if (funcnum)
1042                         tp->phy_addr = 2;
1043                 else
1044                         tp->phy_addr = 1;
1045
1046                 is_serdes = tr32(SG_DIG_STATUS) & SG_DIG_IS_SERDES;
1047                 if (is_serdes)
1048                         tp->phy_addr += 7;
1049         } else
1050                 tp->phy_addr = TG3_PHY_MII_ADDR;
1051
1052         if ((tp->tg3_flags3 & TG3_FLG3_MDIOBUS_INITED) &&
1053             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785)
1054                 tg3_mdio_config_5785(tp);
1055 }
1056
1057 static int tg3_mdio_init(struct tg3 *tp)
1058 {
1059         int i;
1060         u32 reg;
1061         struct phy_device *phydev;
1062
1063         tg3_mdio_start(tp);
1064
1065         if (!(tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) ||
1066             (tp->tg3_flags3 & TG3_FLG3_MDIOBUS_INITED))
1067                 return 0;
1068
1069         tp->mdio_bus = mdiobus_alloc();
1070         if (tp->mdio_bus == NULL)
1071                 return -ENOMEM;
1072
1073         tp->mdio_bus->name     = "tg3 mdio bus";
1074         snprintf(tp->mdio_bus->id, MII_BUS_ID_SIZE, "%x",
1075                  (tp->pdev->bus->number << 8) | tp->pdev->devfn);
1076         tp->mdio_bus->priv     = tp;
1077         tp->mdio_bus->parent   = &tp->pdev->dev;
1078         tp->mdio_bus->read     = &tg3_mdio_read;
1079         tp->mdio_bus->write    = &tg3_mdio_write;
1080         tp->mdio_bus->reset    = &tg3_mdio_reset;
1081         tp->mdio_bus->phy_mask = ~(1 << TG3_PHY_MII_ADDR);
1082         tp->mdio_bus->irq      = &tp->mdio_irq[0];
1083
1084         for (i = 0; i < PHY_MAX_ADDR; i++)
1085                 tp->mdio_bus->irq[i] = PHY_POLL;
1086
1087         /* The bus registration will look for all the PHYs on the mdio bus.
1088          * Unfortunately, it does not ensure the PHY is powered up before
1089          * accessing the PHY ID registers.  A chip reset is the
1090          * quickest way to bring the device back to an operational state..
1091          */
1092         if (tg3_readphy(tp, MII_BMCR, &reg) || (reg & BMCR_PDOWN))
1093                 tg3_bmcr_reset(tp);
1094
1095         i = mdiobus_register(tp->mdio_bus);
1096         if (i) {
1097                 printk(KERN_WARNING "%s: mdiobus_reg failed (0x%x)\n",
1098                         tp->dev->name, i);
1099                 mdiobus_free(tp->mdio_bus);
1100                 return i;
1101         }
1102
1103         phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
1104
1105         if (!phydev || !phydev->drv) {
1106                 printk(KERN_WARNING "%s: No PHY devices\n", tp->dev->name);
1107                 mdiobus_unregister(tp->mdio_bus);
1108                 mdiobus_free(tp->mdio_bus);
1109                 return -ENODEV;
1110         }
1111
1112         switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) {
1113         case TG3_PHY_ID_BCM57780:
1114                 phydev->interface = PHY_INTERFACE_MODE_GMII;
1115                 phydev->dev_flags |= PHY_BRCM_AUTO_PWRDWN_ENABLE;
1116                 break;
1117         case TG3_PHY_ID_BCM50610:
1118         case TG3_PHY_ID_BCM50610M:
1119                 phydev->dev_flags |= PHY_BRCM_CLEAR_RGMII_MODE |
1120                                      PHY_BRCM_RX_REFCLK_UNUSED |
1121                                      PHY_BRCM_DIS_TXCRXC_NOENRGY |
1122                                      PHY_BRCM_AUTO_PWRDWN_ENABLE;
1123                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_STD_IBND_DISABLE)
1124                         phydev->dev_flags |= PHY_BRCM_STD_IBND_DISABLE;
1125                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_RX_EN)
1126                         phydev->dev_flags |= PHY_BRCM_EXT_IBND_RX_ENABLE;
1127                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_TX_EN)
1128                         phydev->dev_flags |= PHY_BRCM_EXT_IBND_TX_ENABLE;
1129                 /* fallthru */
1130         case TG3_PHY_ID_RTL8211C:
1131                 phydev->interface = PHY_INTERFACE_MODE_RGMII;
1132                 break;
1133         case TG3_PHY_ID_RTL8201E:
1134         case TG3_PHY_ID_BCMAC131:
1135                 phydev->interface = PHY_INTERFACE_MODE_MII;
1136                 phydev->dev_flags |= PHY_BRCM_AUTO_PWRDWN_ENABLE;
1137                 tp->tg3_flags3 |= TG3_FLG3_PHY_IS_FET;
1138                 break;
1139         }
1140
1141         tp->tg3_flags3 |= TG3_FLG3_MDIOBUS_INITED;
1142
1143         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785)
1144                 tg3_mdio_config_5785(tp);
1145
1146         return 0;
1147 }
1148
1149 static void tg3_mdio_fini(struct tg3 *tp)
1150 {
1151         if (tp->tg3_flags3 & TG3_FLG3_MDIOBUS_INITED) {
1152                 tp->tg3_flags3 &= ~TG3_FLG3_MDIOBUS_INITED;
1153                 mdiobus_unregister(tp->mdio_bus);
1154                 mdiobus_free(tp->mdio_bus);
1155         }
1156 }
1157
1158 /* tp->lock is held. */
1159 static inline void tg3_generate_fw_event(struct tg3 *tp)
1160 {
1161         u32 val;
1162
1163         val = tr32(GRC_RX_CPU_EVENT);
1164         val |= GRC_RX_CPU_DRIVER_EVENT;
1165         tw32_f(GRC_RX_CPU_EVENT, val);
1166
1167         tp->last_event_jiffies = jiffies;
1168 }
1169
1170 #define TG3_FW_EVENT_TIMEOUT_USEC 2500
1171
1172 /* tp->lock is held. */
1173 static void tg3_wait_for_event_ack(struct tg3 *tp)
1174 {
1175         int i;
1176         unsigned int delay_cnt;
1177         long time_remain;
1178
1179         /* If enough time has passed, no wait is necessary. */
1180         time_remain = (long)(tp->last_event_jiffies + 1 +
1181                       usecs_to_jiffies(TG3_FW_EVENT_TIMEOUT_USEC)) -
1182                       (long)jiffies;
1183         if (time_remain < 0)
1184                 return;
1185
1186         /* Check if we can shorten the wait time. */
1187         delay_cnt = jiffies_to_usecs(time_remain);
1188         if (delay_cnt > TG3_FW_EVENT_TIMEOUT_USEC)
1189                 delay_cnt = TG3_FW_EVENT_TIMEOUT_USEC;
1190         delay_cnt = (delay_cnt >> 3) + 1;
1191
1192         for (i = 0; i < delay_cnt; i++) {
1193                 if (!(tr32(GRC_RX_CPU_EVENT) & GRC_RX_CPU_DRIVER_EVENT))
1194                         break;
1195                 udelay(8);
1196         }
1197 }
1198
1199 /* tp->lock is held. */
1200 static void tg3_ump_link_report(struct tg3 *tp)
1201 {
1202         u32 reg;
1203         u32 val;
1204
1205         if (!(tp->tg3_flags2 & TG3_FLG2_5780_CLASS) ||
1206             !(tp->tg3_flags  & TG3_FLAG_ENABLE_ASF))
1207                 return;
1208
1209         tg3_wait_for_event_ack(tp);
1210
1211         tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_LINK_UPDATE);
1212
1213         tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 14);
1214
1215         val = 0;
1216         if (!tg3_readphy(tp, MII_BMCR, &reg))
1217                 val = reg << 16;
1218         if (!tg3_readphy(tp, MII_BMSR, &reg))
1219                 val |= (reg & 0xffff);
1220         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX, val);
1221
1222         val = 0;
1223         if (!tg3_readphy(tp, MII_ADVERTISE, &reg))
1224                 val = reg << 16;
1225         if (!tg3_readphy(tp, MII_LPA, &reg))
1226                 val |= (reg & 0xffff);
1227         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 4, val);
1228
1229         val = 0;
1230         if (!(tp->tg3_flags2 & TG3_FLG2_MII_SERDES)) {
1231                 if (!tg3_readphy(tp, MII_CTRL1000, &reg))
1232                         val = reg << 16;
1233                 if (!tg3_readphy(tp, MII_STAT1000, &reg))
1234                         val |= (reg & 0xffff);
1235         }
1236         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 8, val);
1237
1238         if (!tg3_readphy(tp, MII_PHYADDR, &reg))
1239                 val = reg << 16;
1240         else
1241                 val = 0;
1242         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 12, val);
1243
1244         tg3_generate_fw_event(tp);
1245 }
1246
1247 static void tg3_link_report(struct tg3 *tp)
1248 {
1249         if (!netif_carrier_ok(tp->dev)) {
1250                 if (netif_msg_link(tp))
1251                         printk(KERN_INFO PFX "%s: Link is down.\n",
1252                                tp->dev->name);
1253                 tg3_ump_link_report(tp);
1254         } else if (netif_msg_link(tp)) {
1255                 printk(KERN_INFO PFX "%s: Link is up at %d Mbps, %s duplex.\n",
1256                        tp->dev->name,
1257                        (tp->link_config.active_speed == SPEED_1000 ?
1258                         1000 :
1259                         (tp->link_config.active_speed == SPEED_100 ?
1260                          100 : 10)),
1261                        (tp->link_config.active_duplex == DUPLEX_FULL ?
1262                         "full" : "half"));
1263
1264                 printk(KERN_INFO PFX
1265                        "%s: Flow control is %s for TX and %s for RX.\n",
1266                        tp->dev->name,
1267                        (tp->link_config.active_flowctrl & FLOW_CTRL_TX) ?
1268                        "on" : "off",
1269                        (tp->link_config.active_flowctrl & FLOW_CTRL_RX) ?
1270                        "on" : "off");
1271                 tg3_ump_link_report(tp);
1272         }
1273 }
1274
1275 static u16 tg3_advert_flowctrl_1000T(u8 flow_ctrl)
1276 {
1277         u16 miireg;
1278
1279         if ((flow_ctrl & FLOW_CTRL_TX) && (flow_ctrl & FLOW_CTRL_RX))
1280                 miireg = ADVERTISE_PAUSE_CAP;
1281         else if (flow_ctrl & FLOW_CTRL_TX)
1282                 miireg = ADVERTISE_PAUSE_ASYM;
1283         else if (flow_ctrl & FLOW_CTRL_RX)
1284                 miireg = ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM;
1285         else
1286                 miireg = 0;
1287
1288         return miireg;
1289 }
1290
1291 static u16 tg3_advert_flowctrl_1000X(u8 flow_ctrl)
1292 {
1293         u16 miireg;
1294
1295         if ((flow_ctrl & FLOW_CTRL_TX) && (flow_ctrl & FLOW_CTRL_RX))
1296                 miireg = ADVERTISE_1000XPAUSE;
1297         else if (flow_ctrl & FLOW_CTRL_TX)
1298                 miireg = ADVERTISE_1000XPSE_ASYM;
1299         else if (flow_ctrl & FLOW_CTRL_RX)
1300                 miireg = ADVERTISE_1000XPAUSE | ADVERTISE_1000XPSE_ASYM;
1301         else
1302                 miireg = 0;
1303
1304         return miireg;
1305 }
1306
1307 static u8 tg3_resolve_flowctrl_1000X(u16 lcladv, u16 rmtadv)
1308 {
1309         u8 cap = 0;
1310
1311         if (lcladv & ADVERTISE_1000XPAUSE) {
1312                 if (lcladv & ADVERTISE_1000XPSE_ASYM) {
1313                         if (rmtadv & LPA_1000XPAUSE)
1314                                 cap = FLOW_CTRL_TX | FLOW_CTRL_RX;
1315                         else if (rmtadv & LPA_1000XPAUSE_ASYM)
1316                                 cap = FLOW_CTRL_RX;
1317                 } else {
1318                         if (rmtadv & LPA_1000XPAUSE)
1319                                 cap = FLOW_CTRL_TX | FLOW_CTRL_RX;
1320                 }
1321         } else if (lcladv & ADVERTISE_1000XPSE_ASYM) {
1322                 if ((rmtadv & LPA_1000XPAUSE) && (rmtadv & LPA_1000XPAUSE_ASYM))
1323                         cap = FLOW_CTRL_TX;
1324         }
1325
1326         return cap;
1327 }
1328
1329 static void tg3_setup_flow_control(struct tg3 *tp, u32 lcladv, u32 rmtadv)
1330 {
1331         u8 autoneg;
1332         u8 flowctrl = 0;
1333         u32 old_rx_mode = tp->rx_mode;
1334         u32 old_tx_mode = tp->tx_mode;
1335
1336         if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB)
1337                 autoneg = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]->autoneg;
1338         else
1339                 autoneg = tp->link_config.autoneg;
1340
1341         if (autoneg == AUTONEG_ENABLE &&
1342             (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG)) {
1343                 if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
1344                         flowctrl = tg3_resolve_flowctrl_1000X(lcladv, rmtadv);
1345                 else
1346                         flowctrl = mii_resolve_flowctrl_fdx(lcladv, rmtadv);
1347         } else
1348                 flowctrl = tp->link_config.flowctrl;
1349
1350         tp->link_config.active_flowctrl = flowctrl;
1351
1352         if (flowctrl & FLOW_CTRL_RX)
1353                 tp->rx_mode |= RX_MODE_FLOW_CTRL_ENABLE;
1354         else
1355                 tp->rx_mode &= ~RX_MODE_FLOW_CTRL_ENABLE;
1356
1357         if (old_rx_mode != tp->rx_mode)
1358                 tw32_f(MAC_RX_MODE, tp->rx_mode);
1359
1360         if (flowctrl & FLOW_CTRL_TX)
1361                 tp->tx_mode |= TX_MODE_FLOW_CTRL_ENABLE;
1362         else
1363                 tp->tx_mode &= ~TX_MODE_FLOW_CTRL_ENABLE;
1364
1365         if (old_tx_mode != tp->tx_mode)
1366                 tw32_f(MAC_TX_MODE, tp->tx_mode);
1367 }
1368
1369 static void tg3_adjust_link(struct net_device *dev)
1370 {
1371         u8 oldflowctrl, linkmesg = 0;
1372         u32 mac_mode, lcl_adv, rmt_adv;
1373         struct tg3 *tp = netdev_priv(dev);
1374         struct phy_device *phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
1375
1376         spin_lock_bh(&tp->lock);
1377
1378         mac_mode = tp->mac_mode & ~(MAC_MODE_PORT_MODE_MASK |
1379                                     MAC_MODE_HALF_DUPLEX);
1380
1381         oldflowctrl = tp->link_config.active_flowctrl;
1382
1383         if (phydev->link) {
1384                 lcl_adv = 0;
1385                 rmt_adv = 0;
1386
1387                 if (phydev->speed == SPEED_100 || phydev->speed == SPEED_10)
1388                         mac_mode |= MAC_MODE_PORT_MODE_MII;
1389                 else if (phydev->speed == SPEED_1000 ||
1390                          GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5785)
1391                         mac_mode |= MAC_MODE_PORT_MODE_GMII;
1392                 else
1393                         mac_mode |= MAC_MODE_PORT_MODE_MII;
1394
1395                 if (phydev->duplex == DUPLEX_HALF)
1396                         mac_mode |= MAC_MODE_HALF_DUPLEX;
1397                 else {
1398                         lcl_adv = tg3_advert_flowctrl_1000T(
1399                                   tp->link_config.flowctrl);
1400
1401                         if (phydev->pause)
1402                                 rmt_adv = LPA_PAUSE_CAP;
1403                         if (phydev->asym_pause)
1404                                 rmt_adv |= LPA_PAUSE_ASYM;
1405                 }
1406
1407                 tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
1408         } else
1409                 mac_mode |= MAC_MODE_PORT_MODE_GMII;
1410
1411         if (mac_mode != tp->mac_mode) {
1412                 tp->mac_mode = mac_mode;
1413                 tw32_f(MAC_MODE, tp->mac_mode);
1414                 udelay(40);
1415         }
1416
1417         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785) {
1418                 if (phydev->speed == SPEED_10)
1419                         tw32(MAC_MI_STAT,
1420                              MAC_MI_STAT_10MBPS_MODE |
1421                              MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
1422                 else
1423                         tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
1424         }
1425
1426         if (phydev->speed == SPEED_1000 && phydev->duplex == DUPLEX_HALF)
1427                 tw32(MAC_TX_LENGTHS,
1428                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
1429                       (6 << TX_LENGTHS_IPG_SHIFT) |
1430                       (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
1431         else
1432                 tw32(MAC_TX_LENGTHS,
1433                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
1434                       (6 << TX_LENGTHS_IPG_SHIFT) |
1435                       (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
1436
1437         if ((phydev->link && tp->link_config.active_speed == SPEED_INVALID) ||
1438             (!phydev->link && tp->link_config.active_speed != SPEED_INVALID) ||
1439             phydev->speed != tp->link_config.active_speed ||
1440             phydev->duplex != tp->link_config.active_duplex ||
1441             oldflowctrl != tp->link_config.active_flowctrl)
1442             linkmesg = 1;
1443
1444         tp->link_config.active_speed = phydev->speed;
1445         tp->link_config.active_duplex = phydev->duplex;
1446
1447         spin_unlock_bh(&tp->lock);
1448
1449         if (linkmesg)
1450                 tg3_link_report(tp);
1451 }
1452
1453 static int tg3_phy_init(struct tg3 *tp)
1454 {
1455         struct phy_device *phydev;
1456
1457         if (tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED)
1458                 return 0;
1459
1460         /* Bring the PHY back to a known state. */
1461         tg3_bmcr_reset(tp);
1462
1463         phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
1464
1465         /* Attach the MAC to the PHY. */
1466         phydev = phy_connect(tp->dev, dev_name(&phydev->dev), tg3_adjust_link,
1467                              phydev->dev_flags, phydev->interface);
1468         if (IS_ERR(phydev)) {
1469                 printk(KERN_ERR "%s: Could not attach to PHY\n", tp->dev->name);
1470                 return PTR_ERR(phydev);
1471         }
1472
1473         /* Mask with MAC supported features. */
1474         switch (phydev->interface) {
1475         case PHY_INTERFACE_MODE_GMII:
1476         case PHY_INTERFACE_MODE_RGMII:
1477                 if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
1478                         phydev->supported &= (PHY_GBIT_FEATURES |
1479                                               SUPPORTED_Pause |
1480                                               SUPPORTED_Asym_Pause);
1481                         break;
1482                 }
1483                 /* fallthru */
1484         case PHY_INTERFACE_MODE_MII:
1485                 phydev->supported &= (PHY_BASIC_FEATURES |
1486                                       SUPPORTED_Pause |
1487                                       SUPPORTED_Asym_Pause);
1488                 break;
1489         default:
1490                 phy_disconnect(tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]);
1491                 return -EINVAL;
1492         }
1493
1494         tp->tg3_flags3 |= TG3_FLG3_PHY_CONNECTED;
1495
1496         phydev->advertising = phydev->supported;
1497
1498         return 0;
1499 }
1500
1501 static void tg3_phy_start(struct tg3 *tp)
1502 {
1503         struct phy_device *phydev;
1504
1505         if (!(tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED))
1506                 return;
1507
1508         phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
1509
1510         if (tp->link_config.phy_is_low_power) {
1511                 tp->link_config.phy_is_low_power = 0;
1512                 phydev->speed = tp->link_config.orig_speed;
1513                 phydev->duplex = tp->link_config.orig_duplex;
1514                 phydev->autoneg = tp->link_config.orig_autoneg;
1515                 phydev->advertising = tp->link_config.orig_advertising;
1516         }
1517
1518         phy_start(phydev);
1519
1520         phy_start_aneg(phydev);
1521 }
1522
1523 static void tg3_phy_stop(struct tg3 *tp)
1524 {
1525         if (!(tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED))
1526                 return;
1527
1528         phy_stop(tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]);
1529 }
1530
1531 static void tg3_phy_fini(struct tg3 *tp)
1532 {
1533         if (tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED) {
1534                 phy_disconnect(tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]);
1535                 tp->tg3_flags3 &= ~TG3_FLG3_PHY_CONNECTED;
1536         }
1537 }
1538
1539 static void tg3_phydsp_write(struct tg3 *tp, u32 reg, u32 val)
1540 {
1541         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, reg);
1542         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, val);
1543 }
1544
1545 static void tg3_phy_fet_toggle_apd(struct tg3 *tp, bool enable)
1546 {
1547         u32 phytest;
1548
1549         if (!tg3_readphy(tp, MII_TG3_FET_TEST, &phytest)) {
1550                 u32 phy;
1551
1552                 tg3_writephy(tp, MII_TG3_FET_TEST,
1553                              phytest | MII_TG3_FET_SHADOW_EN);
1554                 if (!tg3_readphy(tp, MII_TG3_FET_SHDW_AUXSTAT2, &phy)) {
1555                         if (enable)
1556                                 phy |= MII_TG3_FET_SHDW_AUXSTAT2_APD;
1557                         else
1558                                 phy &= ~MII_TG3_FET_SHDW_AUXSTAT2_APD;
1559                         tg3_writephy(tp, MII_TG3_FET_SHDW_AUXSTAT2, phy);
1560                 }
1561                 tg3_writephy(tp, MII_TG3_FET_TEST, phytest);
1562         }
1563 }
1564
1565 static void tg3_phy_toggle_apd(struct tg3 *tp, bool enable)
1566 {
1567         u32 reg;
1568
1569         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ||
1570                 (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 &&
1571              (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)))
1572                 return;
1573
1574         if (tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET) {
1575                 tg3_phy_fet_toggle_apd(tp, enable);
1576                 return;
1577         }
1578
1579         reg = MII_TG3_MISC_SHDW_WREN |
1580               MII_TG3_MISC_SHDW_SCR5_SEL |
1581               MII_TG3_MISC_SHDW_SCR5_LPED |
1582               MII_TG3_MISC_SHDW_SCR5_DLPTLM |
1583               MII_TG3_MISC_SHDW_SCR5_SDTL |
1584               MII_TG3_MISC_SHDW_SCR5_C125OE;
1585         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5784 || !enable)
1586                 reg |= MII_TG3_MISC_SHDW_SCR5_DLLAPD;
1587
1588         tg3_writephy(tp, MII_TG3_MISC_SHDW, reg);
1589
1590
1591         reg = MII_TG3_MISC_SHDW_WREN |
1592               MII_TG3_MISC_SHDW_APD_SEL |
1593               MII_TG3_MISC_SHDW_APD_WKTM_84MS;
1594         if (enable)
1595                 reg |= MII_TG3_MISC_SHDW_APD_ENABLE;
1596
1597         tg3_writephy(tp, MII_TG3_MISC_SHDW, reg);
1598 }
1599
1600 static void tg3_phy_toggle_automdix(struct tg3 *tp, int enable)
1601 {
1602         u32 phy;
1603
1604         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ||
1605             (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES))
1606                 return;
1607
1608         if (tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET) {
1609                 u32 ephy;
1610
1611                 if (!tg3_readphy(tp, MII_TG3_FET_TEST, &ephy)) {
1612                         u32 reg = MII_TG3_FET_SHDW_MISCCTRL;
1613
1614                         tg3_writephy(tp, MII_TG3_FET_TEST,
1615                                      ephy | MII_TG3_FET_SHADOW_EN);
1616                         if (!tg3_readphy(tp, reg, &phy)) {
1617                                 if (enable)
1618                                         phy |= MII_TG3_FET_SHDW_MISCCTRL_MDIX;
1619                                 else
1620                                         phy &= ~MII_TG3_FET_SHDW_MISCCTRL_MDIX;
1621                                 tg3_writephy(tp, reg, phy);
1622                         }
1623                         tg3_writephy(tp, MII_TG3_FET_TEST, ephy);
1624                 }
1625         } else {
1626                 phy = MII_TG3_AUXCTL_MISC_RDSEL_MISC |
1627                       MII_TG3_AUXCTL_SHDWSEL_MISC;
1628                 if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, phy) &&
1629                     !tg3_readphy(tp, MII_TG3_AUX_CTRL, &phy)) {
1630                         if (enable)
1631                                 phy |= MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
1632                         else
1633                                 phy &= ~MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
1634                         phy |= MII_TG3_AUXCTL_MISC_WREN;
1635                         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy);
1636                 }
1637         }
1638 }
1639
1640 static void tg3_phy_set_wirespeed(struct tg3 *tp)
1641 {
1642         u32 val;
1643
1644         if (tp->tg3_flags2 & TG3_FLG2_NO_ETH_WIRE_SPEED)
1645                 return;
1646
1647         if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x7007) &&
1648             !tg3_readphy(tp, MII_TG3_AUX_CTRL, &val))
1649                 tg3_writephy(tp, MII_TG3_AUX_CTRL,
1650                              (val | (1 << 15) | (1 << 4)));
1651 }
1652
1653 static void tg3_phy_apply_otp(struct tg3 *tp)
1654 {
1655         u32 otp, phy;
1656
1657         if (!tp->phy_otp)
1658                 return;
1659
1660         otp = tp->phy_otp;
1661
1662         /* Enable SM_DSP clock and tx 6dB coding. */
1663         phy = MII_TG3_AUXCTL_SHDWSEL_AUXCTL |
1664               MII_TG3_AUXCTL_ACTL_SMDSP_ENA |
1665               MII_TG3_AUXCTL_ACTL_TX_6DB;
1666         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy);
1667
1668         phy = ((otp & TG3_OTP_AGCTGT_MASK) >> TG3_OTP_AGCTGT_SHIFT);
1669         phy |= MII_TG3_DSP_TAP1_AGCTGT_DFLT;
1670         tg3_phydsp_write(tp, MII_TG3_DSP_TAP1, phy);
1671
1672         phy = ((otp & TG3_OTP_HPFFLTR_MASK) >> TG3_OTP_HPFFLTR_SHIFT) |
1673               ((otp & TG3_OTP_HPFOVER_MASK) >> TG3_OTP_HPFOVER_SHIFT);
1674         tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH0, phy);
1675
1676         phy = ((otp & TG3_OTP_LPFDIS_MASK) >> TG3_OTP_LPFDIS_SHIFT);
1677         phy |= MII_TG3_DSP_AADJ1CH3_ADCCKADJ;
1678         tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH3, phy);
1679
1680         phy = ((otp & TG3_OTP_VDAC_MASK) >> TG3_OTP_VDAC_SHIFT);
1681         tg3_phydsp_write(tp, MII_TG3_DSP_EXP75, phy);
1682
1683         phy = ((otp & TG3_OTP_10BTAMP_MASK) >> TG3_OTP_10BTAMP_SHIFT);
1684         tg3_phydsp_write(tp, MII_TG3_DSP_EXP96, phy);
1685
1686         phy = ((otp & TG3_OTP_ROFF_MASK) >> TG3_OTP_ROFF_SHIFT) |
1687               ((otp & TG3_OTP_RCOFF_MASK) >> TG3_OTP_RCOFF_SHIFT);
1688         tg3_phydsp_write(tp, MII_TG3_DSP_EXP97, phy);
1689
1690         /* Turn off SM_DSP clock. */
1691         phy = MII_TG3_AUXCTL_SHDWSEL_AUXCTL |
1692               MII_TG3_AUXCTL_ACTL_TX_6DB;
1693         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy);
1694 }
1695
1696 static int tg3_wait_macro_done(struct tg3 *tp)
1697 {
1698         int limit = 100;
1699
1700         while (limit--) {
1701                 u32 tmp32;
1702
1703                 if (!tg3_readphy(tp, 0x16, &tmp32)) {
1704                         if ((tmp32 & 0x1000) == 0)
1705                                 break;
1706                 }
1707         }
1708         if (limit < 0)
1709                 return -EBUSY;
1710
1711         return 0;
1712 }
1713
1714 static int tg3_phy_write_and_check_testpat(struct tg3 *tp, int *resetp)
1715 {
1716         static const u32 test_pat[4][6] = {
1717         { 0x00005555, 0x00000005, 0x00002aaa, 0x0000000a, 0x00003456, 0x00000003 },
1718         { 0x00002aaa, 0x0000000a, 0x00003333, 0x00000003, 0x0000789a, 0x00000005 },
1719         { 0x00005a5a, 0x00000005, 0x00002a6a, 0x0000000a, 0x00001bcd, 0x00000003 },
1720         { 0x00002a5a, 0x0000000a, 0x000033c3, 0x00000003, 0x00002ef1, 0x00000005 }
1721         };
1722         int chan;
1723
1724         for (chan = 0; chan < 4; chan++) {
1725                 int i;
1726
1727                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
1728                              (chan * 0x2000) | 0x0200);
1729                 tg3_writephy(tp, 0x16, 0x0002);
1730
1731                 for (i = 0; i < 6; i++)
1732                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT,
1733                                      test_pat[chan][i]);
1734
1735                 tg3_writephy(tp, 0x16, 0x0202);
1736                 if (tg3_wait_macro_done(tp)) {
1737                         *resetp = 1;
1738                         return -EBUSY;
1739                 }
1740
1741                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
1742                              (chan * 0x2000) | 0x0200);
1743                 tg3_writephy(tp, 0x16, 0x0082);
1744                 if (tg3_wait_macro_done(tp)) {
1745                         *resetp = 1;
1746                         return -EBUSY;
1747                 }
1748
1749                 tg3_writephy(tp, 0x16, 0x0802);
1750                 if (tg3_wait_macro_done(tp)) {
1751                         *resetp = 1;
1752                         return -EBUSY;
1753                 }
1754
1755                 for (i = 0; i < 6; i += 2) {
1756                         u32 low, high;
1757
1758                         if (tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &low) ||
1759                             tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &high) ||
1760                             tg3_wait_macro_done(tp)) {
1761                                 *resetp = 1;
1762                                 return -EBUSY;
1763                         }
1764                         low &= 0x7fff;
1765                         high &= 0x000f;
1766                         if (low != test_pat[chan][i] ||
1767                             high != test_pat[chan][i+1]) {
1768                                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000b);
1769                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4001);
1770                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4005);
1771
1772                                 return -EBUSY;
1773                         }
1774                 }
1775         }
1776
1777         return 0;
1778 }
1779
1780 static int tg3_phy_reset_chanpat(struct tg3 *tp)
1781 {
1782         int chan;
1783
1784         for (chan = 0; chan < 4; chan++) {
1785                 int i;
1786
1787                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
1788                              (chan * 0x2000) | 0x0200);
1789                 tg3_writephy(tp, 0x16, 0x0002);
1790                 for (i = 0; i < 6; i++)
1791                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x000);
1792                 tg3_writephy(tp, 0x16, 0x0202);
1793                 if (tg3_wait_macro_done(tp))
1794                         return -EBUSY;
1795         }
1796
1797         return 0;
1798 }
1799
1800 static int tg3_phy_reset_5703_4_5(struct tg3 *tp)
1801 {
1802         u32 reg32, phy9_orig;
1803         int retries, do_phy_reset, err;
1804
1805         retries = 10;
1806         do_phy_reset = 1;
1807         do {
1808                 if (do_phy_reset) {
1809                         err = tg3_bmcr_reset(tp);
1810                         if (err)
1811                                 return err;
1812                         do_phy_reset = 0;
1813                 }
1814
1815                 /* Disable transmitter and interrupt.  */
1816                 if (tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32))
1817                         continue;
1818
1819                 reg32 |= 0x3000;
1820                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
1821
1822                 /* Set full-duplex, 1000 mbps.  */
1823                 tg3_writephy(tp, MII_BMCR,
1824                              BMCR_FULLDPLX | TG3_BMCR_SPEED1000);
1825
1826                 /* Set to master mode.  */
1827                 if (tg3_readphy(tp, MII_TG3_CTRL, &phy9_orig))
1828                         continue;
1829
1830                 tg3_writephy(tp, MII_TG3_CTRL,
1831                              (MII_TG3_CTRL_AS_MASTER |
1832                               MII_TG3_CTRL_ENABLE_AS_MASTER));
1833
1834                 /* Enable SM_DSP_CLOCK and 6dB.  */
1835                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1836
1837                 /* Block the PHY control access.  */
1838                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
1839                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0800);
1840
1841                 err = tg3_phy_write_and_check_testpat(tp, &do_phy_reset);
1842                 if (!err)
1843                         break;
1844         } while (--retries);
1845
1846         err = tg3_phy_reset_chanpat(tp);
1847         if (err)
1848                 return err;
1849
1850         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
1851         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0000);
1852
1853         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8200);
1854         tg3_writephy(tp, 0x16, 0x0000);
1855
1856         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1857             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
1858                 /* Set Extended packet length bit for jumbo frames */
1859                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4400);
1860         }
1861         else {
1862                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1863         }
1864
1865         tg3_writephy(tp, MII_TG3_CTRL, phy9_orig);
1866
1867         if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32)) {
1868                 reg32 &= ~0x3000;
1869                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
1870         } else if (!err)
1871                 err = -EBUSY;
1872
1873         return err;
1874 }
1875
1876 /* This will reset the tigon3 PHY if there is no valid
1877  * link unless the FORCE argument is non-zero.
1878  */
1879 static int tg3_phy_reset(struct tg3 *tp)
1880 {
1881         u32 cpmuctrl;
1882         u32 phy_status;
1883         int err;
1884
1885         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1886                 u32 val;
1887
1888                 val = tr32(GRC_MISC_CFG);
1889                 tw32_f(GRC_MISC_CFG, val & ~GRC_MISC_CFG_EPHY_IDDQ);
1890                 udelay(40);
1891         }
1892         err  = tg3_readphy(tp, MII_BMSR, &phy_status);
1893         err |= tg3_readphy(tp, MII_BMSR, &phy_status);
1894         if (err != 0)
1895                 return -EBUSY;
1896
1897         if (netif_running(tp->dev) && netif_carrier_ok(tp->dev)) {
1898                 netif_carrier_off(tp->dev);
1899                 tg3_link_report(tp);
1900         }
1901
1902         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1903             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1904             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
1905                 err = tg3_phy_reset_5703_4_5(tp);
1906                 if (err)
1907                         return err;
1908                 goto out;
1909         }
1910
1911         cpmuctrl = 0;
1912         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 &&
1913             GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5784_AX) {
1914                 cpmuctrl = tr32(TG3_CPMU_CTRL);
1915                 if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY)
1916                         tw32(TG3_CPMU_CTRL,
1917                              cpmuctrl & ~CPMU_CTRL_GPHY_10MB_RXONLY);
1918         }
1919
1920         err = tg3_bmcr_reset(tp);
1921         if (err)
1922                 return err;
1923
1924         if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY) {
1925                 u32 phy;
1926
1927                 phy = MII_TG3_DSP_EXP8_AEDW | MII_TG3_DSP_EXP8_REJ2MHz;
1928                 tg3_phydsp_write(tp, MII_TG3_DSP_EXP8, phy);
1929
1930                 tw32(TG3_CPMU_CTRL, cpmuctrl);
1931         }
1932
1933         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX ||
1934             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5761_AX) {
1935                 u32 val;
1936
1937                 val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
1938                 if ((val & CPMU_LSPD_1000MB_MACCLK_MASK) ==
1939                     CPMU_LSPD_1000MB_MACCLK_12_5) {
1940                         val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
1941                         udelay(40);
1942                         tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
1943                 }
1944         }
1945
1946         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 &&
1947             (tp->tg3_flags2 & TG3_FLG2_MII_SERDES))
1948                 return 0;
1949
1950         tg3_phy_apply_otp(tp);
1951
1952         if (tp->tg3_flags3 & TG3_FLG3_PHY_ENABLE_APD)
1953                 tg3_phy_toggle_apd(tp, true);
1954         else
1955                 tg3_phy_toggle_apd(tp, false);
1956
1957 out:
1958         if (tp->tg3_flags2 & TG3_FLG2_PHY_ADC_BUG) {
1959                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1960                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1961                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x2aaa);
1962                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1963                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0323);
1964                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1965         }
1966         if (tp->tg3_flags2 & TG3_FLG2_PHY_5704_A0_BUG) {
1967                 tg3_writephy(tp, 0x1c, 0x8d68);
1968                 tg3_writephy(tp, 0x1c, 0x8d68);
1969         }
1970         if (tp->tg3_flags2 & TG3_FLG2_PHY_BER_BUG) {
1971                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1972                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1973                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x310b);
1974                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1975                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x9506);
1976                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x401f);
1977                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x14e2);
1978                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1979         }
1980         else if (tp->tg3_flags2 & TG3_FLG2_PHY_JITTER_BUG) {
1981                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1982                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1983                 if (tp->tg3_flags2 & TG3_FLG2_PHY_ADJUST_TRIM) {
1984                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x110b);
1985                         tg3_writephy(tp, MII_TG3_TEST1,
1986                                      MII_TG3_TEST1_TRIM_EN | 0x4);
1987                 } else
1988                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x010b);
1989                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1990         }
1991         /* Set Extended packet length bit (bit 14) on all chips that */
1992         /* support jumbo frames */
1993         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
1994                 /* Cannot do read-modify-write on 5401 */
1995                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1996         } else if (tp->tg3_flags & TG3_FLAG_JUMBO_CAPABLE) {
1997                 u32 phy_reg;
1998
1999                 /* Set bit 14 with read-modify-write to preserve other bits */
2000                 if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0007) &&
2001                     !tg3_readphy(tp, MII_TG3_AUX_CTRL, &phy_reg))
2002                         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy_reg | 0x4000);
2003         }
2004
2005         /* Set phy register 0x10 bit 0 to high fifo elasticity to support
2006          * jumbo frames transmission.
2007          */
2008         if (tp->tg3_flags & TG3_FLAG_JUMBO_CAPABLE) {
2009                 u32 phy_reg;
2010
2011                 if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &phy_reg))
2012                     tg3_writephy(tp, MII_TG3_EXT_CTRL,
2013                                  phy_reg | MII_TG3_EXT_CTRL_FIFO_ELASTIC);
2014         }
2015
2016         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2017                 /* adjust output voltage */
2018                 tg3_writephy(tp, MII_TG3_FET_PTEST, 0x12);
2019         }
2020
2021         tg3_phy_toggle_automdix(tp, 1);
2022         tg3_phy_set_wirespeed(tp);
2023         return 0;
2024 }
2025
2026 static void tg3_frob_aux_power(struct tg3 *tp)
2027 {
2028         struct tg3 *tp_peer = tp;
2029
2030         /* The GPIOs do something completely different on 57765. */
2031         if ((tp->tg3_flags2 & TG3_FLG2_IS_NIC) == 0 ||
2032             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57765)
2033                 return;
2034
2035         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
2036             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714 ||
2037             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717) {
2038                 struct net_device *dev_peer;
2039
2040                 dev_peer = pci_get_drvdata(tp->pdev_peer);
2041                 /* remove_one() may have been run on the peer. */
2042                 if (!dev_peer)
2043                         tp_peer = tp;
2044                 else
2045                         tp_peer = netdev_priv(dev_peer);
2046         }
2047
2048         if ((tp->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
2049             (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0 ||
2050             (tp_peer->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
2051             (tp_peer->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0) {
2052                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2053                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2054                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2055                                     (GRC_LCLCTRL_GPIO_OE0 |
2056                                      GRC_LCLCTRL_GPIO_OE1 |
2057                                      GRC_LCLCTRL_GPIO_OE2 |
2058                                      GRC_LCLCTRL_GPIO_OUTPUT0 |
2059                                      GRC_LCLCTRL_GPIO_OUTPUT1),
2060                                     100);
2061                 } else if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761 ||
2062                            tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761S) {
2063                         /* The 5761 non-e device swaps GPIO 0 and GPIO 2. */
2064                         u32 grc_local_ctrl = GRC_LCLCTRL_GPIO_OE0 |
2065                                              GRC_LCLCTRL_GPIO_OE1 |
2066                                              GRC_LCLCTRL_GPIO_OE2 |
2067                                              GRC_LCLCTRL_GPIO_OUTPUT0 |
2068                                              GRC_LCLCTRL_GPIO_OUTPUT1 |
2069                                              tp->grc_local_ctrl;
2070                         tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl, 100);
2071
2072                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT2;
2073                         tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl, 100);
2074
2075                         grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT0;
2076                         tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl, 100);
2077                 } else {
2078                         u32 no_gpio2;
2079                         u32 grc_local_ctrl = 0;
2080
2081                         if (tp_peer != tp &&
2082                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
2083                                 return;
2084
2085                         /* Workaround to prevent overdrawing Amps. */
2086                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
2087                             ASIC_REV_5714) {
2088                                 grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
2089                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2090                                             grc_local_ctrl, 100);
2091                         }
2092
2093                         /* On 5753 and variants, GPIO2 cannot be used. */
2094                         no_gpio2 = tp->nic_sram_data_cfg &
2095                                     NIC_SRAM_DATA_CFG_NO_GPIO2;
2096
2097                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
2098                                          GRC_LCLCTRL_GPIO_OE1 |
2099                                          GRC_LCLCTRL_GPIO_OE2 |
2100                                          GRC_LCLCTRL_GPIO_OUTPUT1 |
2101                                          GRC_LCLCTRL_GPIO_OUTPUT2;
2102                         if (no_gpio2) {
2103                                 grc_local_ctrl &= ~(GRC_LCLCTRL_GPIO_OE2 |
2104                                                     GRC_LCLCTRL_GPIO_OUTPUT2);
2105                         }
2106                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2107                                                     grc_local_ctrl, 100);
2108
2109                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT0;
2110
2111                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2112                                                     grc_local_ctrl, 100);
2113
2114                         if (!no_gpio2) {
2115                                 grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT2;
2116                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2117                                             grc_local_ctrl, 100);
2118                         }
2119                 }
2120         } else {
2121                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
2122                     GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
2123                         if (tp_peer != tp &&
2124                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
2125                                 return;
2126
2127                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2128                                     (GRC_LCLCTRL_GPIO_OE1 |
2129                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
2130
2131                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2132                                     GRC_LCLCTRL_GPIO_OE1, 100);
2133
2134                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2135                                     (GRC_LCLCTRL_GPIO_OE1 |
2136                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
2137                 }
2138         }
2139 }
2140
2141 static int tg3_5700_link_polarity(struct tg3 *tp, u32 speed)
2142 {
2143         if (tp->led_ctrl == LED_CTRL_MODE_PHY_2)
2144                 return 1;
2145         else if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411) {
2146                 if (speed != SPEED_10)
2147                         return 1;
2148         } else if (speed == SPEED_10)
2149                 return 1;
2150
2151         return 0;
2152 }
2153
2154 static int tg3_setup_phy(struct tg3 *, int);
2155
2156 #define RESET_KIND_SHUTDOWN     0
2157 #define RESET_KIND_INIT         1
2158 #define RESET_KIND_SUSPEND      2
2159
2160 static void tg3_write_sig_post_reset(struct tg3 *, int);
2161 static int tg3_halt_cpu(struct tg3 *, u32);
2162
2163 static void tg3_power_down_phy(struct tg3 *tp, bool do_low_power)
2164 {
2165         u32 val;
2166
2167         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
2168                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
2169                         u32 sg_dig_ctrl = tr32(SG_DIG_CTRL);
2170                         u32 serdes_cfg = tr32(MAC_SERDES_CFG);
2171
2172                         sg_dig_ctrl |=
2173                                 SG_DIG_USING_HW_AUTONEG | SG_DIG_SOFT_RESET;
2174                         tw32(SG_DIG_CTRL, sg_dig_ctrl);
2175                         tw32(MAC_SERDES_CFG, serdes_cfg | (1 << 15));
2176                 }
2177                 return;
2178         }
2179
2180         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2181                 tg3_bmcr_reset(tp);
2182                 val = tr32(GRC_MISC_CFG);
2183                 tw32_f(GRC_MISC_CFG, val | GRC_MISC_CFG_EPHY_IDDQ);
2184                 udelay(40);
2185                 return;
2186         } else if (tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET) {
2187                 u32 phytest;
2188                 if (!tg3_readphy(tp, MII_TG3_FET_TEST, &phytest)) {
2189                         u32 phy;
2190
2191                         tg3_writephy(tp, MII_ADVERTISE, 0);
2192                         tg3_writephy(tp, MII_BMCR,
2193                                      BMCR_ANENABLE | BMCR_ANRESTART);
2194
2195                         tg3_writephy(tp, MII_TG3_FET_TEST,
2196                                      phytest | MII_TG3_FET_SHADOW_EN);
2197                         if (!tg3_readphy(tp, MII_TG3_FET_SHDW_AUXMODE4, &phy)) {
2198                                 phy |= MII_TG3_FET_SHDW_AUXMODE4_SBPD;
2199                                 tg3_writephy(tp,
2200                                              MII_TG3_FET_SHDW_AUXMODE4,
2201                                              phy);
2202                         }
2203                         tg3_writephy(tp, MII_TG3_FET_TEST, phytest);
2204                 }
2205                 return;
2206         } else if (do_low_power) {
2207                 tg3_writephy(tp, MII_TG3_EXT_CTRL,
2208                              MII_TG3_EXT_CTRL_FORCE_LED_OFF);
2209
2210                 tg3_writephy(tp, MII_TG3_AUX_CTRL,
2211                              MII_TG3_AUXCTL_SHDWSEL_PWRCTL |
2212                              MII_TG3_AUXCTL_PCTL_100TX_LPWR |
2213                              MII_TG3_AUXCTL_PCTL_SPR_ISOLATE |
2214                              MII_TG3_AUXCTL_PCTL_VREG_11V);
2215         }
2216
2217         /* The PHY should not be powered down on some chips because
2218          * of bugs.
2219          */
2220         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2221             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
2222             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 &&
2223              (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)))
2224                 return;
2225
2226         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX ||
2227             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5761_AX) {
2228                 val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
2229                 val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
2230                 val |= CPMU_LSPD_1000MB_MACCLK_12_5;
2231                 tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
2232         }
2233
2234         tg3_writephy(tp, MII_BMCR, BMCR_PDOWN);
2235 }
2236
2237 /* tp->lock is held. */
2238 static int tg3_nvram_lock(struct tg3 *tp)
2239 {
2240         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
2241                 int i;
2242
2243                 if (tp->nvram_lock_cnt == 0) {
2244                         tw32(NVRAM_SWARB, SWARB_REQ_SET1);
2245                         for (i = 0; i < 8000; i++) {
2246                                 if (tr32(NVRAM_SWARB) & SWARB_GNT1)
2247                                         break;
2248                                 udelay(20);
2249                         }
2250                         if (i == 8000) {
2251                                 tw32(NVRAM_SWARB, SWARB_REQ_CLR1);
2252                                 return -ENODEV;
2253                         }
2254                 }
2255                 tp->nvram_lock_cnt++;
2256         }
2257         return 0;
2258 }
2259
2260 /* tp->lock is held. */
2261 static void tg3_nvram_unlock(struct tg3 *tp)
2262 {
2263         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
2264                 if (tp->nvram_lock_cnt > 0)
2265                         tp->nvram_lock_cnt--;
2266                 if (tp->nvram_lock_cnt == 0)
2267                         tw32_f(NVRAM_SWARB, SWARB_REQ_CLR1);
2268         }
2269 }
2270
2271 /* tp->lock is held. */
2272 static void tg3_enable_nvram_access(struct tg3 *tp)
2273 {
2274         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
2275             !(tp->tg3_flags3 & TG3_FLG3_PROTECTED_NVRAM)) {
2276                 u32 nvaccess = tr32(NVRAM_ACCESS);
2277
2278                 tw32(NVRAM_ACCESS, nvaccess | ACCESS_ENABLE);
2279         }
2280 }
2281
2282 /* tp->lock is held. */
2283 static void tg3_disable_nvram_access(struct tg3 *tp)
2284 {
2285         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
2286             !(tp->tg3_flags3 & TG3_FLG3_PROTECTED_NVRAM)) {
2287                 u32 nvaccess = tr32(NVRAM_ACCESS);
2288
2289                 tw32(NVRAM_ACCESS, nvaccess & ~ACCESS_ENABLE);
2290         }
2291 }
2292
2293 static int tg3_nvram_read_using_eeprom(struct tg3 *tp,
2294                                         u32 offset, u32 *val)
2295 {
2296         u32 tmp;
2297         int i;
2298
2299         if (offset > EEPROM_ADDR_ADDR_MASK || (offset % 4) != 0)
2300                 return -EINVAL;
2301
2302         tmp = tr32(GRC_EEPROM_ADDR) & ~(EEPROM_ADDR_ADDR_MASK |
2303                                         EEPROM_ADDR_DEVID_MASK |
2304                                         EEPROM_ADDR_READ);
2305         tw32(GRC_EEPROM_ADDR,
2306              tmp |
2307              (0 << EEPROM_ADDR_DEVID_SHIFT) |
2308              ((offset << EEPROM_ADDR_ADDR_SHIFT) &
2309               EEPROM_ADDR_ADDR_MASK) |
2310              EEPROM_ADDR_READ | EEPROM_ADDR_START);
2311
2312         for (i = 0; i < 1000; i++) {
2313                 tmp = tr32(GRC_EEPROM_ADDR);
2314
2315                 if (tmp & EEPROM_ADDR_COMPLETE)
2316                         break;
2317                 msleep(1);
2318         }
2319         if (!(tmp & EEPROM_ADDR_COMPLETE))
2320                 return -EBUSY;
2321
2322         tmp = tr32(GRC_EEPROM_DATA);
2323
2324         /*
2325          * The data will always be opposite the native endian
2326          * format.  Perform a blind byteswap to compensate.
2327          */
2328         *val = swab32(tmp);
2329
2330         return 0;
2331 }
2332
2333 #define NVRAM_CMD_TIMEOUT 10000
2334
2335 static int tg3_nvram_exec_cmd(struct tg3 *tp, u32 nvram_cmd)
2336 {
2337         int i;
2338
2339         tw32(NVRAM_CMD, nvram_cmd);
2340         for (i = 0; i < NVRAM_CMD_TIMEOUT; i++) {
2341                 udelay(10);
2342                 if (tr32(NVRAM_CMD) & NVRAM_CMD_DONE) {
2343                         udelay(10);
2344                         break;
2345                 }
2346         }
2347
2348         if (i == NVRAM_CMD_TIMEOUT)
2349                 return -EBUSY;
2350
2351         return 0;
2352 }
2353
2354 static u32 tg3_nvram_phys_addr(struct tg3 *tp, u32 addr)
2355 {
2356         if ((tp->tg3_flags & TG3_FLAG_NVRAM) &&
2357             (tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) &&
2358             (tp->tg3_flags2 & TG3_FLG2_FLASH) &&
2359            !(tp->tg3_flags3 & TG3_FLG3_NO_NVRAM_ADDR_TRANS) &&
2360             (tp->nvram_jedecnum == JEDEC_ATMEL))
2361
2362                 addr = ((addr / tp->nvram_pagesize) <<
2363                         ATMEL_AT45DB0X1B_PAGE_POS) +
2364                        (addr % tp->nvram_pagesize);
2365
2366         return addr;
2367 }
2368
2369 static u32 tg3_nvram_logical_addr(struct tg3 *tp, u32 addr)
2370 {
2371         if ((tp->tg3_flags & TG3_FLAG_NVRAM) &&
2372             (tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) &&
2373             (tp->tg3_flags2 & TG3_FLG2_FLASH) &&
2374            !(tp->tg3_flags3 & TG3_FLG3_NO_NVRAM_ADDR_TRANS) &&
2375             (tp->nvram_jedecnum == JEDEC_ATMEL))
2376
2377                 addr = ((addr >> ATMEL_AT45DB0X1B_PAGE_POS) *
2378                         tp->nvram_pagesize) +
2379                        (addr & ((1 << ATMEL_AT45DB0X1B_PAGE_POS) - 1));
2380
2381         return addr;
2382 }
2383
2384 /* NOTE: Data read in from NVRAM is byteswapped according to
2385  * the byteswapping settings for all other register accesses.
2386  * tg3 devices are BE devices, so on a BE machine, the data
2387  * returned will be exactly as it is seen in NVRAM.  On a LE
2388  * machine, the 32-bit value will be byteswapped.
2389  */
2390 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val)
2391 {
2392         int ret;
2393
2394         if (!(tp->tg3_flags & TG3_FLAG_NVRAM))
2395                 return tg3_nvram_read_using_eeprom(tp, offset, val);
2396
2397         offset = tg3_nvram_phys_addr(tp, offset);
2398
2399         if (offset > NVRAM_ADDR_MSK)
2400                 return -EINVAL;
2401
2402         ret = tg3_nvram_lock(tp);
2403         if (ret)
2404                 return ret;
2405
2406         tg3_enable_nvram_access(tp);
2407
2408         tw32(NVRAM_ADDR, offset);
2409         ret = tg3_nvram_exec_cmd(tp, NVRAM_CMD_RD | NVRAM_CMD_GO |
2410                 NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_DONE);
2411
2412         if (ret == 0)
2413                 *val = tr32(NVRAM_RDDATA);
2414
2415         tg3_disable_nvram_access(tp);
2416
2417         tg3_nvram_unlock(tp);
2418
2419         return ret;
2420 }
2421
2422 /* Ensures NVRAM data is in bytestream format. */
2423 static int tg3_nvram_read_be32(struct tg3 *tp, u32 offset, __be32 *val)
2424 {
2425         u32 v;
2426         int res = tg3_nvram_read(tp, offset, &v);
2427         if (!res)
2428                 *val = cpu_to_be32(v);
2429         return res;
2430 }
2431
2432 /* tp->lock is held. */
2433 static void __tg3_set_mac_addr(struct tg3 *tp, int skip_mac_1)
2434 {
2435         u32 addr_high, addr_low;
2436         int i;
2437
2438         addr_high = ((tp->dev->dev_addr[0] << 8) |
2439                      tp->dev->dev_addr[1]);
2440         addr_low = ((tp->dev->dev_addr[2] << 24) |
2441                     (tp->dev->dev_addr[3] << 16) |
2442                     (tp->dev->dev_addr[4] <<  8) |
2443                     (tp->dev->dev_addr[5] <<  0));
2444         for (i = 0; i < 4; i++) {
2445                 if (i == 1 && skip_mac_1)
2446                         continue;
2447                 tw32(MAC_ADDR_0_HIGH + (i * 8), addr_high);
2448                 tw32(MAC_ADDR_0_LOW + (i * 8), addr_low);
2449         }
2450
2451         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
2452             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
2453                 for (i = 0; i < 12; i++) {
2454                         tw32(MAC_EXTADDR_0_HIGH + (i * 8), addr_high);
2455                         tw32(MAC_EXTADDR_0_LOW + (i * 8), addr_low);
2456                 }
2457         }
2458
2459         addr_high = (tp->dev->dev_addr[0] +
2460                      tp->dev->dev_addr[1] +
2461                      tp->dev->dev_addr[2] +
2462                      tp->dev->dev_addr[3] +
2463                      tp->dev->dev_addr[4] +
2464                      tp->dev->dev_addr[5]) &
2465                 TX_BACKOFF_SEED_MASK;
2466         tw32(MAC_TX_BACKOFF_SEED, addr_high);
2467 }
2468
2469 static int tg3_set_power_state(struct tg3 *tp, pci_power_t state)
2470 {
2471         u32 misc_host_ctrl;
2472         bool device_should_wake, do_low_power;
2473
2474         /* Make sure register accesses (indirect or otherwise)
2475          * will function correctly.
2476          */
2477         pci_write_config_dword(tp->pdev,
2478                                TG3PCI_MISC_HOST_CTRL,
2479                                tp->misc_host_ctrl);
2480
2481         switch (state) {
2482         case PCI_D0:
2483                 pci_enable_wake(tp->pdev, state, false);
2484                 pci_set_power_state(tp->pdev, PCI_D0);
2485
2486                 /* Switch out of Vaux if it is a NIC */
2487                 if (tp->tg3_flags2 & TG3_FLG2_IS_NIC)
2488                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl, 100);
2489
2490                 return 0;
2491
2492         case PCI_D1:
2493         case PCI_D2:
2494         case PCI_D3hot:
2495                 break;
2496
2497         default:
2498                 printk(KERN_ERR PFX "%s: Invalid power state (D%d) requested\n",
2499                         tp->dev->name, state);
2500                 return -EINVAL;
2501         }
2502
2503         /* Restore the CLKREQ setting. */
2504         if (tp->tg3_flags3 & TG3_FLG3_CLKREQ_BUG) {
2505                 u16 lnkctl;
2506
2507                 pci_read_config_word(tp->pdev,
2508                                      tp->pcie_cap + PCI_EXP_LNKCTL,
2509                                      &lnkctl);
2510                 lnkctl |= PCI_EXP_LNKCTL_CLKREQ_EN;
2511                 pci_write_config_word(tp->pdev,
2512                                       tp->pcie_cap + PCI_EXP_LNKCTL,
2513                                       lnkctl);
2514         }
2515
2516         misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
2517         tw32(TG3PCI_MISC_HOST_CTRL,
2518              misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT);
2519
2520         device_should_wake = pci_pme_capable(tp->pdev, state) &&
2521                              device_may_wakeup(&tp->pdev->dev) &&
2522                              (tp->tg3_flags & TG3_FLAG_WOL_ENABLE);
2523
2524         if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) {
2525                 do_low_power = false;
2526                 if ((tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED) &&
2527                     !tp->link_config.phy_is_low_power) {
2528                         struct phy_device *phydev;
2529                         u32 phyid, advertising;
2530
2531                         phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
2532
2533                         tp->link_config.phy_is_low_power = 1;
2534
2535                         tp->link_config.orig_speed = phydev->speed;
2536                         tp->link_config.orig_duplex = phydev->duplex;
2537                         tp->link_config.orig_autoneg = phydev->autoneg;
2538                         tp->link_config.orig_advertising = phydev->advertising;
2539
2540                         advertising = ADVERTISED_TP |
2541                                       ADVERTISED_Pause |
2542                                       ADVERTISED_Autoneg |
2543                                       ADVERTISED_10baseT_Half;
2544
2545                         if ((tp->tg3_flags & TG3_FLAG_ENABLE_ASF) ||
2546                             device_should_wake) {
2547                                 if (tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB)
2548                                         advertising |=
2549                                                 ADVERTISED_100baseT_Half |
2550                                                 ADVERTISED_100baseT_Full |
2551                                                 ADVERTISED_10baseT_Full;
2552                                 else
2553                                         advertising |= ADVERTISED_10baseT_Full;
2554                         }
2555
2556                         phydev->advertising = advertising;
2557
2558                         phy_start_aneg(phydev);
2559
2560                         phyid = phydev->drv->phy_id & phydev->drv->phy_id_mask;
2561                         if (phyid != TG3_PHY_ID_BCMAC131) {
2562                                 phyid &= TG3_PHY_OUI_MASK;
2563                                 if (phyid == TG3_PHY_OUI_1 ||
2564                                     phyid == TG3_PHY_OUI_2 ||
2565                                     phyid == TG3_PHY_OUI_3)
2566                                         do_low_power = true;
2567                         }
2568                 }
2569         } else {
2570                 do_low_power = true;
2571
2572                 if (tp->link_config.phy_is_low_power == 0) {
2573                         tp->link_config.phy_is_low_power = 1;
2574                         tp->link_config.orig_speed = tp->link_config.speed;
2575                         tp->link_config.orig_duplex = tp->link_config.duplex;
2576                         tp->link_config.orig_autoneg = tp->link_config.autoneg;
2577                 }
2578
2579                 if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)) {
2580                         tp->link_config.speed = SPEED_10;
2581                         tp->link_config.duplex = DUPLEX_HALF;
2582                         tp->link_config.autoneg = AUTONEG_ENABLE;
2583                         tg3_setup_phy(tp, 0);
2584                 }
2585         }
2586
2587         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2588                 u32 val;
2589
2590                 val = tr32(GRC_VCPU_EXT_CTRL);
2591                 tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_DISABLE_WOL);
2592         } else if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
2593                 int i;
2594                 u32 val;
2595
2596                 for (i = 0; i < 200; i++) {
2597                         tg3_read_mem(tp, NIC_SRAM_FW_ASF_STATUS_MBOX, &val);
2598                         if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
2599                                 break;
2600                         msleep(1);
2601                 }
2602         }
2603         if (tp->tg3_flags & TG3_FLAG_WOL_CAP)
2604                 tg3_write_mem(tp, NIC_SRAM_WOL_MBOX, WOL_SIGNATURE |
2605                                                      WOL_DRV_STATE_SHUTDOWN |
2606                                                      WOL_DRV_WOL |
2607                                                      WOL_SET_MAGIC_PKT);
2608
2609         if (device_should_wake) {
2610                 u32 mac_mode;
2611
2612                 if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
2613                         if (do_low_power) {
2614                                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x5a);
2615                                 udelay(40);
2616                         }
2617
2618                         if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
2619                                 mac_mode = MAC_MODE_PORT_MODE_GMII;
2620                         else
2621                                 mac_mode = MAC_MODE_PORT_MODE_MII;
2622
2623                         mac_mode |= tp->mac_mode & MAC_MODE_LINK_POLARITY;
2624                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
2625                             ASIC_REV_5700) {
2626                                 u32 speed = (tp->tg3_flags &
2627                                              TG3_FLAG_WOL_SPEED_100MB) ?
2628                                              SPEED_100 : SPEED_10;
2629                                 if (tg3_5700_link_polarity(tp, speed))
2630                                         mac_mode |= MAC_MODE_LINK_POLARITY;
2631                                 else
2632                                         mac_mode &= ~MAC_MODE_LINK_POLARITY;
2633                         }
2634                 } else {
2635                         mac_mode = MAC_MODE_PORT_MODE_TBI;
2636                 }
2637
2638                 if (!(tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
2639                         tw32(MAC_LED_CTRL, tp->led_ctrl);
2640
2641                 mac_mode |= MAC_MODE_MAGIC_PKT_ENABLE;
2642                 if (((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) &&
2643                     !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) &&
2644                     ((tp->tg3_flags & TG3_FLAG_ENABLE_ASF) ||
2645                      (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)))
2646                         mac_mode |= MAC_MODE_KEEP_FRAME_IN_WOL;
2647
2648                 if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) {
2649                         mac_mode |= tp->mac_mode &
2650                                     (MAC_MODE_APE_TX_EN | MAC_MODE_APE_RX_EN);
2651                         if (mac_mode & MAC_MODE_APE_TX_EN)
2652                                 mac_mode |= MAC_MODE_TDE_ENABLE;
2653                 }
2654
2655                 tw32_f(MAC_MODE, mac_mode);
2656                 udelay(100);
2657
2658                 tw32_f(MAC_RX_MODE, RX_MODE_ENABLE);
2659                 udelay(10);
2660         }
2661
2662         if (!(tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB) &&
2663             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2664              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
2665                 u32 base_val;
2666
2667                 base_val = tp->pci_clock_ctrl;
2668                 base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
2669                              CLOCK_CTRL_TXCLK_DISABLE);
2670
2671                 tw32_wait_f(TG3PCI_CLOCK_CTRL, base_val | CLOCK_CTRL_ALTCLK |
2672                             CLOCK_CTRL_PWRDOWN_PLL133, 40);
2673         } else if ((tp->tg3_flags2 & TG3_FLG2_5780_CLASS) ||
2674                    (tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) ||
2675                    (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)) {
2676                 /* do nothing */
2677         } else if (!((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
2678                      (tp->tg3_flags & TG3_FLAG_ENABLE_ASF))) {
2679                 u32 newbits1, newbits2;
2680
2681                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2682                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2683                         newbits1 = (CLOCK_CTRL_RXCLK_DISABLE |
2684                                     CLOCK_CTRL_TXCLK_DISABLE |
2685                                     CLOCK_CTRL_ALTCLK);
2686                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
2687                 } else if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
2688                         newbits1 = CLOCK_CTRL_625_CORE;
2689                         newbits2 = newbits1 | CLOCK_CTRL_ALTCLK;
2690                 } else {
2691                         newbits1 = CLOCK_CTRL_ALTCLK;
2692                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
2693                 }
2694
2695                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits1,
2696                             40);
2697
2698                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits2,
2699                             40);
2700
2701                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
2702                         u32 newbits3;
2703
2704                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2705                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2706                                 newbits3 = (CLOCK_CTRL_RXCLK_DISABLE |
2707                                             CLOCK_CTRL_TXCLK_DISABLE |
2708                                             CLOCK_CTRL_44MHZ_CORE);
2709                         } else {
2710                                 newbits3 = CLOCK_CTRL_44MHZ_CORE;
2711                         }
2712
2713                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
2714                                     tp->pci_clock_ctrl | newbits3, 40);
2715                 }
2716         }
2717
2718         if (!(device_should_wake) &&
2719             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF))
2720                 tg3_power_down_phy(tp, do_low_power);
2721
2722         tg3_frob_aux_power(tp);
2723
2724         /* Workaround for unstable PLL clock */
2725         if ((GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX) ||
2726             (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX)) {
2727                 u32 val = tr32(0x7d00);
2728
2729                 val &= ~((1 << 16) | (1 << 4) | (1 << 2) | (1 << 1) | 1);
2730                 tw32(0x7d00, val);
2731                 if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
2732                         int err;
2733
2734                         err = tg3_nvram_lock(tp);
2735                         tg3_halt_cpu(tp, RX_CPU_BASE);
2736                         if (!err)
2737                                 tg3_nvram_unlock(tp);
2738                 }
2739         }
2740
2741         tg3_write_sig_post_reset(tp, RESET_KIND_SHUTDOWN);
2742
2743         if (device_should_wake)
2744                 pci_enable_wake(tp->pdev, state, true);
2745
2746         /* Finally, set the new power state. */
2747         pci_set_power_state(tp->pdev, state);
2748
2749         return 0;
2750 }
2751
2752 static void tg3_aux_stat_to_speed_duplex(struct tg3 *tp, u32 val, u16 *speed, u8 *duplex)
2753 {
2754         switch (val & MII_TG3_AUX_STAT_SPDMASK) {
2755         case MII_TG3_AUX_STAT_10HALF:
2756                 *speed = SPEED_10;
2757                 *duplex = DUPLEX_HALF;
2758                 break;
2759
2760         case MII_TG3_AUX_STAT_10FULL:
2761                 *speed = SPEED_10;
2762                 *duplex = DUPLEX_FULL;
2763                 break;
2764
2765         case MII_TG3_AUX_STAT_100HALF:
2766                 *speed = SPEED_100;
2767                 *duplex = DUPLEX_HALF;
2768                 break;
2769
2770         case MII_TG3_AUX_STAT_100FULL:
2771                 *speed = SPEED_100;
2772                 *duplex = DUPLEX_FULL;
2773                 break;
2774
2775         case MII_TG3_AUX_STAT_1000HALF:
2776                 *speed = SPEED_1000;
2777                 *duplex = DUPLEX_HALF;
2778                 break;
2779
2780         case MII_TG3_AUX_STAT_1000FULL:
2781                 *speed = SPEED_1000;
2782                 *duplex = DUPLEX_FULL;
2783                 break;
2784
2785         default:
2786                 if (tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET) {
2787                         *speed = (val & MII_TG3_AUX_STAT_100) ? SPEED_100 :
2788                                  SPEED_10;
2789                         *duplex = (val & MII_TG3_AUX_STAT_FULL) ? DUPLEX_FULL :
2790                                   DUPLEX_HALF;
2791                         break;
2792                 }
2793                 *speed = SPEED_INVALID;
2794                 *duplex = DUPLEX_INVALID;
2795                 break;
2796         }
2797 }
2798
2799 static void tg3_phy_copper_begin(struct tg3 *tp)
2800 {
2801         u32 new_adv;
2802         int i;
2803
2804         if (tp->link_config.phy_is_low_power) {
2805                 /* Entering low power mode.  Disable gigabit and
2806                  * 100baseT advertisements.
2807                  */
2808                 tg3_writephy(tp, MII_TG3_CTRL, 0);
2809
2810                 new_adv = (ADVERTISE_10HALF | ADVERTISE_10FULL |
2811                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
2812                 if (tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB)
2813                         new_adv |= (ADVERTISE_100HALF | ADVERTISE_100FULL);
2814
2815                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
2816         } else if (tp->link_config.speed == SPEED_INVALID) {
2817                 if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
2818                         tp->link_config.advertising &=
2819                                 ~(ADVERTISED_1000baseT_Half |
2820                                   ADVERTISED_1000baseT_Full);
2821
2822                 new_adv = ADVERTISE_CSMA;
2823                 if (tp->link_config.advertising & ADVERTISED_10baseT_Half)
2824                         new_adv |= ADVERTISE_10HALF;
2825                 if (tp->link_config.advertising & ADVERTISED_10baseT_Full)
2826                         new_adv |= ADVERTISE_10FULL;
2827                 if (tp->link_config.advertising & ADVERTISED_100baseT_Half)
2828                         new_adv |= ADVERTISE_100HALF;
2829                 if (tp->link_config.advertising & ADVERTISED_100baseT_Full)
2830                         new_adv |= ADVERTISE_100FULL;
2831
2832                 new_adv |= tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
2833
2834                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
2835
2836                 if (tp->link_config.advertising &
2837                     (ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full)) {
2838                         new_adv = 0;
2839                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
2840                                 new_adv |= MII_TG3_CTRL_ADV_1000_HALF;
2841                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
2842                                 new_adv |= MII_TG3_CTRL_ADV_1000_FULL;
2843                         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY) &&
2844                             (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
2845                              tp->pci_chip_rev_id == CHIPREV_ID_5701_B0))
2846                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
2847                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
2848                         tg3_writephy(tp, MII_TG3_CTRL, new_adv);
2849                 } else {
2850                         tg3_writephy(tp, MII_TG3_CTRL, 0);
2851                 }
2852         } else {
2853                 new_adv = tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
2854                 new_adv |= ADVERTISE_CSMA;
2855
2856                 /* Asking for a specific link mode. */
2857                 if (tp->link_config.speed == SPEED_1000) {
2858                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
2859
2860                         if (tp->link_config.duplex == DUPLEX_FULL)
2861                                 new_adv = MII_TG3_CTRL_ADV_1000_FULL;
2862                         else
2863                                 new_adv = MII_TG3_CTRL_ADV_1000_HALF;
2864                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
2865                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
2866                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
2867                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
2868                 } else {
2869                         if (tp->link_config.speed == SPEED_100) {
2870                                 if (tp->link_config.duplex == DUPLEX_FULL)
2871                                         new_adv |= ADVERTISE_100FULL;
2872                                 else
2873                                         new_adv |= ADVERTISE_100HALF;
2874                         } else {
2875                                 if (tp->link_config.duplex == DUPLEX_FULL)
2876                                         new_adv |= ADVERTISE_10FULL;
2877                                 else
2878                                         new_adv |= ADVERTISE_10HALF;
2879                         }
2880                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
2881
2882                         new_adv = 0;
2883                 }
2884
2885                 tg3_writephy(tp, MII_TG3_CTRL, new_adv);
2886         }
2887
2888         if (tp->link_config.autoneg == AUTONEG_DISABLE &&
2889             tp->link_config.speed != SPEED_INVALID) {
2890                 u32 bmcr, orig_bmcr;
2891
2892                 tp->link_config.active_speed = tp->link_config.speed;
2893                 tp->link_config.active_duplex = tp->link_config.duplex;
2894
2895                 bmcr = 0;
2896                 switch (tp->link_config.speed) {
2897                 default:
2898                 case SPEED_10:
2899                         break;
2900
2901                 case SPEED_100:
2902                         bmcr |= BMCR_SPEED100;
2903                         break;
2904
2905                 case SPEED_1000:
2906                         bmcr |= TG3_BMCR_SPEED1000;
2907                         break;
2908                 }
2909
2910                 if (tp->link_config.duplex == DUPLEX_FULL)
2911                         bmcr |= BMCR_FULLDPLX;
2912
2913                 if (!tg3_readphy(tp, MII_BMCR, &orig_bmcr) &&
2914                     (bmcr != orig_bmcr)) {
2915                         tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK);
2916                         for (i = 0; i < 1500; i++) {
2917                                 u32 tmp;
2918
2919                                 udelay(10);
2920                                 if (tg3_readphy(tp, MII_BMSR, &tmp) ||
2921                                     tg3_readphy(tp, MII_BMSR, &tmp))
2922                                         continue;
2923                                 if (!(tmp & BMSR_LSTATUS)) {
2924                                         udelay(40);
2925                                         break;
2926                                 }
2927                         }
2928                         tg3_writephy(tp, MII_BMCR, bmcr);
2929                         udelay(40);
2930                 }
2931         } else {
2932                 tg3_writephy(tp, MII_BMCR,
2933                              BMCR_ANENABLE | BMCR_ANRESTART);
2934         }
2935 }
2936
2937 static int tg3_init_5401phy_dsp(struct tg3 *tp)
2938 {
2939         int err;
2940
2941         /* Turn off tap power management. */
2942         /* Set Extended packet length bit */
2943         err  = tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
2944
2945         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0012);
2946         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1804);
2947
2948         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0013);
2949         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1204);
2950
2951         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
2952         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0132);
2953
2954         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
2955         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0232);
2956
2957         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
2958         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0a20);
2959
2960         udelay(40);
2961
2962         return err;
2963 }
2964
2965 static int tg3_copper_is_advertising_all(struct tg3 *tp, u32 mask)
2966 {
2967         u32 adv_reg, all_mask = 0;
2968
2969         if (mask & ADVERTISED_10baseT_Half)
2970                 all_mask |= ADVERTISE_10HALF;
2971         if (mask & ADVERTISED_10baseT_Full)
2972                 all_mask |= ADVERTISE_10FULL;
2973         if (mask & ADVERTISED_100baseT_Half)
2974                 all_mask |= ADVERTISE_100HALF;
2975         if (mask & ADVERTISED_100baseT_Full)
2976                 all_mask |= ADVERTISE_100FULL;
2977
2978         if (tg3_readphy(tp, MII_ADVERTISE, &adv_reg))
2979                 return 0;
2980
2981         if ((adv_reg & all_mask) != all_mask)
2982                 return 0;
2983         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
2984                 u32 tg3_ctrl;
2985
2986                 all_mask = 0;
2987                 if (mask & ADVERTISED_1000baseT_Half)
2988                         all_mask |= ADVERTISE_1000HALF;
2989                 if (mask & ADVERTISED_1000baseT_Full)
2990                         all_mask |= ADVERTISE_1000FULL;
2991
2992                 if (tg3_readphy(tp, MII_TG3_CTRL, &tg3_ctrl))
2993                         return 0;
2994
2995                 if ((tg3_ctrl & all_mask) != all_mask)
2996                         return 0;
2997         }
2998         return 1;
2999 }
3000
3001 static int tg3_adv_1000T_flowctrl_ok(struct tg3 *tp, u32 *lcladv, u32 *rmtadv)
3002 {
3003         u32 curadv, reqadv;
3004
3005         if (tg3_readphy(tp, MII_ADVERTISE, lcladv))
3006                 return 1;
3007
3008         curadv = *lcladv & (ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM);
3009         reqadv = tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
3010
3011         if (tp->link_config.active_duplex == DUPLEX_FULL) {
3012                 if (curadv != reqadv)
3013                         return 0;
3014
3015                 if (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG)
3016                         tg3_readphy(tp, MII_LPA, rmtadv);
3017         } else {
3018                 /* Reprogram the advertisement register, even if it
3019                  * does not affect the current link.  If the link
3020                  * gets renegotiated in the future, we can save an
3021                  * additional renegotiation cycle by advertising
3022                  * it correctly in the first place.
3023                  */
3024                 if (curadv != reqadv) {
3025                         *lcladv &= ~(ADVERTISE_PAUSE_CAP |
3026                                      ADVERTISE_PAUSE_ASYM);
3027                         tg3_writephy(tp, MII_ADVERTISE, *lcladv | reqadv);
3028                 }
3029         }
3030
3031         return 1;
3032 }
3033
3034 static int tg3_setup_copper_phy(struct tg3 *tp, int force_reset)
3035 {
3036         int current_link_up;
3037         u32 bmsr, dummy;
3038         u32 lcl_adv, rmt_adv;
3039         u16 current_speed;
3040         u8 current_duplex;
3041         int i, err;
3042
3043         tw32(MAC_EVENT, 0);
3044
3045         tw32_f(MAC_STATUS,
3046              (MAC_STATUS_SYNC_CHANGED |
3047               MAC_STATUS_CFG_CHANGED |
3048               MAC_STATUS_MI_COMPLETION |
3049               MAC_STATUS_LNKSTATE_CHANGED));
3050         udelay(40);
3051
3052         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
3053                 tw32_f(MAC_MI_MODE,
3054                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
3055                 udelay(80);
3056         }
3057
3058         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x02);
3059
3060         /* Some third-party PHYs need to be reset on link going
3061          * down.
3062          */
3063         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
3064              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
3065              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
3066             netif_carrier_ok(tp->dev)) {
3067                 tg3_readphy(tp, MII_BMSR, &bmsr);
3068                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
3069                     !(bmsr & BMSR_LSTATUS))
3070                         force_reset = 1;
3071         }
3072         if (force_reset)
3073                 tg3_phy_reset(tp);
3074
3075         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
3076                 tg3_readphy(tp, MII_BMSR, &bmsr);
3077                 if (tg3_readphy(tp, MII_BMSR, &bmsr) ||
3078                     !(tp->tg3_flags & TG3_FLAG_INIT_COMPLETE))
3079                         bmsr = 0;
3080
3081                 if (!(bmsr & BMSR_LSTATUS)) {
3082                         err = tg3_init_5401phy_dsp(tp);
3083                         if (err)
3084                                 return err;
3085
3086                         tg3_readphy(tp, MII_BMSR, &bmsr);
3087                         for (i = 0; i < 1000; i++) {
3088                                 udelay(10);
3089                                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
3090                                     (bmsr & BMSR_LSTATUS)) {
3091                                         udelay(40);
3092                                         break;
3093                                 }
3094                         }
3095
3096                         if ((tp->phy_id & PHY_ID_REV_MASK) == PHY_REV_BCM5401_B0 &&
3097                             !(bmsr & BMSR_LSTATUS) &&
3098                             tp->link_config.active_speed == SPEED_1000) {
3099                                 err = tg3_phy_reset(tp);
3100                                 if (!err)
3101                                         err = tg3_init_5401phy_dsp(tp);
3102                                 if (err)
3103                                         return err;
3104                         }
3105                 }
3106         } else if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
3107                    tp->pci_chip_rev_id == CHIPREV_ID_5701_B0) {
3108                 /* 5701 {A0,B0} CRC bug workaround */
3109                 tg3_writephy(tp, 0x15, 0x0a75);
3110                 tg3_writephy(tp, 0x1c, 0x8c68);
3111                 tg3_writephy(tp, 0x1c, 0x8d68);
3112                 tg3_writephy(tp, 0x1c, 0x8c68);
3113         }
3114
3115         /* Clear pending interrupts... */
3116         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
3117         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
3118
3119         if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT)
3120                 tg3_writephy(tp, MII_TG3_IMASK, ~MII_TG3_INT_LINKCHG);
3121         else if (!(tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET))
3122                 tg3_writephy(tp, MII_TG3_IMASK, ~0);
3123
3124         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
3125             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
3126                 if (tp->led_ctrl == LED_CTRL_MODE_PHY_1)
3127                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
3128                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
3129                 else
3130                         tg3_writephy(tp, MII_TG3_EXT_CTRL, 0);
3131         }
3132
3133         current_link_up = 0;
3134         current_speed = SPEED_INVALID;
3135         current_duplex = DUPLEX_INVALID;
3136
3137         if (tp->tg3_flags2 & TG3_FLG2_CAPACITIVE_COUPLING) {
3138                 u32 val;
3139
3140                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4007);
3141                 tg3_readphy(tp, MII_TG3_AUX_CTRL, &val);
3142                 if (!(val & (1 << 10))) {
3143                         val |= (1 << 10);
3144                         tg3_writephy(tp, MII_TG3_AUX_CTRL, val);
3145                         goto relink;
3146                 }
3147         }
3148
3149         bmsr = 0;
3150         for (i = 0; i < 100; i++) {
3151                 tg3_readphy(tp, MII_BMSR, &bmsr);
3152                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
3153                     (bmsr & BMSR_LSTATUS))
3154                         break;
3155                 udelay(40);
3156         }
3157
3158         if (bmsr & BMSR_LSTATUS) {
3159                 u32 aux_stat, bmcr;
3160
3161                 tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
3162                 for (i = 0; i < 2000; i++) {
3163                         udelay(10);
3164                         if (!tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat) &&
3165                             aux_stat)
3166                                 break;
3167                 }
3168
3169                 tg3_aux_stat_to_speed_duplex(tp, aux_stat,
3170                                              &current_speed,
3171                                              &current_duplex);
3172
3173                 bmcr = 0;
3174                 for (i = 0; i < 200; i++) {
3175                         tg3_readphy(tp, MII_BMCR, &bmcr);
3176                         if (tg3_readphy(tp, MII_BMCR, &bmcr))
3177                                 continue;
3178                         if (bmcr && bmcr != 0x7fff)
3179                                 break;
3180                         udelay(10);
3181                 }
3182
3183                 lcl_adv = 0;
3184                 rmt_adv = 0;
3185
3186                 tp->link_config.active_speed = current_speed;
3187                 tp->link_config.active_duplex = current_duplex;
3188
3189                 if (tp->link_config.autoneg == AUTONEG_ENABLE) {
3190                         if ((bmcr & BMCR_ANENABLE) &&
3191                             tg3_copper_is_advertising_all(tp,
3192                                                 tp->link_config.advertising)) {
3193                                 if (tg3_adv_1000T_flowctrl_ok(tp, &lcl_adv,
3194                                                                   &rmt_adv))
3195                                         current_link_up = 1;
3196                         }
3197                 } else {
3198                         if (!(bmcr & BMCR_ANENABLE) &&
3199                             tp->link_config.speed == current_speed &&
3200                             tp->link_config.duplex == current_duplex &&
3201                             tp->link_config.flowctrl ==
3202                             tp->link_config.active_flowctrl) {
3203                                 current_link_up = 1;
3204                         }
3205                 }
3206
3207                 if (current_link_up == 1 &&
3208                     tp->link_config.active_duplex == DUPLEX_FULL)
3209                         tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
3210         }
3211
3212 relink:
3213         if (current_link_up == 0 || tp->link_config.phy_is_low_power) {
3214                 u32 tmp;
3215
3216                 tg3_phy_copper_begin(tp);
3217
3218                 tg3_readphy(tp, MII_BMSR, &tmp);
3219                 if (!tg3_readphy(tp, MII_BMSR, &tmp) &&
3220                     (tmp & BMSR_LSTATUS))
3221                         current_link_up = 1;
3222         }
3223
3224         tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
3225         if (current_link_up == 1) {
3226                 if (tp->link_config.active_speed == SPEED_100 ||
3227                     tp->link_config.active_speed == SPEED_10)
3228                         tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
3229                 else
3230                         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
3231         } else if (tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET)
3232                 tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
3233         else
3234                 tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
3235
3236         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
3237         if (tp->link_config.active_duplex == DUPLEX_HALF)
3238                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
3239
3240         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
3241                 if (current_link_up == 1 &&
3242                     tg3_5700_link_polarity(tp, tp->link_config.active_speed))
3243                         tp->mac_mode |= MAC_MODE_LINK_POLARITY;
3244                 else
3245                         tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
3246         }
3247
3248         /* ??? Without this setting Netgear GA302T PHY does not
3249          * ??? send/receive packets...
3250          */
3251         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411 &&
3252             tp->pci_chip_rev_id == CHIPREV_ID_5700_ALTIMA) {
3253                 tp->mi_mode |= MAC_MI_MODE_AUTO_POLL;
3254                 tw32_f(MAC_MI_MODE, tp->mi_mode);
3255                 udelay(80);
3256         }
3257
3258         tw32_f(MAC_MODE, tp->mac_mode);
3259         udelay(40);
3260
3261         if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
3262                 /* Polled via timer. */
3263                 tw32_f(MAC_EVENT, 0);
3264         } else {
3265                 tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
3266         }
3267         udelay(40);
3268
3269         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 &&
3270             current_link_up == 1 &&
3271             tp->link_config.active_speed == SPEED_1000 &&
3272             ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) ||
3273              (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED))) {
3274                 udelay(120);
3275                 tw32_f(MAC_STATUS,
3276                      (MAC_STATUS_SYNC_CHANGED |
3277                       MAC_STATUS_CFG_CHANGED));
3278                 udelay(40);
3279                 tg3_write_mem(tp,
3280                               NIC_SRAM_FIRMWARE_MBOX,
3281                               NIC_SRAM_FIRMWARE_MBOX_MAGIC2);
3282         }
3283
3284         /* Prevent send BD corruption. */
3285         if (tp->tg3_flags3 & TG3_FLG3_CLKREQ_BUG) {
3286                 u16 oldlnkctl, newlnkctl;
3287
3288                 pci_read_config_word(tp->pdev,
3289                                      tp->pcie_cap + PCI_EXP_LNKCTL,
3290                                      &oldlnkctl);
3291                 if (tp->link_config.active_speed == SPEED_100 ||
3292                     tp->link_config.active_speed == SPEED_10)
3293                         newlnkctl = oldlnkctl & ~PCI_EXP_LNKCTL_CLKREQ_EN;
3294                 else
3295                         newlnkctl = oldlnkctl | PCI_EXP_LNKCTL_CLKREQ_EN;
3296                 if (newlnkctl != oldlnkctl)
3297                         pci_write_config_word(tp->pdev,
3298                                               tp->pcie_cap + PCI_EXP_LNKCTL,
3299                                               newlnkctl);
3300         }
3301
3302         if (current_link_up != netif_carrier_ok(tp->dev)) {
3303                 if (current_link_up)
3304                         netif_carrier_on(tp->dev);
3305                 else
3306                         netif_carrier_off(tp->dev);
3307                 tg3_link_report(tp);
3308         }
3309
3310         return 0;
3311 }
3312
3313 struct tg3_fiber_aneginfo {
3314         int state;
3315 #define ANEG_STATE_UNKNOWN              0
3316 #define ANEG_STATE_AN_ENABLE            1
3317 #define ANEG_STATE_RESTART_INIT         2
3318 #define ANEG_STATE_RESTART              3
3319 #define ANEG_STATE_DISABLE_LINK_OK      4
3320 #define ANEG_STATE_ABILITY_DETECT_INIT  5
3321 #define ANEG_STATE_ABILITY_DETECT       6
3322 #define ANEG_STATE_ACK_DETECT_INIT      7
3323 #define ANEG_STATE_ACK_DETECT           8
3324 #define ANEG_STATE_COMPLETE_ACK_INIT    9
3325 #define ANEG_STATE_COMPLETE_ACK         10
3326 #define ANEG_STATE_IDLE_DETECT_INIT     11
3327 #define ANEG_STATE_IDLE_DETECT          12
3328 #define ANEG_STATE_LINK_OK              13
3329 #define ANEG_STATE_NEXT_PAGE_WAIT_INIT  14
3330 #define ANEG_STATE_NEXT_PAGE_WAIT       15
3331
3332         u32 flags;
3333 #define MR_AN_ENABLE            0x00000001
3334 #define MR_RESTART_AN           0x00000002
3335 #define MR_AN_COMPLETE          0x00000004
3336 #define MR_PAGE_RX              0x00000008
3337 #define MR_NP_LOADED            0x00000010
3338 #define MR_TOGGLE_TX            0x00000020
3339 #define MR_LP_ADV_FULL_DUPLEX   0x00000040
3340 #define MR_LP_ADV_HALF_DUPLEX   0x00000080
3341 #define MR_LP_ADV_SYM_PAUSE     0x00000100
3342 #define MR_LP_ADV_ASYM_PAUSE    0x00000200
3343 #define MR_LP_ADV_REMOTE_FAULT1 0x00000400
3344 #define MR_LP_ADV_REMOTE_FAULT2 0x00000800
3345 #define MR_LP_ADV_NEXT_PAGE     0x00001000
3346 #define MR_TOGGLE_RX            0x00002000
3347 #define MR_NP_RX                0x00004000
3348
3349 #define MR_LINK_OK              0x80000000
3350
3351         unsigned long link_time, cur_time;
3352
3353         u32 ability_match_cfg;
3354         int ability_match_count;
3355
3356         char ability_match, idle_match, ack_match;
3357
3358         u32 txconfig, rxconfig;
3359 #define ANEG_CFG_NP             0x00000080
3360 #define ANEG_CFG_ACK            0x00000040
3361 #define ANEG_CFG_RF2            0x00000020
3362 #define ANEG_CFG_RF1            0x00000010
3363 #define ANEG_CFG_PS2            0x00000001
3364 #define ANEG_CFG_PS1            0x00008000
3365 #define ANEG_CFG_HD             0x00004000
3366 #define ANEG_CFG_FD             0x00002000
3367 #define ANEG_CFG_INVAL          0x00001f06
3368
3369 };
3370 #define ANEG_OK         0
3371 #define ANEG_DONE       1
3372 #define ANEG_TIMER_ENAB 2
3373 #define ANEG_FAILED     -1
3374
3375 #define ANEG_STATE_SETTLE_TIME  10000
3376
3377 static int tg3_fiber_aneg_smachine(struct tg3 *tp,
3378                                    struct tg3_fiber_aneginfo *ap)
3379 {
3380         u16 flowctrl;
3381         unsigned long delta;
3382         u32 rx_cfg_reg;
3383         int ret;
3384
3385         if (ap->state == ANEG_STATE_UNKNOWN) {
3386                 ap->rxconfig = 0;
3387                 ap->link_time = 0;
3388                 ap->cur_time = 0;
3389                 ap->ability_match_cfg = 0;
3390                 ap->ability_match_count = 0;
3391                 ap->ability_match = 0;
3392                 ap->idle_match = 0;
3393                 ap->ack_match = 0;
3394         }
3395         ap->cur_time++;
3396
3397         if (tr32(MAC_STATUS) & MAC_STATUS_RCVD_CFG) {
3398                 rx_cfg_reg = tr32(MAC_RX_AUTO_NEG);
3399
3400                 if (rx_cfg_reg != ap->ability_match_cfg) {
3401                         ap->ability_match_cfg = rx_cfg_reg;
3402                         ap->ability_match = 0;
3403                         ap->ability_match_count = 0;
3404                 } else {
3405                         if (++ap->ability_match_count > 1) {
3406                                 ap->ability_match = 1;
3407                                 ap->ability_match_cfg = rx_cfg_reg;
3408                         }
3409                 }
3410                 if (rx_cfg_reg & ANEG_CFG_ACK)
3411                         ap->ack_match = 1;
3412                 else
3413                         ap->ack_match = 0;
3414
3415                 ap->idle_match = 0;
3416         } else {
3417                 ap->idle_match = 1;
3418                 ap->ability_match_cfg = 0;
3419                 ap->ability_match_count = 0;
3420                 ap->ability_match = 0;
3421                 ap->ack_match = 0;
3422
3423                 rx_cfg_reg = 0;
3424         }
3425
3426         ap->rxconfig = rx_cfg_reg;
3427         ret = ANEG_OK;
3428
3429         switch(ap->state) {
3430         case ANEG_STATE_UNKNOWN:
3431                 if (ap->flags & (MR_AN_ENABLE | MR_RESTART_AN))
3432                         ap->state = ANEG_STATE_AN_ENABLE;
3433
3434                 /* fallthru */
3435         case ANEG_STATE_AN_ENABLE:
3436                 ap->flags &= ~(MR_AN_COMPLETE | MR_PAGE_RX);
3437                 if (ap->flags & MR_AN_ENABLE) {
3438                         ap->link_time = 0;
3439                         ap->cur_time = 0;
3440                         ap->ability_match_cfg = 0;
3441                         ap->ability_match_count = 0;
3442                         ap->ability_match = 0;
3443                         ap->idle_match = 0;
3444                         ap->ack_match = 0;
3445
3446                         ap->state = ANEG_STATE_RESTART_INIT;
3447                 } else {
3448                         ap->state = ANEG_STATE_DISABLE_LINK_OK;
3449                 }
3450                 break;
3451
3452         case ANEG_STATE_RESTART_INIT:
3453                 ap->link_time = ap->cur_time;
3454                 ap->flags &= ~(MR_NP_LOADED);
3455                 ap->txconfig = 0;
3456                 tw32(MAC_TX_AUTO_NEG, 0);
3457                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
3458                 tw32_f(MAC_MODE, tp->mac_mode);
3459                 udelay(40);
3460
3461                 ret = ANEG_TIMER_ENAB;
3462                 ap->state = ANEG_STATE_RESTART;
3463
3464                 /* fallthru */
3465         case ANEG_STATE_RESTART:
3466                 delta = ap->cur_time - ap->link_time;
3467                 if (delta > ANEG_STATE_SETTLE_TIME) {
3468                         ap->state = ANEG_STATE_ABILITY_DETECT_INIT;
3469                 } else {
3470                         ret = ANEG_TIMER_ENAB;
3471                 }
3472                 break;
3473
3474         case ANEG_STATE_DISABLE_LINK_OK:
3475                 ret = ANEG_DONE;
3476                 break;
3477
3478         case ANEG_STATE_ABILITY_DETECT_INIT:
3479                 ap->flags &= ~(MR_TOGGLE_TX);
3480                 ap->txconfig = ANEG_CFG_FD;
3481                 flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
3482                 if (flowctrl & ADVERTISE_1000XPAUSE)
3483                         ap->txconfig |= ANEG_CFG_PS1;
3484                 if (flowctrl & ADVERTISE_1000XPSE_ASYM)
3485                         ap->txconfig |= ANEG_CFG_PS2;
3486                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
3487                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
3488                 tw32_f(MAC_MODE, tp->mac_mode);
3489                 udelay(40);
3490
3491                 ap->state = ANEG_STATE_ABILITY_DETECT;
3492                 break;
3493
3494         case ANEG_STATE_ABILITY_DETECT:
3495                 if (ap->ability_match != 0 && ap->rxconfig != 0) {
3496                         ap->state = ANEG_STATE_ACK_DETECT_INIT;
3497                 }
3498                 break;
3499
3500         case ANEG_STATE_ACK_DETECT_INIT:
3501                 ap->txconfig |= ANEG_CFG_ACK;
3502                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
3503                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
3504                 tw32_f(MAC_MODE, tp->mac_mode);
3505                 udelay(40);
3506
3507                 ap->state = ANEG_STATE_ACK_DETECT;
3508
3509                 /* fallthru */
3510         case ANEG_STATE_ACK_DETECT:
3511                 if (ap->ack_match != 0) {
3512                         if ((ap->rxconfig & ~ANEG_CFG_ACK) ==
3513                             (ap->ability_match_cfg & ~ANEG_CFG_ACK)) {
3514                                 ap->state = ANEG_STATE_COMPLETE_ACK_INIT;
3515                         } else {
3516                                 ap->state = ANEG_STATE_AN_ENABLE;
3517                         }
3518                 } else if (ap->ability_match != 0 &&
3519                            ap->rxconfig == 0) {
3520                         ap->state = ANEG_STATE_AN_ENABLE;
3521                 }
3522                 break;
3523
3524         case ANEG_STATE_COMPLETE_ACK_INIT:
3525                 if (ap->rxconfig & ANEG_CFG_INVAL) {
3526                         ret = ANEG_FAILED;
3527                         break;
3528                 }
3529                 ap->flags &= ~(MR_LP_ADV_FULL_DUPLEX |
3530                                MR_LP_ADV_HALF_DUPLEX |
3531                                MR_LP_ADV_SYM_PAUSE |
3532                                MR_LP_ADV_ASYM_PAUSE |
3533                                MR_LP_ADV_REMOTE_FAULT1 |
3534                                MR_LP_ADV_REMOTE_FAULT2 |
3535                                MR_LP_ADV_NEXT_PAGE |
3536                                MR_TOGGLE_RX |
3537                                MR_NP_RX);
3538                 if (ap->rxconfig & ANEG_CFG_FD)
3539                         ap->flags |= MR_LP_ADV_FULL_DUPLEX;
3540                 if (ap->rxconfig & ANEG_CFG_HD)
3541                         ap->flags |= MR_LP_ADV_HALF_DUPLEX;
3542                 if (ap->rxconfig & ANEG_CFG_PS1)
3543                         ap->flags |= MR_LP_ADV_SYM_PAUSE;
3544                 if (ap->rxconfig & ANEG_CFG_PS2)
3545                         ap->flags |= MR_LP_ADV_ASYM_PAUSE;
3546                 if (ap->rxconfig & ANEG_CFG_RF1)
3547                         ap->flags |= MR_LP_ADV_REMOTE_FAULT1;
3548                 if (ap->rxconfig & ANEG_CFG_RF2)
3549                         ap->flags |= MR_LP_ADV_REMOTE_FAULT2;
3550                 if (ap->rxconfig & ANEG_CFG_NP)
3551                         ap->flags |= MR_LP_ADV_NEXT_PAGE;
3552
3553                 ap->link_time = ap->cur_time;
3554
3555                 ap->flags ^= (MR_TOGGLE_TX);
3556                 if (ap->rxconfig & 0x0008)
3557                         ap->flags |= MR_TOGGLE_RX;
3558                 if (ap->rxconfig & ANEG_CFG_NP)
3559                         ap->flags |= MR_NP_RX;
3560                 ap->flags |= MR_PAGE_RX;
3561
3562                 ap->state = ANEG_STATE_COMPLETE_ACK;
3563                 ret = ANEG_TIMER_ENAB;
3564                 break;
3565
3566         case ANEG_STATE_COMPLETE_ACK:
3567                 if (ap->ability_match != 0 &&
3568                     ap->rxconfig == 0) {
3569                         ap->state = ANEG_STATE_AN_ENABLE;
3570                         break;
3571                 }
3572                 delta = ap->cur_time - ap->link_time;
3573                 if (delta > ANEG_STATE_SETTLE_TIME) {
3574                         if (!(ap->flags & (MR_LP_ADV_NEXT_PAGE))) {
3575                                 ap->state = ANEG_STATE_IDLE_DETECT_INIT;
3576                         } else {
3577                                 if ((ap->txconfig & ANEG_CFG_NP) == 0 &&
3578                                     !(ap->flags & MR_NP_RX)) {
3579                                         ap->state = ANEG_STATE_IDLE_DETECT_INIT;
3580                                 } else {
3581                                         ret = ANEG_FAILED;
3582                                 }
3583                         }
3584                 }
3585                 break;
3586
3587         case ANEG_STATE_IDLE_DETECT_INIT:
3588                 ap->link_time = ap->cur_time;
3589                 tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
3590                 tw32_f(MAC_MODE, tp->mac_mode);
3591                 udelay(40);
3592
3593                 ap->state = ANEG_STATE_IDLE_DETECT;
3594                 ret = ANEG_TIMER_ENAB;
3595                 break;
3596
3597         case ANEG_STATE_IDLE_DETECT:
3598                 if (ap->ability_match != 0 &&
3599                     ap->rxconfig == 0) {
3600                         ap->state = ANEG_STATE_AN_ENABLE;
3601                         break;
3602                 }
3603                 delta = ap->cur_time - ap->link_time;
3604                 if (delta > ANEG_STATE_SETTLE_TIME) {
3605                         /* XXX another gem from the Broadcom driver :( */
3606                         ap->state = ANEG_STATE_LINK_OK;
3607                 }
3608                 break;
3609
3610         case ANEG_STATE_LINK_OK:
3611                 ap->flags |= (MR_AN_COMPLETE | MR_LINK_OK);
3612                 ret = ANEG_DONE;
3613                 break;
3614
3615         case ANEG_STATE_NEXT_PAGE_WAIT_INIT:
3616                 /* ??? unimplemented */
3617                 break;
3618
3619         case ANEG_STATE_NEXT_PAGE_WAIT:
3620                 /* ??? unimplemented */
3621                 break;
3622
3623         default:
3624                 ret = ANEG_FAILED;
3625                 break;
3626         }
3627
3628         return ret;
3629 }
3630
3631 static int fiber_autoneg(struct tg3 *tp, u32 *txflags, u32 *rxflags)
3632 {
3633         int res = 0;
3634         struct tg3_fiber_aneginfo aninfo;
3635         int status = ANEG_FAILED;
3636         unsigned int tick;
3637         u32 tmp;
3638
3639         tw32_f(MAC_TX_AUTO_NEG, 0);
3640
3641         tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
3642         tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
3643         udelay(40);
3644
3645         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
3646         udelay(40);
3647
3648         memset(&aninfo, 0, sizeof(aninfo));
3649         aninfo.flags |= MR_AN_ENABLE;
3650         aninfo.state = ANEG_STATE_UNKNOWN;
3651         aninfo.cur_time = 0;
3652         tick = 0;
3653         while (++tick < 195000) {
3654                 status = tg3_fiber_aneg_smachine(tp, &aninfo);
3655                 if (status == ANEG_DONE || status == ANEG_FAILED)
3656                         break;
3657
3658                 udelay(1);
3659         }
3660
3661         tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
3662         tw32_f(MAC_MODE, tp->mac_mode);
3663         udelay(40);
3664
3665         *txflags = aninfo.txconfig;
3666         *rxflags = aninfo.flags;
3667
3668         if (status == ANEG_DONE &&
3669             (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
3670                              MR_LP_ADV_FULL_DUPLEX)))
3671                 res = 1;
3672
3673         return res;
3674 }
3675
3676 static void tg3_init_bcm8002(struct tg3 *tp)
3677 {
3678         u32 mac_status = tr32(MAC_STATUS);
3679         int i;
3680
3681         /* Reset when initting first time or we have a link. */
3682         if ((tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) &&
3683             !(mac_status & MAC_STATUS_PCS_SYNCED))
3684                 return;
3685
3686         /* Set PLL lock range. */
3687         tg3_writephy(tp, 0x16, 0x8007);
3688
3689         /* SW reset */
3690         tg3_writephy(tp, MII_BMCR, BMCR_RESET);
3691
3692         /* Wait for reset to complete. */
3693         /* XXX schedule_timeout() ... */
3694         for (i = 0; i < 500; i++)
3695                 udelay(10);
3696
3697         /* Config mode; select PMA/Ch 1 regs. */
3698         tg3_writephy(tp, 0x10, 0x8411);
3699
3700         /* Enable auto-lock and comdet, select txclk for tx. */
3701         tg3_writephy(tp, 0x11, 0x0a10);
3702
3703         tg3_writephy(tp, 0x18, 0x00a0);
3704         tg3_writephy(tp, 0x16, 0x41ff);
3705
3706         /* Assert and deassert POR. */
3707         tg3_writephy(tp, 0x13, 0x0400);
3708         udelay(40);
3709         tg3_writephy(tp, 0x13, 0x0000);
3710
3711         tg3_writephy(tp, 0x11, 0x0a50);
3712         udelay(40);
3713         tg3_writephy(tp, 0x11, 0x0a10);
3714
3715         /* Wait for signal to stabilize */
3716         /* XXX schedule_timeout() ... */
3717         for (i = 0; i < 15000; i++)
3718                 udelay(10);
3719
3720         /* Deselect the channel register so we can read the PHYID
3721          * later.
3722          */
3723         tg3_writephy(tp, 0x10, 0x8011);
3724 }
3725
3726 static int tg3_setup_fiber_hw_autoneg(struct tg3 *tp, u32 mac_status)
3727 {
3728         u16 flowctrl;
3729         u32 sg_dig_ctrl, sg_dig_status;
3730         u32 serdes_cfg, expected_sg_dig_ctrl;
3731         int workaround, port_a;
3732         int current_link_up;
3733
3734         serdes_cfg = 0;
3735         expected_sg_dig_ctrl = 0;
3736         workaround = 0;
3737         port_a = 1;
3738         current_link_up = 0;
3739
3740         if (tp->pci_chip_rev_id != CHIPREV_ID_5704_A0 &&
3741             tp->pci_chip_rev_id != CHIPREV_ID_5704_A1) {
3742                 workaround = 1;
3743                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
3744                         port_a = 0;
3745
3746                 /* preserve bits 0-11,13,14 for signal pre-emphasis */
3747                 /* preserve bits 20-23 for voltage regulator */
3748                 serdes_cfg = tr32(MAC_SERDES_CFG) & 0x00f06fff;
3749         }
3750
3751         sg_dig_ctrl = tr32(SG_DIG_CTRL);
3752
3753         if (tp->link_config.autoneg != AUTONEG_ENABLE) {
3754                 if (sg_dig_ctrl & SG_DIG_USING_HW_AUTONEG) {
3755                         if (workaround) {
3756                                 u32 val = serdes_cfg;
3757
3758                                 if (port_a)
3759                                         val |= 0xc010000;
3760                                 else
3761                                         val |= 0x4010000;
3762                                 tw32_f(MAC_SERDES_CFG, val);
3763                         }
3764
3765                         tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
3766                 }
3767                 if (mac_status & MAC_STATUS_PCS_SYNCED) {
3768                         tg3_setup_flow_control(tp, 0, 0);
3769                         current_link_up = 1;
3770                 }
3771                 goto out;
3772         }
3773
3774         /* Want auto-negotiation.  */
3775         expected_sg_dig_ctrl = SG_DIG_USING_HW_AUTONEG | SG_DIG_COMMON_SETUP;
3776
3777         flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
3778         if (flowctrl & ADVERTISE_1000XPAUSE)
3779                 expected_sg_dig_ctrl |= SG_DIG_PAUSE_CAP;
3780         if (flowctrl & ADVERTISE_1000XPSE_ASYM)
3781                 expected_sg_dig_ctrl |= SG_DIG_ASYM_PAUSE;
3782
3783         if (sg_dig_ctrl != expected_sg_dig_ctrl) {
3784                 if ((tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT) &&
3785                     tp->serdes_counter &&
3786                     ((mac_status & (MAC_STATUS_PCS_SYNCED |
3787                                     MAC_STATUS_RCVD_CFG)) ==
3788                      MAC_STATUS_PCS_SYNCED)) {
3789                         tp->serdes_counter--;
3790                         current_link_up = 1;
3791                         goto out;
3792                 }
3793 restart_autoneg:
3794                 if (workaround)
3795                         tw32_f(MAC_SERDES_CFG, serdes_cfg | 0xc011000);
3796                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl | SG_DIG_SOFT_RESET);
3797                 udelay(5);
3798                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl);
3799
3800                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
3801                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3802         } else if (mac_status & (MAC_STATUS_PCS_SYNCED |
3803                                  MAC_STATUS_SIGNAL_DET)) {
3804                 sg_dig_status = tr32(SG_DIG_STATUS);
3805                 mac_status = tr32(MAC_STATUS);
3806
3807                 if ((sg_dig_status & SG_DIG_AUTONEG_COMPLETE) &&
3808                     (mac_status & MAC_STATUS_PCS_SYNCED)) {
3809                         u32 local_adv = 0, remote_adv = 0;
3810
3811                         if (sg_dig_ctrl & SG_DIG_PAUSE_CAP)
3812                                 local_adv |= ADVERTISE_1000XPAUSE;
3813                         if (sg_dig_ctrl & SG_DIG_ASYM_PAUSE)
3814                                 local_adv |= ADVERTISE_1000XPSE_ASYM;
3815
3816                         if (sg_dig_status & SG_DIG_PARTNER_PAUSE_CAPABLE)
3817                                 remote_adv |= LPA_1000XPAUSE;
3818                         if (sg_dig_status & SG_DIG_PARTNER_ASYM_PAUSE)
3819                                 remote_adv |= LPA_1000XPAUSE_ASYM;
3820
3821                         tg3_setup_flow_control(tp, local_adv, remote_adv);
3822                         current_link_up = 1;
3823                         tp->serdes_counter = 0;
3824                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3825                 } else if (!(sg_dig_status & SG_DIG_AUTONEG_COMPLETE)) {
3826                         if (tp->serdes_counter)
3827                                 tp->serdes_counter--;
3828                         else {
3829                                 if (workaround) {
3830                                         u32 val = serdes_cfg;
3831
3832                                         if (port_a)
3833                                                 val |= 0xc010000;
3834                                         else
3835                                                 val |= 0x4010000;
3836
3837                                         tw32_f(MAC_SERDES_CFG, val);
3838                                 }
3839
3840                                 tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
3841                                 udelay(40);
3842
3843                                 /* Link parallel detection - link is up */
3844                                 /* only if we have PCS_SYNC and not */
3845                                 /* receiving config code words */
3846                                 mac_status = tr32(MAC_STATUS);
3847                                 if ((mac_status & MAC_STATUS_PCS_SYNCED) &&
3848                                     !(mac_status & MAC_STATUS_RCVD_CFG)) {
3849                                         tg3_setup_flow_control(tp, 0, 0);
3850                                         current_link_up = 1;
3851                                         tp->tg3_flags2 |=
3852                                                 TG3_FLG2_PARALLEL_DETECT;
3853                                         tp->serdes_counter =
3854                                                 SERDES_PARALLEL_DET_TIMEOUT;
3855                                 } else
3856                                         goto restart_autoneg;
3857                         }
3858                 }
3859         } else {
3860                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
3861                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3862         }
3863
3864 out:
3865         return current_link_up;
3866 }
3867
3868 static int tg3_setup_fiber_by_hand(struct tg3 *tp, u32 mac_status)
3869 {
3870         int current_link_up = 0;
3871
3872         if (!(mac_status & MAC_STATUS_PCS_SYNCED))
3873                 goto out;
3874
3875         if (tp->link_config.autoneg == AUTONEG_ENABLE) {
3876                 u32 txflags, rxflags;
3877                 int i;
3878
3879                 if (fiber_autoneg(tp, &txflags, &rxflags)) {
3880                         u32 local_adv = 0, remote_adv = 0;
3881
3882                         if (txflags & ANEG_CFG_PS1)
3883                                 local_adv |= ADVERTISE_1000XPAUSE;
3884                         if (txflags & ANEG_CFG_PS2)
3885                                 local_adv |= ADVERTISE_1000XPSE_ASYM;
3886
3887                         if (rxflags & MR_LP_ADV_SYM_PAUSE)
3888                                 remote_adv |= LPA_1000XPAUSE;
3889                         if (rxflags & MR_LP_ADV_ASYM_PAUSE)
3890                                 remote_adv |= LPA_1000XPAUSE_ASYM;
3891
3892                         tg3_setup_flow_control(tp, local_adv, remote_adv);
3893
3894                         current_link_up = 1;
3895                 }
3896                 for (i = 0; i < 30; i++) {
3897                         udelay(20);
3898                         tw32_f(MAC_STATUS,
3899                                (MAC_STATUS_SYNC_CHANGED |
3900                                 MAC_STATUS_CFG_CHANGED));
3901                         udelay(40);
3902                         if ((tr32(MAC_STATUS) &
3903                              (MAC_STATUS_SYNC_CHANGED |
3904                               MAC_STATUS_CFG_CHANGED)) == 0)
3905                                 break;
3906                 }
3907
3908                 mac_status = tr32(MAC_STATUS);
3909                 if (current_link_up == 0 &&
3910                     (mac_status & MAC_STATUS_PCS_SYNCED) &&
3911                     !(mac_status & MAC_STATUS_RCVD_CFG))
3912                         current_link_up = 1;
3913         } else {
3914                 tg3_setup_flow_control(tp, 0, 0);
3915
3916                 /* Forcing 1000FD link up. */
3917                 current_link_up = 1;
3918
3919                 tw32_f(MAC_MODE, (tp->mac_mode | MAC_MODE_SEND_CONFIGS));
3920                 udelay(40);
3921
3922                 tw32_f(MAC_MODE, tp->mac_mode);
3923                 udelay(40);
3924         }
3925
3926 out:
3927         return current_link_up;
3928 }
3929
3930 static int tg3_setup_fiber_phy(struct tg3 *tp, int force_reset)
3931 {
3932         u32 orig_pause_cfg;
3933         u16 orig_active_speed;
3934         u8 orig_active_duplex;
3935         u32 mac_status;
3936         int current_link_up;
3937         int i;
3938
3939         orig_pause_cfg = tp->link_config.active_flowctrl;
3940         orig_active_speed = tp->link_config.active_speed;
3941         orig_active_duplex = tp->link_config.active_duplex;
3942
3943         if (!(tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG) &&
3944             netif_carrier_ok(tp->dev) &&
3945             (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE)) {
3946                 mac_status = tr32(MAC_STATUS);
3947                 mac_status &= (MAC_STATUS_PCS_SYNCED |
3948                                MAC_STATUS_SIGNAL_DET |
3949                                MAC_STATUS_CFG_CHANGED |
3950                                MAC_STATUS_RCVD_CFG);
3951                 if (mac_status == (MAC_STATUS_PCS_SYNCED |
3952                                    MAC_STATUS_SIGNAL_DET)) {
3953                         tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
3954                                             MAC_STATUS_CFG_CHANGED));
3955                         return 0;
3956                 }
3957         }
3958
3959         tw32_f(MAC_TX_AUTO_NEG, 0);
3960
3961         tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
3962         tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
3963         tw32_f(MAC_MODE, tp->mac_mode);
3964         udelay(40);
3965
3966         if (tp->phy_id == PHY_ID_BCM8002)
3967                 tg3_init_bcm8002(tp);
3968
3969         /* Enable link change event even when serdes polling.  */
3970         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
3971         udelay(40);
3972
3973         current_link_up = 0;
3974         mac_status = tr32(MAC_STATUS);
3975
3976         if (tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG)
3977                 current_link_up = tg3_setup_fiber_hw_autoneg(tp, mac_status);
3978         else
3979                 current_link_up = tg3_setup_fiber_by_hand(tp, mac_status);
3980
3981         tp->napi[0].hw_status->status =
3982                 (SD_STATUS_UPDATED |
3983                  (tp->napi[0].hw_status->status & ~SD_STATUS_LINK_CHG));
3984
3985         for (i = 0; i < 100; i++) {
3986                 tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
3987                                     MAC_STATUS_CFG_CHANGED));
3988                 udelay(5);
3989                 if ((tr32(MAC_STATUS) & (MAC_STATUS_SYNC_CHANGED |
3990                                          MAC_STATUS_CFG_CHANGED |
3991                                          MAC_STATUS_LNKSTATE_CHANGED)) == 0)
3992                         break;
3993         }
3994
3995         mac_status = tr32(MAC_STATUS);
3996         if ((mac_status & MAC_STATUS_PCS_SYNCED) == 0) {
3997                 current_link_up = 0;
3998                 if (tp->link_config.autoneg == AUTONEG_ENABLE &&
3999                     tp->serdes_counter == 0) {
4000                         tw32_f(MAC_MODE, (tp->mac_mode |
4001                                           MAC_MODE_SEND_CONFIGS));
4002                         udelay(1);
4003                         tw32_f(MAC_MODE, tp->mac_mode);
4004                 }
4005         }
4006
4007         if (current_link_up == 1) {
4008                 tp->link_config.active_speed = SPEED_1000;
4009                 tp->link_config.active_duplex = DUPLEX_FULL;
4010                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
4011                                     LED_CTRL_LNKLED_OVERRIDE |
4012                                     LED_CTRL_1000MBPS_ON));
4013         } else {
4014                 tp->link_config.active_speed = SPEED_INVALID;
4015                 tp->link_config.active_duplex = DUPLEX_INVALID;
4016                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
4017                                     LED_CTRL_LNKLED_OVERRIDE |
4018                                     LED_CTRL_TRAFFIC_OVERRIDE));
4019         }
4020
4021         if (current_link_up != netif_carrier_ok(tp->dev)) {
4022                 if (current_link_up)
4023                         netif_carrier_on(tp->dev);
4024                 else
4025                         netif_carrier_off(tp->dev);
4026                 tg3_link_report(tp);
4027         } else {
4028                 u32 now_pause_cfg = tp->link_config.active_flowctrl;
4029                 if (orig_pause_cfg != now_pause_cfg ||
4030                     orig_active_speed != tp->link_config.active_speed ||
4031                     orig_active_duplex != tp->link_config.active_duplex)
4032                         tg3_link_report(tp);
4033         }
4034
4035         return 0;
4036 }
4037
4038 static int tg3_setup_fiber_mii_phy(struct tg3 *tp, int force_reset)
4039 {
4040         int current_link_up, err = 0;
4041         u32 bmsr, bmcr;
4042         u16 current_speed;
4043         u8 current_duplex;
4044         u32 local_adv, remote_adv;
4045
4046         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
4047         tw32_f(MAC_MODE, tp->mac_mode);
4048         udelay(40);
4049
4050         tw32(MAC_EVENT, 0);
4051
4052         tw32_f(MAC_STATUS,
4053              (MAC_STATUS_SYNC_CHANGED |
4054               MAC_STATUS_CFG_CHANGED |
4055               MAC_STATUS_MI_COMPLETION |
4056               MAC_STATUS_LNKSTATE_CHANGED));
4057         udelay(40);
4058
4059         if (force_reset)
4060                 tg3_phy_reset(tp);
4061
4062         current_link_up = 0;
4063         current_speed = SPEED_INVALID;
4064         current_duplex = DUPLEX_INVALID;
4065
4066         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4067         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4068         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
4069                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
4070                         bmsr |= BMSR_LSTATUS;
4071                 else
4072                         bmsr &= ~BMSR_LSTATUS;
4073         }
4074
4075         err |= tg3_readphy(tp, MII_BMCR, &bmcr);
4076
4077         if ((tp->link_config.autoneg == AUTONEG_ENABLE) && !force_reset &&
4078             (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
4079                 /* do nothing, just check for link up at the end */
4080         } else if (tp->link_config.autoneg == AUTONEG_ENABLE) {
4081                 u32 adv, new_adv;
4082
4083                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
4084                 new_adv = adv & ~(ADVERTISE_1000XFULL | ADVERTISE_1000XHALF |
4085                                   ADVERTISE_1000XPAUSE |
4086                                   ADVERTISE_1000XPSE_ASYM |
4087                                   ADVERTISE_SLCT);
4088
4089                 new_adv |= tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
4090
4091                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
4092                         new_adv |= ADVERTISE_1000XHALF;
4093                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
4094                         new_adv |= ADVERTISE_1000XFULL;
4095
4096                 if ((new_adv != adv) || !(bmcr & BMCR_ANENABLE)) {
4097                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
4098                         bmcr |= BMCR_ANENABLE | BMCR_ANRESTART;
4099                         tg3_writephy(tp, MII_BMCR, bmcr);
4100
4101                         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
4102                         tp->serdes_counter = SERDES_AN_TIMEOUT_5714S;
4103                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
4104
4105                         return err;
4106                 }
4107         } else {
4108                 u32 new_bmcr;
4109
4110                 bmcr &= ~BMCR_SPEED1000;
4111                 new_bmcr = bmcr & ~(BMCR_ANENABLE | BMCR_FULLDPLX);
4112
4113                 if (tp->link_config.duplex == DUPLEX_FULL)
4114                         new_bmcr |= BMCR_FULLDPLX;
4115
4116                 if (new_bmcr != bmcr) {
4117                         /* BMCR_SPEED1000 is a reserved bit that needs
4118                          * to be set on write.
4119                          */
4120                         new_bmcr |= BMCR_SPEED1000;
4121
4122                         /* Force a linkdown */
4123                         if (netif_carrier_ok(tp->dev)) {
4124                                 u32 adv;
4125
4126                                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
4127                                 adv &= ~(ADVERTISE_1000XFULL |
4128                                          ADVERTISE_1000XHALF |
4129                                          ADVERTISE_SLCT);
4130                                 tg3_writephy(tp, MII_ADVERTISE, adv);
4131                                 tg3_writephy(tp, MII_BMCR, bmcr |
4132                                                            BMCR_ANRESTART |
4133                                                            BMCR_ANENABLE);
4134                                 udelay(10);
4135                                 netif_carrier_off(tp->dev);
4136                         }
4137                         tg3_writephy(tp, MII_BMCR, new_bmcr);
4138                         bmcr = new_bmcr;
4139                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4140                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4141                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
4142                             ASIC_REV_5714) {
4143                                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
4144                                         bmsr |= BMSR_LSTATUS;
4145                                 else
4146                                         bmsr &= ~BMSR_LSTATUS;
4147                         }
4148                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
4149                 }
4150         }
4151
4152         if (bmsr & BMSR_LSTATUS) {
4153                 current_speed = SPEED_1000;
4154                 current_link_up = 1;
4155                 if (bmcr & BMCR_FULLDPLX)
4156                         current_duplex = DUPLEX_FULL;
4157                 else
4158                         current_duplex = DUPLEX_HALF;
4159
4160                 local_adv = 0;
4161                 remote_adv = 0;
4162
4163                 if (bmcr & BMCR_ANENABLE) {
4164                         u32 common;
4165
4166                         err |= tg3_readphy(tp, MII_ADVERTISE, &local_adv);
4167                         err |= tg3_readphy(tp, MII_LPA, &remote_adv);
4168                         common = local_adv & remote_adv;
4169                         if (common & (ADVERTISE_1000XHALF |
4170                                       ADVERTISE_1000XFULL)) {
4171                                 if (common & ADVERTISE_1000XFULL)
4172                                         current_duplex = DUPLEX_FULL;
4173                                 else
4174                                         current_duplex = DUPLEX_HALF;
4175                         }
4176                         else
4177                                 current_link_up = 0;
4178                 }
4179         }
4180
4181         if (current_link_up == 1 && current_duplex == DUPLEX_FULL)
4182                 tg3_setup_flow_control(tp, local_adv, remote_adv);
4183
4184         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
4185         if (tp->link_config.active_duplex == DUPLEX_HALF)
4186                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
4187
4188         tw32_f(MAC_MODE, tp->mac_mode);
4189         udelay(40);
4190
4191         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
4192
4193         tp->link_config.active_speed = current_speed;
4194         tp->link_config.active_duplex = current_duplex;
4195
4196         if (current_link_up != netif_carrier_ok(tp->dev)) {
4197                 if (current_link_up)
4198                         netif_carrier_on(tp->dev);
4199                 else {
4200                         netif_carrier_off(tp->dev);
4201                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
4202                 }
4203                 tg3_link_report(tp);
4204         }
4205         return err;
4206 }
4207
4208 static void tg3_serdes_parallel_detect(struct tg3 *tp)
4209 {
4210         if (tp->serdes_counter) {
4211                 /* Give autoneg time to complete. */
4212                 tp->serdes_counter--;
4213                 return;
4214         }
4215         if (!netif_carrier_ok(tp->dev) &&
4216             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
4217                 u32 bmcr;
4218
4219                 tg3_readphy(tp, MII_BMCR, &bmcr);
4220                 if (bmcr & BMCR_ANENABLE) {
4221                         u32 phy1, phy2;
4222
4223                         /* Select shadow register 0x1f */
4224                         tg3_writephy(tp, 0x1c, 0x7c00);
4225                         tg3_readphy(tp, 0x1c, &phy1);
4226
4227                         /* Select expansion interrupt status register */
4228                         tg3_writephy(tp, 0x17, 0x0f01);
4229                         tg3_readphy(tp, 0x15, &phy2);
4230                         tg3_readphy(tp, 0x15, &phy2);
4231
4232                         if ((phy1 & 0x10) && !(phy2 & 0x20)) {
4233                                 /* We have signal detect and not receiving
4234                                  * config code words, link is up by parallel
4235                                  * detection.
4236                                  */
4237
4238                                 bmcr &= ~BMCR_ANENABLE;
4239                                 bmcr |= BMCR_SPEED1000 | BMCR_FULLDPLX;
4240                                 tg3_writephy(tp, MII_BMCR, bmcr);
4241                                 tp->tg3_flags2 |= TG3_FLG2_PARALLEL_DETECT;
4242                         }
4243                 }
4244         }
4245         else if (netif_carrier_ok(tp->dev) &&
4246                  (tp->link_config.autoneg == AUTONEG_ENABLE) &&
4247                  (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
4248                 u32 phy2;
4249
4250                 /* Select expansion interrupt status register */
4251                 tg3_writephy(tp, 0x17, 0x0f01);
4252                 tg3_readphy(tp, 0x15, &phy2);
4253                 if (phy2 & 0x20) {
4254                         u32 bmcr;
4255
4256                         /* Config code words received, turn on autoneg. */
4257                         tg3_readphy(tp, MII_BMCR, &bmcr);
4258                         tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANENABLE);
4259
4260                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
4261
4262                 }
4263         }
4264 }
4265
4266 static int tg3_setup_phy(struct tg3 *tp, int force_reset)
4267 {
4268         int err;
4269
4270         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
4271                 err = tg3_setup_fiber_phy(tp, force_reset);
4272         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
4273                 err = tg3_setup_fiber_mii_phy(tp, force_reset);
4274         } else {
4275                 err = tg3_setup_copper_phy(tp, force_reset);
4276         }
4277
4278         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX) {
4279                 u32 val, scale;
4280
4281                 val = tr32(TG3_CPMU_CLCK_STAT) & CPMU_CLCK_STAT_MAC_CLCK_MASK;
4282                 if (val == CPMU_CLCK_STAT_MAC_CLCK_62_5)
4283                         scale = 65;
4284                 else if (val == CPMU_CLCK_STAT_MAC_CLCK_6_25)
4285                         scale = 6;
4286                 else
4287                         scale = 12;
4288
4289                 val = tr32(GRC_MISC_CFG) & ~GRC_MISC_CFG_PRESCALAR_MASK;
4290                 val |= (scale << GRC_MISC_CFG_PRESCALAR_SHIFT);
4291                 tw32(GRC_MISC_CFG, val);
4292         }
4293
4294         if (tp->link_config.active_speed == SPEED_1000 &&
4295             tp->link_config.active_duplex == DUPLEX_HALF)
4296                 tw32(MAC_TX_LENGTHS,
4297                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
4298                       (6 << TX_LENGTHS_IPG_SHIFT) |
4299                       (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
4300         else
4301                 tw32(MAC_TX_LENGTHS,
4302                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
4303                       (6 << TX_LENGTHS_IPG_SHIFT) |
4304                       (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
4305
4306         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
4307                 if (netif_carrier_ok(tp->dev)) {
4308                         tw32(HOSTCC_STAT_COAL_TICKS,
4309                              tp->coal.stats_block_coalesce_usecs);
4310                 } else {
4311                         tw32(HOSTCC_STAT_COAL_TICKS, 0);
4312                 }
4313         }
4314
4315         if (tp->tg3_flags & TG3_FLAG_ASPM_WORKAROUND) {
4316                 u32 val = tr32(PCIE_PWR_MGMT_THRESH);
4317                 if (!netif_carrier_ok(tp->dev))
4318                         val = (val & ~PCIE_PWR_MGMT_L1_THRESH_MSK) |
4319                               tp->pwrmgmt_thresh;
4320                 else
4321                         val |= PCIE_PWR_MGMT_L1_THRESH_MSK;
4322                 tw32(PCIE_PWR_MGMT_THRESH, val);
4323         }
4324
4325         return err;
4326 }
4327
4328 /* This is called whenever we suspect that the system chipset is re-
4329  * ordering the sequence of MMIO to the tx send mailbox. The symptom
4330  * is bogus tx completions. We try to recover by setting the
4331  * TG3_FLAG_MBOX_WRITE_REORDER flag and resetting the chip later
4332  * in the workqueue.
4333  */
4334 static void tg3_tx_recover(struct tg3 *tp)
4335 {
4336         BUG_ON((tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) ||
4337                tp->write32_tx_mbox == tg3_write_indirect_mbox);
4338
4339         printk(KERN_WARNING PFX "%s: The system may be re-ordering memory-"
4340                "mapped I/O cycles to the network device, attempting to "
4341                "recover. Please report the problem to the driver maintainer "
4342                "and include system chipset information.\n", tp->dev->name);
4343
4344         spin_lock(&tp->lock);
4345         tp->tg3_flags |= TG3_FLAG_TX_RECOVERY_PENDING;
4346         spin_unlock(&tp->lock);
4347 }
4348
4349 static inline u32 tg3_tx_avail(struct tg3_napi *tnapi)
4350 {
4351         smp_mb();
4352         return tnapi->tx_pending -
4353                ((tnapi->tx_prod - tnapi->tx_cons) & (TG3_TX_RING_SIZE - 1));
4354 }
4355
4356 /* Tigon3 never reports partial packet sends.  So we do not
4357  * need special logic to handle SKBs that have not had all
4358  * of their frags sent yet, like SunGEM does.
4359  */
4360 static void tg3_tx(struct tg3_napi *tnapi)
4361 {
4362         struct tg3 *tp = tnapi->tp;
4363         u32 hw_idx = tnapi->hw_status->idx[0].tx_consumer;
4364         u32 sw_idx = tnapi->tx_cons;
4365         struct netdev_queue *txq;
4366         int index = tnapi - tp->napi;
4367
4368         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_TSS)
4369                 index--;
4370
4371         txq = netdev_get_tx_queue(tp->dev, index);
4372
4373         while (sw_idx != hw_idx) {
4374                 struct ring_info *ri = &tnapi->tx_buffers[sw_idx];
4375                 struct sk_buff *skb = ri->skb;
4376                 int i, tx_bug = 0;
4377
4378                 if (unlikely(skb == NULL)) {
4379                         tg3_tx_recover(tp);
4380                         return;
4381                 }
4382
4383                 pci_unmap_single(tp->pdev,
4384                                  pci_unmap_addr(ri, mapping),
4385                                  skb_headlen(skb),
4386                                  PCI_DMA_TODEVICE);
4387
4388                 ri->skb = NULL;
4389
4390                 sw_idx = NEXT_TX(sw_idx);
4391
4392                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4393                         ri = &tnapi->tx_buffers[sw_idx];
4394                         if (unlikely(ri->skb != NULL || sw_idx == hw_idx))
4395                                 tx_bug = 1;
4396
4397                         pci_unmap_page(tp->pdev,
4398                                        pci_unmap_addr(ri, mapping),
4399                                        skb_shinfo(skb)->frags[i].size,
4400                                        PCI_DMA_TODEVICE);
4401                         sw_idx = NEXT_TX(sw_idx);
4402                 }
4403
4404                 dev_kfree_skb(skb);
4405
4406                 if (unlikely(tx_bug)) {
4407                         tg3_tx_recover(tp);
4408                         return;
4409                 }
4410         }
4411
4412         tnapi->tx_cons = sw_idx;
4413
4414         /* Need to make the tx_cons update visible to tg3_start_xmit()
4415          * before checking for netif_queue_stopped().  Without the
4416          * memory barrier, there is a small possibility that tg3_start_xmit()
4417          * will miss it and cause the queue to be stopped forever.
4418          */
4419         smp_mb();
4420
4421         if (unlikely(netif_tx_queue_stopped(txq) &&
4422                      (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi)))) {
4423                 __netif_tx_lock(txq, smp_processor_id());
4424                 if (netif_tx_queue_stopped(txq) &&
4425                     (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi)))
4426                         netif_tx_wake_queue(txq);
4427                 __netif_tx_unlock(txq);
4428         }
4429 }
4430
4431 static void tg3_rx_skb_free(struct tg3 *tp, struct ring_info *ri, u32 map_sz)
4432 {
4433         if (!ri->skb)
4434                 return;
4435
4436         pci_unmap_single(tp->pdev, pci_unmap_addr(ri, mapping),
4437                          map_sz, PCI_DMA_FROMDEVICE);
4438         dev_kfree_skb_any(ri->skb);
4439         ri->skb = NULL;
4440 }
4441
4442 /* Returns size of skb allocated or < 0 on error.
4443  *
4444  * We only need to fill in the address because the other members
4445  * of the RX descriptor are invariant, see tg3_init_rings.
4446  *
4447  * Note the purposeful assymetry of cpu vs. chip accesses.  For
4448  * posting buffers we only dirty the first cache line of the RX
4449  * descriptor (containing the address).  Whereas for the RX status
4450  * buffers the cpu only reads the last cacheline of the RX descriptor
4451  * (to fetch the error flags, vlan tag, checksum, and opaque cookie).
4452  */
4453 static int tg3_alloc_rx_skb(struct tg3 *tp, struct tg3_rx_prodring_set *tpr,
4454                             u32 opaque_key, u32 dest_idx_unmasked)
4455 {
4456         struct tg3_rx_buffer_desc *desc;
4457         struct ring_info *map, *src_map;
4458         struct sk_buff *skb;
4459         dma_addr_t mapping;
4460         int skb_size, dest_idx;
4461
4462         src_map = NULL;
4463         switch (opaque_key) {
4464         case RXD_OPAQUE_RING_STD:
4465                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
4466                 desc = &tpr->rx_std[dest_idx];
4467                 map = &tpr->rx_std_buffers[dest_idx];
4468                 skb_size = tp->rx_pkt_map_sz;
4469                 break;
4470
4471         case RXD_OPAQUE_RING_JUMBO:
4472                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
4473                 desc = &tpr->rx_jmb[dest_idx].std;
4474                 map = &tpr->rx_jmb_buffers[dest_idx];
4475                 skb_size = TG3_RX_JMB_MAP_SZ;
4476                 break;
4477
4478         default:
4479                 return -EINVAL;
4480         }
4481
4482         /* Do not overwrite any of the map or rp information
4483          * until we are sure we can commit to a new buffer.
4484          *
4485          * Callers depend upon this behavior and assume that
4486          * we leave everything unchanged if we fail.
4487          */
4488         skb = netdev_alloc_skb(tp->dev, skb_size + tp->rx_offset);
4489         if (skb == NULL)
4490                 return -ENOMEM;
4491
4492         skb_reserve(skb, tp->rx_offset);
4493
4494         mapping = pci_map_single(tp->pdev, skb->data, skb_size,
4495                                  PCI_DMA_FROMDEVICE);
4496         if (pci_dma_mapping_error(tp->pdev, mapping)) {
4497                 dev_kfree_skb(skb);
4498                 return -EIO;
4499         }
4500
4501         map->skb = skb;
4502         pci_unmap_addr_set(map, mapping, mapping);
4503
4504         desc->addr_hi = ((u64)mapping >> 32);
4505         desc->addr_lo = ((u64)mapping & 0xffffffff);
4506
4507         return skb_size;
4508 }
4509
4510 /* We only need to move over in the address because the other
4511  * members of the RX descriptor are invariant.  See notes above
4512  * tg3_alloc_rx_skb for full details.
4513  */
4514 static void tg3_recycle_rx(struct tg3_napi *tnapi,
4515                            struct tg3_rx_prodring_set *dpr,
4516                            u32 opaque_key, int src_idx,
4517                            u32 dest_idx_unmasked)
4518 {
4519         struct tg3 *tp = tnapi->tp;
4520         struct tg3_rx_buffer_desc *src_desc, *dest_desc;
4521         struct ring_info *src_map, *dest_map;
4522         int dest_idx;
4523         struct tg3_rx_prodring_set *spr = &tp->prodring[0];
4524
4525         switch (opaque_key) {
4526         case RXD_OPAQUE_RING_STD:
4527                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
4528                 dest_desc = &dpr->rx_std[dest_idx];
4529                 dest_map = &dpr->rx_std_buffers[dest_idx];
4530                 src_desc = &spr->rx_std[src_idx];
4531                 src_map = &spr->rx_std_buffers[src_idx];
4532                 break;
4533
4534         case RXD_OPAQUE_RING_JUMBO:
4535                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
4536                 dest_desc = &dpr->rx_jmb[dest_idx].std;
4537                 dest_map = &dpr->rx_jmb_buffers[dest_idx];
4538                 src_desc = &spr->rx_jmb[src_idx].std;
4539                 src_map = &spr->rx_jmb_buffers[src_idx];
4540                 break;
4541
4542         default:
4543                 return;
4544         }
4545
4546         dest_map->skb = src_map->skb;
4547         pci_unmap_addr_set(dest_map, mapping,
4548                            pci_unmap_addr(src_map, mapping));
4549         dest_desc->addr_hi = src_desc->addr_hi;
4550         dest_desc->addr_lo = src_desc->addr_lo;
4551         src_map->skb = NULL;
4552 }
4553
4554 /* The RX ring scheme is composed of multiple rings which post fresh
4555  * buffers to the chip, and one special ring the chip uses to report
4556  * status back to the host.
4557  *
4558  * The special ring reports the status of received packets to the
4559  * host.  The chip does not write into the original descriptor the
4560  * RX buffer was obtained from.  The chip simply takes the original
4561  * descriptor as provided by the host, updates the status and length
4562  * field, then writes this into the next status ring entry.
4563  *
4564  * Each ring the host uses to post buffers to the chip is described
4565  * by a TG3_BDINFO entry in the chips SRAM area.  When a packet arrives,
4566  * it is first placed into the on-chip ram.  When the packet's length
4567  * is known, it walks down the TG3_BDINFO entries to select the ring.
4568  * Each TG3_BDINFO specifies a MAXLEN field and the first TG3_BDINFO
4569  * which is within the range of the new packet's length is chosen.
4570  *
4571  * The "separate ring for rx status" scheme may sound queer, but it makes
4572  * sense from a cache coherency perspective.  If only the host writes
4573  * to the buffer post rings, and only the chip writes to the rx status
4574  * rings, then cache lines never move beyond shared-modified state.
4575  * If both the host and chip were to write into the same ring, cache line
4576  * eviction could occur since both entities want it in an exclusive state.
4577  */
4578 static int tg3_rx(struct tg3_napi *tnapi, int budget)
4579 {
4580         struct tg3 *tp = tnapi->tp;
4581         u32 work_mask, rx_std_posted = 0;
4582         u32 std_prod_idx, jmb_prod_idx;
4583         u32 sw_idx = tnapi->rx_rcb_ptr;
4584         u16 hw_idx;
4585         int received;
4586         struct tg3_rx_prodring_set *tpr = tnapi->prodring;
4587
4588         hw_idx = *(tnapi->rx_rcb_prod_idx);
4589         /*
4590          * We need to order the read of hw_idx and the read of
4591          * the opaque cookie.
4592          */
4593         rmb();
4594         work_mask = 0;
4595         received = 0;
4596         std_prod_idx = tpr->rx_std_prod_idx;
4597         jmb_prod_idx = tpr->rx_jmb_prod_idx;
4598         while (sw_idx != hw_idx && budget > 0) {
4599                 struct ring_info *ri;
4600                 struct tg3_rx_buffer_desc *desc = &tnapi->rx_rcb[sw_idx];
4601                 unsigned int len;
4602                 struct sk_buff *skb;
4603                 dma_addr_t dma_addr;
4604                 u32 opaque_key, desc_idx, *post_ptr;
4605
4606                 desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
4607                 opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
4608                 if (opaque_key == RXD_OPAQUE_RING_STD) {
4609                         ri = &tp->prodring[0].rx_std_buffers[desc_idx];
4610                         dma_addr = pci_unmap_addr(ri, mapping);
4611                         skb = ri->skb;
4612                         post_ptr = &std_prod_idx;
4613                         rx_std_posted++;
4614                 } else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
4615                         ri = &tp->prodring[0].rx_jmb_buffers[desc_idx];
4616                         dma_addr = pci_unmap_addr(ri, mapping);
4617                         skb = ri->skb;
4618                         post_ptr = &jmb_prod_idx;
4619                 } else
4620                         goto next_pkt_nopost;
4621
4622                 work_mask |= opaque_key;
4623
4624                 if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
4625                     (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII)) {
4626                 drop_it:
4627                         tg3_recycle_rx(tnapi, tpr, opaque_key,
4628                                        desc_idx, *post_ptr);
4629                 drop_it_no_recycle:
4630                         /* Other statistics kept track of by card. */
4631                         tp->net_stats.rx_dropped++;
4632                         goto next_pkt;
4633                 }
4634
4635                 len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) -
4636                       ETH_FCS_LEN;
4637
4638                 if (len > RX_COPY_THRESHOLD &&
4639                     tp->rx_offset == NET_IP_ALIGN) {
4640                     /* rx_offset will likely not equal NET_IP_ALIGN
4641                      * if this is a 5701 card running in PCI-X mode
4642                      * [see tg3_get_invariants()]
4643                      */
4644                         int skb_size;
4645
4646                         skb_size = tg3_alloc_rx_skb(tp, tpr, opaque_key,
4647                                                     *post_ptr);
4648                         if (skb_size < 0)
4649                                 goto drop_it;
4650
4651                         ri->skb = NULL;
4652
4653                         pci_unmap_single(tp->pdev, dma_addr, skb_size,
4654                                          PCI_DMA_FROMDEVICE);
4655
4656                         skb_put(skb, len);
4657                 } else {
4658                         struct sk_buff *copy_skb;
4659
4660                         tg3_recycle_rx(tnapi, tpr, opaque_key,
4661                                        desc_idx, *post_ptr);
4662
4663                         copy_skb = netdev_alloc_skb(tp->dev,
4664                                                     len + TG3_RAW_IP_ALIGN);
4665                         if (copy_skb == NULL)
4666                                 goto drop_it_no_recycle;
4667
4668                         skb_reserve(copy_skb, TG3_RAW_IP_ALIGN);
4669                         skb_put(copy_skb, len);
4670                         pci_dma_sync_single_for_cpu(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
4671                         skb_copy_from_linear_data(skb, copy_skb->data, len);
4672                         pci_dma_sync_single_for_device(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
4673
4674                         /* We'll reuse the original ring buffer. */
4675                         skb = copy_skb;
4676                 }
4677
4678                 if ((tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) &&
4679                     (desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
4680                     (((desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
4681                       >> RXD_TCPCSUM_SHIFT) == 0xffff))
4682                         skb->ip_summed = CHECKSUM_UNNECESSARY;
4683                 else
4684                         skb->ip_summed = CHECKSUM_NONE;
4685
4686                 skb->protocol = eth_type_trans(skb, tp->dev);
4687
4688                 if (len > (tp->dev->mtu + ETH_HLEN) &&
4689                     skb->protocol != htons(ETH_P_8021Q)) {
4690                         dev_kfree_skb(skb);
4691                         goto next_pkt;
4692                 }
4693
4694 #if TG3_VLAN_TAG_USED
4695                 if (tp->vlgrp != NULL &&
4696                     desc->type_flags & RXD_FLAG_VLAN) {
4697                         vlan_gro_receive(&tnapi->napi, tp->vlgrp,
4698                                          desc->err_vlan & RXD_VLAN_MASK, skb);
4699                 } else
4700 #endif
4701                         napi_gro_receive(&tnapi->napi, skb);
4702
4703                 received++;
4704                 budget--;
4705
4706 next_pkt:
4707                 (*post_ptr)++;
4708
4709                 if (unlikely(rx_std_posted >= tp->rx_std_max_post)) {
4710                         u32 idx = *post_ptr % TG3_RX_RING_SIZE;
4711                         tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG, idx);
4712                         work_mask &= ~RXD_OPAQUE_RING_STD;
4713                         rx_std_posted = 0;
4714                 }
4715 next_pkt_nopost:
4716                 sw_idx++;
4717                 sw_idx &= (TG3_RX_RCB_RING_SIZE(tp) - 1);
4718
4719                 /* Refresh hw_idx to see if there is new work */
4720                 if (sw_idx == hw_idx) {
4721                         hw_idx = *(tnapi->rx_rcb_prod_idx);
4722                         rmb();
4723                 }
4724         }
4725
4726         /* ACK the status ring. */
4727         tnapi->rx_rcb_ptr = sw_idx;
4728         tw32_rx_mbox(tnapi->consmbox, sw_idx);
4729
4730         /* Refill RX ring(s). */
4731         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_RSS) || tnapi == &tp->napi[1]) {
4732                 if (work_mask & RXD_OPAQUE_RING_STD) {
4733                         tpr->rx_std_prod_idx = std_prod_idx % TG3_RX_RING_SIZE;
4734                         tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG,
4735                                      tpr->rx_std_prod_idx);
4736                 }
4737                 if (work_mask & RXD_OPAQUE_RING_JUMBO) {
4738                         tpr->rx_jmb_prod_idx = jmb_prod_idx %
4739                                                TG3_RX_JUMBO_RING_SIZE;
4740                         tw32_rx_mbox(TG3_RX_JMB_PROD_IDX_REG,
4741                                      tpr->rx_jmb_prod_idx);
4742                 }
4743                 mmiowb();
4744         } else if (work_mask) {
4745                 /* rx_std_buffers[] and rx_jmb_buffers[] entries must be
4746                  * updated before the producer indices can be updated.
4747                  */
4748                 smp_wmb();
4749
4750                 tpr->rx_std_prod_idx = std_prod_idx % TG3_RX_RING_SIZE;
4751                 tpr->rx_jmb_prod_idx = jmb_prod_idx % TG3_RX_JUMBO_RING_SIZE;
4752
4753                 napi_schedule(&tp->napi[1].napi);
4754         }
4755
4756         return received;
4757 }
4758
4759 static void tg3_poll_link(struct tg3 *tp)
4760 {
4761         /* handle link change and other phy events */
4762         if (!(tp->tg3_flags &
4763               (TG3_FLAG_USE_LINKCHG_REG |
4764                TG3_FLAG_POLL_SERDES))) {
4765                 struct tg3_hw_status *sblk = tp->napi[0].hw_status;
4766
4767                 if (sblk->status & SD_STATUS_LINK_CHG) {
4768                         sblk->status = SD_STATUS_UPDATED |
4769                                        (sblk->status & ~SD_STATUS_LINK_CHG);
4770                         spin_lock(&tp->lock);
4771                         if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) {
4772                                 tw32_f(MAC_STATUS,
4773                                      (MAC_STATUS_SYNC_CHANGED |
4774                                       MAC_STATUS_CFG_CHANGED |
4775                                       MAC_STATUS_MI_COMPLETION |
4776                                       MAC_STATUS_LNKSTATE_CHANGED));
4777                                 udelay(40);
4778                         } else
4779                                 tg3_setup_phy(tp, 0);
4780                         spin_unlock(&tp->lock);
4781                 }
4782         }
4783 }
4784
4785 static void tg3_rx_prodring_xfer(struct tg3 *tp,
4786                                  struct tg3_rx_prodring_set *dpr,
4787                                  struct tg3_rx_prodring_set *spr)
4788 {
4789         u32 si, di, cpycnt, src_prod_idx;
4790         int i;
4791
4792         while (1) {
4793                 src_prod_idx = spr->rx_std_prod_idx;
4794
4795                 /* Make sure updates to the rx_std_buffers[] entries and the
4796                  * standard producer index are seen in the correct order.
4797                  */
4798                 smp_rmb();
4799
4800                 if (spr->rx_std_cons_idx == src_prod_idx)
4801                         break;
4802
4803                 if (spr->rx_std_cons_idx < src_prod_idx)
4804                         cpycnt = src_prod_idx - spr->rx_std_cons_idx;
4805                 else
4806                         cpycnt = TG3_RX_RING_SIZE - spr->rx_std_cons_idx;
4807
4808                 cpycnt = min(cpycnt, TG3_RX_RING_SIZE - dpr->rx_std_prod_idx);
4809
4810                 si = spr->rx_std_cons_idx;
4811                 di = dpr->rx_std_prod_idx;
4812
4813                 memcpy(&dpr->rx_std_buffers[di],
4814                        &spr->rx_std_buffers[si],
4815                        cpycnt * sizeof(struct ring_info));
4816
4817                 for (i = 0; i < cpycnt; i++, di++, si++) {
4818                         struct tg3_rx_buffer_desc *sbd, *dbd;
4819                         sbd = &spr->rx_std[si];
4820                         dbd = &dpr->rx_std[di];
4821                         dbd->addr_hi = sbd->addr_hi;
4822                         dbd->addr_lo = sbd->addr_lo;
4823                 }
4824
4825                 spr->rx_std_cons_idx = (spr->rx_std_cons_idx + cpycnt) %
4826                                        TG3_RX_RING_SIZE;
4827                 dpr->rx_std_prod_idx = (dpr->rx_std_prod_idx + cpycnt) %
4828                                        TG3_RX_RING_SIZE;
4829         }
4830
4831         while (1) {
4832                 src_prod_idx = spr->rx_jmb_prod_idx;
4833
4834                 /* Make sure updates to the rx_jmb_buffers[] entries and
4835                  * the jumbo producer index are seen in the correct order.
4836                  */
4837                 smp_rmb();
4838
4839                 if (spr->rx_jmb_cons_idx == src_prod_idx)
4840                         break;
4841
4842                 if (spr->rx_jmb_cons_idx < src_prod_idx)
4843                         cpycnt = src_prod_idx - spr->rx_jmb_cons_idx;
4844                 else
4845                         cpycnt = TG3_RX_JUMBO_RING_SIZE - spr->rx_jmb_cons_idx;
4846
4847                 cpycnt = min(cpycnt,
4848                              TG3_RX_JUMBO_RING_SIZE - dpr->rx_jmb_prod_idx);
4849
4850                 si = spr->rx_jmb_cons_idx;
4851                 di = dpr->rx_jmb_prod_idx;
4852
4853                 memcpy(&dpr->rx_jmb_buffers[di],
4854                        &spr->rx_jmb_buffers[si],
4855                        cpycnt * sizeof(struct ring_info));
4856
4857                 for (i = 0; i < cpycnt; i++, di++, si++) {
4858                         struct tg3_rx_buffer_desc *sbd, *dbd;
4859                         sbd = &spr->rx_jmb[si].std;
4860                         dbd = &dpr->rx_jmb[di].std;
4861                         dbd->addr_hi = sbd->addr_hi;
4862                         dbd->addr_lo = sbd->addr_lo;
4863                 }
4864
4865                 spr->rx_jmb_cons_idx = (spr->rx_jmb_cons_idx + cpycnt) %
4866                                        TG3_RX_JUMBO_RING_SIZE;
4867                 dpr->rx_jmb_prod_idx = (dpr->rx_jmb_prod_idx + cpycnt) %
4868                                        TG3_RX_JUMBO_RING_SIZE;
4869         }
4870 }
4871
4872 static int tg3_poll_work(struct tg3_napi *tnapi, int work_done, int budget)
4873 {
4874         struct tg3 *tp = tnapi->tp;
4875
4876         /* run TX completion thread */
4877         if (tnapi->hw_status->idx[0].tx_consumer != tnapi->tx_cons) {
4878                 tg3_tx(tnapi);
4879                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING))
4880                         return work_done;
4881         }
4882
4883         /* run RX thread, within the bounds set by NAPI.
4884          * All RX "locking" is done by ensuring outside
4885          * code synchronizes with tg3->napi.poll()
4886          */
4887         if (*(tnapi->rx_rcb_prod_idx) != tnapi->rx_rcb_ptr)
4888                 work_done += tg3_rx(tnapi, budget - work_done);
4889
4890         if ((tp->tg3_flags3 & TG3_FLG3_ENABLE_RSS) && tnapi == &tp->napi[1]) {
4891                 int i;
4892                 u32 std_prod_idx = tp->prodring[0].rx_std_prod_idx;
4893                 u32 jmb_prod_idx = tp->prodring[0].rx_jmb_prod_idx;
4894
4895                 for (i = 2; i < tp->irq_cnt; i++)
4896                         tg3_rx_prodring_xfer(tp, tnapi->prodring,
4897                                              tp->napi[i].prodring);
4898
4899                 wmb();
4900
4901                 if (std_prod_idx != tp->prodring[0].rx_std_prod_idx) {
4902                         u32 mbox = TG3_RX_STD_PROD_IDX_REG;
4903                         tw32_rx_mbox(mbox, tp->prodring[0].rx_std_prod_idx);
4904                 }
4905
4906                 if (jmb_prod_idx != tp->prodring[0].rx_jmb_prod_idx) {
4907                         u32 mbox = TG3_RX_JMB_PROD_IDX_REG;
4908                         tw32_rx_mbox(mbox, tp->prodring[0].rx_jmb_prod_idx);
4909                 }
4910
4911                 mmiowb();
4912         }
4913
4914         return work_done;
4915 }
4916
4917 static int tg3_poll_msix(struct napi_struct *napi, int budget)
4918 {
4919         struct tg3_napi *tnapi = container_of(napi, struct tg3_napi, napi);
4920         struct tg3 *tp = tnapi->tp;
4921         int work_done = 0;
4922         struct tg3_hw_status *sblk = tnapi->hw_status;
4923
4924         while (1) {
4925                 work_done = tg3_poll_work(tnapi, work_done, budget);
4926
4927                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING))
4928                         goto tx_recovery;
4929
4930                 if (unlikely(work_done >= budget))
4931                         break;
4932
4933                 /* tp->last_tag is used in tg3_restart_ints() below
4934                  * to tell the hw how much work has been processed,
4935                  * so we must read it before checking for more work.
4936                  */
4937                 tnapi->last_tag = sblk->status_tag;
4938                 tnapi->last_irq_tag = tnapi->last_tag;
4939                 rmb();
4940
4941                 /* check for RX/TX work to do */
4942                 if (sblk->idx[0].tx_consumer == tnapi->tx_cons &&
4943                     *(tnapi->rx_rcb_prod_idx) == tnapi->rx_rcb_ptr) {
4944                         napi_complete(napi);
4945                         /* Reenable interrupts. */
4946                         tw32_mailbox(tnapi->int_mbox, tnapi->last_tag << 24);
4947                         mmiowb();
4948                         break;
4949                 }
4950         }
4951
4952         return work_done;
4953
4954 tx_recovery:
4955         /* work_done is guaranteed to be less than budget. */
4956         napi_complete(napi);
4957         schedule_work(&tp->reset_task);
4958         return work_done;
4959 }
4960
4961 static int tg3_poll(struct napi_struct *napi, int budget)
4962 {
4963         struct tg3_napi *tnapi = container_of(napi, struct tg3_napi, napi);
4964         struct tg3 *tp = tnapi->tp;
4965         int work_done = 0;
4966         struct tg3_hw_status *sblk = tnapi->hw_status;
4967
4968         while (1) {
4969                 tg3_poll_link(tp);
4970
4971                 work_done = tg3_poll_work(tnapi, work_done, budget);
4972
4973                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING))
4974                         goto tx_recovery;
4975
4976                 if (unlikely(work_done >= budget))
4977                         break;
4978
4979                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) {
4980                         /* tp->last_tag is used in tg3_int_reenable() below
4981                          * to tell the hw how much work has been processed,
4982                          * so we must read it before checking for more work.
4983                          */
4984                         tnapi->last_tag = sblk->status_tag;
4985                         tnapi->last_irq_tag = tnapi->last_tag;
4986                         rmb();
4987                 } else
4988                         sblk->status &= ~SD_STATUS_UPDATED;
4989
4990                 if (likely(!tg3_has_work(tnapi))) {
4991                         napi_complete(napi);
4992                         tg3_int_reenable(tnapi);
4993                         break;
4994                 }
4995         }
4996
4997         return work_done;
4998
4999 tx_recovery:
5000         /* work_done is guaranteed to be less than budget. */
5001         napi_complete(napi);
5002         schedule_work(&tp->reset_task);
5003         return work_done;
5004 }
5005
5006 static void tg3_irq_quiesce(struct tg3 *tp)
5007 {
5008         int i;
5009
5010         BUG_ON(tp->irq_sync);
5011
5012         tp->irq_sync = 1;
5013         smp_mb();
5014
5015         for (i = 0; i < tp->irq_cnt; i++)
5016                 synchronize_irq(tp->napi[i].irq_vec);
5017 }
5018
5019 static inline int tg3_irq_sync(struct tg3 *tp)
5020 {
5021         return tp->irq_sync;
5022 }
5023
5024 /* Fully shutdown all tg3 driver activity elsewhere in the system.
5025  * If irq_sync is non-zero, then the IRQ handler must be synchronized
5026  * with as well.  Most of the time, this is not necessary except when
5027  * shutting down the device.
5028  */
5029 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync)
5030 {
5031         spin_lock_bh(&tp->lock);
5032         if (irq_sync)
5033                 tg3_irq_quiesce(tp);
5034 }
5035
5036 static inline void tg3_full_unlock(struct tg3 *tp)
5037 {
5038         spin_unlock_bh(&tp->lock);
5039 }
5040
5041 /* One-shot MSI handler - Chip automatically disables interrupt
5042  * after sending MSI so driver doesn't have to do it.
5043  */
5044 static irqreturn_t tg3_msi_1shot(int irq, void *dev_id)
5045 {
5046         struct tg3_napi *tnapi = dev_id;
5047         struct tg3 *tp = tnapi->tp;
5048
5049         prefetch(tnapi->hw_status);
5050         if (tnapi->rx_rcb)
5051                 prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
5052
5053         if (likely(!tg3_irq_sync(tp)))
5054                 napi_schedule(&tnapi->napi);
5055
5056         return IRQ_HANDLED;
5057 }
5058
5059 /* MSI ISR - No need to check for interrupt sharing and no need to
5060  * flush status block and interrupt mailbox. PCI ordering rules
5061  * guarantee that MSI will arrive after the status block.
5062  */
5063 static irqreturn_t tg3_msi(int irq, void *dev_id)
5064 {
5065         struct tg3_napi *tnapi = dev_id;
5066         struct tg3 *tp = tnapi->tp;
5067
5068         prefetch(tnapi->hw_status);
5069         if (tnapi->rx_rcb)
5070                 prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
5071         /*
5072          * Writing any value to intr-mbox-0 clears PCI INTA# and
5073          * chip-internal interrupt pending events.
5074          * Writing non-zero to intr-mbox-0 additional tells the
5075          * NIC to stop sending us irqs, engaging "in-intr-handler"
5076          * event coalescing.
5077          */
5078         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
5079         if (likely(!tg3_irq_sync(tp)))
5080                 napi_schedule(&tnapi->napi);
5081
5082         return IRQ_RETVAL(1);
5083 }
5084
5085 static irqreturn_t tg3_interrupt(int irq, void *dev_id)
5086 {
5087         struct tg3_napi *tnapi = dev_id;
5088         struct tg3 *tp = tnapi->tp;
5089         struct tg3_hw_status *sblk = tnapi->hw_status;
5090         unsigned int handled = 1;
5091
5092         /* In INTx mode, it is possible for the interrupt to arrive at
5093          * the CPU before the status block posted prior to the interrupt.
5094          * Reading the PCI State register will confirm whether the
5095          * interrupt is ours and will flush the status block.
5096          */
5097         if (unlikely(!(sblk->status & SD_STATUS_UPDATED))) {
5098                 if ((tp->tg3_flags & TG3_FLAG_CHIP_RESETTING) ||
5099                     (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
5100                         handled = 0;
5101                         goto out;
5102                 }
5103         }
5104
5105         /*
5106          * Writing any value to intr-mbox-0 clears PCI INTA# and
5107          * chip-internal interrupt pending events.
5108          * Writing non-zero to intr-mbox-0 additional tells the
5109          * NIC to stop sending us irqs, engaging "in-intr-handler"
5110          * event coalescing.
5111          *
5112          * Flush the mailbox to de-assert the IRQ immediately to prevent
5113          * spurious interrupts.  The flush impacts performance but
5114          * excessive spurious interrupts can be worse in some cases.
5115          */
5116         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
5117         if (tg3_irq_sync(tp))
5118                 goto out;
5119         sblk->status &= ~SD_STATUS_UPDATED;
5120         if (likely(tg3_has_work(tnapi))) {
5121                 prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
5122                 napi_schedule(&tnapi->napi);
5123         } else {
5124                 /* No work, shared interrupt perhaps?  re-enable
5125                  * interrupts, and flush that PCI write
5126                  */
5127                 tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
5128                                0x00000000);
5129         }
5130 out:
5131         return IRQ_RETVAL(handled);
5132 }
5133
5134 static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id)
5135 {
5136         struct tg3_napi *tnapi = dev_id;
5137         struct tg3 *tp = tnapi->tp;
5138         struct tg3_hw_status *sblk = tnapi->hw_status;
5139         unsigned int handled = 1;
5140
5141         /* In INTx mode, it is possible for the interrupt to arrive at
5142          * the CPU before the status block posted prior to the interrupt.
5143          * Reading the PCI State register will confirm whether the
5144          * interrupt is ours and will flush the status block.
5145          */
5146         if (unlikely(sblk->status_tag == tnapi->last_irq_tag)) {
5147                 if ((tp->tg3_flags & TG3_FLAG_CHIP_RESETTING) ||
5148                     (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
5149                         handled = 0;
5150                         goto out;
5151                 }
5152         }
5153
5154         /*
5155          * writing any value to intr-mbox-0 clears PCI INTA# and
5156          * chip-internal interrupt pending events.
5157          * writing non-zero to intr-mbox-0 additional tells the
5158          * NIC to stop sending us irqs, engaging "in-intr-handler"
5159          * event coalescing.
5160          *
5161          * Flush the mailbox to de-assert the IRQ immediately to prevent
5162          * spurious interrupts.  The flush impacts performance but
5163          * excessive spurious interrupts can be worse in some cases.
5164          */
5165         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
5166
5167         /*
5168          * In a shared interrupt configuration, sometimes other devices'
5169          * interrupts will scream.  We record the current status tag here
5170          * so that the above check can report that the screaming interrupts
5171          * are unhandled.  Eventually they will be silenced.
5172          */
5173         tnapi->last_irq_tag = sblk->status_tag;
5174
5175         if (tg3_irq_sync(tp))
5176                 goto out;
5177
5178         prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
5179
5180         napi_schedule(&tnapi->napi);
5181
5182 out:
5183         return IRQ_RETVAL(handled);
5184 }
5185
5186 /* ISR for interrupt test */
5187 static irqreturn_t tg3_test_isr(int irq, void *dev_id)
5188 {
5189         struct tg3_napi *tnapi = dev_id;
5190         struct tg3 *tp = tnapi->tp;
5191         struct tg3_hw_status *sblk = tnapi->hw_status;
5192
5193         if ((sblk->status & SD_STATUS_UPDATED) ||
5194             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
5195                 tg3_disable_ints(tp);
5196                 return IRQ_RETVAL(1);
5197         }
5198         return IRQ_RETVAL(0);
5199 }
5200
5201 static int tg3_init_hw(struct tg3 *, int);
5202 static int tg3_halt(struct tg3 *, int, int);
5203
5204 /* Restart hardware after configuration changes, self-test, etc.
5205  * Invoked with tp->lock held.
5206  */
5207 static int tg3_restart_hw(struct tg3 *tp, int reset_phy)
5208         __releases(tp->lock)
5209         __acquires(tp->lock)
5210 {
5211         int err;
5212
5213         err = tg3_init_hw(tp, reset_phy);
5214         if (err) {
5215                 printk(KERN_ERR PFX "%s: Failed to re-initialize device, "
5216                        "aborting.\n", tp->dev->name);
5217                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
5218                 tg3_full_unlock(tp);
5219                 del_timer_sync(&tp->timer);
5220                 tp->irq_sync = 0;
5221                 tg3_napi_enable(tp);
5222                 dev_close(tp->dev);
5223                 tg3_full_lock(tp, 0);
5224         }
5225         return err;
5226 }
5227
5228 #ifdef CONFIG_NET_POLL_CONTROLLER
5229 static void tg3_poll_controller(struct net_device *dev)
5230 {
5231         int i;
5232         struct tg3 *tp = netdev_priv(dev);
5233
5234         for (i = 0; i < tp->irq_cnt; i++)
5235                 tg3_interrupt(tp->napi[i].irq_vec, dev);
5236 }
5237 #endif
5238
5239 static void tg3_reset_task(struct work_struct *work)
5240 {
5241         struct tg3 *tp = container_of(work, struct tg3, reset_task);
5242         int err;
5243         unsigned int restart_timer;
5244
5245         tg3_full_lock(tp, 0);
5246
5247         if (!netif_running(tp->dev)) {
5248                 tg3_full_unlock(tp);
5249                 return;
5250         }
5251
5252         tg3_full_unlock(tp);
5253
5254         tg3_phy_stop(tp);
5255
5256         tg3_netif_stop(tp);
5257
5258         tg3_full_lock(tp, 1);
5259
5260         restart_timer = tp->tg3_flags2 & TG3_FLG2_RESTART_TIMER;
5261         tp->tg3_flags2 &= ~TG3_FLG2_RESTART_TIMER;
5262
5263         if (tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING) {
5264                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
5265                 tp->write32_rx_mbox = tg3_write_flush_reg32;
5266                 tp->tg3_flags |= TG3_FLAG_MBOX_WRITE_REORDER;
5267                 tp->tg3_flags &= ~TG3_FLAG_TX_RECOVERY_PENDING;
5268         }
5269
5270         tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
5271         err = tg3_init_hw(tp, 1);
5272         if (err)
5273                 goto out;
5274
5275         tg3_netif_start(tp);
5276
5277         if (restart_timer)
5278                 mod_timer(&tp->timer, jiffies + 1);
5279
5280 out:
5281         tg3_full_unlock(tp);
5282
5283         if (!err)
5284                 tg3_phy_start(tp);
5285 }
5286
5287 static void tg3_dump_short_state(struct tg3 *tp)
5288 {
5289         printk(KERN_ERR PFX "DEBUG: MAC_TX_STATUS[%08x] MAC_RX_STATUS[%08x]\n",
5290                tr32(MAC_TX_STATUS), tr32(MAC_RX_STATUS));
5291         printk(KERN_ERR PFX "DEBUG: RDMAC_STATUS[%08x] WDMAC_STATUS[%08x]\n",
5292                tr32(RDMAC_STATUS), tr32(WDMAC_STATUS));
5293 }
5294
5295 static void tg3_tx_timeout(struct net_device *dev)
5296 {
5297         struct tg3 *tp = netdev_priv(dev);
5298
5299         if (netif_msg_tx_err(tp)) {
5300                 printk(KERN_ERR PFX "%s: transmit timed out, resetting\n",
5301                        dev->name);
5302                 tg3_dump_short_state(tp);
5303         }
5304
5305         schedule_work(&tp->reset_task);
5306 }
5307
5308 /* Test for DMA buffers crossing any 4GB boundaries: 4G, 8G, etc */
5309 static inline int tg3_4g_overflow_test(dma_addr_t mapping, int len)
5310 {
5311         u32 base = (u32) mapping & 0xffffffff;
5312
5313         return ((base > 0xffffdcc0) &&
5314                 (base + len + 8 < base));
5315 }
5316
5317 /* Test for DMA addresses > 40-bit */
5318 static inline int tg3_40bit_overflow_test(struct tg3 *tp, dma_addr_t mapping,
5319                                           int len)
5320 {
5321 #if defined(CONFIG_HIGHMEM) && (BITS_PER_LONG == 64)
5322         if (tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG)
5323                 return (((u64) mapping + len) > DMA_BIT_MASK(40));
5324         return 0;
5325 #else
5326         return 0;
5327 #endif
5328 }
5329
5330 static void tg3_set_txd(struct tg3_napi *, int, dma_addr_t, int, u32, u32);
5331
5332 /* Workaround 4GB and 40-bit hardware DMA bugs. */
5333 static int tigon3_dma_hwbug_workaround(struct tg3_napi *tnapi,
5334                                        struct sk_buff *skb, u32 last_plus_one,
5335                                        u32 *start, u32 base_flags, u32 mss)
5336 {
5337         struct tg3 *tp = tnapi->tp;
5338         struct sk_buff *new_skb;
5339         dma_addr_t new_addr = 0;
5340         u32 entry = *start;
5341         int i, ret = 0;
5342
5343         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701)
5344                 new_skb = skb_copy(skb, GFP_ATOMIC);
5345         else {
5346                 int more_headroom = 4 - ((unsigned long)skb->data & 3);
5347
5348                 new_skb = skb_copy_expand(skb,
5349                                           skb_headroom(skb) + more_headroom,
5350                                           skb_tailroom(skb), GFP_ATOMIC);
5351         }
5352
5353         if (!new_skb) {
5354                 ret = -1;
5355         } else {
5356                 /* New SKB is guaranteed to be linear. */
5357                 entry = *start;
5358                 new_addr = pci_map_single(tp->pdev, new_skb->data, new_skb->len,
5359                                           PCI_DMA_TODEVICE);
5360                 /* Make sure the mapping succeeded */
5361                 if (pci_dma_mapping_error(tp->pdev, new_addr)) {
5362                         ret = -1;
5363                         dev_kfree_skb(new_skb);
5364                         new_skb = NULL;
5365
5366                 /* Make sure new skb does not cross any 4G boundaries.
5367                  * Drop the packet if it does.
5368                  */
5369                 } else if ((tp->tg3_flags3 & TG3_FLG3_4G_DMA_BNDRY_BUG) &&
5370                             tg3_4g_overflow_test(new_addr, new_skb->len)) {
5371                         pci_unmap_single(tp->pdev, new_addr, new_skb->len,
5372                                          PCI_DMA_TODEVICE);
5373                         ret = -1;
5374                         dev_kfree_skb(new_skb);
5375                         new_skb = NULL;
5376                 } else {
5377                         tg3_set_txd(tnapi, entry, new_addr, new_skb->len,
5378                                     base_flags, 1 | (mss << 1));
5379                         *start = NEXT_TX(entry);
5380                 }
5381         }
5382
5383         /* Now clean up the sw ring entries. */
5384         i = 0;
5385         while (entry != last_plus_one) {
5386                 int len;
5387
5388                 if (i == 0)
5389                         len = skb_headlen(skb);
5390                 else
5391                         len = skb_shinfo(skb)->frags[i-1].size;
5392
5393                 pci_unmap_single(tp->pdev,
5394                                  pci_unmap_addr(&tnapi->tx_buffers[entry],
5395                                                 mapping),
5396                                  len, PCI_DMA_TODEVICE);
5397                 if (i == 0) {
5398                         tnapi->tx_buffers[entry].skb = new_skb;
5399                         pci_unmap_addr_set(&tnapi->tx_buffers[entry], mapping,
5400                                            new_addr);
5401                 } else {
5402                         tnapi->tx_buffers[entry].skb = NULL;
5403                 }
5404                 entry = NEXT_TX(entry);
5405                 i++;
5406         }
5407
5408         dev_kfree_skb(skb);
5409
5410         return ret;
5411 }
5412
5413 static void tg3_set_txd(struct tg3_napi *tnapi, int entry,
5414                         dma_addr_t mapping, int len, u32 flags,
5415                         u32 mss_and_is_end)
5416 {
5417         struct tg3_tx_buffer_desc *txd = &tnapi->tx_ring[entry];
5418         int is_end = (mss_and_is_end & 0x1);
5419         u32 mss = (mss_and_is_end >> 1);
5420         u32 vlan_tag = 0;
5421
5422         if (is_end)
5423                 flags |= TXD_FLAG_END;
5424         if (flags & TXD_FLAG_VLAN) {
5425                 vlan_tag = flags >> 16;
5426                 flags &= 0xffff;
5427         }
5428         vlan_tag |= (mss << TXD_MSS_SHIFT);
5429
5430         txd->addr_hi = ((u64) mapping >> 32);
5431         txd->addr_lo = ((u64) mapping & 0xffffffff);
5432         txd->len_flags = (len << TXD_LEN_SHIFT) | flags;
5433         txd->vlan_tag = vlan_tag << TXD_VLAN_TAG_SHIFT;
5434 }
5435
5436 /* hard_start_xmit for devices that don't have any bugs and
5437  * support TG3_FLG2_HW_TSO_2 and TG3_FLG2_HW_TSO_3 only.
5438  */
5439 static netdev_tx_t tg3_start_xmit(struct sk_buff *skb,
5440                                   struct net_device *dev)
5441 {
5442         struct tg3 *tp = netdev_priv(dev);
5443         u32 len, entry, base_flags, mss;
5444         dma_addr_t mapping;
5445         struct tg3_napi *tnapi;
5446         struct netdev_queue *txq;
5447         unsigned int i, last;
5448
5449
5450         txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
5451         tnapi = &tp->napi[skb_get_queue_mapping(skb)];
5452         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_TSS)
5453                 tnapi++;
5454
5455         /* We are running in BH disabled context with netif_tx_lock
5456          * and TX reclaim runs via tp->napi.poll inside of a software
5457          * interrupt.  Furthermore, IRQ processing runs lockless so we have
5458          * no IRQ context deadlocks to worry about either.  Rejoice!
5459          */
5460         if (unlikely(tg3_tx_avail(tnapi) <= (skb_shinfo(skb)->nr_frags + 1))) {
5461                 if (!netif_tx_queue_stopped(txq)) {
5462                         netif_tx_stop_queue(txq);
5463
5464                         /* This is a hard error, log it. */
5465                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
5466                                "queue awake!\n", dev->name);
5467                 }
5468                 return NETDEV_TX_BUSY;
5469         }
5470
5471         entry = tnapi->tx_prod;
5472         base_flags = 0;
5473         mss = 0;
5474         if ((mss = skb_shinfo(skb)->gso_size) != 0) {
5475                 int tcp_opt_len, ip_tcp_len;
5476                 u32 hdrlen;
5477
5478                 if (skb_header_cloned(skb) &&
5479                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
5480                         dev_kfree_skb(skb);
5481                         goto out_unlock;
5482                 }
5483
5484                 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
5485                         hdrlen = skb_headlen(skb) - ETH_HLEN;
5486                 else {
5487                         struct iphdr *iph = ip_hdr(skb);
5488
5489                         tcp_opt_len = tcp_optlen(skb);
5490                         ip_tcp_len = ip_hdrlen(skb) + sizeof(struct tcphdr);
5491
5492                         iph->check = 0;
5493                         iph->tot_len = htons(mss + ip_tcp_len + tcp_opt_len);
5494                         hdrlen = ip_tcp_len + tcp_opt_len;
5495                 }
5496
5497                 if (tp->tg3_flags2 & TG3_FLG2_HW_TSO_3) {
5498                         mss |= (hdrlen & 0xc) << 12;
5499                         if (hdrlen & 0x10)
5500                                 base_flags |= 0x00000010;
5501                         base_flags |= (hdrlen & 0x3e0) << 5;
5502                 } else
5503                         mss |= hdrlen << 9;
5504
5505                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
5506                                TXD_FLAG_CPU_POST_DMA);
5507
5508                 tcp_hdr(skb)->check = 0;
5509
5510         }
5511         else if (skb->ip_summed == CHECKSUM_PARTIAL)
5512                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
5513 #if TG3_VLAN_TAG_USED
5514         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
5515                 base_flags |= (TXD_FLAG_VLAN |
5516                                (vlan_tx_tag_get(skb) << 16));
5517 #endif
5518
5519         len = skb_headlen(skb);
5520
5521         /* Queue skb data, a.k.a. the main skb fragment. */
5522         mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
5523         if (pci_dma_mapping_error(tp->pdev, mapping)) {
5524                 dev_kfree_skb(skb);
5525                 goto out_unlock;
5526         }
5527
5528         tnapi->tx_buffers[entry].skb = skb;
5529         pci_unmap_addr_set(&tnapi->tx_buffers[entry], mapping, mapping);
5530
5531         if ((tp->tg3_flags3 & TG3_FLG3_USE_JUMBO_BDFLAG) &&
5532             !mss && skb->len > ETH_DATA_LEN)
5533                 base_flags |= TXD_FLAG_JMB_PKT;
5534
5535         tg3_set_txd(tnapi, entry, mapping, len, base_flags,
5536                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
5537
5538         entry = NEXT_TX(entry);
5539
5540         /* Now loop through additional data fragments, and queue them. */
5541         if (skb_shinfo(skb)->nr_frags > 0) {
5542                 last = skb_shinfo(skb)->nr_frags - 1;
5543                 for (i = 0; i <= last; i++) {
5544                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
5545
5546                         len = frag->size;
5547                         mapping = pci_map_page(tp->pdev,
5548                                                frag->page,
5549                                                frag->page_offset,
5550                                                len, PCI_DMA_TODEVICE);
5551                         if (pci_dma_mapping_error(tp->pdev, mapping))
5552                                 goto dma_error;
5553
5554                         tnapi->tx_buffers[entry].skb = NULL;
5555                         pci_unmap_addr_set(&tnapi->tx_buffers[entry], mapping,
5556                                            mapping);
5557
5558                         tg3_set_txd(tnapi, entry, mapping, len,
5559                                     base_flags, (i == last) | (mss << 1));
5560
5561                         entry = NEXT_TX(entry);
5562                 }
5563         }
5564
5565         /* Packets are ready, update Tx producer idx local and on card. */
5566         tw32_tx_mbox(tnapi->prodmbox, entry);
5567
5568         tnapi->tx_prod = entry;
5569         if (unlikely(tg3_tx_avail(tnapi) <= (MAX_SKB_FRAGS + 1))) {
5570                 netif_tx_stop_queue(txq);
5571                 if (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi))
5572                         netif_tx_wake_queue(txq);
5573         }
5574
5575 out_unlock:
5576         mmiowb();
5577
5578         return NETDEV_TX_OK;
5579
5580 dma_error:
5581         last = i;
5582         entry = tnapi->tx_prod;
5583         tnapi->tx_buffers[entry].skb = NULL;
5584         pci_unmap_single(tp->pdev,
5585                          pci_unmap_addr(&tnapi->tx_buffers[entry], mapping),
5586                          skb_headlen(skb),
5587                          PCI_DMA_TODEVICE);
5588         for (i = 0; i <= last; i++) {
5589                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
5590                 entry = NEXT_TX(entry);
5591
5592                 pci_unmap_page(tp->pdev,
5593                                pci_unmap_addr(&tnapi->tx_buffers[entry],
5594                                               mapping),
5595                                frag->size, PCI_DMA_TODEVICE);
5596         }
5597
5598         dev_kfree_skb(skb);
5599         return NETDEV_TX_OK;
5600 }
5601
5602 static netdev_tx_t tg3_start_xmit_dma_bug(struct sk_buff *,
5603                                           struct net_device *);
5604
5605 /* Use GSO to workaround a rare TSO bug that may be triggered when the
5606  * TSO header is greater than 80 bytes.
5607  */
5608 static int tg3_tso_bug(struct tg3 *tp, struct sk_buff *skb)
5609 {
5610         struct sk_buff *segs, *nskb;
5611         u32 frag_cnt_est = skb_shinfo(skb)->gso_segs * 3;
5612
5613         /* Estimate the number of fragments in the worst case */
5614         if (unlikely(tg3_tx_avail(&tp->napi[0]) <= frag_cnt_est)) {
5615                 netif_stop_queue(tp->dev);
5616                 if (tg3_tx_avail(&tp->napi[0]) <= frag_cnt_est)
5617                         return NETDEV_TX_BUSY;
5618
5619                 netif_wake_queue(tp->dev);
5620         }
5621
5622         segs = skb_gso_segment(skb, tp->dev->features & ~NETIF_F_TSO);
5623         if (IS_ERR(segs))
5624                 goto tg3_tso_bug_end;
5625
5626         do {
5627                 nskb = segs;
5628                 segs = segs->next;
5629                 nskb->next = NULL;
5630                 tg3_start_xmit_dma_bug(nskb, tp->dev);
5631         } while (segs);
5632
5633 tg3_tso_bug_end:
5634         dev_kfree_skb(skb);
5635
5636         return NETDEV_TX_OK;
5637 }
5638
5639 /* hard_start_xmit for devices that have the 4G bug and/or 40-bit bug and
5640  * support TG3_FLG2_HW_TSO_1 or firmware TSO only.
5641  */
5642 static netdev_tx_t tg3_start_xmit_dma_bug(struct sk_buff *skb,
5643                                           struct net_device *dev)
5644 {
5645         struct tg3 *tp = netdev_priv(dev);
5646         u32 len, entry, base_flags, mss;
5647         int would_hit_hwbug;
5648         dma_addr_t mapping;
5649         struct tg3_napi *tnapi;
5650         struct netdev_queue *txq;
5651         unsigned int i, last;
5652
5653
5654         txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
5655         tnapi = &tp->napi[skb_get_queue_mapping(skb)];
5656         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_TSS)
5657                 tnapi++;
5658
5659         /* We are running in BH disabled context with netif_tx_lock
5660          * and TX reclaim runs via tp->napi.poll inside of a software
5661          * interrupt.  Furthermore, IRQ processing runs lockless so we have
5662          * no IRQ context deadlocks to worry about either.  Rejoice!
5663          */
5664         if (unlikely(tg3_tx_avail(tnapi) <= (skb_shinfo(skb)->nr_frags + 1))) {
5665                 if (!netif_tx_queue_stopped(txq)) {
5666                         netif_tx_stop_queue(txq);
5667
5668                         /* This is a hard error, log it. */
5669                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
5670                                "queue awake!\n", dev->name);
5671                 }
5672                 return NETDEV_TX_BUSY;
5673         }
5674
5675         entry = tnapi->tx_prod;
5676         base_flags = 0;
5677         if (skb->ip_summed == CHECKSUM_PARTIAL)
5678                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
5679
5680         if ((mss = skb_shinfo(skb)->gso_size) != 0) {
5681                 struct iphdr *iph;
5682                 u32 tcp_opt_len, ip_tcp_len, hdr_len;
5683
5684                 if (skb_header_cloned(skb) &&
5685                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
5686                         dev_kfree_skb(skb);
5687                         goto out_unlock;
5688                 }
5689
5690                 tcp_opt_len = tcp_optlen(skb);
5691                 ip_tcp_len = ip_hdrlen(skb) + sizeof(struct tcphdr);
5692
5693                 hdr_len = ip_tcp_len + tcp_opt_len;
5694                 if (unlikely((ETH_HLEN + hdr_len) > 80) &&
5695                              (tp->tg3_flags2 & TG3_FLG2_TSO_BUG))
5696                         return (tg3_tso_bug(tp, skb));
5697
5698                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
5699                                TXD_FLAG_CPU_POST_DMA);
5700
5701                 iph = ip_hdr(skb);
5702                 iph->check = 0;
5703                 iph->tot_len = htons(mss + hdr_len);
5704                 if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
5705                         tcp_hdr(skb)->check = 0;
5706                         base_flags &= ~TXD_FLAG_TCPUDP_CSUM;
5707                 } else
5708                         tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
5709                                                                  iph->daddr, 0,
5710                                                                  IPPROTO_TCP,
5711                                                                  0);
5712
5713                 if (tp->tg3_flags2 & TG3_FLG2_HW_TSO_3) {
5714                         mss |= (hdr_len & 0xc) << 12;
5715                         if (hdr_len & 0x10)
5716                                 base_flags |= 0x00000010;
5717                         base_flags |= (hdr_len & 0x3e0) << 5;
5718                 } else if (tp->tg3_flags2 & TG3_FLG2_HW_TSO_2)
5719                         mss |= hdr_len << 9;
5720                 else if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO_1) ||
5721                          GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
5722                         if (tcp_opt_len || iph->ihl > 5) {
5723                                 int tsflags;
5724
5725                                 tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
5726                                 mss |= (tsflags << 11);
5727                         }
5728                 } else {
5729                         if (tcp_opt_len || iph->ihl > 5) {
5730                                 int tsflags;
5731
5732                                 tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
5733                                 base_flags |= tsflags << 12;
5734                         }
5735                 }
5736         }
5737 #if TG3_VLAN_TAG_USED
5738         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
5739                 base_flags |= (TXD_FLAG_VLAN |
5740                                (vlan_tx_tag_get(skb) << 16));
5741 #endif
5742
5743         if ((tp->tg3_flags3 & TG3_FLG3_USE_JUMBO_BDFLAG) &&
5744             !mss && skb->len > ETH_DATA_LEN)
5745                 base_flags |= TXD_FLAG_JMB_PKT;
5746
5747         len = skb_headlen(skb);
5748
5749         mapping = pci_map_single(tp->pdev,