e3e380f90f86dec173b0966936d64951aa80ba40
[linux-3.10.git] / drivers / net / tg3.c
1 /*
2  * tg3.c: Broadcom Tigon3 ethernet driver.
3  *
4  * Copyright (C) 2001, 2002, 2003, 2004 David S. Miller (davem@redhat.com)
5  * Copyright (C) 2001, 2002, 2003 Jeff Garzik (jgarzik@pobox.com)
6  * Copyright (C) 2004 Sun Microsystems Inc.
7  * Copyright (C) 2005 Broadcom Corporation.
8  *
9  * Firmware is:
10  *      Derived from proprietary unpublished source code,
11  *      Copyright (C) 2000-2003 Broadcom Corporation.
12  *
13  *      Permission is hereby granted for the distribution of this firmware
14  *      data in hexadecimal or equivalent format, provided this copyright
15  *      notice is accompanying it.
16  */
17
18 #include <linux/config.h>
19
20 #include <linux/module.h>
21 #include <linux/moduleparam.h>
22 #include <linux/kernel.h>
23 #include <linux/types.h>
24 #include <linux/compiler.h>
25 #include <linux/slab.h>
26 #include <linux/delay.h>
27 #include <linux/in.h>
28 #include <linux/init.h>
29 #include <linux/ioport.h>
30 #include <linux/pci.h>
31 #include <linux/netdevice.h>
32 #include <linux/etherdevice.h>
33 #include <linux/skbuff.h>
34 #include <linux/ethtool.h>
35 #include <linux/mii.h>
36 #include <linux/if_vlan.h>
37 #include <linux/ip.h>
38 #include <linux/tcp.h>
39 #include <linux/workqueue.h>
40 #include <linux/prefetch.h>
41 #include <linux/dma-mapping.h>
42
43 #include <net/checksum.h>
44
45 #include <asm/system.h>
46 #include <asm/io.h>
47 #include <asm/byteorder.h>
48 #include <asm/uaccess.h>
49
50 #ifdef CONFIG_SPARC64
51 #include <asm/idprom.h>
52 #include <asm/oplib.h>
53 #include <asm/pbm.h>
54 #endif
55
56 #if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
57 #define TG3_VLAN_TAG_USED 1
58 #else
59 #define TG3_VLAN_TAG_USED 0
60 #endif
61
62 #ifdef NETIF_F_TSO
63 #define TG3_TSO_SUPPORT 1
64 #else
65 #define TG3_TSO_SUPPORT 0
66 #endif
67
68 #include "tg3.h"
69
70 #define DRV_MODULE_NAME         "tg3"
71 #define PFX DRV_MODULE_NAME     ": "
72 #define DRV_MODULE_VERSION      "3.60"
73 #define DRV_MODULE_RELDATE      "June 17, 2006"
74
75 #define TG3_DEF_MAC_MODE        0
76 #define TG3_DEF_RX_MODE         0
77 #define TG3_DEF_TX_MODE         0
78 #define TG3_DEF_MSG_ENABLE        \
79         (NETIF_MSG_DRV          | \
80          NETIF_MSG_PROBE        | \
81          NETIF_MSG_LINK         | \
82          NETIF_MSG_TIMER        | \
83          NETIF_MSG_IFDOWN       | \
84          NETIF_MSG_IFUP         | \
85          NETIF_MSG_RX_ERR       | \
86          NETIF_MSG_TX_ERR)
87
88 /* length of time before we decide the hardware is borked,
89  * and dev->tx_timeout() should be called to fix the problem
90  */
91 #define TG3_TX_TIMEOUT                  (5 * HZ)
92
93 /* hardware minimum and maximum for a single frame's data payload */
94 #define TG3_MIN_MTU                     60
95 #define TG3_MAX_MTU(tp) \
96         ((tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) ? 9000 : 1500)
97
98 /* These numbers seem to be hard coded in the NIC firmware somehow.
99  * You can't change the ring sizes, but you can change where you place
100  * them in the NIC onboard memory.
101  */
102 #define TG3_RX_RING_SIZE                512
103 #define TG3_DEF_RX_RING_PENDING         200
104 #define TG3_RX_JUMBO_RING_SIZE          256
105 #define TG3_DEF_RX_JUMBO_RING_PENDING   100
106
107 /* Do not place this n-ring entries value into the tp struct itself,
108  * we really want to expose these constants to GCC so that modulo et
109  * al.  operations are done with shifts and masks instead of with
110  * hw multiply/modulo instructions.  Another solution would be to
111  * replace things like '% foo' with '& (foo - 1)'.
112  */
113 #define TG3_RX_RCB_RING_SIZE(tp)        \
114         ((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ?  512 : 1024)
115
116 #define TG3_TX_RING_SIZE                512
117 #define TG3_DEF_TX_RING_PENDING         (TG3_TX_RING_SIZE - 1)
118
119 #define TG3_RX_RING_BYTES       (sizeof(struct tg3_rx_buffer_desc) * \
120                                  TG3_RX_RING_SIZE)
121 #define TG3_RX_JUMBO_RING_BYTES (sizeof(struct tg3_rx_buffer_desc) * \
122                                  TG3_RX_JUMBO_RING_SIZE)
123 #define TG3_RX_RCB_RING_BYTES(tp) (sizeof(struct tg3_rx_buffer_desc) * \
124                                    TG3_RX_RCB_RING_SIZE(tp))
125 #define TG3_TX_RING_BYTES       (sizeof(struct tg3_tx_buffer_desc) * \
126                                  TG3_TX_RING_SIZE)
127 #define TX_BUFFS_AVAIL(TP)                                              \
128         ((TP)->tx_pending -                                             \
129          (((TP)->tx_prod - (TP)->tx_cons) & (TG3_TX_RING_SIZE - 1)))
130 #define NEXT_TX(N)              (((N) + 1) & (TG3_TX_RING_SIZE - 1))
131
132 #define RX_PKT_BUF_SZ           (1536 + tp->rx_offset + 64)
133 #define RX_JUMBO_PKT_BUF_SZ     (9046 + tp->rx_offset + 64)
134
135 /* minimum number of free TX descriptors required to wake up TX process */
136 #define TG3_TX_WAKEUP_THRESH            (TG3_TX_RING_SIZE / 4)
137
138 /* number of ETHTOOL_GSTATS u64's */
139 #define TG3_NUM_STATS           (sizeof(struct tg3_ethtool_stats)/sizeof(u64))
140
141 #define TG3_NUM_TEST            6
142
143 static char version[] __devinitdata =
144         DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
145
146 MODULE_AUTHOR("David S. Miller (davem@redhat.com) and Jeff Garzik (jgarzik@pobox.com)");
147 MODULE_DESCRIPTION("Broadcom Tigon3 ethernet driver");
148 MODULE_LICENSE("GPL");
149 MODULE_VERSION(DRV_MODULE_VERSION);
150
151 static int tg3_debug = -1;      /* -1 == use TG3_DEF_MSG_ENABLE as value */
152 module_param(tg3_debug, int, 0);
153 MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");
154
155 static struct pci_device_id tg3_pci_tbl[] = {
156         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700,
157           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
158         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701,
159           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
160         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702,
161           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
162         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703,
163           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
164         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704,
165           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
166         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE,
167           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
168         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705,
169           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
170         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2,
171           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
172         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M,
173           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
174         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2,
175           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
176         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X,
177           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
178         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X,
179           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
180         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S,
181           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
182         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3,
183           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
184         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3,
185           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
186         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782,
187           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
188         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788,
189           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
190         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789,
191           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
192         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901,
193           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
194         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2,
195           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
196         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2,
197           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
198         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F,
199           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
200         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5720,
201           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
202         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721,
203           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
204         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750,
205           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
206         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751,
207           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
208         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750M,
209           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
210         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M,
211           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
212         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F,
213           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
214         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752,
215           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
216         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M,
217           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
218         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753,
219           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
220         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M,
221           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
222         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F,
223           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
224         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754,
225           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
226         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M,
227           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
228         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755,
229           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
230         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M,
231           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
232         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786,
233           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
234         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787,
235           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
236         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M,
237           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
238         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714,
239           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
240         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S,
241           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
242         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715,
243           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
244         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S,
245           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
246         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780,
247           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
248         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S,
249           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
250         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781,
251           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
252         { PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX,
253           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
254         { PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX,
255           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
256         { PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000,
257           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
258         { PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001,
259           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
260         { PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003,
261           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
262         { PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100,
263           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
264         { PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3,
265           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
266         { 0, }
267 };
268
269 MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);
270
271 static struct {
272         const char string[ETH_GSTRING_LEN];
273 } ethtool_stats_keys[TG3_NUM_STATS] = {
274         { "rx_octets" },
275         { "rx_fragments" },
276         { "rx_ucast_packets" },
277         { "rx_mcast_packets" },
278         { "rx_bcast_packets" },
279         { "rx_fcs_errors" },
280         { "rx_align_errors" },
281         { "rx_xon_pause_rcvd" },
282         { "rx_xoff_pause_rcvd" },
283         { "rx_mac_ctrl_rcvd" },
284         { "rx_xoff_entered" },
285         { "rx_frame_too_long_errors" },
286         { "rx_jabbers" },
287         { "rx_undersize_packets" },
288         { "rx_in_length_errors" },
289         { "rx_out_length_errors" },
290         { "rx_64_or_less_octet_packets" },
291         { "rx_65_to_127_octet_packets" },
292         { "rx_128_to_255_octet_packets" },
293         { "rx_256_to_511_octet_packets" },
294         { "rx_512_to_1023_octet_packets" },
295         { "rx_1024_to_1522_octet_packets" },
296         { "rx_1523_to_2047_octet_packets" },
297         { "rx_2048_to_4095_octet_packets" },
298         { "rx_4096_to_8191_octet_packets" },
299         { "rx_8192_to_9022_octet_packets" },
300
301         { "tx_octets" },
302         { "tx_collisions" },
303
304         { "tx_xon_sent" },
305         { "tx_xoff_sent" },
306         { "tx_flow_control" },
307         { "tx_mac_errors" },
308         { "tx_single_collisions" },
309         { "tx_mult_collisions" },
310         { "tx_deferred" },
311         { "tx_excessive_collisions" },
312         { "tx_late_collisions" },
313         { "tx_collide_2times" },
314         { "tx_collide_3times" },
315         { "tx_collide_4times" },
316         { "tx_collide_5times" },
317         { "tx_collide_6times" },
318         { "tx_collide_7times" },
319         { "tx_collide_8times" },
320         { "tx_collide_9times" },
321         { "tx_collide_10times" },
322         { "tx_collide_11times" },
323         { "tx_collide_12times" },
324         { "tx_collide_13times" },
325         { "tx_collide_14times" },
326         { "tx_collide_15times" },
327         { "tx_ucast_packets" },
328         { "tx_mcast_packets" },
329         { "tx_bcast_packets" },
330         { "tx_carrier_sense_errors" },
331         { "tx_discards" },
332         { "tx_errors" },
333
334         { "dma_writeq_full" },
335         { "dma_write_prioq_full" },
336         { "rxbds_empty" },
337         { "rx_discards" },
338         { "rx_errors" },
339         { "rx_threshold_hit" },
340
341         { "dma_readq_full" },
342         { "dma_read_prioq_full" },
343         { "tx_comp_queue_full" },
344
345         { "ring_set_send_prod_index" },
346         { "ring_status_update" },
347         { "nic_irqs" },
348         { "nic_avoided_irqs" },
349         { "nic_tx_threshold_hit" }
350 };
351
352 static struct {
353         const char string[ETH_GSTRING_LEN];
354 } ethtool_test_keys[TG3_NUM_TEST] = {
355         { "nvram test     (online) " },
356         { "link test      (online) " },
357         { "register test  (offline)" },
358         { "memory test    (offline)" },
359         { "loopback test  (offline)" },
360         { "interrupt test (offline)" },
361 };
362
363 static void tg3_write32(struct tg3 *tp, u32 off, u32 val)
364 {
365         writel(val, tp->regs + off);
366 }
367
368 static u32 tg3_read32(struct tg3 *tp, u32 off)
369 {
370         return (readl(tp->regs + off)); 
371 }
372
373 static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val)
374 {
375         unsigned long flags;
376
377         spin_lock_irqsave(&tp->indirect_lock, flags);
378         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
379         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
380         spin_unlock_irqrestore(&tp->indirect_lock, flags);
381 }
382
383 static void tg3_write_flush_reg32(struct tg3 *tp, u32 off, u32 val)
384 {
385         writel(val, tp->regs + off);
386         readl(tp->regs + off);
387 }
388
389 static u32 tg3_read_indirect_reg32(struct tg3 *tp, u32 off)
390 {
391         unsigned long flags;
392         u32 val;
393
394         spin_lock_irqsave(&tp->indirect_lock, flags);
395         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
396         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
397         spin_unlock_irqrestore(&tp->indirect_lock, flags);
398         return val;
399 }
400
401 static void tg3_write_indirect_mbox(struct tg3 *tp, u32 off, u32 val)
402 {
403         unsigned long flags;
404
405         if (off == (MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW)) {
406                 pci_write_config_dword(tp->pdev, TG3PCI_RCV_RET_RING_CON_IDX +
407                                        TG3_64BIT_REG_LOW, val);
408                 return;
409         }
410         if (off == (MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW)) {
411                 pci_write_config_dword(tp->pdev, TG3PCI_STD_RING_PROD_IDX +
412                                        TG3_64BIT_REG_LOW, val);
413                 return;
414         }
415
416         spin_lock_irqsave(&tp->indirect_lock, flags);
417         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
418         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
419         spin_unlock_irqrestore(&tp->indirect_lock, flags);
420
421         /* In indirect mode when disabling interrupts, we also need
422          * to clear the interrupt bit in the GRC local ctrl register.
423          */
424         if ((off == (MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW)) &&
425             (val == 0x1)) {
426                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_LOCAL_CTRL,
427                                        tp->grc_local_ctrl|GRC_LCLCTRL_CLEARINT);
428         }
429 }
430
431 static u32 tg3_read_indirect_mbox(struct tg3 *tp, u32 off)
432 {
433         unsigned long flags;
434         u32 val;
435
436         spin_lock_irqsave(&tp->indirect_lock, flags);
437         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
438         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
439         spin_unlock_irqrestore(&tp->indirect_lock, flags);
440         return val;
441 }
442
443 /* usec_wait specifies the wait time in usec when writing to certain registers
444  * where it is unsafe to read back the register without some delay.
445  * GRC_LOCAL_CTRL is one example if the GPIOs are toggled to switch power.
446  * TG3PCI_CLOCK_CTRL is another example if the clock frequencies are changed.
447  */
448 static void _tw32_flush(struct tg3 *tp, u32 off, u32 val, u32 usec_wait)
449 {
450         if ((tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) ||
451             (tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
452                 /* Non-posted methods */
453                 tp->write32(tp, off, val);
454         else {
455                 /* Posted method */
456                 tg3_write32(tp, off, val);
457                 if (usec_wait)
458                         udelay(usec_wait);
459                 tp->read32(tp, off);
460         }
461         /* Wait again after the read for the posted method to guarantee that
462          * the wait time is met.
463          */
464         if (usec_wait)
465                 udelay(usec_wait);
466 }
467
468 static inline void tw32_mailbox_flush(struct tg3 *tp, u32 off, u32 val)
469 {
470         tp->write32_mbox(tp, off, val);
471         if (!(tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) &&
472             !(tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
473                 tp->read32_mbox(tp, off);
474 }
475
476 static void tg3_write32_tx_mbox(struct tg3 *tp, u32 off, u32 val)
477 {
478         void __iomem *mbox = tp->regs + off;
479         writel(val, mbox);
480         if (tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG)
481                 writel(val, mbox);
482         if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
483                 readl(mbox);
484 }
485
486 #define tw32_mailbox(reg, val)  tp->write32_mbox(tp, reg, val)
487 #define tw32_mailbox_f(reg, val)        tw32_mailbox_flush(tp, (reg), (val))
488 #define tw32_rx_mbox(reg, val)  tp->write32_rx_mbox(tp, reg, val)
489 #define tw32_tx_mbox(reg, val)  tp->write32_tx_mbox(tp, reg, val)
490 #define tr32_mailbox(reg)       tp->read32_mbox(tp, reg)
491
492 #define tw32(reg,val)           tp->write32(tp, reg, val)
493 #define tw32_f(reg,val)         _tw32_flush(tp,(reg),(val), 0)
494 #define tw32_wait_f(reg,val,us) _tw32_flush(tp,(reg),(val), (us))
495 #define tr32(reg)               tp->read32(tp, reg)
496
497 static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val)
498 {
499         unsigned long flags;
500
501         spin_lock_irqsave(&tp->indirect_lock, flags);
502         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
503                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
504                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
505
506                 /* Always leave this as zero. */
507                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
508         } else {
509                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
510                 tw32_f(TG3PCI_MEM_WIN_DATA, val);
511
512                 /* Always leave this as zero. */
513                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
514         }
515         spin_unlock_irqrestore(&tp->indirect_lock, flags);
516 }
517
518 static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val)
519 {
520         unsigned long flags;
521
522         spin_lock_irqsave(&tp->indirect_lock, flags);
523         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
524                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
525                 pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
526
527                 /* Always leave this as zero. */
528                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
529         } else {
530                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
531                 *val = tr32(TG3PCI_MEM_WIN_DATA);
532
533                 /* Always leave this as zero. */
534                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
535         }
536         spin_unlock_irqrestore(&tp->indirect_lock, flags);
537 }
538
539 static void tg3_disable_ints(struct tg3 *tp)
540 {
541         tw32(TG3PCI_MISC_HOST_CTRL,
542              (tp->misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT));
543         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
544 }
545
546 static inline void tg3_cond_int(struct tg3 *tp)
547 {
548         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
549             (tp->hw_status->status & SD_STATUS_UPDATED))
550                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
551 }
552
553 static void tg3_enable_ints(struct tg3 *tp)
554 {
555         tp->irq_sync = 0;
556         wmb();
557
558         tw32(TG3PCI_MISC_HOST_CTRL,
559              (tp->misc_host_ctrl & ~MISC_HOST_CTRL_MASK_PCI_INT));
560         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
561                        (tp->last_tag << 24));
562         if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)
563                 tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
564                                (tp->last_tag << 24));
565         tg3_cond_int(tp);
566 }
567
568 static inline unsigned int tg3_has_work(struct tg3 *tp)
569 {
570         struct tg3_hw_status *sblk = tp->hw_status;
571         unsigned int work_exists = 0;
572
573         /* check for phy events */
574         if (!(tp->tg3_flags &
575               (TG3_FLAG_USE_LINKCHG_REG |
576                TG3_FLAG_POLL_SERDES))) {
577                 if (sblk->status & SD_STATUS_LINK_CHG)
578                         work_exists = 1;
579         }
580         /* check for RX/TX work to do */
581         if (sblk->idx[0].tx_consumer != tp->tx_cons ||
582             sblk->idx[0].rx_producer != tp->rx_rcb_ptr)
583                 work_exists = 1;
584
585         return work_exists;
586 }
587
588 /* tg3_restart_ints
589  *  similar to tg3_enable_ints, but it accurately determines whether there
590  *  is new work pending and can return without flushing the PIO write
591  *  which reenables interrupts 
592  */
593 static void tg3_restart_ints(struct tg3 *tp)
594 {
595         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
596                      tp->last_tag << 24);
597         mmiowb();
598
599         /* When doing tagged status, this work check is unnecessary.
600          * The last_tag we write above tells the chip which piece of
601          * work we've completed.
602          */
603         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
604             tg3_has_work(tp))
605                 tw32(HOSTCC_MODE, tp->coalesce_mode |
606                      (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
607 }
608
609 static inline void tg3_netif_stop(struct tg3 *tp)
610 {
611         tp->dev->trans_start = jiffies; /* prevent tx timeout */
612         netif_poll_disable(tp->dev);
613         netif_tx_disable(tp->dev);
614 }
615
616 static inline void tg3_netif_start(struct tg3 *tp)
617 {
618         netif_wake_queue(tp->dev);
619         /* NOTE: unconditional netif_wake_queue is only appropriate
620          * so long as all callers are assured to have free tx slots
621          * (such as after tg3_init_hw)
622          */
623         netif_poll_enable(tp->dev);
624         tp->hw_status->status |= SD_STATUS_UPDATED;
625         tg3_enable_ints(tp);
626 }
627
628 static void tg3_switch_clocks(struct tg3 *tp)
629 {
630         u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL);
631         u32 orig_clock_ctrl;
632
633         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
634                 return;
635
636         orig_clock_ctrl = clock_ctrl;
637         clock_ctrl &= (CLOCK_CTRL_FORCE_CLKRUN |
638                        CLOCK_CTRL_CLKRUN_OENABLE |
639                        0x1f);
640         tp->pci_clock_ctrl = clock_ctrl;
641
642         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
643                 if (orig_clock_ctrl & CLOCK_CTRL_625_CORE) {
644                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
645                                     clock_ctrl | CLOCK_CTRL_625_CORE, 40);
646                 }
647         } else if ((orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
648                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
649                             clock_ctrl |
650                             (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK),
651                             40);
652                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
653                             clock_ctrl | (CLOCK_CTRL_ALTCLK),
654                             40);
655         }
656         tw32_wait_f(TG3PCI_CLOCK_CTRL, clock_ctrl, 40);
657 }
658
659 #define PHY_BUSY_LOOPS  5000
660
661 static int tg3_readphy(struct tg3 *tp, int reg, u32 *val)
662 {
663         u32 frame_val;
664         unsigned int loops;
665         int ret;
666
667         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
668                 tw32_f(MAC_MI_MODE,
669                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
670                 udelay(80);
671         }
672
673         *val = 0x0;
674
675         frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
676                       MI_COM_PHY_ADDR_MASK);
677         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
678                       MI_COM_REG_ADDR_MASK);
679         frame_val |= (MI_COM_CMD_READ | MI_COM_START);
680         
681         tw32_f(MAC_MI_COM, frame_val);
682
683         loops = PHY_BUSY_LOOPS;
684         while (loops != 0) {
685                 udelay(10);
686                 frame_val = tr32(MAC_MI_COM);
687
688                 if ((frame_val & MI_COM_BUSY) == 0) {
689                         udelay(5);
690                         frame_val = tr32(MAC_MI_COM);
691                         break;
692                 }
693                 loops -= 1;
694         }
695
696         ret = -EBUSY;
697         if (loops != 0) {
698                 *val = frame_val & MI_COM_DATA_MASK;
699                 ret = 0;
700         }
701
702         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
703                 tw32_f(MAC_MI_MODE, tp->mi_mode);
704                 udelay(80);
705         }
706
707         return ret;
708 }
709
710 static int tg3_writephy(struct tg3 *tp, int reg, u32 val)
711 {
712         u32 frame_val;
713         unsigned int loops;
714         int ret;
715
716         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
717                 tw32_f(MAC_MI_MODE,
718                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
719                 udelay(80);
720         }
721
722         frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
723                       MI_COM_PHY_ADDR_MASK);
724         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
725                       MI_COM_REG_ADDR_MASK);
726         frame_val |= (val & MI_COM_DATA_MASK);
727         frame_val |= (MI_COM_CMD_WRITE | MI_COM_START);
728         
729         tw32_f(MAC_MI_COM, frame_val);
730
731         loops = PHY_BUSY_LOOPS;
732         while (loops != 0) {
733                 udelay(10);
734                 frame_val = tr32(MAC_MI_COM);
735                 if ((frame_val & MI_COM_BUSY) == 0) {
736                         udelay(5);
737                         frame_val = tr32(MAC_MI_COM);
738                         break;
739                 }
740                 loops -= 1;
741         }
742
743         ret = -EBUSY;
744         if (loops != 0)
745                 ret = 0;
746
747         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
748                 tw32_f(MAC_MI_MODE, tp->mi_mode);
749                 udelay(80);
750         }
751
752         return ret;
753 }
754
755 static void tg3_phy_set_wirespeed(struct tg3 *tp)
756 {
757         u32 val;
758
759         if (tp->tg3_flags2 & TG3_FLG2_NO_ETH_WIRE_SPEED)
760                 return;
761
762         if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x7007) &&
763             !tg3_readphy(tp, MII_TG3_AUX_CTRL, &val))
764                 tg3_writephy(tp, MII_TG3_AUX_CTRL,
765                              (val | (1 << 15) | (1 << 4)));
766 }
767
768 static int tg3_bmcr_reset(struct tg3 *tp)
769 {
770         u32 phy_control;
771         int limit, err;
772
773         /* OK, reset it, and poll the BMCR_RESET bit until it
774          * clears or we time out.
775          */
776         phy_control = BMCR_RESET;
777         err = tg3_writephy(tp, MII_BMCR, phy_control);
778         if (err != 0)
779                 return -EBUSY;
780
781         limit = 5000;
782         while (limit--) {
783                 err = tg3_readphy(tp, MII_BMCR, &phy_control);
784                 if (err != 0)
785                         return -EBUSY;
786
787                 if ((phy_control & BMCR_RESET) == 0) {
788                         udelay(40);
789                         break;
790                 }
791                 udelay(10);
792         }
793         if (limit <= 0)
794                 return -EBUSY;
795
796         return 0;
797 }
798
799 static int tg3_wait_macro_done(struct tg3 *tp)
800 {
801         int limit = 100;
802
803         while (limit--) {
804                 u32 tmp32;
805
806                 if (!tg3_readphy(tp, 0x16, &tmp32)) {
807                         if ((tmp32 & 0x1000) == 0)
808                                 break;
809                 }
810         }
811         if (limit <= 0)
812                 return -EBUSY;
813
814         return 0;
815 }
816
817 static int tg3_phy_write_and_check_testpat(struct tg3 *tp, int *resetp)
818 {
819         static const u32 test_pat[4][6] = {
820         { 0x00005555, 0x00000005, 0x00002aaa, 0x0000000a, 0x00003456, 0x00000003 },
821         { 0x00002aaa, 0x0000000a, 0x00003333, 0x00000003, 0x0000789a, 0x00000005 },
822         { 0x00005a5a, 0x00000005, 0x00002a6a, 0x0000000a, 0x00001bcd, 0x00000003 },
823         { 0x00002a5a, 0x0000000a, 0x000033c3, 0x00000003, 0x00002ef1, 0x00000005 }
824         };
825         int chan;
826
827         for (chan = 0; chan < 4; chan++) {
828                 int i;
829
830                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
831                              (chan * 0x2000) | 0x0200);
832                 tg3_writephy(tp, 0x16, 0x0002);
833
834                 for (i = 0; i < 6; i++)
835                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT,
836                                      test_pat[chan][i]);
837
838                 tg3_writephy(tp, 0x16, 0x0202);
839                 if (tg3_wait_macro_done(tp)) {
840                         *resetp = 1;
841                         return -EBUSY;
842                 }
843
844                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
845                              (chan * 0x2000) | 0x0200);
846                 tg3_writephy(tp, 0x16, 0x0082);
847                 if (tg3_wait_macro_done(tp)) {
848                         *resetp = 1;
849                         return -EBUSY;
850                 }
851
852                 tg3_writephy(tp, 0x16, 0x0802);
853                 if (tg3_wait_macro_done(tp)) {
854                         *resetp = 1;
855                         return -EBUSY;
856                 }
857
858                 for (i = 0; i < 6; i += 2) {
859                         u32 low, high;
860
861                         if (tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &low) ||
862                             tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &high) ||
863                             tg3_wait_macro_done(tp)) {
864                                 *resetp = 1;
865                                 return -EBUSY;
866                         }
867                         low &= 0x7fff;
868                         high &= 0x000f;
869                         if (low != test_pat[chan][i] ||
870                             high != test_pat[chan][i+1]) {
871                                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000b);
872                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4001);
873                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4005);
874
875                                 return -EBUSY;
876                         }
877                 }
878         }
879
880         return 0;
881 }
882
883 static int tg3_phy_reset_chanpat(struct tg3 *tp)
884 {
885         int chan;
886
887         for (chan = 0; chan < 4; chan++) {
888                 int i;
889
890                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
891                              (chan * 0x2000) | 0x0200);
892                 tg3_writephy(tp, 0x16, 0x0002);
893                 for (i = 0; i < 6; i++)
894                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x000);
895                 tg3_writephy(tp, 0x16, 0x0202);
896                 if (tg3_wait_macro_done(tp))
897                         return -EBUSY;
898         }
899
900         return 0;
901 }
902
903 static int tg3_phy_reset_5703_4_5(struct tg3 *tp)
904 {
905         u32 reg32, phy9_orig;
906         int retries, do_phy_reset, err;
907
908         retries = 10;
909         do_phy_reset = 1;
910         do {
911                 if (do_phy_reset) {
912                         err = tg3_bmcr_reset(tp);
913                         if (err)
914                                 return err;
915                         do_phy_reset = 0;
916                 }
917
918                 /* Disable transmitter and interrupt.  */
919                 if (tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32))
920                         continue;
921
922                 reg32 |= 0x3000;
923                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
924
925                 /* Set full-duplex, 1000 mbps.  */
926                 tg3_writephy(tp, MII_BMCR,
927                              BMCR_FULLDPLX | TG3_BMCR_SPEED1000);
928
929                 /* Set to master mode.  */
930                 if (tg3_readphy(tp, MII_TG3_CTRL, &phy9_orig))
931                         continue;
932
933                 tg3_writephy(tp, MII_TG3_CTRL,
934                              (MII_TG3_CTRL_AS_MASTER |
935                               MII_TG3_CTRL_ENABLE_AS_MASTER));
936
937                 /* Enable SM_DSP_CLOCK and 6dB.  */
938                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
939
940                 /* Block the PHY control access.  */
941                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
942                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0800);
943
944                 err = tg3_phy_write_and_check_testpat(tp, &do_phy_reset);
945                 if (!err)
946                         break;
947         } while (--retries);
948
949         err = tg3_phy_reset_chanpat(tp);
950         if (err)
951                 return err;
952
953         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
954         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0000);
955
956         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8200);
957         tg3_writephy(tp, 0x16, 0x0000);
958
959         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
960             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
961                 /* Set Extended packet length bit for jumbo frames */
962                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4400);
963         }
964         else {
965                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
966         }
967
968         tg3_writephy(tp, MII_TG3_CTRL, phy9_orig);
969
970         if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32)) {
971                 reg32 &= ~0x3000;
972                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
973         } else if (!err)
974                 err = -EBUSY;
975
976         return err;
977 }
978
979 static void tg3_link_report(struct tg3 *);
980
981 /* This will reset the tigon3 PHY if there is no valid
982  * link unless the FORCE argument is non-zero.
983  */
984 static int tg3_phy_reset(struct tg3 *tp)
985 {
986         u32 phy_status;
987         int err;
988
989         err  = tg3_readphy(tp, MII_BMSR, &phy_status);
990         err |= tg3_readphy(tp, MII_BMSR, &phy_status);
991         if (err != 0)
992                 return -EBUSY;
993
994         if (netif_running(tp->dev) && netif_carrier_ok(tp->dev)) {
995                 netif_carrier_off(tp->dev);
996                 tg3_link_report(tp);
997         }
998
999         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1000             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1001             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
1002                 err = tg3_phy_reset_5703_4_5(tp);
1003                 if (err)
1004                         return err;
1005                 goto out;
1006         }
1007
1008         err = tg3_bmcr_reset(tp);
1009         if (err)
1010                 return err;
1011
1012 out:
1013         if (tp->tg3_flags2 & TG3_FLG2_PHY_ADC_BUG) {
1014                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1015                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1016                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x2aaa);
1017                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1018                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0323);
1019                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1020         }
1021         if (tp->tg3_flags2 & TG3_FLG2_PHY_5704_A0_BUG) {
1022                 tg3_writephy(tp, 0x1c, 0x8d68);
1023                 tg3_writephy(tp, 0x1c, 0x8d68);
1024         }
1025         if (tp->tg3_flags2 & TG3_FLG2_PHY_BER_BUG) {
1026                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1027                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1028                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x310b);
1029                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1030                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x9506);
1031                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x401f);
1032                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x14e2);
1033                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1034         }
1035         else if (tp->tg3_flags2 & TG3_FLG2_PHY_JITTER_BUG) {
1036                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1037                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1038                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x010b);
1039                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1040         }
1041         /* Set Extended packet length bit (bit 14) on all chips that */
1042         /* support jumbo frames */
1043         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
1044                 /* Cannot do read-modify-write on 5401 */
1045                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1046         } else if (tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) {
1047                 u32 phy_reg;
1048
1049                 /* Set bit 14 with read-modify-write to preserve other bits */
1050                 if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0007) &&
1051                     !tg3_readphy(tp, MII_TG3_AUX_CTRL, &phy_reg))
1052                         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy_reg | 0x4000);
1053         }
1054
1055         /* Set phy register 0x10 bit 0 to high fifo elasticity to support
1056          * jumbo frames transmission.
1057          */
1058         if (tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) {
1059                 u32 phy_reg;
1060
1061                 if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &phy_reg))
1062                     tg3_writephy(tp, MII_TG3_EXT_CTRL,
1063                                  phy_reg | MII_TG3_EXT_CTRL_FIFO_ELASTIC);
1064         }
1065
1066         tg3_phy_set_wirespeed(tp);
1067         return 0;
1068 }
1069
1070 static void tg3_frob_aux_power(struct tg3 *tp)
1071 {
1072         struct tg3 *tp_peer = tp;
1073
1074         if ((tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) != 0)
1075                 return;
1076
1077         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
1078             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714)) {
1079                 struct net_device *dev_peer;
1080
1081                 dev_peer = pci_get_drvdata(tp->pdev_peer);
1082                 /* remove_one() may have been run on the peer. */
1083                 if (!dev_peer)
1084                         tp_peer = tp;
1085                 else
1086                         tp_peer = netdev_priv(dev_peer);
1087         }
1088
1089         if ((tp->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
1090             (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0 ||
1091             (tp_peer->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
1092             (tp_peer->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0) {
1093                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1094                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1095                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1096                                     (GRC_LCLCTRL_GPIO_OE0 |
1097                                      GRC_LCLCTRL_GPIO_OE1 |
1098                                      GRC_LCLCTRL_GPIO_OE2 |
1099                                      GRC_LCLCTRL_GPIO_OUTPUT0 |
1100                                      GRC_LCLCTRL_GPIO_OUTPUT1),
1101                                     100);
1102                 } else {
1103                         u32 no_gpio2;
1104                         u32 grc_local_ctrl = 0;
1105
1106                         if (tp_peer != tp &&
1107                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
1108                                 return;
1109
1110                         /* Workaround to prevent overdrawing Amps. */
1111                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
1112                             ASIC_REV_5714) {
1113                                 grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
1114                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1115                                             grc_local_ctrl, 100);
1116                         }
1117
1118                         /* On 5753 and variants, GPIO2 cannot be used. */
1119                         no_gpio2 = tp->nic_sram_data_cfg &
1120                                     NIC_SRAM_DATA_CFG_NO_GPIO2;
1121
1122                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
1123                                          GRC_LCLCTRL_GPIO_OE1 |
1124                                          GRC_LCLCTRL_GPIO_OE2 |
1125                                          GRC_LCLCTRL_GPIO_OUTPUT1 |
1126                                          GRC_LCLCTRL_GPIO_OUTPUT2;
1127                         if (no_gpio2) {
1128                                 grc_local_ctrl &= ~(GRC_LCLCTRL_GPIO_OE2 |
1129                                                     GRC_LCLCTRL_GPIO_OUTPUT2);
1130                         }
1131                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1132                                                     grc_local_ctrl, 100);
1133
1134                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT0;
1135
1136                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1137                                                     grc_local_ctrl, 100);
1138
1139                         if (!no_gpio2) {
1140                                 grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT2;
1141                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1142                                             grc_local_ctrl, 100);
1143                         }
1144                 }
1145         } else {
1146                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
1147                     GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
1148                         if (tp_peer != tp &&
1149                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
1150                                 return;
1151
1152                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1153                                     (GRC_LCLCTRL_GPIO_OE1 |
1154                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
1155
1156                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1157                                     GRC_LCLCTRL_GPIO_OE1, 100);
1158
1159                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1160                                     (GRC_LCLCTRL_GPIO_OE1 |
1161                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
1162                 }
1163         }
1164 }
1165
1166 static int tg3_setup_phy(struct tg3 *, int);
1167
1168 #define RESET_KIND_SHUTDOWN     0
1169 #define RESET_KIND_INIT         1
1170 #define RESET_KIND_SUSPEND      2
1171
1172 static void tg3_write_sig_post_reset(struct tg3 *, int);
1173 static int tg3_halt_cpu(struct tg3 *, u32);
1174 static int tg3_nvram_lock(struct tg3 *);
1175 static void tg3_nvram_unlock(struct tg3 *);
1176
1177 static void tg3_power_down_phy(struct tg3 *tp)
1178 {
1179         /* The PHY should not be powered down on some chips because
1180          * of bugs.
1181          */
1182         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1183             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1184             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 &&
1185              (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)))
1186                 return;
1187         tg3_writephy(tp, MII_BMCR, BMCR_PDOWN);
1188 }
1189
1190 static int tg3_set_power_state(struct tg3 *tp, pci_power_t state)
1191 {
1192         u32 misc_host_ctrl;
1193         u16 power_control, power_caps;
1194         int pm = tp->pm_cap;
1195
1196         /* Make sure register accesses (indirect or otherwise)
1197          * will function correctly.
1198          */
1199         pci_write_config_dword(tp->pdev,
1200                                TG3PCI_MISC_HOST_CTRL,
1201                                tp->misc_host_ctrl);
1202
1203         pci_read_config_word(tp->pdev,
1204                              pm + PCI_PM_CTRL,
1205                              &power_control);
1206         power_control |= PCI_PM_CTRL_PME_STATUS;
1207         power_control &= ~(PCI_PM_CTRL_STATE_MASK);
1208         switch (state) {
1209         case PCI_D0:
1210                 power_control |= 0;
1211                 pci_write_config_word(tp->pdev,
1212                                       pm + PCI_PM_CTRL,
1213                                       power_control);
1214                 udelay(100);    /* Delay after power state change */
1215
1216                 /* Switch out of Vaux if it is not a LOM */
1217                 if (!(tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT))
1218                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl, 100);
1219
1220                 return 0;
1221
1222         case PCI_D1:
1223                 power_control |= 1;
1224                 break;
1225
1226         case PCI_D2:
1227                 power_control |= 2;
1228                 break;
1229
1230         case PCI_D3hot:
1231                 power_control |= 3;
1232                 break;
1233
1234         default:
1235                 printk(KERN_WARNING PFX "%s: Invalid power state (%d) "
1236                        "requested.\n",
1237                        tp->dev->name, state);
1238                 return -EINVAL;
1239         };
1240
1241         power_control |= PCI_PM_CTRL_PME_ENABLE;
1242
1243         misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
1244         tw32(TG3PCI_MISC_HOST_CTRL,
1245              misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT);
1246
1247         if (tp->link_config.phy_is_low_power == 0) {
1248                 tp->link_config.phy_is_low_power = 1;
1249                 tp->link_config.orig_speed = tp->link_config.speed;
1250                 tp->link_config.orig_duplex = tp->link_config.duplex;
1251                 tp->link_config.orig_autoneg = tp->link_config.autoneg;
1252         }
1253
1254         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)) {
1255                 tp->link_config.speed = SPEED_10;
1256                 tp->link_config.duplex = DUPLEX_HALF;
1257                 tp->link_config.autoneg = AUTONEG_ENABLE;
1258                 tg3_setup_phy(tp, 0);
1259         }
1260
1261         if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1262                 int i;
1263                 u32 val;
1264
1265                 for (i = 0; i < 200; i++) {
1266                         tg3_read_mem(tp, NIC_SRAM_FW_ASF_STATUS_MBOX, &val);
1267                         if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
1268                                 break;
1269                         msleep(1);
1270                 }
1271         }
1272         tg3_write_mem(tp, NIC_SRAM_WOL_MBOX, WOL_SIGNATURE |
1273                                              WOL_DRV_STATE_SHUTDOWN |
1274                                              WOL_DRV_WOL | WOL_SET_MAGIC_PKT);
1275
1276         pci_read_config_word(tp->pdev, pm + PCI_PM_PMC, &power_caps);
1277
1278         if (tp->tg3_flags & TG3_FLAG_WOL_ENABLE) {
1279                 u32 mac_mode;
1280
1281                 if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
1282                         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x5a);
1283                         udelay(40);
1284
1285                         mac_mode = MAC_MODE_PORT_MODE_MII;
1286
1287                         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 ||
1288                             !(tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB))
1289                                 mac_mode |= MAC_MODE_LINK_POLARITY;
1290                 } else {
1291                         mac_mode = MAC_MODE_PORT_MODE_TBI;
1292                 }
1293
1294                 if (!(tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
1295                         tw32(MAC_LED_CTRL, tp->led_ctrl);
1296
1297                 if (((power_caps & PCI_PM_CAP_PME_D3cold) &&
1298                      (tp->tg3_flags & TG3_FLAG_WOL_ENABLE)))
1299                         mac_mode |= MAC_MODE_MAGIC_PKT_ENABLE;
1300
1301                 tw32_f(MAC_MODE, mac_mode);
1302                 udelay(100);
1303
1304                 tw32_f(MAC_RX_MODE, RX_MODE_ENABLE);
1305                 udelay(10);
1306         }
1307
1308         if (!(tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB) &&
1309             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1310              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
1311                 u32 base_val;
1312
1313                 base_val = tp->pci_clock_ctrl;
1314                 base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
1315                              CLOCK_CTRL_TXCLK_DISABLE);
1316
1317                 tw32_wait_f(TG3PCI_CLOCK_CTRL, base_val | CLOCK_CTRL_ALTCLK |
1318                             CLOCK_CTRL_PWRDOWN_PLL133, 40);
1319         } else if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
1320                 /* do nothing */
1321         } else if (!((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
1322                      (tp->tg3_flags & TG3_FLAG_ENABLE_ASF))) {
1323                 u32 newbits1, newbits2;
1324
1325                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1326                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1327                         newbits1 = (CLOCK_CTRL_RXCLK_DISABLE |
1328                                     CLOCK_CTRL_TXCLK_DISABLE |
1329                                     CLOCK_CTRL_ALTCLK);
1330                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
1331                 } else if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
1332                         newbits1 = CLOCK_CTRL_625_CORE;
1333                         newbits2 = newbits1 | CLOCK_CTRL_ALTCLK;
1334                 } else {
1335                         newbits1 = CLOCK_CTRL_ALTCLK;
1336                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
1337                 }
1338
1339                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits1,
1340                             40);
1341
1342                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits2,
1343                             40);
1344
1345                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
1346                         u32 newbits3;
1347
1348                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1349                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1350                                 newbits3 = (CLOCK_CTRL_RXCLK_DISABLE |
1351                                             CLOCK_CTRL_TXCLK_DISABLE |
1352                                             CLOCK_CTRL_44MHZ_CORE);
1353                         } else {
1354                                 newbits3 = CLOCK_CTRL_44MHZ_CORE;
1355                         }
1356
1357                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
1358                                     tp->pci_clock_ctrl | newbits3, 40);
1359                 }
1360         }
1361
1362         if (!(tp->tg3_flags & TG3_FLAG_WOL_ENABLE) &&
1363             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1364                 /* Turn off the PHY */
1365                 if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
1366                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
1367                                      MII_TG3_EXT_CTRL_FORCE_LED_OFF);
1368                         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x01b2);
1369                         tg3_power_down_phy(tp);
1370                 }
1371         }
1372
1373         tg3_frob_aux_power(tp);
1374
1375         /* Workaround for unstable PLL clock */
1376         if ((GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX) ||
1377             (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX)) {
1378                 u32 val = tr32(0x7d00);
1379
1380                 val &= ~((1 << 16) | (1 << 4) | (1 << 2) | (1 << 1) | 1);
1381                 tw32(0x7d00, val);
1382                 if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1383                         int err;
1384
1385                         err = tg3_nvram_lock(tp);
1386                         tg3_halt_cpu(tp, RX_CPU_BASE);
1387                         if (!err)
1388                                 tg3_nvram_unlock(tp);
1389                 }
1390         }
1391
1392         tg3_write_sig_post_reset(tp, RESET_KIND_SHUTDOWN);
1393
1394         /* Finally, set the new power state. */
1395         pci_write_config_word(tp->pdev, pm + PCI_PM_CTRL, power_control);
1396         udelay(100);    /* Delay after power state change */
1397
1398         return 0;
1399 }
1400
1401 static void tg3_link_report(struct tg3 *tp)
1402 {
1403         if (!netif_carrier_ok(tp->dev)) {
1404                 printk(KERN_INFO PFX "%s: Link is down.\n", tp->dev->name);
1405         } else {
1406                 printk(KERN_INFO PFX "%s: Link is up at %d Mbps, %s duplex.\n",
1407                        tp->dev->name,
1408                        (tp->link_config.active_speed == SPEED_1000 ?
1409                         1000 :
1410                         (tp->link_config.active_speed == SPEED_100 ?
1411                          100 : 10)),
1412                        (tp->link_config.active_duplex == DUPLEX_FULL ?
1413                         "full" : "half"));
1414
1415                 printk(KERN_INFO PFX "%s: Flow control is %s for TX and "
1416                        "%s for RX.\n",
1417                        tp->dev->name,
1418                        (tp->tg3_flags & TG3_FLAG_TX_PAUSE) ? "on" : "off",
1419                        (tp->tg3_flags & TG3_FLAG_RX_PAUSE) ? "on" : "off");
1420         }
1421 }
1422
1423 static void tg3_setup_flow_control(struct tg3 *tp, u32 local_adv, u32 remote_adv)
1424 {
1425         u32 new_tg3_flags = 0;
1426         u32 old_rx_mode = tp->rx_mode;
1427         u32 old_tx_mode = tp->tx_mode;
1428
1429         if (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG) {
1430
1431                 /* Convert 1000BaseX flow control bits to 1000BaseT
1432                  * bits before resolving flow control.
1433                  */
1434                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
1435                         local_adv &= ~(ADVERTISE_PAUSE_CAP |
1436                                        ADVERTISE_PAUSE_ASYM);
1437                         remote_adv &= ~(LPA_PAUSE_CAP | LPA_PAUSE_ASYM);
1438
1439                         if (local_adv & ADVERTISE_1000XPAUSE)
1440                                 local_adv |= ADVERTISE_PAUSE_CAP;
1441                         if (local_adv & ADVERTISE_1000XPSE_ASYM)
1442                                 local_adv |= ADVERTISE_PAUSE_ASYM;
1443                         if (remote_adv & LPA_1000XPAUSE)
1444                                 remote_adv |= LPA_PAUSE_CAP;
1445                         if (remote_adv & LPA_1000XPAUSE_ASYM)
1446                                 remote_adv |= LPA_PAUSE_ASYM;
1447                 }
1448
1449                 if (local_adv & ADVERTISE_PAUSE_CAP) {
1450                         if (local_adv & ADVERTISE_PAUSE_ASYM) {
1451                                 if (remote_adv & LPA_PAUSE_CAP)
1452                                         new_tg3_flags |=
1453                                                 (TG3_FLAG_RX_PAUSE |
1454                                                 TG3_FLAG_TX_PAUSE);
1455                                 else if (remote_adv & LPA_PAUSE_ASYM)
1456                                         new_tg3_flags |=
1457                                                 (TG3_FLAG_RX_PAUSE);
1458                         } else {
1459                                 if (remote_adv & LPA_PAUSE_CAP)
1460                                         new_tg3_flags |=
1461                                                 (TG3_FLAG_RX_PAUSE |
1462                                                 TG3_FLAG_TX_PAUSE);
1463                         }
1464                 } else if (local_adv & ADVERTISE_PAUSE_ASYM) {
1465                         if ((remote_adv & LPA_PAUSE_CAP) &&
1466                         (remote_adv & LPA_PAUSE_ASYM))
1467                                 new_tg3_flags |= TG3_FLAG_TX_PAUSE;
1468                 }
1469
1470                 tp->tg3_flags &= ~(TG3_FLAG_RX_PAUSE | TG3_FLAG_TX_PAUSE);
1471                 tp->tg3_flags |= new_tg3_flags;
1472         } else {
1473                 new_tg3_flags = tp->tg3_flags;
1474         }
1475
1476         if (new_tg3_flags & TG3_FLAG_RX_PAUSE)
1477                 tp->rx_mode |= RX_MODE_FLOW_CTRL_ENABLE;
1478         else
1479                 tp->rx_mode &= ~RX_MODE_FLOW_CTRL_ENABLE;
1480
1481         if (old_rx_mode != tp->rx_mode) {
1482                 tw32_f(MAC_RX_MODE, tp->rx_mode);
1483         }
1484         
1485         if (new_tg3_flags & TG3_FLAG_TX_PAUSE)
1486                 tp->tx_mode |= TX_MODE_FLOW_CTRL_ENABLE;
1487         else
1488                 tp->tx_mode &= ~TX_MODE_FLOW_CTRL_ENABLE;
1489
1490         if (old_tx_mode != tp->tx_mode) {
1491                 tw32_f(MAC_TX_MODE, tp->tx_mode);
1492         }
1493 }
1494
1495 static void tg3_aux_stat_to_speed_duplex(struct tg3 *tp, u32 val, u16 *speed, u8 *duplex)
1496 {
1497         switch (val & MII_TG3_AUX_STAT_SPDMASK) {
1498         case MII_TG3_AUX_STAT_10HALF:
1499                 *speed = SPEED_10;
1500                 *duplex = DUPLEX_HALF;
1501                 break;
1502
1503         case MII_TG3_AUX_STAT_10FULL:
1504                 *speed = SPEED_10;
1505                 *duplex = DUPLEX_FULL;
1506                 break;
1507
1508         case MII_TG3_AUX_STAT_100HALF:
1509                 *speed = SPEED_100;
1510                 *duplex = DUPLEX_HALF;
1511                 break;
1512
1513         case MII_TG3_AUX_STAT_100FULL:
1514                 *speed = SPEED_100;
1515                 *duplex = DUPLEX_FULL;
1516                 break;
1517
1518         case MII_TG3_AUX_STAT_1000HALF:
1519                 *speed = SPEED_1000;
1520                 *duplex = DUPLEX_HALF;
1521                 break;
1522
1523         case MII_TG3_AUX_STAT_1000FULL:
1524                 *speed = SPEED_1000;
1525                 *duplex = DUPLEX_FULL;
1526                 break;
1527
1528         default:
1529                 *speed = SPEED_INVALID;
1530                 *duplex = DUPLEX_INVALID;
1531                 break;
1532         };
1533 }
1534
1535 static void tg3_phy_copper_begin(struct tg3 *tp)
1536 {
1537         u32 new_adv;
1538         int i;
1539
1540         if (tp->link_config.phy_is_low_power) {
1541                 /* Entering low power mode.  Disable gigabit and
1542                  * 100baseT advertisements.
1543                  */
1544                 tg3_writephy(tp, MII_TG3_CTRL, 0);
1545
1546                 new_adv = (ADVERTISE_10HALF | ADVERTISE_10FULL |
1547                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
1548                 if (tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB)
1549                         new_adv |= (ADVERTISE_100HALF | ADVERTISE_100FULL);
1550
1551                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
1552         } else if (tp->link_config.speed == SPEED_INVALID) {
1553                 tp->link_config.advertising =
1554                         (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full |
1555                          ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full |
1556                          ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full |
1557                          ADVERTISED_Autoneg | ADVERTISED_MII);
1558
1559                 if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
1560                         tp->link_config.advertising &=
1561                                 ~(ADVERTISED_1000baseT_Half |
1562                                   ADVERTISED_1000baseT_Full);
1563
1564                 new_adv = (ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
1565                 if (tp->link_config.advertising & ADVERTISED_10baseT_Half)
1566                         new_adv |= ADVERTISE_10HALF;
1567                 if (tp->link_config.advertising & ADVERTISED_10baseT_Full)
1568                         new_adv |= ADVERTISE_10FULL;
1569                 if (tp->link_config.advertising & ADVERTISED_100baseT_Half)
1570                         new_adv |= ADVERTISE_100HALF;
1571                 if (tp->link_config.advertising & ADVERTISED_100baseT_Full)
1572                         new_adv |= ADVERTISE_100FULL;
1573                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
1574
1575                 if (tp->link_config.advertising &
1576                     (ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full)) {
1577                         new_adv = 0;
1578                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
1579                                 new_adv |= MII_TG3_CTRL_ADV_1000_HALF;
1580                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
1581                                 new_adv |= MII_TG3_CTRL_ADV_1000_FULL;
1582                         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY) &&
1583                             (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1584                              tp->pci_chip_rev_id == CHIPREV_ID_5701_B0))
1585                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
1586                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
1587                         tg3_writephy(tp, MII_TG3_CTRL, new_adv);
1588                 } else {
1589                         tg3_writephy(tp, MII_TG3_CTRL, 0);
1590                 }
1591         } else {
1592                 /* Asking for a specific link mode. */
1593                 if (tp->link_config.speed == SPEED_1000) {
1594                         new_adv = ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP;
1595                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
1596
1597                         if (tp->link_config.duplex == DUPLEX_FULL)
1598                                 new_adv = MII_TG3_CTRL_ADV_1000_FULL;
1599                         else
1600                                 new_adv = MII_TG3_CTRL_ADV_1000_HALF;
1601                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1602                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
1603                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
1604                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
1605                         tg3_writephy(tp, MII_TG3_CTRL, new_adv);
1606                 } else {
1607                         tg3_writephy(tp, MII_TG3_CTRL, 0);
1608
1609                         new_adv = ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP;
1610                         if (tp->link_config.speed == SPEED_100) {
1611                                 if (tp->link_config.duplex == DUPLEX_FULL)
1612                                         new_adv |= ADVERTISE_100FULL;
1613                                 else
1614                                         new_adv |= ADVERTISE_100HALF;
1615                         } else {
1616                                 if (tp->link_config.duplex == DUPLEX_FULL)
1617                                         new_adv |= ADVERTISE_10FULL;
1618                                 else
1619                                         new_adv |= ADVERTISE_10HALF;
1620                         }
1621                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
1622                 }
1623         }
1624
1625         if (tp->link_config.autoneg == AUTONEG_DISABLE &&
1626             tp->link_config.speed != SPEED_INVALID) {
1627                 u32 bmcr, orig_bmcr;
1628
1629                 tp->link_config.active_speed = tp->link_config.speed;
1630                 tp->link_config.active_duplex = tp->link_config.duplex;
1631
1632                 bmcr = 0;
1633                 switch (tp->link_config.speed) {
1634                 default:
1635                 case SPEED_10:
1636                         break;
1637
1638                 case SPEED_100:
1639                         bmcr |= BMCR_SPEED100;
1640                         break;
1641
1642                 case SPEED_1000:
1643                         bmcr |= TG3_BMCR_SPEED1000;
1644                         break;
1645                 };
1646
1647                 if (tp->link_config.duplex == DUPLEX_FULL)
1648                         bmcr |= BMCR_FULLDPLX;
1649
1650                 if (!tg3_readphy(tp, MII_BMCR, &orig_bmcr) &&
1651                     (bmcr != orig_bmcr)) {
1652                         tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK);
1653                         for (i = 0; i < 1500; i++) {
1654                                 u32 tmp;
1655
1656                                 udelay(10);
1657                                 if (tg3_readphy(tp, MII_BMSR, &tmp) ||
1658                                     tg3_readphy(tp, MII_BMSR, &tmp))
1659                                         continue;
1660                                 if (!(tmp & BMSR_LSTATUS)) {
1661                                         udelay(40);
1662                                         break;
1663                                 }
1664                         }
1665                         tg3_writephy(tp, MII_BMCR, bmcr);
1666                         udelay(40);
1667                 }
1668         } else {
1669                 tg3_writephy(tp, MII_BMCR,
1670                              BMCR_ANENABLE | BMCR_ANRESTART);
1671         }
1672 }
1673
1674 static int tg3_init_5401phy_dsp(struct tg3 *tp)
1675 {
1676         int err;
1677
1678         /* Turn off tap power management. */
1679         /* Set Extended packet length bit */
1680         err  = tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1681
1682         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0012);
1683         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1804);
1684
1685         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0013);
1686         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1204);
1687
1688         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
1689         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0132);
1690
1691         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
1692         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0232);
1693
1694         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1695         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0a20);
1696
1697         udelay(40);
1698
1699         return err;
1700 }
1701
1702 static int tg3_copper_is_advertising_all(struct tg3 *tp)
1703 {
1704         u32 adv_reg, all_mask;
1705
1706         if (tg3_readphy(tp, MII_ADVERTISE, &adv_reg))
1707                 return 0;
1708
1709         all_mask = (ADVERTISE_10HALF | ADVERTISE_10FULL |
1710                     ADVERTISE_100HALF | ADVERTISE_100FULL);
1711         if ((adv_reg & all_mask) != all_mask)
1712                 return 0;
1713         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
1714                 u32 tg3_ctrl;
1715
1716                 if (tg3_readphy(tp, MII_TG3_CTRL, &tg3_ctrl))
1717                         return 0;
1718
1719                 all_mask = (MII_TG3_CTRL_ADV_1000_HALF |
1720                             MII_TG3_CTRL_ADV_1000_FULL);
1721                 if ((tg3_ctrl & all_mask) != all_mask)
1722                         return 0;
1723         }
1724         return 1;
1725 }
1726
1727 static int tg3_setup_copper_phy(struct tg3 *tp, int force_reset)
1728 {
1729         int current_link_up;
1730         u32 bmsr, dummy;
1731         u16 current_speed;
1732         u8 current_duplex;
1733         int i, err;
1734
1735         tw32(MAC_EVENT, 0);
1736
1737         tw32_f(MAC_STATUS,
1738              (MAC_STATUS_SYNC_CHANGED |
1739               MAC_STATUS_CFG_CHANGED |
1740               MAC_STATUS_MI_COMPLETION |
1741               MAC_STATUS_LNKSTATE_CHANGED));
1742         udelay(40);
1743
1744         tp->mi_mode = MAC_MI_MODE_BASE;
1745         tw32_f(MAC_MI_MODE, tp->mi_mode);
1746         udelay(80);
1747
1748         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x02);
1749
1750         /* Some third-party PHYs need to be reset on link going
1751          * down.
1752          */
1753         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1754              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1755              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
1756             netif_carrier_ok(tp->dev)) {
1757                 tg3_readphy(tp, MII_BMSR, &bmsr);
1758                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
1759                     !(bmsr & BMSR_LSTATUS))
1760                         force_reset = 1;
1761         }
1762         if (force_reset)
1763                 tg3_phy_reset(tp);
1764
1765         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
1766                 tg3_readphy(tp, MII_BMSR, &bmsr);
1767                 if (tg3_readphy(tp, MII_BMSR, &bmsr) ||
1768                     !(tp->tg3_flags & TG3_FLAG_INIT_COMPLETE))
1769                         bmsr = 0;
1770
1771                 if (!(bmsr & BMSR_LSTATUS)) {
1772                         err = tg3_init_5401phy_dsp(tp);
1773                         if (err)
1774                                 return err;
1775
1776                         tg3_readphy(tp, MII_BMSR, &bmsr);
1777                         for (i = 0; i < 1000; i++) {
1778                                 udelay(10);
1779                                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
1780                                     (bmsr & BMSR_LSTATUS)) {
1781                                         udelay(40);
1782                                         break;
1783                                 }
1784                         }
1785
1786                         if ((tp->phy_id & PHY_ID_REV_MASK) == PHY_REV_BCM5401_B0 &&
1787                             !(bmsr & BMSR_LSTATUS) &&
1788                             tp->link_config.active_speed == SPEED_1000) {
1789                                 err = tg3_phy_reset(tp);
1790                                 if (!err)
1791                                         err = tg3_init_5401phy_dsp(tp);
1792                                 if (err)
1793                                         return err;
1794                         }
1795                 }
1796         } else if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1797                    tp->pci_chip_rev_id == CHIPREV_ID_5701_B0) {
1798                 /* 5701 {A0,B0} CRC bug workaround */
1799                 tg3_writephy(tp, 0x15, 0x0a75);
1800                 tg3_writephy(tp, 0x1c, 0x8c68);
1801                 tg3_writephy(tp, 0x1c, 0x8d68);
1802                 tg3_writephy(tp, 0x1c, 0x8c68);
1803         }
1804
1805         /* Clear pending interrupts... */
1806         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
1807         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
1808
1809         if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT)
1810                 tg3_writephy(tp, MII_TG3_IMASK, ~MII_TG3_INT_LINKCHG);
1811         else
1812                 tg3_writephy(tp, MII_TG3_IMASK, ~0);
1813
1814         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1815             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1816                 if (tp->led_ctrl == LED_CTRL_MODE_PHY_1)
1817                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
1818                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
1819                 else
1820                         tg3_writephy(tp, MII_TG3_EXT_CTRL, 0);
1821         }
1822
1823         current_link_up = 0;
1824         current_speed = SPEED_INVALID;
1825         current_duplex = DUPLEX_INVALID;
1826
1827         if (tp->tg3_flags2 & TG3_FLG2_CAPACITIVE_COUPLING) {
1828                 u32 val;
1829
1830                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4007);
1831                 tg3_readphy(tp, MII_TG3_AUX_CTRL, &val);
1832                 if (!(val & (1 << 10))) {
1833                         val |= (1 << 10);
1834                         tg3_writephy(tp, MII_TG3_AUX_CTRL, val);
1835                         goto relink;
1836                 }
1837         }
1838
1839         bmsr = 0;
1840         for (i = 0; i < 100; i++) {
1841                 tg3_readphy(tp, MII_BMSR, &bmsr);
1842                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
1843                     (bmsr & BMSR_LSTATUS))
1844                         break;
1845                 udelay(40);
1846         }
1847
1848         if (bmsr & BMSR_LSTATUS) {
1849                 u32 aux_stat, bmcr;
1850
1851                 tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
1852                 for (i = 0; i < 2000; i++) {
1853                         udelay(10);
1854                         if (!tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat) &&
1855                             aux_stat)
1856                                 break;
1857                 }
1858
1859                 tg3_aux_stat_to_speed_duplex(tp, aux_stat,
1860                                              &current_speed,
1861                                              &current_duplex);
1862
1863                 bmcr = 0;
1864                 for (i = 0; i < 200; i++) {
1865                         tg3_readphy(tp, MII_BMCR, &bmcr);
1866                         if (tg3_readphy(tp, MII_BMCR, &bmcr))
1867                                 continue;
1868                         if (bmcr && bmcr != 0x7fff)
1869                                 break;
1870                         udelay(10);
1871                 }
1872
1873                 if (tp->link_config.autoneg == AUTONEG_ENABLE) {
1874                         if (bmcr & BMCR_ANENABLE) {
1875                                 current_link_up = 1;
1876
1877                                 /* Force autoneg restart if we are exiting
1878                                  * low power mode.
1879                                  */
1880                                 if (!tg3_copper_is_advertising_all(tp))
1881                                         current_link_up = 0;
1882                         } else {
1883                                 current_link_up = 0;
1884                         }
1885                 } else {
1886                         if (!(bmcr & BMCR_ANENABLE) &&
1887                             tp->link_config.speed == current_speed &&
1888                             tp->link_config.duplex == current_duplex) {
1889                                 current_link_up = 1;
1890                         } else {
1891                                 current_link_up = 0;
1892                         }
1893                 }
1894
1895                 tp->link_config.active_speed = current_speed;
1896                 tp->link_config.active_duplex = current_duplex;
1897         }
1898
1899         if (current_link_up == 1 &&
1900             (tp->link_config.active_duplex == DUPLEX_FULL) &&
1901             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
1902                 u32 local_adv, remote_adv;
1903
1904                 if (tg3_readphy(tp, MII_ADVERTISE, &local_adv))
1905                         local_adv = 0;
1906                 local_adv &= (ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM);
1907
1908                 if (tg3_readphy(tp, MII_LPA, &remote_adv))
1909                         remote_adv = 0;
1910
1911                 remote_adv &= (LPA_PAUSE_CAP | LPA_PAUSE_ASYM);
1912
1913                 /* If we are not advertising full pause capability,
1914                  * something is wrong.  Bring the link down and reconfigure.
1915                  */
1916                 if (local_adv != ADVERTISE_PAUSE_CAP) {
1917                         current_link_up = 0;
1918                 } else {
1919                         tg3_setup_flow_control(tp, local_adv, remote_adv);
1920                 }
1921         }
1922 relink:
1923         if (current_link_up == 0 || tp->link_config.phy_is_low_power) {
1924                 u32 tmp;
1925
1926                 tg3_phy_copper_begin(tp);
1927
1928                 tg3_readphy(tp, MII_BMSR, &tmp);
1929                 if (!tg3_readphy(tp, MII_BMSR, &tmp) &&
1930                     (tmp & BMSR_LSTATUS))
1931                         current_link_up = 1;
1932         }
1933
1934         tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
1935         if (current_link_up == 1) {
1936                 if (tp->link_config.active_speed == SPEED_100 ||
1937                     tp->link_config.active_speed == SPEED_10)
1938                         tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
1939                 else
1940                         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
1941         } else
1942                 tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
1943
1944         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
1945         if (tp->link_config.active_duplex == DUPLEX_HALF)
1946                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
1947
1948         tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
1949         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
1950                 if ((tp->led_ctrl == LED_CTRL_MODE_PHY_2) ||
1951                     (current_link_up == 1 &&
1952                      tp->link_config.active_speed == SPEED_10))
1953                         tp->mac_mode |= MAC_MODE_LINK_POLARITY;
1954         } else {
1955                 if (current_link_up == 1)
1956                         tp->mac_mode |= MAC_MODE_LINK_POLARITY;
1957         }
1958
1959         /* ??? Without this setting Netgear GA302T PHY does not
1960          * ??? send/receive packets...
1961          */
1962         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411 &&
1963             tp->pci_chip_rev_id == CHIPREV_ID_5700_ALTIMA) {
1964                 tp->mi_mode |= MAC_MI_MODE_AUTO_POLL;
1965                 tw32_f(MAC_MI_MODE, tp->mi_mode);
1966                 udelay(80);
1967         }
1968
1969         tw32_f(MAC_MODE, tp->mac_mode);
1970         udelay(40);
1971
1972         if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
1973                 /* Polled via timer. */
1974                 tw32_f(MAC_EVENT, 0);
1975         } else {
1976                 tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
1977         }
1978         udelay(40);
1979
1980         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 &&
1981             current_link_up == 1 &&
1982             tp->link_config.active_speed == SPEED_1000 &&
1983             ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) ||
1984              (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED))) {
1985                 udelay(120);
1986                 tw32_f(MAC_STATUS,
1987                      (MAC_STATUS_SYNC_CHANGED |
1988                       MAC_STATUS_CFG_CHANGED));
1989                 udelay(40);
1990                 tg3_write_mem(tp,
1991                               NIC_SRAM_FIRMWARE_MBOX,
1992                               NIC_SRAM_FIRMWARE_MBOX_MAGIC2);
1993         }
1994
1995         if (current_link_up != netif_carrier_ok(tp->dev)) {
1996                 if (current_link_up)
1997                         netif_carrier_on(tp->dev);
1998                 else
1999                         netif_carrier_off(tp->dev);
2000                 tg3_link_report(tp);
2001         }
2002
2003         return 0;
2004 }
2005
2006 struct tg3_fiber_aneginfo {
2007         int state;
2008 #define ANEG_STATE_UNKNOWN              0
2009 #define ANEG_STATE_AN_ENABLE            1
2010 #define ANEG_STATE_RESTART_INIT         2
2011 #define ANEG_STATE_RESTART              3
2012 #define ANEG_STATE_DISABLE_LINK_OK      4
2013 #define ANEG_STATE_ABILITY_DETECT_INIT  5
2014 #define ANEG_STATE_ABILITY_DETECT       6
2015 #define ANEG_STATE_ACK_DETECT_INIT      7
2016 #define ANEG_STATE_ACK_DETECT           8
2017 #define ANEG_STATE_COMPLETE_ACK_INIT    9
2018 #define ANEG_STATE_COMPLETE_ACK         10
2019 #define ANEG_STATE_IDLE_DETECT_INIT     11
2020 #define ANEG_STATE_IDLE_DETECT          12
2021 #define ANEG_STATE_LINK_OK              13
2022 #define ANEG_STATE_NEXT_PAGE_WAIT_INIT  14
2023 #define ANEG_STATE_NEXT_PAGE_WAIT       15
2024
2025         u32 flags;
2026 #define MR_AN_ENABLE            0x00000001
2027 #define MR_RESTART_AN           0x00000002
2028 #define MR_AN_COMPLETE          0x00000004
2029 #define MR_PAGE_RX              0x00000008
2030 #define MR_NP_LOADED            0x00000010
2031 #define MR_TOGGLE_TX            0x00000020
2032 #define MR_LP_ADV_FULL_DUPLEX   0x00000040
2033 #define MR_LP_ADV_HALF_DUPLEX   0x00000080
2034 #define MR_LP_ADV_SYM_PAUSE     0x00000100
2035 #define MR_LP_ADV_ASYM_PAUSE    0x00000200
2036 #define MR_LP_ADV_REMOTE_FAULT1 0x00000400
2037 #define MR_LP_ADV_REMOTE_FAULT2 0x00000800
2038 #define MR_LP_ADV_NEXT_PAGE     0x00001000
2039 #define MR_TOGGLE_RX            0x00002000
2040 #define MR_NP_RX                0x00004000
2041
2042 #define MR_LINK_OK              0x80000000
2043
2044         unsigned long link_time, cur_time;
2045
2046         u32 ability_match_cfg;
2047         int ability_match_count;
2048
2049         char ability_match, idle_match, ack_match;
2050
2051         u32 txconfig, rxconfig;
2052 #define ANEG_CFG_NP             0x00000080
2053 #define ANEG_CFG_ACK            0x00000040
2054 #define ANEG_CFG_RF2            0x00000020
2055 #define ANEG_CFG_RF1            0x00000010
2056 #define ANEG_CFG_PS2            0x00000001
2057 #define ANEG_CFG_PS1            0x00008000
2058 #define ANEG_CFG_HD             0x00004000
2059 #define ANEG_CFG_FD             0x00002000
2060 #define ANEG_CFG_INVAL          0x00001f06
2061
2062 };
2063 #define ANEG_OK         0
2064 #define ANEG_DONE       1
2065 #define ANEG_TIMER_ENAB 2
2066 #define ANEG_FAILED     -1
2067
2068 #define ANEG_STATE_SETTLE_TIME  10000
2069
2070 static int tg3_fiber_aneg_smachine(struct tg3 *tp,
2071                                    struct tg3_fiber_aneginfo *ap)
2072 {
2073         unsigned long delta;
2074         u32 rx_cfg_reg;
2075         int ret;
2076
2077         if (ap->state == ANEG_STATE_UNKNOWN) {
2078                 ap->rxconfig = 0;
2079                 ap->link_time = 0;
2080                 ap->cur_time = 0;
2081                 ap->ability_match_cfg = 0;
2082                 ap->ability_match_count = 0;
2083                 ap->ability_match = 0;
2084                 ap->idle_match = 0;
2085                 ap->ack_match = 0;
2086         }
2087         ap->cur_time++;
2088
2089         if (tr32(MAC_STATUS) & MAC_STATUS_RCVD_CFG) {
2090                 rx_cfg_reg = tr32(MAC_RX_AUTO_NEG);
2091
2092                 if (rx_cfg_reg != ap->ability_match_cfg) {
2093                         ap->ability_match_cfg = rx_cfg_reg;
2094                         ap->ability_match = 0;
2095                         ap->ability_match_count = 0;
2096                 } else {
2097                         if (++ap->ability_match_count > 1) {
2098                                 ap->ability_match = 1;
2099                                 ap->ability_match_cfg = rx_cfg_reg;
2100                         }
2101                 }
2102                 if (rx_cfg_reg & ANEG_CFG_ACK)
2103                         ap->ack_match = 1;
2104                 else
2105                         ap->ack_match = 0;
2106
2107                 ap->idle_match = 0;
2108         } else {
2109                 ap->idle_match = 1;
2110                 ap->ability_match_cfg = 0;
2111                 ap->ability_match_count = 0;
2112                 ap->ability_match = 0;
2113                 ap->ack_match = 0;
2114
2115                 rx_cfg_reg = 0;
2116         }
2117
2118         ap->rxconfig = rx_cfg_reg;
2119         ret = ANEG_OK;
2120
2121         switch(ap->state) {
2122         case ANEG_STATE_UNKNOWN:
2123                 if (ap->flags & (MR_AN_ENABLE | MR_RESTART_AN))
2124                         ap->state = ANEG_STATE_AN_ENABLE;
2125
2126                 /* fallthru */
2127         case ANEG_STATE_AN_ENABLE:
2128                 ap->flags &= ~(MR_AN_COMPLETE | MR_PAGE_RX);
2129                 if (ap->flags & MR_AN_ENABLE) {
2130                         ap->link_time = 0;
2131                         ap->cur_time = 0;
2132                         ap->ability_match_cfg = 0;
2133                         ap->ability_match_count = 0;
2134                         ap->ability_match = 0;
2135                         ap->idle_match = 0;
2136                         ap->ack_match = 0;
2137
2138                         ap->state = ANEG_STATE_RESTART_INIT;
2139                 } else {
2140                         ap->state = ANEG_STATE_DISABLE_LINK_OK;
2141                 }
2142                 break;
2143
2144         case ANEG_STATE_RESTART_INIT:
2145                 ap->link_time = ap->cur_time;
2146                 ap->flags &= ~(MR_NP_LOADED);
2147                 ap->txconfig = 0;
2148                 tw32(MAC_TX_AUTO_NEG, 0);
2149                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2150                 tw32_f(MAC_MODE, tp->mac_mode);
2151                 udelay(40);
2152
2153                 ret = ANEG_TIMER_ENAB;
2154                 ap->state = ANEG_STATE_RESTART;
2155
2156                 /* fallthru */
2157         case ANEG_STATE_RESTART:
2158                 delta = ap->cur_time - ap->link_time;
2159                 if (delta > ANEG_STATE_SETTLE_TIME) {
2160                         ap->state = ANEG_STATE_ABILITY_DETECT_INIT;
2161                 } else {
2162                         ret = ANEG_TIMER_ENAB;
2163                 }
2164                 break;
2165
2166         case ANEG_STATE_DISABLE_LINK_OK:
2167                 ret = ANEG_DONE;
2168                 break;
2169
2170         case ANEG_STATE_ABILITY_DETECT_INIT:
2171                 ap->flags &= ~(MR_TOGGLE_TX);
2172                 ap->txconfig = (ANEG_CFG_FD | ANEG_CFG_PS1);
2173                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
2174                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2175                 tw32_f(MAC_MODE, tp->mac_mode);
2176                 udelay(40);
2177
2178                 ap->state = ANEG_STATE_ABILITY_DETECT;
2179                 break;
2180
2181         case ANEG_STATE_ABILITY_DETECT:
2182                 if (ap->ability_match != 0 && ap->rxconfig != 0) {
2183                         ap->state = ANEG_STATE_ACK_DETECT_INIT;
2184                 }
2185                 break;
2186
2187         case ANEG_STATE_ACK_DETECT_INIT:
2188                 ap->txconfig |= ANEG_CFG_ACK;
2189                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
2190                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2191                 tw32_f(MAC_MODE, tp->mac_mode);
2192                 udelay(40);
2193
2194                 ap->state = ANEG_STATE_ACK_DETECT;
2195
2196                 /* fallthru */
2197         case ANEG_STATE_ACK_DETECT:
2198                 if (ap->ack_match != 0) {
2199                         if ((ap->rxconfig & ~ANEG_CFG_ACK) ==
2200                             (ap->ability_match_cfg & ~ANEG_CFG_ACK)) {
2201                                 ap->state = ANEG_STATE_COMPLETE_ACK_INIT;
2202                         } else {
2203                                 ap->state = ANEG_STATE_AN_ENABLE;
2204                         }
2205                 } else if (ap->ability_match != 0 &&
2206                            ap->rxconfig == 0) {
2207                         ap->state = ANEG_STATE_AN_ENABLE;
2208                 }
2209                 break;
2210
2211         case ANEG_STATE_COMPLETE_ACK_INIT:
2212                 if (ap->rxconfig & ANEG_CFG_INVAL) {
2213                         ret = ANEG_FAILED;
2214                         break;
2215                 }
2216                 ap->flags &= ~(MR_LP_ADV_FULL_DUPLEX |
2217                                MR_LP_ADV_HALF_DUPLEX |
2218                                MR_LP_ADV_SYM_PAUSE |
2219                                MR_LP_ADV_ASYM_PAUSE |
2220                                MR_LP_ADV_REMOTE_FAULT1 |
2221                                MR_LP_ADV_REMOTE_FAULT2 |
2222                                MR_LP_ADV_NEXT_PAGE |
2223                                MR_TOGGLE_RX |
2224                                MR_NP_RX);
2225                 if (ap->rxconfig & ANEG_CFG_FD)
2226                         ap->flags |= MR_LP_ADV_FULL_DUPLEX;
2227                 if (ap->rxconfig & ANEG_CFG_HD)
2228                         ap->flags |= MR_LP_ADV_HALF_DUPLEX;
2229                 if (ap->rxconfig & ANEG_CFG_PS1)
2230                         ap->flags |= MR_LP_ADV_SYM_PAUSE;
2231                 if (ap->rxconfig & ANEG_CFG_PS2)
2232                         ap->flags |= MR_LP_ADV_ASYM_PAUSE;
2233                 if (ap->rxconfig & ANEG_CFG_RF1)
2234                         ap->flags |= MR_LP_ADV_REMOTE_FAULT1;
2235                 if (ap->rxconfig & ANEG_CFG_RF2)
2236                         ap->flags |= MR_LP_ADV_REMOTE_FAULT2;
2237                 if (ap->rxconfig & ANEG_CFG_NP)
2238                         ap->flags |= MR_LP_ADV_NEXT_PAGE;
2239
2240                 ap->link_time = ap->cur_time;
2241
2242                 ap->flags ^= (MR_TOGGLE_TX);
2243                 if (ap->rxconfig & 0x0008)
2244                         ap->flags |= MR_TOGGLE_RX;
2245                 if (ap->rxconfig & ANEG_CFG_NP)
2246                         ap->flags |= MR_NP_RX;
2247                 ap->flags |= MR_PAGE_RX;
2248
2249                 ap->state = ANEG_STATE_COMPLETE_ACK;
2250                 ret = ANEG_TIMER_ENAB;
2251                 break;
2252
2253         case ANEG_STATE_COMPLETE_ACK:
2254                 if (ap->ability_match != 0 &&
2255                     ap->rxconfig == 0) {
2256                         ap->state = ANEG_STATE_AN_ENABLE;
2257                         break;
2258                 }
2259                 delta = ap->cur_time - ap->link_time;
2260                 if (delta > ANEG_STATE_SETTLE_TIME) {
2261                         if (!(ap->flags & (MR_LP_ADV_NEXT_PAGE))) {
2262                                 ap->state = ANEG_STATE_IDLE_DETECT_INIT;
2263                         } else {
2264                                 if ((ap->txconfig & ANEG_CFG_NP) == 0 &&
2265                                     !(ap->flags & MR_NP_RX)) {
2266                                         ap->state = ANEG_STATE_IDLE_DETECT_INIT;
2267                                 } else {
2268                                         ret = ANEG_FAILED;
2269                                 }
2270                         }
2271                 }
2272                 break;
2273
2274         case ANEG_STATE_IDLE_DETECT_INIT:
2275                 ap->link_time = ap->cur_time;
2276                 tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
2277                 tw32_f(MAC_MODE, tp->mac_mode);
2278                 udelay(40);
2279
2280                 ap->state = ANEG_STATE_IDLE_DETECT;
2281                 ret = ANEG_TIMER_ENAB;
2282                 break;
2283
2284         case ANEG_STATE_IDLE_DETECT:
2285                 if (ap->ability_match != 0 &&
2286                     ap->rxconfig == 0) {
2287                         ap->state = ANEG_STATE_AN_ENABLE;
2288                         break;
2289                 }
2290                 delta = ap->cur_time - ap->link_time;
2291                 if (delta > ANEG_STATE_SETTLE_TIME) {
2292                         /* XXX another gem from the Broadcom driver :( */
2293                         ap->state = ANEG_STATE_LINK_OK;
2294                 }
2295                 break;
2296
2297         case ANEG_STATE_LINK_OK:
2298                 ap->flags |= (MR_AN_COMPLETE | MR_LINK_OK);
2299                 ret = ANEG_DONE;
2300                 break;
2301
2302         case ANEG_STATE_NEXT_PAGE_WAIT_INIT:
2303                 /* ??? unimplemented */
2304                 break;
2305
2306         case ANEG_STATE_NEXT_PAGE_WAIT:
2307                 /* ??? unimplemented */
2308                 break;
2309
2310         default:
2311                 ret = ANEG_FAILED;
2312                 break;
2313         };
2314
2315         return ret;
2316 }
2317
2318 static int fiber_autoneg(struct tg3 *tp, u32 *flags)
2319 {
2320         int res = 0;
2321         struct tg3_fiber_aneginfo aninfo;
2322         int status = ANEG_FAILED;
2323         unsigned int tick;
2324         u32 tmp;
2325
2326         tw32_f(MAC_TX_AUTO_NEG, 0);
2327
2328         tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
2329         tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
2330         udelay(40);
2331
2332         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
2333         udelay(40);
2334
2335         memset(&aninfo, 0, sizeof(aninfo));
2336         aninfo.flags |= MR_AN_ENABLE;
2337         aninfo.state = ANEG_STATE_UNKNOWN;
2338         aninfo.cur_time = 0;
2339         tick = 0;
2340         while (++tick < 195000) {
2341                 status = tg3_fiber_aneg_smachine(tp, &aninfo);
2342                 if (status == ANEG_DONE || status == ANEG_FAILED)
2343                         break;
2344
2345                 udelay(1);
2346         }
2347
2348         tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
2349         tw32_f(MAC_MODE, tp->mac_mode);
2350         udelay(40);
2351
2352         *flags = aninfo.flags;
2353
2354         if (status == ANEG_DONE &&
2355             (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
2356                              MR_LP_ADV_FULL_DUPLEX)))
2357                 res = 1;
2358
2359         return res;
2360 }
2361
2362 static void tg3_init_bcm8002(struct tg3 *tp)
2363 {
2364         u32 mac_status = tr32(MAC_STATUS);
2365         int i;
2366
2367         /* Reset when initting first time or we have a link. */
2368         if ((tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) &&
2369             !(mac_status & MAC_STATUS_PCS_SYNCED))
2370                 return;
2371
2372         /* Set PLL lock range. */
2373         tg3_writephy(tp, 0x16, 0x8007);
2374
2375         /* SW reset */
2376         tg3_writephy(tp, MII_BMCR, BMCR_RESET);
2377
2378         /* Wait for reset to complete. */
2379         /* XXX schedule_timeout() ... */
2380         for (i = 0; i < 500; i++)
2381                 udelay(10);
2382
2383         /* Config mode; select PMA/Ch 1 regs. */
2384         tg3_writephy(tp, 0x10, 0x8411);
2385
2386         /* Enable auto-lock and comdet, select txclk for tx. */
2387         tg3_writephy(tp, 0x11, 0x0a10);
2388
2389         tg3_writephy(tp, 0x18, 0x00a0);
2390         tg3_writephy(tp, 0x16, 0x41ff);
2391
2392         /* Assert and deassert POR. */
2393         tg3_writephy(tp, 0x13, 0x0400);
2394         udelay(40);
2395         tg3_writephy(tp, 0x13, 0x0000);
2396
2397         tg3_writephy(tp, 0x11, 0x0a50);
2398         udelay(40);
2399         tg3_writephy(tp, 0x11, 0x0a10);
2400
2401         /* Wait for signal to stabilize */
2402         /* XXX schedule_timeout() ... */
2403         for (i = 0; i < 15000; i++)
2404                 udelay(10);
2405
2406         /* Deselect the channel register so we can read the PHYID
2407          * later.
2408          */
2409         tg3_writephy(tp, 0x10, 0x8011);
2410 }
2411
2412 static int tg3_setup_fiber_hw_autoneg(struct tg3 *tp, u32 mac_status)
2413 {
2414         u32 sg_dig_ctrl, sg_dig_status;
2415         u32 serdes_cfg, expected_sg_dig_ctrl;
2416         int workaround, port_a;
2417         int current_link_up;
2418
2419         serdes_cfg = 0;
2420         expected_sg_dig_ctrl = 0;
2421         workaround = 0;
2422         port_a = 1;
2423         current_link_up = 0;
2424
2425         if (tp->pci_chip_rev_id != CHIPREV_ID_5704_A0 &&
2426             tp->pci_chip_rev_id != CHIPREV_ID_5704_A1) {
2427                 workaround = 1;
2428                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
2429                         port_a = 0;
2430
2431                 /* preserve bits 0-11,13,14 for signal pre-emphasis */
2432                 /* preserve bits 20-23 for voltage regulator */
2433                 serdes_cfg = tr32(MAC_SERDES_CFG) & 0x00f06fff;
2434         }
2435
2436         sg_dig_ctrl = tr32(SG_DIG_CTRL);
2437
2438         if (tp->link_config.autoneg != AUTONEG_ENABLE) {
2439                 if (sg_dig_ctrl & (1 << 31)) {
2440                         if (workaround) {
2441                                 u32 val = serdes_cfg;
2442
2443                                 if (port_a)
2444                                         val |= 0xc010000;
2445                                 else
2446                                         val |= 0x4010000;
2447                                 tw32_f(MAC_SERDES_CFG, val);
2448                         }
2449                         tw32_f(SG_DIG_CTRL, 0x01388400);
2450                 }
2451                 if (mac_status & MAC_STATUS_PCS_SYNCED) {
2452                         tg3_setup_flow_control(tp, 0, 0);
2453                         current_link_up = 1;
2454                 }
2455                 goto out;
2456         }
2457
2458         /* Want auto-negotiation.  */
2459         expected_sg_dig_ctrl = 0x81388400;
2460
2461         /* Pause capability */
2462         expected_sg_dig_ctrl |= (1 << 11);
2463
2464         /* Asymettric pause */
2465         expected_sg_dig_ctrl |= (1 << 12);
2466
2467         if (sg_dig_ctrl != expected_sg_dig_ctrl) {
2468                 if (workaround)
2469                         tw32_f(MAC_SERDES_CFG, serdes_cfg | 0xc011000);
2470                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl | (1 << 30));
2471                 udelay(5);
2472                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl);
2473
2474                 tp->tg3_flags2 |= TG3_FLG2_PHY_JUST_INITTED;
2475         } else if (mac_status & (MAC_STATUS_PCS_SYNCED |
2476                                  MAC_STATUS_SIGNAL_DET)) {
2477                 int i;
2478
2479                 /* Giver time to negotiate (~200ms) */
2480                 for (i = 0; i < 40000; i++) {
2481                         sg_dig_status = tr32(SG_DIG_STATUS);
2482                         if (sg_dig_status & (0x3))
2483                                 break;
2484                         udelay(5);
2485                 }
2486                 mac_status = tr32(MAC_STATUS);
2487
2488                 if ((sg_dig_status & (1 << 1)) &&
2489                     (mac_status & MAC_STATUS_PCS_SYNCED)) {
2490                         u32 local_adv, remote_adv;
2491
2492                         local_adv = ADVERTISE_PAUSE_CAP;
2493                         remote_adv = 0;
2494                         if (sg_dig_status & (1 << 19))
2495                                 remote_adv |= LPA_PAUSE_CAP;
2496                         if (sg_dig_status & (1 << 20))
2497                                 remote_adv |= LPA_PAUSE_ASYM;
2498
2499                         tg3_setup_flow_control(tp, local_adv, remote_adv);
2500                         current_link_up = 1;
2501                         tp->tg3_flags2 &= ~TG3_FLG2_PHY_JUST_INITTED;
2502                 } else if (!(sg_dig_status & (1 << 1))) {
2503                         if (tp->tg3_flags2 & TG3_FLG2_PHY_JUST_INITTED)
2504                                 tp->tg3_flags2 &= ~TG3_FLG2_PHY_JUST_INITTED;
2505                         else {
2506                                 if (workaround) {
2507                                         u32 val = serdes_cfg;
2508
2509                                         if (port_a)
2510                                                 val |= 0xc010000;
2511                                         else
2512                                                 val |= 0x4010000;
2513
2514                                         tw32_f(MAC_SERDES_CFG, val);
2515                                 }
2516
2517                                 tw32_f(SG_DIG_CTRL, 0x01388400);
2518                                 udelay(40);
2519
2520                                 /* Link parallel detection - link is up */
2521                                 /* only if we have PCS_SYNC and not */
2522                                 /* receiving config code words */
2523                                 mac_status = tr32(MAC_STATUS);
2524                                 if ((mac_status & MAC_STATUS_PCS_SYNCED) &&
2525                                     !(mac_status & MAC_STATUS_RCVD_CFG)) {
2526                                         tg3_setup_flow_control(tp, 0, 0);
2527                                         current_link_up = 1;
2528                                 }
2529                         }
2530                 }
2531         }
2532
2533 out:
2534         return current_link_up;
2535 }
2536
2537 static int tg3_setup_fiber_by_hand(struct tg3 *tp, u32 mac_status)
2538 {
2539         int current_link_up = 0;
2540
2541         if (!(mac_status & MAC_STATUS_PCS_SYNCED)) {
2542                 tp->tg3_flags &= ~TG3_FLAG_GOT_SERDES_FLOWCTL;
2543                 goto out;
2544         }
2545
2546         if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2547                 u32 flags;
2548                 int i;
2549   
2550                 if (fiber_autoneg(tp, &flags)) {
2551                         u32 local_adv, remote_adv;
2552
2553                         local_adv = ADVERTISE_PAUSE_CAP;
2554                         remote_adv = 0;
2555                         if (flags & MR_LP_ADV_SYM_PAUSE)
2556                                 remote_adv |= LPA_PAUSE_CAP;
2557                         if (flags & MR_LP_ADV_ASYM_PAUSE)
2558                                 remote_adv |= LPA_PAUSE_ASYM;
2559
2560                         tg3_setup_flow_control(tp, local_adv, remote_adv);
2561
2562                         tp->tg3_flags |= TG3_FLAG_GOT_SERDES_FLOWCTL;
2563                         current_link_up = 1;
2564                 }
2565                 for (i = 0; i < 30; i++) {
2566                         udelay(20);
2567                         tw32_f(MAC_STATUS,
2568                                (MAC_STATUS_SYNC_CHANGED |
2569                                 MAC_STATUS_CFG_CHANGED));
2570                         udelay(40);
2571                         if ((tr32(MAC_STATUS) &
2572                              (MAC_STATUS_SYNC_CHANGED |
2573                               MAC_STATUS_CFG_CHANGED)) == 0)
2574                                 break;
2575                 }
2576
2577                 mac_status = tr32(MAC_STATUS);
2578                 if (current_link_up == 0 &&
2579                     (mac_status & MAC_STATUS_PCS_SYNCED) &&
2580                     !(mac_status & MAC_STATUS_RCVD_CFG))
2581                         current_link_up = 1;
2582         } else {
2583                 /* Forcing 1000FD link up. */
2584                 current_link_up = 1;
2585                 tp->tg3_flags |= TG3_FLAG_GOT_SERDES_FLOWCTL;
2586
2587                 tw32_f(MAC_MODE, (tp->mac_mode | MAC_MODE_SEND_CONFIGS));
2588                 udelay(40);
2589         }
2590
2591 out:
2592         return current_link_up;
2593 }
2594
2595 static int tg3_setup_fiber_phy(struct tg3 *tp, int force_reset)
2596 {
2597         u32 orig_pause_cfg;
2598         u16 orig_active_speed;
2599         u8 orig_active_duplex;
2600         u32 mac_status;
2601         int current_link_up;
2602         int i;
2603
2604         orig_pause_cfg =
2605                 (tp->tg3_flags & (TG3_FLAG_RX_PAUSE |
2606                                   TG3_FLAG_TX_PAUSE));
2607         orig_active_speed = tp->link_config.active_speed;
2608         orig_active_duplex = tp->link_config.active_duplex;
2609
2610         if (!(tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG) &&
2611             netif_carrier_ok(tp->dev) &&
2612             (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE)) {
2613                 mac_status = tr32(MAC_STATUS);
2614                 mac_status &= (MAC_STATUS_PCS_SYNCED |
2615                                MAC_STATUS_SIGNAL_DET |
2616                                MAC_STATUS_CFG_CHANGED |
2617                                MAC_STATUS_RCVD_CFG);
2618                 if (mac_status == (MAC_STATUS_PCS_SYNCED |
2619                                    MAC_STATUS_SIGNAL_DET)) {
2620                         tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
2621                                             MAC_STATUS_CFG_CHANGED));
2622                         return 0;
2623                 }
2624         }
2625
2626         tw32_f(MAC_TX_AUTO_NEG, 0);
2627
2628         tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
2629         tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
2630         tw32_f(MAC_MODE, tp->mac_mode);
2631         udelay(40);
2632
2633         if (tp->phy_id == PHY_ID_BCM8002)
2634                 tg3_init_bcm8002(tp);
2635
2636         /* Enable link change event even when serdes polling.  */
2637         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2638         udelay(40);
2639
2640         current_link_up = 0;
2641         mac_status = tr32(MAC_STATUS);
2642
2643         if (tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG)
2644                 current_link_up = tg3_setup_fiber_hw_autoneg(tp, mac_status);
2645         else
2646                 current_link_up = tg3_setup_fiber_by_hand(tp, mac_status);
2647
2648         tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
2649         tw32_f(MAC_MODE, tp->mac_mode);
2650         udelay(40);
2651
2652         tp->hw_status->status =
2653                 (SD_STATUS_UPDATED |
2654                  (tp->hw_status->status & ~SD_STATUS_LINK_CHG));
2655
2656         for (i = 0; i < 100; i++) {
2657                 tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
2658                                     MAC_STATUS_CFG_CHANGED));
2659                 udelay(5);
2660                 if ((tr32(MAC_STATUS) & (MAC_STATUS_SYNC_CHANGED |
2661                                          MAC_STATUS_CFG_CHANGED)) == 0)
2662                         break;
2663         }
2664
2665         mac_status = tr32(MAC_STATUS);
2666         if ((mac_status & MAC_STATUS_PCS_SYNCED) == 0) {
2667                 current_link_up = 0;
2668                 if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2669                         tw32_f(MAC_MODE, (tp->mac_mode |
2670                                           MAC_MODE_SEND_CONFIGS));
2671                         udelay(1);
2672                         tw32_f(MAC_MODE, tp->mac_mode);
2673                 }
2674         }
2675
2676         if (current_link_up == 1) {
2677                 tp->link_config.active_speed = SPEED_1000;
2678                 tp->link_config.active_duplex = DUPLEX_FULL;
2679                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
2680                                     LED_CTRL_LNKLED_OVERRIDE |
2681                                     LED_CTRL_1000MBPS_ON));
2682         } else {
2683                 tp->link_config.active_speed = SPEED_INVALID;
2684                 tp->link_config.active_duplex = DUPLEX_INVALID;
2685                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
2686                                     LED_CTRL_LNKLED_OVERRIDE |
2687                                     LED_CTRL_TRAFFIC_OVERRIDE));
2688         }
2689
2690         if (current_link_up != netif_carrier_ok(tp->dev)) {
2691                 if (current_link_up)
2692                         netif_carrier_on(tp->dev);
2693                 else
2694                         netif_carrier_off(tp->dev);
2695                 tg3_link_report(tp);
2696         } else {
2697                 u32 now_pause_cfg =
2698                         tp->tg3_flags & (TG3_FLAG_RX_PAUSE |
2699                                          TG3_FLAG_TX_PAUSE);
2700                 if (orig_pause_cfg != now_pause_cfg ||
2701                     orig_active_speed != tp->link_config.active_speed ||
2702                     orig_active_duplex != tp->link_config.active_duplex)
2703                         tg3_link_report(tp);
2704         }
2705
2706         return 0;
2707 }
2708
2709 static int tg3_setup_fiber_mii_phy(struct tg3 *tp, int force_reset)
2710 {
2711         int current_link_up, err = 0;
2712         u32 bmsr, bmcr;
2713         u16 current_speed;
2714         u8 current_duplex;
2715
2716         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
2717         tw32_f(MAC_MODE, tp->mac_mode);
2718         udelay(40);
2719
2720         tw32(MAC_EVENT, 0);
2721
2722         tw32_f(MAC_STATUS,
2723              (MAC_STATUS_SYNC_CHANGED |
2724               MAC_STATUS_CFG_CHANGED |
2725               MAC_STATUS_MI_COMPLETION |
2726               MAC_STATUS_LNKSTATE_CHANGED));
2727         udelay(40);
2728
2729         if (force_reset)
2730                 tg3_phy_reset(tp);
2731
2732         current_link_up = 0;
2733         current_speed = SPEED_INVALID;
2734         current_duplex = DUPLEX_INVALID;
2735
2736         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2737         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2738         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
2739                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
2740                         bmsr |= BMSR_LSTATUS;
2741                 else
2742                         bmsr &= ~BMSR_LSTATUS;
2743         }
2744
2745         err |= tg3_readphy(tp, MII_BMCR, &bmcr);
2746
2747         if ((tp->link_config.autoneg == AUTONEG_ENABLE) && !force_reset &&
2748             (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
2749                 /* do nothing, just check for link up at the end */
2750         } else if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2751                 u32 adv, new_adv;
2752
2753                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
2754                 new_adv = adv & ~(ADVERTISE_1000XFULL | ADVERTISE_1000XHALF |
2755                                   ADVERTISE_1000XPAUSE |
2756                                   ADVERTISE_1000XPSE_ASYM |
2757                                   ADVERTISE_SLCT);
2758
2759                 /* Always advertise symmetric PAUSE just like copper */
2760                 new_adv |= ADVERTISE_1000XPAUSE;
2761
2762                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
2763                         new_adv |= ADVERTISE_1000XHALF;
2764                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
2765                         new_adv |= ADVERTISE_1000XFULL;
2766
2767                 if ((new_adv != adv) || !(bmcr & BMCR_ANENABLE)) {
2768                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
2769                         bmcr |= BMCR_ANENABLE | BMCR_ANRESTART;
2770                         tg3_writephy(tp, MII_BMCR, bmcr);
2771
2772                         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2773                         tp->tg3_flags2 |= TG3_FLG2_PHY_JUST_INITTED;
2774                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2775
2776                         return err;
2777                 }
2778         } else {
2779                 u32 new_bmcr;
2780
2781                 bmcr &= ~BMCR_SPEED1000;
2782                 new_bmcr = bmcr & ~(BMCR_ANENABLE | BMCR_FULLDPLX);
2783
2784                 if (tp->link_config.duplex == DUPLEX_FULL)
2785                         new_bmcr |= BMCR_FULLDPLX;
2786
2787                 if (new_bmcr != bmcr) {
2788                         /* BMCR_SPEED1000 is a reserved bit that needs
2789                          * to be set on write.
2790                          */
2791                         new_bmcr |= BMCR_SPEED1000;
2792
2793                         /* Force a linkdown */
2794                         if (netif_carrier_ok(tp->dev)) {
2795                                 u32 adv;
2796
2797                                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
2798                                 adv &= ~(ADVERTISE_1000XFULL |
2799                                          ADVERTISE_1000XHALF |
2800                                          ADVERTISE_SLCT);
2801                                 tg3_writephy(tp, MII_ADVERTISE, adv);
2802                                 tg3_writephy(tp, MII_BMCR, bmcr |
2803                                                            BMCR_ANRESTART |
2804                                                            BMCR_ANENABLE);
2805                                 udelay(10);
2806                                 netif_carrier_off(tp->dev);
2807                         }
2808                         tg3_writephy(tp, MII_BMCR, new_bmcr);
2809                         bmcr = new_bmcr;
2810                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2811                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2812                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
2813                             ASIC_REV_5714) {
2814                                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
2815                                         bmsr |= BMSR_LSTATUS;
2816                                 else
2817                                         bmsr &= ~BMSR_LSTATUS;
2818                         }
2819                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2820                 }
2821         }
2822
2823         if (bmsr & BMSR_LSTATUS) {
2824                 current_speed = SPEED_1000;
2825                 current_link_up = 1;
2826                 if (bmcr & BMCR_FULLDPLX)
2827                         current_duplex = DUPLEX_FULL;
2828                 else
2829                         current_duplex = DUPLEX_HALF;
2830
2831                 if (bmcr & BMCR_ANENABLE) {
2832                         u32 local_adv, remote_adv, common;
2833
2834                         err |= tg3_readphy(tp, MII_ADVERTISE, &local_adv);
2835                         err |= tg3_readphy(tp, MII_LPA, &remote_adv);
2836                         common = local_adv & remote_adv;
2837                         if (common & (ADVERTISE_1000XHALF |
2838                                       ADVERTISE_1000XFULL)) {
2839                                 if (common & ADVERTISE_1000XFULL)
2840                                         current_duplex = DUPLEX_FULL;
2841                                 else
2842                                         current_duplex = DUPLEX_HALF;
2843
2844                                 tg3_setup_flow_control(tp, local_adv,
2845                                                        remote_adv);
2846                         }
2847                         else
2848                                 current_link_up = 0;
2849                 }
2850         }
2851
2852         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
2853         if (tp->link_config.active_duplex == DUPLEX_HALF)
2854                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
2855
2856         tw32_f(MAC_MODE, tp->mac_mode);
2857         udelay(40);
2858
2859         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2860
2861         tp->link_config.active_speed = current_speed;
2862         tp->link_config.active_duplex = current_duplex;
2863
2864         if (current_link_up != netif_carrier_ok(tp->dev)) {
2865                 if (current_link_up)
2866                         netif_carrier_on(tp->dev);
2867                 else {
2868                         netif_carrier_off(tp->dev);
2869                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2870                 }
2871                 tg3_link_report(tp);
2872         }
2873         return err;
2874 }
2875
2876 static void tg3_serdes_parallel_detect(struct tg3 *tp)
2877 {
2878         if (tp->tg3_flags2 & TG3_FLG2_PHY_JUST_INITTED) {
2879                 /* Give autoneg time to complete. */
2880                 tp->tg3_flags2 &= ~TG3_FLG2_PHY_JUST_INITTED;
2881                 return;
2882         }
2883         if (!netif_carrier_ok(tp->dev) &&
2884             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
2885                 u32 bmcr;
2886
2887                 tg3_readphy(tp, MII_BMCR, &bmcr);
2888                 if (bmcr & BMCR_ANENABLE) {
2889                         u32 phy1, phy2;
2890
2891                         /* Select shadow register 0x1f */
2892                         tg3_writephy(tp, 0x1c, 0x7c00);
2893                         tg3_readphy(tp, 0x1c, &phy1);
2894
2895                         /* Select expansion interrupt status register */
2896                         tg3_writephy(tp, 0x17, 0x0f01);
2897                         tg3_readphy(tp, 0x15, &phy2);
2898                         tg3_readphy(tp, 0x15, &phy2);
2899
2900                         if ((phy1 & 0x10) && !(phy2 & 0x20)) {
2901                                 /* We have signal detect and not receiving
2902                                  * config code words, link is up by parallel
2903                                  * detection.
2904                                  */
2905
2906                                 bmcr &= ~BMCR_ANENABLE;
2907                                 bmcr |= BMCR_SPEED1000 | BMCR_FULLDPLX;
2908                                 tg3_writephy(tp, MII_BMCR, bmcr);
2909                                 tp->tg3_flags2 |= TG3_FLG2_PARALLEL_DETECT;
2910                         }
2911                 }
2912         }
2913         else if (netif_carrier_ok(tp->dev) &&
2914                  (tp->link_config.autoneg == AUTONEG_ENABLE) &&
2915                  (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
2916                 u32 phy2;
2917
2918                 /* Select expansion interrupt status register */
2919                 tg3_writephy(tp, 0x17, 0x0f01);
2920                 tg3_readphy(tp, 0x15, &phy2);
2921                 if (phy2 & 0x20) {
2922                         u32 bmcr;
2923
2924                         /* Config code words received, turn on autoneg. */
2925                         tg3_readphy(tp, MII_BMCR, &bmcr);
2926                         tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANENABLE);
2927
2928                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2929
2930                 }
2931         }
2932 }
2933
2934 static int tg3_setup_phy(struct tg3 *tp, int force_reset)
2935 {
2936         int err;
2937
2938         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
2939                 err = tg3_setup_fiber_phy(tp, force_reset);
2940         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
2941                 err = tg3_setup_fiber_mii_phy(tp, force_reset);
2942         } else {
2943                 err = tg3_setup_copper_phy(tp, force_reset);
2944         }
2945
2946         if (tp->link_config.active_speed == SPEED_1000 &&
2947             tp->link_config.active_duplex == DUPLEX_HALF)
2948                 tw32(MAC_TX_LENGTHS,
2949                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
2950                       (6 << TX_LENGTHS_IPG_SHIFT) |
2951                       (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
2952         else
2953                 tw32(MAC_TX_LENGTHS,
2954                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
2955                       (6 << TX_LENGTHS_IPG_SHIFT) |
2956                       (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
2957
2958         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
2959                 if (netif_carrier_ok(tp->dev)) {
2960                         tw32(HOSTCC_STAT_COAL_TICKS,
2961                              tp->coal.stats_block_coalesce_usecs);
2962                 } else {
2963                         tw32(HOSTCC_STAT_COAL_TICKS, 0);
2964                 }
2965         }
2966
2967         return err;
2968 }
2969
2970 /* This is called whenever we suspect that the system chipset is re-
2971  * ordering the sequence of MMIO to the tx send mailbox. The symptom
2972  * is bogus tx completions. We try to recover by setting the
2973  * TG3_FLAG_MBOX_WRITE_REORDER flag and resetting the chip later
2974  * in the workqueue.
2975  */
2976 static void tg3_tx_recover(struct tg3 *tp)
2977 {
2978         BUG_ON((tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) ||
2979                tp->write32_tx_mbox == tg3_write_indirect_mbox);
2980
2981         printk(KERN_WARNING PFX "%s: The system may be re-ordering memory-"
2982                "mapped I/O cycles to the network device, attempting to "
2983                "recover. Please report the problem to the driver maintainer "
2984                "and include system chipset information.\n", tp->dev->name);
2985
2986         spin_lock(&tp->lock);
2987         tp->tg3_flags |= TG3_FLAG_TX_RECOVERY_PENDING;
2988         spin_unlock(&tp->lock);
2989 }
2990
2991 /* Tigon3 never reports partial packet sends.  So we do not
2992  * need special logic to handle SKBs that have not had all
2993  * of their frags sent yet, like SunGEM does.
2994  */
2995 static void tg3_tx(struct tg3 *tp)
2996 {
2997         u32 hw_idx = tp->hw_status->idx[0].tx_consumer;
2998         u32 sw_idx = tp->tx_cons;
2999
3000         while (sw_idx != hw_idx) {
3001                 struct tx_ring_info *ri = &tp->tx_buffers[sw_idx];
3002                 struct sk_buff *skb = ri->skb;
3003                 int i, tx_bug = 0;
3004
3005                 if (unlikely(skb == NULL)) {
3006                         tg3_tx_recover(tp);
3007                         return;
3008                 }
3009
3010                 pci_unmap_single(tp->pdev,
3011                                  pci_unmap_addr(ri, mapping),
3012                                  skb_headlen(skb),
3013                                  PCI_DMA_TODEVICE);
3014
3015                 ri->skb = NULL;
3016
3017                 sw_idx = NEXT_TX(sw_idx);
3018
3019                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3020                         ri = &tp->tx_buffers[sw_idx];
3021                         if (unlikely(ri->skb != NULL || sw_idx == hw_idx))
3022                                 tx_bug = 1;
3023
3024                         pci_unmap_page(tp->pdev,
3025                                        pci_unmap_addr(ri, mapping),
3026                                        skb_shinfo(skb)->frags[i].size,
3027                                        PCI_DMA_TODEVICE);
3028
3029                         sw_idx = NEXT_TX(sw_idx);
3030                 }
3031
3032                 dev_kfree_skb(skb);
3033
3034                 if (unlikely(tx_bug)) {
3035                         tg3_tx_recover(tp);
3036                         return;
3037                 }
3038         }
3039
3040         tp->tx_cons = sw_idx;
3041
3042         if (unlikely(netif_queue_stopped(tp->dev))) {
3043                 spin_lock(&tp->tx_lock);
3044                 if (netif_queue_stopped(tp->dev) &&
3045                     (TX_BUFFS_AVAIL(tp) > TG3_TX_WAKEUP_THRESH))
3046                         netif_wake_queue(tp->dev);
3047                 spin_unlock(&tp->tx_lock);
3048         }
3049 }
3050
3051 /* Returns size of skb allocated or < 0 on error.
3052  *
3053  * We only need to fill in the address because the other members
3054  * of the RX descriptor are invariant, see tg3_init_rings.
3055  *
3056  * Note the purposeful assymetry of cpu vs. chip accesses.  For
3057  * posting buffers we only dirty the first cache line of the RX
3058  * descriptor (containing the address).  Whereas for the RX status
3059  * buffers the cpu only reads the last cacheline of the RX descriptor
3060  * (to fetch the error flags, vlan tag, checksum, and opaque cookie).
3061  */
3062 static int tg3_alloc_rx_skb(struct tg3 *tp, u32 opaque_key,
3063                             int src_idx, u32 dest_idx_unmasked)
3064 {
3065         struct tg3_rx_buffer_desc *desc;
3066         struct ring_info *map, *src_map;
3067         struct sk_buff *skb;
3068         dma_addr_t mapping;
3069         int skb_size, dest_idx;
3070
3071         src_map = NULL;
3072         switch (opaque_key) {
3073         case RXD_OPAQUE_RING_STD:
3074                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
3075                 desc = &tp->rx_std[dest_idx];
3076                 map = &tp->rx_std_buffers[dest_idx];
3077                 if (src_idx >= 0)
3078                         src_map = &tp->rx_std_buffers[src_idx];
3079                 skb_size = tp->rx_pkt_buf_sz;
3080                 break;
3081
3082         case RXD_OPAQUE_RING_JUMBO:
3083                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
3084                 desc = &tp->rx_jumbo[dest_idx];
3085                 map = &tp->rx_jumbo_buffers[dest_idx];
3086                 if (src_idx >= 0)
3087                         src_map = &tp->rx_jumbo_buffers[src_idx];
3088                 skb_size = RX_JUMBO_PKT_BUF_SZ;
3089                 break;
3090
3091         default:
3092                 return -EINVAL;
3093         };
3094
3095         /* Do not overwrite any of the map or rp information
3096          * until we are sure we can commit to a new buffer.
3097          *
3098          * Callers depend upon this behavior and assume that
3099          * we leave everything unchanged if we fail.
3100          */
3101         skb = dev_alloc_skb(skb_size);
3102         if (skb == NULL)
3103                 return -ENOMEM;
3104
3105         skb->dev = tp->dev;
3106         skb_reserve(skb, tp->rx_offset);
3107
3108         mapping = pci_map_single(tp->pdev, skb->data,
3109                                  skb_size - tp->rx_offset,
3110                                  PCI_DMA_FROMDEVICE);
3111
3112         map->skb = skb;
3113         pci_unmap_addr_set(map, mapping, mapping);
3114
3115         if (src_map != NULL)
3116                 src_map->skb = NULL;
3117
3118         desc->addr_hi = ((u64)mapping >> 32);
3119         desc->addr_lo = ((u64)mapping & 0xffffffff);
3120
3121         return skb_size;
3122 }
3123
3124 /* We only need to move over in the address because the other
3125  * members of the RX descriptor are invariant.  See notes above
3126  * tg3_alloc_rx_skb for full details.
3127  */
3128 static void tg3_recycle_rx(struct tg3 *tp, u32 opaque_key,
3129                            int src_idx, u32 dest_idx_unmasked)
3130 {
3131         struct tg3_rx_buffer_desc *src_desc, *dest_desc;
3132         struct ring_info *src_map, *dest_map;
3133         int dest_idx;
3134
3135         switch (opaque_key) {
3136         case RXD_OPAQUE_RING_STD:
3137                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
3138                 dest_desc = &tp->rx_std[dest_idx];
3139                 dest_map = &tp->rx_std_buffers[dest_idx];
3140                 src_desc = &tp->rx_std[src_idx];
3141                 src_map = &tp->rx_std_buffers[src_idx];
3142                 break;
3143
3144         case RXD_OPAQUE_RING_JUMBO:
3145                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
3146                 dest_desc = &tp->rx_jumbo[dest_idx];
3147                 dest_map = &tp->rx_jumbo_buffers[dest_idx];
3148                 src_desc = &tp->rx_jumbo[src_idx];
3149                 src_map = &tp->rx_jumbo_buffers[src_idx];
3150                 break;
3151
3152         default:
3153                 return;
3154         };
3155
3156         dest_map->skb = src_map->skb;
3157         pci_unmap_addr_set(dest_map, mapping,
3158                            pci_unmap_addr(src_map, mapping));
3159         dest_desc->addr_hi = src_desc->addr_hi;
3160         dest_desc->addr_lo = src_desc->addr_lo;
3161
3162         src_map->skb = NULL;
3163 }
3164
3165 #if TG3_VLAN_TAG_USED
3166 static int tg3_vlan_rx(struct tg3 *tp, struct sk_buff *skb, u16 vlan_tag)
3167 {
3168         return vlan_hwaccel_receive_skb(skb, tp->vlgrp, vlan_tag);
3169 }
3170 #endif
3171
3172 /* The RX ring scheme is composed of multiple rings which post fresh
3173  * buffers to the chip, and one special ring the chip uses to report
3174  * status back to the host.
3175  *
3176  * The special ring reports the status of received packets to the
3177  * host.  The chip does not write into the original descriptor the
3178  * RX buffer was obtained from.  The chip simply takes the original
3179  * descriptor as provided by the host, updates the status and length
3180  * field, then writes this into the next status ring entry.
3181  *
3182  * Each ring the host uses to post buffers to the chip is described
3183  * by a TG3_BDINFO entry in the chips SRAM area.  When a packet arrives,
3184  * it is first placed into the on-chip ram.  When the packet's length
3185  * is known, it walks down the TG3_BDINFO entries to select the ring.
3186  * Each TG3_BDINFO specifies a MAXLEN field and the first TG3_BDINFO
3187  * which is within the range of the new packet's length is chosen.
3188  *
3189  * The "separate ring for rx status" scheme may sound queer, but it makes
3190  * sense from a cache coherency perspective.  If only the host writes
3191  * to the buffer post rings, and only the chip writes to the rx status
3192  * rings, then cache lines never move beyond shared-modified state.
3193  * If both the host and chip were to write into the same ring, cache line
3194  * eviction could occur since both entities want it in an exclusive state.
3195  */
3196 static int tg3_rx(struct tg3 *tp, int budget)
3197 {
3198         u32 work_mask;
3199         u32 sw_idx = tp->rx_rcb_ptr;
3200         u16 hw_idx;
3201         int received;
3202
3203         hw_idx = tp->hw_status->idx[0].rx_producer;
3204         /*
3205          * We need to order the read of hw_idx and the read of
3206          * the opaque cookie.
3207          */
3208         rmb();
3209         work_mask = 0;
3210         received = 0;
3211         while (sw_idx != hw_idx && budget > 0) {
3212                 struct tg3_rx_buffer_desc *desc = &tp->rx_rcb[sw_idx];
3213                 unsigned int len;
3214                 struct sk_buff *skb;
3215                 dma_addr_t dma_addr;
3216                 u32 opaque_key, desc_idx, *post_ptr;
3217
3218                 desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
3219                 opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
3220                 if (opaque_key == RXD_OPAQUE_RING_STD) {
3221                         dma_addr = pci_unmap_addr(&tp->rx_std_buffers[desc_idx],
3222                                                   mapping);
3223                         skb = tp->rx_std_buffers[desc_idx].skb;
3224                         post_ptr = &tp->rx_std_ptr;
3225                 } else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
3226                         dma_addr = pci_unmap_addr(&tp->rx_jumbo_buffers[desc_idx],
3227                                                   mapping);
3228                         skb = tp->rx_jumbo_buffers[desc_idx].skb;
3229                         post_ptr = &tp->rx_jumbo_ptr;
3230                 }
3231                 else {
3232                         goto next_pkt_nopost;
3233                 }
3234
3235                 work_mask |= opaque_key;
3236
3237                 if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
3238                     (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII)) {
3239                 drop_it:
3240                         tg3_recycle_rx(tp, opaque_key,
3241                                        desc_idx, *post_ptr);
3242                 drop_it_no_recycle:
3243                         /* Other statistics kept track of by card. */
3244                         tp->net_stats.rx_dropped++;
3245                         goto next_pkt;
3246                 }
3247
3248                 len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) - 4; /* omit crc */
3249
3250                 if (len > RX_COPY_THRESHOLD 
3251                         && tp->rx_offset == 2
3252                         /* rx_offset != 2 iff this is a 5701 card running
3253                          * in PCI-X mode [see tg3_get_invariants()] */
3254                 ) {
3255                         int skb_size;
3256
3257                         skb_size = tg3_alloc_rx_skb(tp, opaque_key,
3258                                                     desc_idx, *post_ptr);
3259                         if (skb_size < 0)
3260                                 goto drop_it;
3261
3262                         pci_unmap_single(tp->pdev, dma_addr,
3263                                          skb_size - tp->rx_offset,
3264                                          PCI_DMA_FROMDEVICE);
3265
3266                         skb_put(skb, len);
3267                 } else {
3268                         struct sk_buff *copy_skb;
3269
3270                         tg3_recycle_rx(tp, opaque_key,
3271                                        desc_idx, *post_ptr);
3272
3273                         copy_skb = dev_alloc_skb(len + 2);
3274                         if (copy_skb == NULL)
3275                                 goto drop_it_no_recycle;
3276
3277                         copy_skb->dev = tp->dev;
3278                         skb_reserve(copy_skb, 2);
3279                         skb_put(copy_skb, len);
3280                         pci_dma_sync_single_for_cpu(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
3281                         memcpy(copy_skb->data, skb->data, len);
3282                         pci_dma_sync_single_for_device(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
3283
3284                         /* We'll reuse the original ring buffer. */
3285                         skb = copy_skb;
3286                 }
3287
3288                 if ((tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) &&
3289                     (desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
3290                     (((desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
3291                       >> RXD_TCPCSUM_SHIFT) == 0xffff))
3292                         skb->ip_summed = CHECKSUM_UNNECESSARY;
3293                 else
3294                         skb->ip_summed = CHECKSUM_NONE;
3295
3296                 skb->protocol = eth_type_trans(skb, tp->dev);
3297 #if TG3_VLAN_TAG_USED
3298                 if (tp->vlgrp != NULL &&
3299                     desc->type_flags & RXD_FLAG_VLAN) {
3300                         tg3_vlan_rx(tp, skb,
3301                                     desc->err_vlan & RXD_VLAN_MASK);
3302                 } else
3303 #endif
3304                         netif_receive_skb(skb);
3305
3306                 tp->dev->last_rx = jiffies;
3307                 received++;
3308                 budget--;
3309
3310 next_pkt:
3311                 (*post_ptr)++;
3312 next_pkt_nopost:
3313                 sw_idx++;
3314                 sw_idx %= TG3_RX_RCB_RING_SIZE(tp);
3315
3316                 /* Refresh hw_idx to see if there is new work */
3317                 if (sw_idx == hw_idx) {
3318                         hw_idx = tp->hw_status->idx[0].rx_producer;
3319                         rmb();
3320                 }
3321         }
3322
3323         /* ACK the status ring. */
3324         tp->rx_rcb_ptr = sw_idx;
3325         tw32_rx_mbox(MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW, sw_idx);
3326
3327         /* Refill RX ring(s). */
3328         if (work_mask & RXD_OPAQUE_RING_STD) {
3329                 sw_idx = tp->rx_std_ptr % TG3_RX_RING_SIZE;
3330                 tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW,
3331                              sw_idx);
3332         }
3333         if (work_mask & RXD_OPAQUE_RING_JUMBO) {
3334                 sw_idx = tp->rx_jumbo_ptr % TG3_RX_JUMBO_RING_SIZE;
3335                 tw32_rx_mbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW,
3336                              sw_idx);
3337         }
3338         mmiowb();
3339
3340         return received;
3341 }
3342
3343 static int tg3_poll(struct net_device *netdev, int *budget)
3344 {
3345         struct tg3 *tp = netdev_priv(netdev);
3346         struct tg3_hw_status *sblk = tp->hw_status;
3347         int done;
3348
3349         /* handle link change and other phy events */
3350         if (!(tp->tg3_flags &
3351               (TG3_FLAG_USE_LINKCHG_REG |
3352                TG3_FLAG_POLL_SERDES))) {
3353                 if (sblk->status & SD_STATUS_LINK_CHG) {
3354                         sblk->status = SD_STATUS_UPDATED |
3355                                 (sblk->status & ~SD_STATUS_LINK_CHG);
3356                         spin_lock(&tp->lock);
3357                         tg3_setup_phy(tp, 0);
3358                         spin_unlock(&tp->lock);
3359                 }
3360         }
3361
3362         /* run TX completion thread */
3363         if (sblk->idx[0].tx_consumer != tp->tx_cons) {
3364                 tg3_tx(tp);
3365                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING)) {
3366                         netif_rx_complete(netdev);
3367                         schedule_work(&tp->reset_task);
3368                         return 0;
3369                 }
3370         }
3371
3372         /* run RX thread, within the bounds set by NAPI.
3373          * All RX "locking" is done by ensuring outside
3374          * code synchronizes with dev->poll()
3375          */
3376         if (sblk->idx[0].rx_producer != tp->rx_rcb_ptr) {
3377                 int orig_budget = *budget;
3378                 int work_done;
3379
3380                 if (orig_budget > netdev->quota)
3381                         orig_budget = netdev->quota;
3382
3383                 work_done = tg3_rx(tp, orig_budget);
3384
3385                 *budget -= work_done;
3386                 netdev->quota -= work_done;
3387         }
3388
3389         if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) {
3390                 tp->last_tag = sblk->status_tag;
3391                 rmb();
3392         } else
3393                 sblk->status &= ~SD_STATUS_UPDATED;
3394
3395         /* if no more work, tell net stack and NIC we're done */
3396         done = !tg3_has_work(tp);
3397         if (done) {
3398                 netif_rx_complete(netdev);
3399                 tg3_restart_ints(tp);
3400         }
3401
3402         return (done ? 0 : 1);
3403 }
3404
3405 static void tg3_irq_quiesce(struct tg3 *tp)
3406 {
3407         BUG_ON(tp->irq_sync);
3408
3409         tp->irq_sync = 1;
3410         smp_mb();
3411
3412         synchronize_irq(tp->pdev->irq);
3413 }
3414
3415 static inline int tg3_irq_sync(struct tg3 *tp)
3416 {
3417         return tp->irq_sync;
3418 }
3419
3420 /* Fully shutdown all tg3 driver activity elsewhere in the system.
3421  * If irq_sync is non-zero, then the IRQ handler must be synchronized
3422  * with as well.  Most of the time, this is not necessary except when
3423  * shutting down the device.
3424  */
3425 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync)
3426 {
3427         if (irq_sync)
3428                 tg3_irq_quiesce(tp);
3429         spin_lock_bh(&tp->lock);
3430 }
3431
3432 static inline void tg3_full_unlock(struct tg3 *tp)
3433 {
3434         spin_unlock_bh(&tp->lock);
3435 }
3436
3437 /* One-shot MSI handler - Chip automatically disables interrupt
3438  * after sending MSI so driver doesn't have to do it.
3439  */
3440 static irqreturn_t tg3_msi_1shot(int irq, void *dev_id, struct pt_regs *regs)
3441 {
3442         struct net_device *dev = dev_id;
3443         struct tg3 *tp = netdev_priv(dev);
3444
3445         prefetch(tp->hw_status);
3446         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3447
3448         if (likely(!tg3_irq_sync(tp)))
3449                 netif_rx_schedule(dev);         /* schedule NAPI poll */
3450
3451         return IRQ_HANDLED;
3452 }
3453
3454 /* MSI ISR - No need to check for interrupt sharing and no need to
3455  * flush status block and interrupt mailbox. PCI ordering rules
3456  * guarantee that MSI will arrive after the status block.
3457  */
3458 static irqreturn_t tg3_msi(int irq, void *dev_id, struct pt_regs *regs)
3459 {
3460         struct net_device *dev = dev_id;
3461         struct tg3 *tp = netdev_priv(dev);
3462
3463         prefetch(tp->hw_status);
3464         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3465         /*
3466          * Writing any value to intr-mbox-0 clears PCI INTA# and
3467          * chip-internal interrupt pending events.
3468          * Writing non-zero to intr-mbox-0 additional tells the
3469          * NIC to stop sending us irqs, engaging "in-intr-handler"
3470          * event coalescing.
3471          */
3472         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
3473         if (likely(!tg3_irq_sync(tp)))
3474                 netif_rx_schedule(dev);         /* schedule NAPI poll */
3475
3476         return IRQ_RETVAL(1);
3477 }
3478
3479 static irqreturn_t tg3_interrupt(int irq, void *dev_id, struct pt_regs *regs)
3480 {
3481         struct net_device *dev = dev_id;
3482         struct tg3 *tp = netdev_priv(dev);
3483         struct tg3_hw_status *sblk = tp->hw_status;
3484         unsigned int handled = 1;
3485
3486         /* In INTx mode, it is possible for the interrupt to arrive at
3487          * the CPU before the status block posted prior to the interrupt.
3488          * Reading the PCI State register will confirm whether the
3489          * interrupt is ours and will flush the status block.
3490          */
3491         if ((sblk->status & SD_STATUS_UPDATED) ||
3492             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3493                 /*
3494                  * Writing any value to intr-mbox-0 clears PCI INTA# and
3495                  * chip-internal interrupt pending events.
3496                  * Writing non-zero to intr-mbox-0 additional tells the
3497                  * NIC to stop sending us irqs, engaging "in-intr-handler"
3498                  * event coalescing.
3499                  */
3500                 tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3501                              0x00000001);
3502                 if (tg3_irq_sync(tp))
3503                         goto out;
3504                 sblk->status &= ~SD_STATUS_UPDATED;
3505                 if (likely(tg3_has_work(tp))) {
3506                         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3507                         netif_rx_schedule(dev);         /* schedule NAPI poll */
3508                 } else {
3509                         /* No work, shared interrupt perhaps?  re-enable
3510                          * interrupts, and flush that PCI write
3511                          */
3512                         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3513                                 0x00000000);
3514                 }
3515         } else {        /* shared interrupt */
3516                 handled = 0;
3517         }
3518 out:
3519         return IRQ_RETVAL(handled);
3520 }
3521
3522 static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id, struct pt_regs *regs)
3523 {
3524         struct net_device *dev = dev_id;
3525         struct tg3 *tp = netdev_priv(dev);
3526         struct tg3_hw_status *sblk = tp->hw_status;
3527         unsigned int handled = 1;
3528
3529         /* In INTx mode, it is possible for the interrupt to arrive at
3530          * the CPU before the status block posted prior to the interrupt.
3531          * Reading the PCI State register will confirm whether the
3532          * interrupt is ours and will flush the status block.
3533          */
3534         if ((sblk->status_tag != tp->last_tag) ||
3535             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3536                 /*
3537                  * writing any value to intr-mbox-0 clears PCI INTA# and
3538                  * chip-internal interrupt pending events.
3539                  * writing non-zero to intr-mbox-0 additional tells the
3540                  * NIC to stop sending us irqs, engaging "in-intr-handler"
3541                  * event coalescing.
3542                  */
3543                 tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3544                              0x00000001);
3545                 if (tg3_irq_sync(tp))
3546                         goto out;
3547                 if (netif_rx_schedule_prep(dev)) {
3548                         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3549                         /* Update last_tag to mark that this status has been
3550                          * seen. Because interrupt may be shared, we may be
3551                          * racing with tg3_poll(), so only update last_tag
3552                          * if tg3_poll() is not scheduled.
3553                          */
3554                         tp->last_tag = sblk->status_tag;
3555                         __netif_rx_schedule(dev);
3556                 }
3557         } else {        /* shared interrupt */
3558                 handled = 0;
3559         }
3560 out:
3561         return IRQ_RETVAL(handled);
3562 }
3563
3564 /* ISR for interrupt test */
3565 static irqreturn_t tg3_test_isr(int irq, void *dev_id,
3566                 struct pt_regs *regs)
3567 {
3568         struct net_device *dev = dev_id;
3569         struct tg3 *tp = netdev_priv(dev);
3570         struct tg3_hw_status *sblk = tp->hw_status;
3571
3572         if ((sblk->status & SD_STATUS_UPDATED) ||
3573             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3574                 tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3575                              0x00000001);
3576                 return IRQ_RETVAL(1);
3577         }
3578         return IRQ_RETVAL(0);
3579 }
3580
3581 static int tg3_init_hw(struct tg3 *, int);
3582 static int tg3_halt(struct tg3 *, int, int);
3583
3584 #ifdef CONFIG_NET_POLL_CONTROLLER
3585 static void tg3_poll_controller(struct net_device *dev)
3586 {
3587         struct tg3 *tp = netdev_priv(dev);
3588
3589         tg3_interrupt(tp->pdev->irq, dev, NULL);
3590 }
3591 #endif
3592
3593 static void tg3_reset_task(void *_data)
3594 {
3595         struct tg3 *tp = _data;
3596         unsigned int restart_timer;
3597
3598         tg3_full_lock(tp, 0);
3599         tp->tg3_flags |= TG3_FLAG_IN_RESET_TASK;
3600
3601         if (!netif_running(tp->dev)) {
3602                 tp->tg3_flags &= ~TG3_FLAG_IN_RESET_TASK;
3603                 tg3_full_unlock(tp);
3604                 return;
3605         }
3606
3607         tg3_full_unlock(tp);
3608
3609         tg3_netif_stop(tp);
3610
3611         tg3_full_lock(tp, 1);
3612
3613         restart_timer = tp->tg3_flags2 & TG3_FLG2_RESTART_TIMER;
3614         tp->tg3_flags2 &= ~TG3_FLG2_RESTART_TIMER;
3615
3616         if (tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING) {
3617                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
3618                 tp->write32_rx_mbox = tg3_write_flush_reg32;
3619                 tp->tg3_flags |= TG3_FLAG_MBOX_WRITE_REORDER;
3620                 tp->tg3_flags &= ~TG3_FLAG_TX_RECOVERY_PENDING;
3621         }
3622
3623         tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
3624         tg3_init_hw(tp, 1);
3625
3626         tg3_netif_start(tp);
3627
3628         if (restart_timer)
3629                 mod_timer(&tp->timer, jiffies + 1);
3630
3631         tp->tg3_flags &= ~TG3_FLAG_IN_RESET_TASK;
3632
3633         tg3_full_unlock(tp);
3634 }
3635
3636 static void tg3_tx_timeout(struct net_device *dev)
3637 {
3638         struct tg3 *tp = netdev_priv(dev);
3639
3640         printk(KERN_ERR PFX "%s: transmit timed out, resetting\n",
3641                dev->name);
3642
3643         schedule_work(&tp->reset_task);
3644 }
3645
3646 /* Test for DMA buffers crossing any 4GB boundaries: 4G, 8G, etc */
3647 static inline int tg3_4g_overflow_test(dma_addr_t mapping, int len)
3648 {
3649         u32 base = (u32) mapping & 0xffffffff;
3650
3651         return ((base > 0xffffdcc0) &&
3652                 (base + len + 8 < base));
3653 }
3654
3655 /* Test for DMA addresses > 40-bit */
3656 static inline int tg3_40bit_overflow_test(struct tg3 *tp, dma_addr_t mapping,
3657                                           int len)
3658 {
3659 #if defined(CONFIG_HIGHMEM) && (BITS_PER_LONG == 64)
3660         if (tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG)
3661                 return (((u64) mapping + len) > DMA_40BIT_MASK);
3662         return 0;
3663 #else
3664         return 0;
3665 #endif
3666 }
3667
3668 static void tg3_set_txd(struct tg3 *, int, dma_addr_t, int, u32, u32);
3669
3670 /* Workaround 4GB and 40-bit hardware DMA bugs. */
3671 static int tigon3_dma_hwbug_workaround(struct tg3 *tp, struct sk_buff *skb,
3672                                        u32 last_plus_one, u32 *start,
3673                                        u32 base_flags, u32 mss)
3674 {
3675         struct sk_buff *new_skb = skb_copy(skb, GFP_ATOMIC);
3676         dma_addr_t new_addr = 0;
3677         u32 entry = *start;
3678         int i, ret = 0;
3679
3680         if (!new_skb) {
3681                 ret = -1;
3682         } else {
3683                 /* New SKB is guaranteed to be linear. */
3684                 entry = *start;
3685                 new_addr = pci_map_single(tp->pdev, new_skb->data, new_skb->len,
3686                                           PCI_DMA_TODEVICE);
3687                 /* Make sure new skb does not cross any 4G boundaries.
3688                  * Drop the packet if it does.
3689                  */
3690                 if (tg3_4g_overflow_test(new_addr, new_skb->len)) {
3691                         ret = -1;
3692                         dev_kfree_skb(new_skb);
3693                         new_skb = NULL;
3694                 } else {
3695                         tg3_set_txd(tp, entry, new_addr, new_skb->len,
3696                                     base_flags, 1 | (mss << 1));
3697                         *start = NEXT_TX(entry);
3698                 }
3699         }
3700
3701         /* Now clean up the sw ring entries. */
3702         i = 0;
3703         while (entry != last_plus_one) {
3704                 int len;
3705
3706                 if (i == 0)
3707                         len = skb_headlen(skb);
3708                 else
3709                         len = skb_shinfo(skb)->frags[i-1].size;
3710                 pci_unmap_single(tp->pdev,
3711                                  pci_unmap_addr(&tp->tx_buffers[entry], mapping),
3712                                  len, PCI_DMA_TODEVICE);
3713                 if (i == 0) {
3714                         tp->tx_buffers[entry].skb = new_skb;
3715                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, new_addr);
3716                 } else {
3717                         tp->tx_buffers[entry].skb = NULL;
3718                 }
3719                 entry = NEXT_TX(entry);
3720                 i++;
3721         }
3722
3723         dev_kfree_skb(skb);
3724
3725         return ret;
3726 }
3727
3728 static void tg3_set_txd(struct tg3 *tp, int entry,
3729                         dma_addr_t mapping, int len, u32 flags,
3730                         u32 mss_and_is_end)
3731 {
3732         struct tg3_tx_buffer_desc *txd = &tp->tx_ring[entry];
3733         int is_end = (mss_and_is_end & 0x1);
3734         u32 mss = (mss_and_is_end >> 1);
3735         u32 vlan_tag = 0;
3736
3737         if (is_end)
3738                 flags |= TXD_FLAG_END;
3739         if (flags & TXD_FLAG_VLAN) {
3740                 vlan_tag = flags >> 16;
3741                 flags &= 0xffff;
3742         }
3743         vlan_tag |= (mss << TXD_MSS_SHIFT);
3744
3745         txd->addr_hi = ((u64) mapping >> 32);
3746         txd->addr_lo = ((u64) mapping & 0xffffffff);
3747         txd->len_flags = (len << TXD_LEN_SHIFT) | flags;
3748         txd->vlan_tag = vlan_tag << TXD_VLAN_TAG_SHIFT;
3749 }
3750
3751 /* hard_start_xmit for devices that don't have any bugs and
3752  * support TG3_FLG2_HW_TSO_2 only.
3753  */
3754 static int tg3_start_xmit(struct sk_buff *skb, struct net_device *dev)
3755 {
3756         struct tg3 *tp = netdev_priv(dev);
3757         dma_addr_t mapping;
3758         u32 len, entry, base_flags, mss;
3759
3760         len = skb_headlen(skb);
3761
3762         /* We are running in BH disabled context with netif_tx_lock
3763          * and TX reclaim runs via tp->poll inside of a software
3764          * interrupt.  Furthermore, IRQ processing runs lockless so we have
3765          * no IRQ context deadlocks to worry about either.  Rejoice!
3766          */
3767         if (unlikely(TX_BUFFS_AVAIL(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
3768                 if (!netif_queue_stopped(dev)) {
3769                         netif_stop_queue(dev);
3770
3771                         /* This is a hard error, log it. */
3772                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
3773                                "queue awake!\n", dev->name);
3774                 }
3775                 return NETDEV_TX_BUSY;
3776         }
3777
3778         entry = tp->tx_prod;
3779         base_flags = 0;
3780 #if TG3_TSO_SUPPORT != 0
3781         mss = 0;
3782         if (skb->len > (tp->dev->mtu + ETH_HLEN) &&
3783             (mss = skb_shinfo(skb)->gso_size) != 0) {
3784                 int tcp_opt_len, ip_tcp_len;
3785
3786                 if (skb_header_cloned(skb) &&
3787                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
3788                         dev_kfree_skb(skb);
3789                         goto out_unlock;
3790                 }
3791
3792                 tcp_opt_len = ((skb->h.th->doff - 5) * 4);
3793                 ip_tcp_len = (skb->nh.iph->ihl * 4) + sizeof(struct tcphdr);
3794
3795                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
3796                                TXD_FLAG_CPU_POST_DMA);
3797
3798                 skb->nh.iph->check = 0;
3799                 skb->nh.iph->tot_len = htons(mss + ip_tcp_len + tcp_opt_len);
3800
3801                 skb->h.th->check = 0;
3802
3803                 mss |= (ip_tcp_len + tcp_opt_len) << 9;
3804         }
3805         else if (skb->ip_summed == CHECKSUM_HW)
3806                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
3807 #else
3808         mss = 0;
3809         if (skb->ip_summed == CHECKSUM_HW)
3810                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
3811 #endif
3812 #if TG3_VLAN_TAG_USED
3813         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
3814                 base_flags |= (TXD_FLAG_VLAN |
3815                                (vlan_tx_tag_get(skb) << 16));
3816 #endif
3817
3818         /* Queue skb data, a.k.a. the main skb fragment. */
3819         mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
3820
3821         tp->tx_buffers[entry].skb = skb;
3822         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
3823
3824         tg3_set_txd(tp, entry, mapping, len, base_flags,
3825                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
3826
3827         entry = NEXT_TX(entry);
3828
3829         /* Now loop through additional data fragments, and queue them. */
3830         if (skb_shinfo(skb)->nr_frags > 0) {
3831                 unsigned int i, last;
3832
3833                 last = skb_shinfo(skb)->nr_frags - 1;
3834                 for (i = 0; i <= last; i++) {
3835                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3836
3837                         len = frag->size;
3838                         mapping = pci_map_page(tp->pdev,
3839                                                frag->page,
3840                                                frag->page_offset,
3841                                                len, PCI_DMA_TODEVICE);
3842
3843                         tp->tx_buffers[entry].skb = NULL;
3844                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
3845
3846                         tg3_set_txd(tp, entry, mapping, len,
3847                                     base_flags, (i == last) | (mss << 1));
3848
3849                         entry = NEXT_TX(entry);
3850                 }
3851         }
3852
3853         /* Packets are ready, update Tx producer idx local and on card. */
3854         tw32_tx_mbox((MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW), entry);
3855
3856         tp->tx_prod = entry;
3857         if (unlikely(TX_BUFFS_AVAIL(tp) <= (MAX_SKB_FRAGS + 1))) {
3858                 spin_lock(&tp->tx_lock);
3859                 netif_stop_queue(dev);
3860                 if (TX_BUFFS_AVAIL(tp) > TG3_TX_WAKEUP_THRESH)
3861                         netif_wake_queue(tp->dev);
3862                 spin_unlock(&tp->tx_lock);
3863         }
3864
3865 out_unlock:
3866         mmiowb();
3867
3868         dev->trans_start = jiffies;
3869
3870         return NETDEV_TX_OK;
3871 }
3872
3873 /* hard_start_xmit for devices that have the 4G bug and/or 40-bit bug and
3874  * support TG3_FLG2_HW_TSO_1 or firmware TSO only.
3875  */
3876 static int tg3_start_xmit_dma_bug(struct sk_buff *skb, struct net_device *dev)
3877 {
3878         struct tg3 *tp = netdev_priv(dev);
3879         dma_addr_t mapping;
3880         u32 len, entry, base_flags, mss;
3881         int would_hit_hwbug;
3882
3883         len = skb_headlen(skb);
3884
3885         /* We are running in BH disabled context with netif_tx_lock
3886          * and TX reclaim runs via tp->poll inside of a software
3887          * interrupt.  Furthermore, IRQ processing runs lockless so we have
3888          * no IRQ context deadlocks to worry about either.  Rejoice!
3889          */
3890         if (unlikely(TX_BUFFS_AVAIL(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
3891                 if (!netif_queue_stopped(dev)) {
3892                         netif_stop_queue(dev);
3893
3894                         /* This is a hard error, log it. */
3895                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
3896                                "queue awake!\n", dev->name);
3897                 }
3898                 return NETDEV_TX_BUSY;
3899         }
3900
3901         entry = tp->tx_prod;
3902         base_flags = 0;
3903         if (skb->ip_summed == CHECKSUM_HW)
3904                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
3905 #if TG3_TSO_SUPPORT != 0
3906         mss = 0;
3907         if (skb->len > (tp->dev->mtu + ETH_HLEN) &&
3908             (mss = skb_shinfo(skb)->gso_size) != 0) {
3909                 int tcp_opt_len, ip_tcp_len;
3910
3911                 if (skb_header_cloned(skb) &&
3912                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
3913                         dev_kfree_skb(skb);
3914                         goto out_unlock;
3915                 }
3916
3917                 tcp_opt_len = ((skb->h.th->doff - 5) * 4);
3918                 ip_tcp_len = (skb->nh.iph->ihl * 4) + sizeof(struct tcphdr);
3919
3920                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
3921                                TXD_FLAG_CPU_POST_DMA);
3922
3923                 skb->nh.iph->check = 0;
3924                 skb->nh.iph->tot_len = htons(mss + ip_tcp_len + tcp_opt_len);
3925                 if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
3926                         skb->h.th->check = 0;
3927                         base_flags &= ~TXD_FLAG_TCPUDP_CSUM;
3928                 }
3929                 else {
3930                         skb->h.th->check =
3931                                 ~csum_tcpudp_magic(skb->nh.iph->saddr,
3932                                                    skb->nh.iph->daddr,
3933                                                    0, IPPROTO_TCP, 0);
3934                 }
3935
3936                 if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO) ||
3937                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705)) {
3938                         if (tcp_opt_len || skb->nh.iph->ihl > 5) {
3939                                 int tsflags;
3940
3941                                 tsflags = ((skb->nh.iph->ihl - 5) +
3942                                            (tcp_opt_len >> 2));
3943                                 mss |= (tsflags << 11);
3944                         }
3945                 } else {
3946                         if (tcp_opt_len || skb->nh.iph->ihl > 5) {
3947                                 int tsflags;
3948
3949                                 tsflags = ((skb->nh.iph->ihl - 5) +
3950                                            (tcp_opt_len >> 2));
3951                                 base_flags |= tsflags << 12;
3952                         }
3953                 }
3954         }
3955 #else
3956         mss = 0;
3957 #endif
3958 #if TG3_VLAN_TAG_USED
3959         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
3960                 base_flags |= (TXD_FLAG_VLAN |
3961                                (vlan_tx_tag_get(skb) << 16));
3962 #endif
3963
3964         /* Queue skb data, a.k.a. the main skb fragment. */
3965         mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
3966
3967         tp->tx_buffers[entry].skb = skb;
3968         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
3969
3970         would_hit_hwbug = 0;
3971
3972         if (tg3_4g_overflow_test(mapping, len))
3973                 would_hit_hwbug = 1;
3974
3975         tg3_set_txd(tp, entry, mapping, len, base_flags,
3976                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
3977
3978         entry = NEXT_TX(entry);
3979
3980         /* Now loop through additional data fragments, and queue them. */
3981         if (skb_shinfo(skb)->nr_frags > 0) {
3982                 unsigned int i, last;
3983
3984                 last = skb_shinfo(skb)->nr_frags - 1;
3985                 for (i = 0; i <= last; i++) {
3986                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3987
3988                         len = frag->size;
3989                         mapping = pci_map_page(tp->pdev,
3990                                                frag->page,
3991                                                frag->page_offset,
3992                                                len, PCI_DMA_TODEVICE);
3993
3994                         tp->tx_buffers[entry].skb = NULL;
3995                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
3996
3997                         if (tg3_4g_overflow_test(mapping, len))
3998                                 would_hit_hwbug = 1;
3999
4000                         if (tg3_40bit_overflow_test(tp, mapping, len))
4001                                 would_hit_hwbug = 1;
4002
4003                         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
4004                                 tg3_set_txd(tp, entry, mapping, len,
4005                                             base_flags, (i == last)|(mss << 1));
4006                         else
4007                                 tg3_set_txd(tp, entry, mapping, len,
4008                                             base_flags, (i == last));
4009
4010                         entry = NEXT_TX(entry);
4011                 }
4012         }
4013
4014         if (would_hit_hwbug) {
4015                 u32 last_plus_one = entry;
4016                 u32 start;
4017
4018                 start = entry - 1 - skb_shinfo(skb)->nr_frags;
4019                 start &= (TG3_TX_RING_SIZE - 1);
4020
4021                 /* If the workaround fails due to memory/mapping
4022                  * failure, silently drop this packet.
4023                  */
4024                 if (tigon3_dma_hwbug_workaround(tp, skb, last_plus_one,
4025                                                 &start, base_flags, mss))
4026                         goto out_unlock;
4027
4028                 entry = start;
4029         }
4030
4031         /* Packets are ready, update Tx producer idx local and on card. */
4032         tw32_tx_mbox((MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW), entry);
4033
4034         tp->tx_prod = entry;
4035         if (unlikely(TX_BUFFS_AVAIL(tp) <= (MAX_SKB_FRAGS + 1))) {
4036                 spin_lock(&tp->tx_lock);
4037                 netif_stop_queue(dev);
4038                 if (TX_BUFFS_AVAIL(tp) > TG3_TX_WAKEUP_THRESH)
4039                         netif_wake_queue(tp->dev);
4040                 spin_unlock(&tp->tx_lock);
4041         }
4042
4043 out_unlock:
4044         mmiowb();
4045
4046         dev->trans_start = jiffies;
4047
4048         return NETDEV_TX_OK;
4049 }
4050
4051 static inline void tg3_set_mtu(struct net_device *dev, struct tg3 *tp,
4052                                int new_mtu)
4053 {
4054         dev->mtu = new_mtu;
4055
4056         if (new_mtu > ETH_DATA_LEN) {
4057                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
4058                         tp->tg3_flags2 &= ~TG3_FLG2_TSO_CAPABLE;
4059                         ethtool_op_set_tso(dev, 0);
4060                 }
4061                 else
4062                         tp->tg3_flags |= TG3_FLAG_JUMBO_RING_ENABLE;
4063         } else {
4064                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
4065                         tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
4066                 tp->tg3_flags &= ~TG3_FLAG_JUMBO_RING_ENABLE;
4067         }
4068 }
4069
4070 static int tg3_change_mtu(struct net_device *dev, int new_mtu)
4071 {
4072         struct tg3 *tp = netdev_priv(dev);
4073
4074         if (new_mtu < TG3_MIN_MTU || new_mtu > TG3_MAX_MTU(tp))
4075                 return -EINVAL;
4076
4077         if (!netif_running(dev)) {
4078                 /* We'll just catch it later when the
4079                  * device is up'd.
4080                  */
4081                 tg3_set_mtu(dev, tp, new_mtu);
4082                 return 0;
4083         }
4084
4085         tg3_netif_stop(tp);
4086
4087         tg3_full_lock(tp, 1);
4088
4089         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
4090
4091         tg3_set_mtu(dev, tp, new_mtu);
4092
4093         tg3_init_hw(tp, 0);
4094
4095         tg3_netif_start(tp);
4096
4097         tg3_full_unlock(tp);
4098
4099         return 0;
4100 }
4101
4102 /* Free up pending packets in all rx/tx rings.
4103  *
4104  * The chip has been shut down and the driver detached from
4105  * the networking, so no interrupts or new tx packets will
4106  * end up in the driver.  tp->{tx,}lock is not held and we are not
4107  * in an interrupt context and thus may sleep.
4108  */
4109 static void tg3_free_rings(struct tg3 *tp)
4110 {
4111         struct ring_info *rxp;
4112         int i;
4113
4114         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
4115                 rxp = &tp->rx_std_buffers[i];
4116
4117                 if (rxp->skb == NULL)
4118                         continue;
4119                 pci_unmap_single(tp->pdev,
4120                                  pci_unmap_addr(rxp, mapping),
4121                                  tp->rx_pkt_buf_sz - tp->rx_offset,
4122                                  PCI_DMA_FROMDEVICE);
4123                 dev_kfree_skb_any(rxp->skb);
4124                 rxp->skb = NULL;
4125         }
4126
4127         for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
4128                 rxp = &tp->rx_jumbo_buffers[i];
4129
4130                 if (rxp->skb == NULL)
4131                         continue;
4132                 pci_unmap_single(tp->pdev,
4133                                  pci_unmap_addr(rxp, mapping),
4134                                  RX_JUMBO_PKT_BUF_SZ - tp->rx_offset,
4135                                  PCI_DMA_FROMDEVICE);
4136                 dev_kfree_skb_any(rxp->skb);
4137                 rxp->skb = NULL;
4138         }
4139
4140         for (i = 0; i < TG3_TX_RING_SIZE; ) {
4141                 struct tx_ring_info *txp;
4142                 struct sk_buff *skb;
4143                 int j;
4144
4145                 txp = &tp->tx_buffers[i];
4146                 skb = txp->skb;
4147
4148                 if (skb == NULL) {
4149                         i++;
4150                         continue;
4151                 }
4152
4153                 pci_unmap_single(tp->pdev,
4154                                  pci_unmap_addr(txp, mapping),
4155                                  skb_headlen(skb),
4156                                  PCI_DMA_TODEVICE);
4157                 txp->skb = NULL;
4158
4159                 i++;
4160
4161                 for (j = 0; j < skb_shinfo(skb)->nr_frags; j++) {
4162                         txp = &tp->tx_buffers[i & (TG3_TX_RING_SIZE - 1)];
4163                         pci_unmap_page(tp->pdev,
4164                                        pci_unmap_addr(txp, mapping),
4165                                        skb_shinfo(skb)->frags[j].size,
4166                                        PCI_DMA_TODEVICE);
4167                         i++;
4168                 }
4169
4170                 dev_kfree_skb_any(skb);
4171         }
4172 }
4173
4174 /* Initialize tx/rx rings for packet processing.
4175  *
4176  * The chip has been shut down and the driver detached from
4177  * the networking, so no interrupts or new tx packets will
4178  * end up in the driver.  tp->{tx,}lock are held and thus
4179  * we may not sleep.
4180  */
4181 static void tg3_init_rings(struct tg3 *tp)
4182 {
4183         u32 i;
4184
4185         /* Free up all the SKBs. */
4186         tg3_free_rings(tp);
4187
4188         /* Zero out all descriptors. */
4189         memset(tp->rx_std, 0, TG3_RX_RING_BYTES);
4190         memset(tp->rx_jumbo, 0, TG3_RX_JUMBO_RING_BYTES);
4191         memset(tp->rx_rcb, 0, TG3_RX_RCB_RING_BYTES(tp));
4192         memset(tp->tx_ring, 0, TG3_TX_RING_BYTES);
4193
4194         tp->rx_pkt_buf_sz = RX_PKT_BUF_SZ;
4195         if ((tp->tg3_flags2 & TG3_FLG2_5780_CLASS) &&
4196             (tp->dev->mtu > ETH_DATA_LEN))
4197                 tp->rx_pkt_buf_sz = RX_JUMBO_PKT_BUF_SZ;
4198
4199         /* Initialize invariants of the rings, we only set this
4200          * stuff once.  This works because the card does not
4201          * write into the rx buffer posting rings.
4202          */
4203         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
4204                 struct tg3_rx_buffer_desc *rxd;
4205
4206                 rxd = &tp->rx_std[i];
4207                 rxd->idx_len = (tp->rx_pkt_buf_sz - tp->rx_offset - 64)
4208                         << RXD_LEN_SHIFT;
4209                 rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT);
4210                 rxd->opaque = (RXD_OPAQUE_RING_STD |
4211                                (i << RXD_OPAQUE_INDEX_SHIFT));
4212         }
4213
4214         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
4215                 for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
4216                         struct tg3_rx_buffer_desc *rxd;
4217
4218                         rxd = &tp->rx_jumbo[i];
4219                         rxd->idx_len = (RX_JUMBO_PKT_BUF_SZ - tp->rx_offset - 64)
4220                                 << RXD_LEN_SHIFT;
4221                         rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT) |
4222                                 RXD_FLAG_JUMBO;
4223                         rxd->opaque = (RXD_OPAQUE_RING_JUMBO |
4224                                (i << RXD_OPAQUE_INDEX_SHIFT));
4225                 }
4226         }
4227
4228         /* Now allocate fresh SKBs for each rx ring. */
4229         for (i = 0; i < tp->rx_pending; i++) {
4230                 if (tg3_alloc_rx_skb(tp, RXD_OPAQUE_RING_STD,
4231                                      -1, i) < 0)
4232                         break;
4233         }
4234
4235         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
4236                 for (i = 0; i < tp->rx_jumbo_pending; i++) {
4237                         if (tg3_alloc_rx_skb(tp, RXD_OPAQUE_RING_JUMBO,
4238                                              -1, i) < 0)
4239                                 break;
4240                 }
4241         }
4242 }
4243
4244 /*
4245  * Must not be invoked with interrupt sources disabled and
4246  * the hardware shutdown down.
4247  */
4248 static void tg3_free_consistent(struct tg3 *tp)
4249 {
4250         kfree(tp->rx_std_buffers);
4251         tp->rx_std_buffers = NULL;
4252         if (tp->rx_std) {
4253                 pci_free_consistent(tp->pdev, TG3_RX_RING_BYTES,
4254                                     tp->rx_std, tp->rx_std_mapping);
4255                 tp->rx_std = NULL;
4256         }
4257         if (tp->rx_jumbo) {
4258                 pci_free_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
4259                                     tp->rx_jumbo, tp->rx_jumbo_mapping);
4260                 tp->rx_jumbo = NULL;
4261         }
4262         if (tp->rx_rcb) {
4263                 pci_free_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
4264                                     tp->rx_rcb, tp->rx_rcb_mapping);
4265                 tp->rx_rcb = NULL;
4266         }
4267         if (tp->tx_ring) {
4268                 pci_free_consistent(tp->pdev, TG3_TX_RING_BYTES,
4269                         tp->tx_ring, tp->tx_desc_mapping);
4270                 tp->tx_ring = NULL;
4271         }
4272         if (tp->hw_status) {
4273                 pci_free_consistent(tp->pdev, TG3_HW_STATUS_SIZE,
4274                                     tp->hw_status, tp->status_mapping);
4275                 tp->hw_status = NULL;
4276         }
4277         if (tp->hw_stats) {
4278                 pci_free_consistent(tp->pdev, sizeof(struct tg3_hw_stats),
4279                                     tp->hw_stats, tp->stats_mapping);
4280                 tp->hw_stats = NULL;
4281         }
4282 }
4283
4284 /*
4285  * Must not be invoked with interrupt sources disabled and
4286  * the hardware shutdown down.  Can sleep.
4287  */
4288 static int tg3_alloc_consistent(struct tg3 *tp)
4289 {
4290         tp->rx_std_buffers = kmalloc((sizeof(struct ring_info) *
4291                                       (TG3_RX_RING_SIZE +
4292                                        TG3_RX_JUMBO_RING_SIZE)) +
4293                                      (sizeof(struct tx_ring_info) *
4294                                       TG3_TX_RING_SIZE),
4295                                      GFP_KERNEL);
4296         if (!tp->rx_std_buffers)
4297                 return -ENOMEM;
4298
4299         memset(tp->rx_std_buffers, 0,
4300                (sizeof(struct ring_info) *
4301                 (TG3_RX_RING_SIZE +
4302                  TG3_RX_JUMBO_RING_SIZE)) +
4303                (sizeof(struct tx_ring_info) *
4304                 TG3_TX_RING_SIZE));
4305
4306         tp->rx_jumbo_buffers = &tp->rx_std_buffers[TG3_RX_RING_SIZE];
4307         tp->tx_buffers = (struct tx_ring_info *)
4308                 &tp->rx_jumbo_buffers[TG3_RX_JUMBO_RING_SIZE];
4309
4310         tp->rx_std = pci_alloc_consistent(tp->pdev, TG3_RX_RING_BYTES,
4311                                           &tp->rx_std_mapping);
4312         if (!tp->rx_std)
4313                 goto err_out;
4314
4315         tp->rx_jumbo = pci_alloc_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
4316                                             &tp->rx_jumbo_mapping);
4317
4318         if (!tp->rx_jumbo)
4319                 goto err_out;
4320
4321         tp->rx_rcb = pci_alloc_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
4322                                           &tp->rx_rcb_mapping);
4323         if (!tp->rx_rcb)
4324                 goto err_out;
4325
4326         tp->tx_ring = pci_alloc_consistent(tp->pdev, TG3_TX_RING_BYTES,
4327                                            &tp->tx_desc_mapping);
4328         if (!tp->tx_ring)
4329                 goto err_out;
4330
4331         tp->hw_status = pci_alloc_consistent(tp->pdev,
4332                                              TG3_HW_STATUS_SIZE,
4333                                              &tp->status_mapping);
4334         if (!tp->hw_status)
4335                 goto err_out;
4336
4337         tp->hw_stats = pci_alloc_consistent(tp->pdev,
4338                                             sizeof(struct tg3_hw_stats),
4339                                             &tp->stats_mapping);
4340         if (!tp->hw_stats)
4341                 goto err_out;
4342
4343         memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
4344         memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
4345
4346         return 0;
4347
4348 err_out:
4349         tg3_free_consistent(tp);
4350         return -ENOMEM;
4351 }
4352
4353 #define MAX_WAIT_CNT 1000
4354
4355 /* To stop a block, clear the enable bit and poll till it
4356  * clears.  tp->lock is held.
4357  */
4358 static int tg3_stop_block(struct tg3 *tp, unsigned long ofs, u32 enable_bit, int silent)
4359 {
4360         unsigned int i;
4361         u32 val;
4362
4363         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
4364                 switch (ofs) {
4365                 case RCVLSC_MODE:
4366                 case DMAC_MODE:
4367                 case MBFREE_MODE:
4368                 case BUFMGR_MODE:
4369                 case MEMARB_MODE:
4370                         /* We can't enable/disable these bits of the
4371                          * 5705/5750, just say success.
4372                          */
4373                         return 0;
4374
4375                 default:
4376                         break;
4377                 };
4378         }
4379
4380         val = tr32(ofs);
4381         val &= ~enable_bit;
4382         tw32_f(ofs, val);
4383
4384         for (i = 0; i < MAX_WAIT_CNT; i++) {
4385                 udelay(100);
4386                 val = tr32(ofs);
4387                 if ((val & enable_bit) == 0)
4388                         break;
4389         }
4390
4391         if (i == MAX_WAIT_CNT && !silent) {
4392                 printk(KERN_ERR PFX "tg3_stop_block timed out, "
4393                        "ofs=%lx enable_bit=%x\n",
4394                        ofs, enable_bit);
4395                 return -ENODEV;
4396         }
4397
4398         return 0;
4399 }
4400
4401 /* tp->lock is held. */
4402 static int tg3_abort_hw(struct tg3 *tp, int silent)
4403 {
4404         int i, err;
4405
4406         tg3_disable_ints(tp);
4407
4408         tp->rx_mode &= ~RX_MODE_ENABLE;
4409         tw32_f(MAC_RX_MODE, tp->rx_mode);
4410         udelay(10);
4411
4412         err  = tg3_stop_block(tp, RCVBDI_MODE, RCVBDI_MODE_ENABLE, silent);
4413         err |= tg3_stop_block(tp, RCVLPC_MODE, RCVLPC_MODE_ENABLE, silent);
4414         err |= tg3_stop_block(tp, RCVLSC_MODE, RCVLSC_MODE_ENABLE, silent);
4415         err |= tg3_stop_block(tp, RCVDBDI_MODE, RCVDBDI_MODE_ENABLE, silent);
4416         err |= tg3_stop_block(tp, RCVDCC_MODE, RCVDCC_MODE_ENABLE, silent);
4417         err |= tg3_stop_block(tp, RCVCC_MODE, RCVCC_MODE_ENABLE, silent);
4418
4419         err |= tg3_stop_block(tp, SNDBDS_MODE, SNDBDS_MODE_ENABLE, silent);
4420         err |= tg3_stop_block(tp, SNDBDI_MODE, SNDBDI_MODE_ENABLE, silent);
4421         err |= tg3_stop_block(tp, SNDDATAI_MODE, SNDDATAI_MODE_ENABLE, silent);
4422         err |= tg3_stop_block(tp, RDMAC_MODE, RDMAC_MODE_ENABLE, silent);
4423         err |= tg3_stop_block(tp, SNDDATAC_MODE, SNDDATAC_MODE_ENABLE, silent);
4424         err |= tg3_stop_block(tp, DMAC_MODE, DMAC_MODE_ENABLE, silent);
4425         err |= tg3_stop_block(tp, SNDBDC_MODE, SNDBDC_MODE_ENABLE, silent);
4426
4427         tp->mac_mode &= ~MAC_MODE_TDE_ENABLE;
4428         tw32_f(MAC_MODE, tp->mac_mode);
4429         udelay(40);
4430
4431         tp->tx_mode &= ~TX_MODE_ENABLE;
4432         tw32_f(MAC_TX_MODE, tp->tx_mode);
4433
4434         for (i = 0; i < MAX_WAIT_CNT; i++) {
4435                 udelay(100);
4436                 if (!(tr32(MAC_TX_MODE) & TX_MODE_ENABLE))
4437                         break;
4438         }
4439         if (i >= MAX_WAIT_CNT) {
4440                 printk(KERN_ERR PFX "tg3_abort_hw timed out for %s, "
4441                        "TX_MODE_ENABLE will not clear MAC_TX_MODE=%08x\n",
4442                        tp->dev->name, tr32(MAC_TX_MODE));
4443                 err |= -ENODEV;
4444         }
4445
4446         err |= tg3_stop_block(tp, HOSTCC_MODE, HOSTCC_MODE_ENABLE, silent);
4447         err |= tg3_stop_block(tp, WDMAC_MODE, WDMAC_MODE_ENABLE, silent);
4448         err |= tg3_stop_block(tp, MBFREE_MODE, MBFREE_MODE_ENABLE, silent);
4449
4450         tw32(FTQ_RESET, 0xffffffff);
4451         tw32(FTQ_RESET, 0x00000000);
4452
4453         err |= tg3_stop_block(tp, BUFMGR_MODE, BUFMGR_MODE_ENABLE, silent);
4454         err |= tg3_stop_block(tp, MEMARB_MODE, MEMARB_MODE_ENABLE, silent);
4455
4456         if (tp->hw_status)
4457                 memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
4458         if (tp->hw_stats)
4459                 memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
4460
4461         return err;
4462 }
4463
4464 /* tp->lock is held. */
4465 static int tg3_nvram_lock(struct tg3 *tp)
4466 {
4467         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
4468                 int i;
4469
4470                 if (tp->nvram_lock_cnt == 0) {
4471                         tw32(NVRAM_SWARB, SWARB_REQ_SET1);
4472                         for (i = 0; i < 8000; i++) {
4473                                 if (tr32(NVRAM_SWARB) & SWARB_GNT1)
4474                                         break;
4475                                 udelay(20);
4476                         }
4477                         if (i == 8000) {
4478                                 tw32(NVRAM_SWARB, SWARB_REQ_CLR1);
4479                                 return -ENODEV;
4480                         }
4481                 }
4482                 tp->nvram_lock_cnt++;
4483         }
4484         return 0;
4485 }
4486
4487 /* tp->lock is held. */
4488 static void tg3_nvram_unlock(struct tg3 *tp)
4489 {
4490         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
4491                 if (tp->nvram_lock_cnt > 0)
4492                         tp->nvram_lock_cnt--;
4493                 if (tp->nvram_lock_cnt == 0)
4494                         tw32_f(NVRAM_SWARB, SWARB_REQ_CLR1);
4495         }
4496 }
4497
4498 /* tp->lock is held. */
4499 static void tg3_enable_nvram_access(struct tg3 *tp)
4500 {
4501         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
4502             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
4503                 u32 nvaccess = tr32(NVRAM_ACCESS);
4504
4505                 tw32(NVRAM_ACCESS, nvaccess | ACCESS_ENABLE);
4506         }
4507 }
4508
4509 /* tp->lock is held. */
4510 static void tg3_disable_nvram_access(struct tg3 *tp)
4511 {
4512         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
4513             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
4514                 u32 nvaccess = tr32(NVRAM_ACCESS);
4515
4516                 tw32(NVRAM_ACCESS, nvaccess & ~ACCESS_ENABLE);
4517         }
4518 }
4519
4520 /* tp->lock is held. */
4521 static void tg3_write_sig_pre_reset(struct tg3 *tp, int kind)
4522 {
4523         tg3_write_mem(tp, NIC_SRAM_FIRMWARE_MBOX,
4524                       NIC_SRAM_FIRMWARE_MBOX_MAGIC1);
4525
4526         if (tp->tg3_flags2 & TG3_FLG2_ASF_NEW_HANDSHAKE) {
4527                 switch (kind) {
4528                 case RESET_KIND_INIT:
4529                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4530                                       DRV_STATE_START);
4531                         break;
4532
4533                 case RESET_KIND_SHUTDOWN:
4534                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4535                                       DRV_STATE_UNLOAD);
4536                         break;
4537
4538                 case RESET_KIND_SUSPEND:
4539                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4540                                       DRV_STATE_SUSPEND);
4541                         break;
4542
4543                 default:
4544                         break;
4545                 };
4546         }
4547 }
4548
4549 /* tp->lock is held. */
4550 static void tg3_write_sig_post_reset(struct tg3 *tp, int kind)
4551 {
4552         if (tp->tg3_flags2 & TG3_FLG2_ASF_NEW_HANDSHAKE) {
4553                 switch (kind) {
4554                 case RESET_KIND_INIT:
4555                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4556                                       DRV_STATE_START_DONE);
4557                         break;
4558
4559                 case RESET_KIND_SHUTDOWN:
4560                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4561                                       DRV_STATE_UNLOAD_DONE);
4562                         break;
4563
4564                 default:
4565                         break;
4566                 };
4567         }
4568 }
4569
4570 /* tp->lock is held. */
4571 static void tg3_write_sig_legacy(struct tg3 *tp, int kind)
4572 {
4573         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
4574                 switch (kind) {
4575                 case RESET_KIND_INIT:
4576                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4577                                       DRV_STATE_START);
4578                         break;
4579
4580                 case RESET_KIND_SHUTDOWN:
4581                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4582                                       DRV_STATE_UNLOAD);
4583                         break;
4584
4585                 case RESET_KIND_SUSPEND:
4586                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4587                                       DRV_STATE_SUSPEND);
4588                         break;
4589
4590                 default:
4591                         break;
4592                 };
4593         }
4594 }
4595
4596 static void tg3_stop_fw(struct tg3 *);
4597
4598 /* tp->lock is held. */
4599 static int tg3_chip_reset(struct tg3 *tp)
4600 {
4601         u32 val;
4602         void (*write_op)(struct tg3 *, u32, u32);
4603         int i;
4604
4605         tg3_nvram_lock(tp);
4606
4607         /* No matching tg3_nvram_unlock() after this because
4608          * chip reset below will undo the nvram lock.
4609          */
4610         tp->nvram_lock_cnt = 0;
4611
4612         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
4613             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
4614             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
4615                 tw32(GRC_FASTBOOT_PC, 0);
4616
4617         /*
4618          * We must avoid the readl() that normally takes place.
4619          * It locks machines, causes machine checks, and other
4620          * fun things.  So, temporarily disable the 5701
4621          * hardware workaround, while we do the reset.
4622          */
4623         write_op = tp->write32;
4624         if (write_op == tg3_write_flush_reg32)
4625                 tp->write32 = tg3_write32;
4626
4627         /* do the reset */
4628         val = GRC_MISC_CFG_CORECLK_RESET;
4629
4630         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
4631                 if (tr32(0x7e2c) == 0x60) {
4632                         tw32(0x7e2c, 0x20);
4633                 }
4634                 if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0) {
4635                         tw32(GRC_MISC_CFG, (1 << 29));
4636                         val |= (1 << 29);
4637                 }
4638         }
4639
4640         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
4641                 val |= GRC_MISC_CFG_KEEP_GPHY_POWER;
4642         tw32(GRC_MISC_CFG, val);
4643
4644         /* restore 5701 hardware bug workaround write method */
4645         tp->write32 = write_op;
4646
4647         /* Unfortunately, we have to delay before the PCI read back.
4648          * Some 575X chips even will not respond to a PCI cfg access
4649          * when the reset command is given to the chip.
4650          *
4651          * How do these hardware designers expect things to work
4652          * properly if the PCI write is posted for a long period
4653          * of time?  It is always necessary to have some method by
4654          * which a register read back can occur to push the write
4655          * out which does the reset.
4656          *
4657          * For most tg3 variants the trick below was working.
4658          * Ho hum...
4659          */
4660         udelay(120);
4661
4662         /* Flush PCI posted writes.  The normal MMIO registers
4663          * are inaccessible at this time so this is the only
4664          * way to make this reliably (actually, this is no longer
4665          * the case, see above).  I tried to use indirect
4666          * register read/write but this upset some 5701 variants.
4667          */
4668         pci_read_config_dword(tp->pdev, PCI_COMMAND, &val);
4669
4670         udelay(120);
4671
4672         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
4673                 if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A0) {
4674                         int i;
4675                         u32 cfg_val;
4676
4677                         /* Wait for link training to complete.  */
4678                         for (i = 0; i < 5000; i++)
4679                                 udelay(100);
4680
4681                         pci_read_config_dword(tp->pdev, 0xc4, &cfg_val);
4682                         pci_write_config_dword(tp->pdev, 0xc4,
4683                                                cfg_val | (1 << 15));
4684                 }
4685                 /* Set PCIE max payload size and clear error status.  */
4686                 pci_write_config_dword(tp->pdev, 0xd8, 0xf5000);
4687         }
4688
4689         /* Re-enable indirect register accesses. */
4690         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
4691                                tp->misc_host_ctrl);
4692
4693         /* Set MAX PCI retry to zero. */
4694         val = (PCISTATE_ROM_ENABLE | PCISTATE_ROM_RETRY_ENABLE);
4695         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0 &&
4696             (tp->tg3_flags & TG3_FLAG_PCIX_MODE))
4697                 val |= PCISTATE_RETRY_SAME_DMA;
4698         pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, val);
4699
4700         pci_restore_state(tp->pdev);
4701
4702         /* Make sure PCI-X relaxed ordering bit is clear. */
4703         pci_read_config_dword(tp->pdev, TG3PCI_X_CAPS, &val);
4704         val &= ~PCIX_CAPS_RELAXED_ORDERING;
4705         pci_write_config_dword(tp->pdev, TG3PCI_X_CAPS, val);
4706
4707         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
4708                 u32 val;
4709
4710                 /* Chip reset on 5780 will reset MSI enable bit,
4711                  * so need to restore it.
4712                  */
4713                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
4714                         u16 ctrl;
4715
4716                         pci_read_config_word(tp->pdev,
4717                                              tp->msi_cap + PCI_MSI_FLAGS,
4718                                              &ctrl);
4719                         pci_write_config_word(tp->pdev,
4720                                               tp->msi_cap + PCI_MSI_FLAGS,
4721                                               ctrl | PCI_MSI_FLAGS_ENABLE);
4722                         val = tr32(MSGINT_MODE);
4723                         tw32(MSGINT_MODE, val | MSGINT_MODE_ENABLE);
4724                 }
4725
4726                 val = tr32(MEMARB_MODE);
4727                 tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
4728
4729         } else
4730                 tw32(MEMARB_MODE, MEMARB_MODE_ENABLE);
4731
4732         if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A3) {
4733                 tg3_stop_fw(tp);
4734                 tw32(0x5000, 0x400);
4735         }
4736
4737         tw32(GRC_MODE, tp->grc_mode);
4738
4739         if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0) {
4740                 u32 val = tr32(0xc4);
4741
4742                 tw32(0xc4, val | (1 << 15));
4743         }
4744
4745         if ((tp->nic_sram_data_cfg & NIC_SRAM_DATA_CFG_MINI_PCI) != 0 &&
4746             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
4747                 tp->pci_clock_ctrl |= CLOCK_CTRL_CLKRUN_OENABLE;
4748                 if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0)
4749                         tp->pci_clock_ctrl |= CLOCK_CTRL_FORCE_CLKRUN;
4750                 tw32(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
4751         }
4752
4753         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
4754                 tp->mac_mode = MAC_MODE_PORT_MODE_TBI;
4755                 tw32_f(MAC_MODE, tp->mac_mode);
4756         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
4757                 tp->mac_mode = MAC_MODE_PORT_MODE_GMII;
4758                 tw32_f(MAC_MODE, tp->mac_mode);
4759         } else
4760                 tw32_f(MAC_MODE, 0);
4761         udelay(40);
4762
4763         /* Wait for firmware initialization to complete. */
4764         for (i = 0; i < 100000; i++) {
4765                 tg3_read_mem(tp, NIC_SRAM_FIRMWARE_MBOX, &val);
4766                 if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
4767                         break;
4768                 udelay(10);
4769         }
4770
4771         /* Chip might not be fitted with firmare.  Some Sun onboard
4772          * parts are configured like that.  So don't signal the timeout
4773          * of the above loop as an error, but do report the lack of
4774          * running firmware once.
4775          */
4776         if (i >= 100000 &&
4777             !(tp->tg3_flags2 & TG3_FLG2_NO_FWARE_REPORTED)) {
4778                 tp->tg3_flags2 |= TG3_FLG2_NO_FWARE_REPORTED;
4779
4780                 printk(KERN_INFO PFX "%s: No firmware running.\n",
4781                        tp->dev->name);
4782         }
4783
4784         if ((tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) &&
4785             tp->pci_chip_rev_id != CHIPREV_ID_5750_A0) {
4786                 u32 val = tr32(0x7c00);
4787
4788                 tw32(0x7c00, val | (1 << 25));
4789         }
4790
4791         /* Reprobe ASF enable state.  */
4792         tp->tg3_flags &= ~TG3_FLAG_ENABLE_ASF;
4793         tp->tg3_flags2 &= ~TG3_FLG2_ASF_NEW_HANDSHAKE;
4794         tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
4795         if (val == NIC_SRAM_DATA_SIG_MAGIC) {
4796                 u32 nic_cfg;
4797
4798                 tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
4799                 if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
4800                         tp->tg3_flags |= TG3_FLAG_ENABLE_ASF;
4801                         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
4802                                 tp->tg3_flags2 |= TG3_FLG2_ASF_NEW_HANDSHAKE;
4803                 }
4804         }
4805
4806         return 0;
4807 }
4808
4809 /* tp->lock is held. */
4810 static void tg3_stop_fw(struct tg3 *tp)
4811 {
4812         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
4813                 u32 val;
4814                 int i;
4815
4816                 tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_PAUSE_FW);
4817                 val = tr32(GRC_RX_CPU_EVENT);
4818                 val |= (1 << 14);
4819                 tw32(GRC_RX_CPU_EVENT, val);
4820
4821                 /* Wait for RX cpu to ACK the event.  */
4822                 for (i = 0; i < 100; i++) {
4823                         if (!(tr32(GRC_RX_CPU_EVENT) & (1 << 14)))
4824                                 break;
4825                         udelay(1);
4826                 }
4827         }
4828 }
4829
4830 /* tp->lock is held. */
4831 static int tg3_halt(struct tg3 *tp, int kind, int silent)
4832 {
4833         int err;
4834
4835         tg3_stop_fw(tp);
4836
4837         tg3_write_sig_pre_reset(tp, kind);
4838
4839         tg3_abort_hw(tp, silent);
4840         err = tg3_chip_reset(tp);
4841
4842         tg3_write_sig_legacy(tp, kind);
4843         tg3_write_sig_post_reset(tp, kind);
4844
4845         if (err)
4846                 return err;
4847
4848         return 0;
4849 }
4850
4851 #define TG3_FW_RELEASE_MAJOR    0x0
4852 #define TG3_FW_RELASE_MINOR     0x0
4853 #define TG3_FW_RELEASE_FIX      0x0
4854 #define TG3_FW_START_ADDR       0x08000000
4855 #define TG3_FW_TEXT_ADDR        0x08000000
4856 #define TG3_FW_TEXT_LEN         0x9c0
4857 #define TG3_FW_RODATA_ADDR      0x080009c0
4858 #define TG3_FW_RODATA_LEN       0x60
4859 #define TG3_FW_DATA_ADDR        0x08000a40
4860 #define TG3_FW_DATA_LEN         0x20
4861 #define TG3_FW_SBSS_ADDR        0x08000a60
4862 #define TG3_FW_SBSS_LEN         0xc
4863 #define TG3_FW_BSS_ADDR         0x08000a70
4864 #define TG3_FW_BSS_LEN          0x10
4865
4866 static u32 tg3FwText[(TG3_FW_TEXT_LEN / sizeof(u32)) + 1] = {
4867         0x00000000, 0x10000003, 0x00000000, 0x0000000d, 0x0000000d, 0x3c1d0800,
4868         0x37bd3ffc, 0x03a0f021, 0x3c100800, 0x26100000, 0x0e000018, 0x00000000,
4869         0x0000000d, 0x3c1d0800, 0x37bd3ffc, 0x03a0f021, 0x3c100800, 0x26100034,
4870         0x0e00021c, 0x00000000, 0x0000000d, 0x00000000, 0x00000000, 0x00000000,
4871         0x27bdffe0, 0x3c1cc000, 0xafbf0018, 0xaf80680c, 0x0e00004c, 0x241b2105,
4872         0x97850000, 0x97870002, 0x9782002c, 0x9783002e, 0x3c040800, 0x248409c0,
4873         0xafa00014, 0x00021400, 0x00621825, 0x00052c00, 0xafa30010, 0x8f860010,
4874         0x00e52825, 0x0e000060, 0x24070102, 0x3c02ac00, 0x34420100, 0x3c03ac01,
4875         0x34630100, 0xaf820490, 0x3c02ffff, 0xaf820494, 0xaf830498, 0xaf82049c,
4876         0x24020001, 0xaf825ce0, 0x0e00003f, 0xaf825d00, 0x0e000140, 0x00000000,
4877         0x8fbf0018, 0x03e00008, 0x27bd0020, 0x2402ffff, 0xaf825404, 0x8f835400,
4878         0x34630400, 0xaf835400, 0xaf825404, 0x3c020800, 0x24420034, 0xaf82541c,
4879         0x03e00008, 0xaf805400, 0x00000000, 0x00000000, 0x3c020800, 0x34423000,
4880         0x3c030800, 0x34633000, 0x3c040800, 0x348437ff, 0x3c010800, 0xac220a64,
4881         0x24020040, 0x3c010800, 0xac220a68, 0x3c010800, 0xac200a60, 0xac600000,
4882         0x24630004, 0x0083102b, 0x5040fffd, 0xac600000, 0x03e00008, 0x00000000,
4883         0x00804821, 0x8faa0010, 0x3c020800, 0x8c420a60, 0x3c040800, 0x8c840a68,
4884         0x8fab0014, 0x24430001, 0x0044102b, 0x3c010800, 0xac230a60, 0x14400003,
4885         0x00004021, 0x3c010800, 0xac200a60, 0x3c020800, 0x8c420a60, 0x3c030800,
4886         0x8c630a64, 0x91240000, 0x00021140, 0x00431021, 0x00481021, 0x25080001,
4887         0xa0440000, 0x29020008, 0x1440fff4, 0x25290001, 0x3c020800, 0x8c420a60,
4888         0x3c030800, 0x8c630a64, 0x8f84680c, 0x00021140, 0x00431021, 0xac440008,
4889         0xac45000c, 0xac460010, 0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c,
4890         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4891         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4892         0, 0, 0, 0, 0, 0,
4893         0x02000008, 0x00000000, 0x0a0001e3, 0x3c0a0001, 0x0a0001e3, 0x3c0a0002,
4894         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
4895         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
4896         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
4897         0x0a0001e3, 0x3c0a0007, 0x0a0001e3, 0x3c0a0008, 0x0a0001e3, 0x3c0a0009,
4898         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a000b,
4899         0x0a0001e3, 0x3c0a000c, 0x0a0001e3, 0x3c0a000d, 0x0a0001e3, 0x00000000,
4900         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a000e, 0x0a0001e3, 0x00000000,
4901         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
4902         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
4903         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a0013, 0x0a0001e3, 0x3c0a0014,
4904         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4905         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4906         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4907         0x27bdffe0, 0x00001821, 0x00001021, 0xafbf0018, 0xafb10014, 0xafb00010,
4908         0x3c010800, 0x00220821, 0xac200a70, 0x3c010800, 0x00220821, 0xac200a74,
4909         0x3c010800, 0x00220821, 0xac200a78, 0x24630001, 0x1860fff5, 0x2442000c,
4910         0x24110001, 0x8f906810, 0x32020004, 0x14400005, 0x24040001, 0x3c020800,
4911         0x8c420a78, 0x18400003, 0x00002021, 0x0e000182, 0x00000000, 0x32020001,
4912         0x10400003, 0x00000000, 0x0e000169, 0x00000000, 0x0a000153, 0xaf915028,
4913         0x8fbf0018, 0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0020, 0x3c050800,
4914         0x8ca50a70, 0x3c060800, 0x8cc60a80, 0x3c070800, 0x8ce70a78, 0x27bdffe0,
4915         0x3c040800, 0x248409d0, 0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014,
4916         0x0e00017b, 0x00002021, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x24020001,
4917         0x8f836810, 0x00821004, 0x00021027, 0x00621824, 0x03e00008, 0xaf836810,
4918         0x27bdffd8, 0xafbf0024, 0x1080002e, 0xafb00020, 0x8f825cec, 0xafa20018,
4919         0x8f825cec, 0x3c100800, 0x26100a78, 0xafa2001c, 0x34028000, 0xaf825cec,
4920         0x8e020000, 0x18400016, 0x00000000, 0x3c020800, 0x94420a74, 0x8fa3001c,
4921         0x000221c0, 0xac830004, 0x8fa2001c, 0x3c010800, 0x0e000201, 0xac220a74,
4922         0x10400005, 0x00000000, 0x8e020000, 0x24420001, 0x0a0001df, 0xae020000,
4923         0x3c020800, 0x8c420a70, 0x00021c02, 0x000321c0, 0x0a0001c5, 0xafa2001c,
4924         0x0e000201, 0x00000000, 0x1040001f, 0x00000000, 0x8e020000, 0x8fa3001c,
4925         0x24420001, 0x3c010800, 0xac230a70, 0x3c010800, 0xac230a74, 0x0a0001df,
4926         0xae020000, 0x3c100800, 0x26100a78, 0x8e020000, 0x18400028, 0x00000000,
4927         0x0e000201, 0x00000000, 0x14400024, 0x00000000, 0x8e020000, 0x3c030800,
4928         0x8c630a70, 0x2442ffff, 0xafa3001c, 0x18400006, 0xae020000, 0x00031402,
4929         0x000221c0, 0x8c820004, 0x3c010800, 0xac220a70, 0x97a2001e, 0x2442ff00,
4930         0x2c420300, 0x1440000b, 0x24024000, 0x3c040800, 0x248409dc, 0xafa00010,
4931         0xafa00014, 0x8fa6001c, 0x24050008, 0x0e000060, 0x00003821, 0x0a0001df,
4932         0x00000000, 0xaf825cf8, 0x3c020800, 0x8c420a40, 0x8fa3001c, 0x24420001,
4933         0xaf835cf8, 0x3c010800, 0xac220a40, 0x8fbf0024, 0x8fb00020, 0x03e00008,
4934         0x27bd0028, 0x27bdffe0, 0x3c040800, 0x248409e8, 0x00002821, 0x00003021,
4935         0x00003821, 0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014, 0x8fbf0018,
4936         0x03e00008, 0x27bd0020, 0x8f82680c, 0x8f85680c, 0x00021827, 0x0003182b,
4937         0x00031823, 0x00431024, 0x00441021, 0x00a2282b, 0x10a00006, 0x00000000,
4938         0x00401821, 0x8f82680c, 0x0043102b, 0x1440fffd, 0x00000000, 0x03e00008,
4939         0x00000000, 0x3c040800, 0x8c840000, 0x3c030800, 0x8c630a40, 0x0064102b,
4940         0x54400002, 0x00831023, 0x00641023, 0x2c420008, 0x03e00008, 0x38420001,
4941         0x27bdffe0, 0x00802821, 0x3c040800, 0x24840a00, 0x00003021, 0x00003821,
4942         0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014, 0x0a000216, 0x00000000,
4943         0x8fbf0018, 0x03e00008, 0x27bd0020, 0x00000000, 0x27bdffe0, 0x3c1cc000,
4944         0xafbf0018, 0x0e00004c, 0xaf80680c, 0x3c040800, 0x24840a10, 0x03802821,
4945         0x00003021, 0x00003821, 0xafa00010, 0x0e000060, 0xafa00014, 0x2402ffff,
4946         0xaf825404, 0x3c0200aa, 0x0e000234, 0xaf825434, 0x8fbf0018, 0x03e00008,
4947         0x27bd0020, 0x00000000, 0x00000000, 0x00000000, 0x27bdffe8, 0xafb00010,
4948         0x24100001, 0xafbf0014, 0x3c01c003, 0xac200000, 0x8f826810, 0x30422000,
4949         0x10400003, 0x00000000, 0x0e000246, 0x00000000, 0x0a00023a, 0xaf905428,
4950         0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x27bdfff8, 0x8f845d0c,
4951         0x3c0200ff, 0x3c030800, 0x8c630a50, 0x3442fff8, 0x00821024, 0x1043001e,
4952         0x3c0500ff, 0x34a5fff8, 0x3c06c003, 0x3c074000, 0x00851824, 0x8c620010,
4953         0x3c010800, 0xac230a50, 0x30420008, 0x10400005, 0x00871025, 0x8cc20000,
4954         0x24420001, 0xacc20000, 0x00871025, 0xaf825d0c, 0x8fa20000, 0x24420001,
4955         0xafa20000, 0x8fa20000, 0x8fa20000, 0x24420001, 0xafa20000, 0x8fa20000,
4956         0x8f845d0c, 0x3c030800, 0x8c630a50, 0x00851024, 0x1443ffe8, 0x00851824,
4957         0x27bd0008, 0x03e00008, 0x00000000, 0x00000000, 0x00000000
4958 };
4959
4960 static u32 tg3FwRodata[(TG3_FW_RODATA_LEN / sizeof(u32)) + 1] = {
4961         0x35373031, 0x726c7341, 0x00000000, 0x00000000, 0x53774576, 0x656e7430,
4962         0x00000000, 0x726c7045, 0x76656e74, 0x31000000, 0x556e6b6e, 0x45766e74,
4963         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x66617461, 0x6c457272,
4964         0x00000000, 0x00000000, 0x4d61696e, 0x43707542, 0x00000000, 0x00000000,
4965         0x00000000
4966 };
4967
4968 #if 0 /* All zeros, don't eat up space with it. */
4969 u32 tg3FwData[(TG3_FW_DATA_LEN / sizeof(u32)) + 1] = {
4970         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
4971         0x00000000, 0x00000000, 0x00000000, 0x00000000
4972 };
4973 #endif
4974
4975 #define RX_CPU_SCRATCH_BASE     0x30000
4976 #define RX_CPU_SCRATCH_SIZE     0x04000
4977 #define TX_CPU_SCRATCH_BASE     0x34000
4978 #define TX_CPU_SCRATCH_SIZE     0x04000
4979
4980 /* tp->lock is held. */
4981 static int tg3_halt_cpu(struct tg3 *tp, u32 offset)
4982 {
4983         int i;
4984
4985         BUG_ON(offset == TX_CPU_BASE &&
4986             (tp->tg3_flags2 & TG3_FLG2_5705_PLUS));
4987
4988         if (offset == RX_CPU_BASE) {
4989                 for (i = 0; i < 10000; i++) {
4990                         tw32(offset + CPU_STATE, 0xffffffff);
4991                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
4992                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
4993                                 break;
4994                 }
4995
4996                 tw32(offset + CPU_STATE, 0xffffffff);
4997                 tw32_f(offset + CPU_MODE,  CPU_MODE_HALT);
4998                 udelay(10);
4999         } else {
5000                 for (i = 0; i < 10000; i++) {
5001                         tw32(offset + CPU_STATE, 0xffffffff);
5002                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
5003                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
5004                                 break;
5005                 }
5006         }
5007
5008         if (i >= 10000) {
5009                 printk(KERN_ERR PFX "tg3_reset_cpu timed out for %s, "
5010                        "and %s CPU\n",
5011                        tp->dev->name,
5012                        (offset == RX_CPU_BASE ? "RX" : "TX"));
5013                 return -ENODEV;
5014         }
5015
5016         /* Clear firmware's nvram arbitration. */
5017         if (tp->tg3_flags & TG3_FLAG_NVRAM)
5018                 tw32(NVRAM_SWARB, SWARB_REQ_CLR0);
5019         return 0;
5020 }
5021
5022 struct fw_info {
5023         unsigned int text_base;
5024         unsigned int text_len;
5025         u32 *text_data;
5026         unsigned int rodata_base;
5027         unsigned int rodata_len;
5028         u32 *rodata_data;
5029         unsigned int data_base;
5030         unsigned int data_len;
5031         u32 *data_data;
5032 };
5033
5034 /* tp->lock is held. */
5035 static int tg3_load_firmware_cpu(struct tg3 *tp, u32 cpu_base, u32 cpu_scratch_base,
5036                                  int cpu_scratch_size, struct fw_info *info)
5037 {
5038         int err, lock_err, i;
5039         void (*write_op)(struct tg3 *, u32, u32);
5040
5041         if (cpu_base == TX_CPU_BASE &&
5042             (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
5043                 printk(KERN_ERR PFX "tg3_load_firmware_cpu: Trying to load "
5044                        "TX cpu firmware on %s which is 5705.\n",
5045                        tp->dev->name);
5046                 return -EINVAL;
5047         }
5048
5049         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
5050                 write_op = tg3_write_mem;
5051         else
5052                 write_op = tg3_write_indirect_reg32;
5053
5054         /* It is possible that bootcode is still loading at this point.
5055          * Get the nvram lock first before halting the cpu.
5056          */
5057         lock_err = tg3_nvram_lock(tp);
5058         err = tg3_halt_cpu(tp, cpu_base);
5059         if (!lock_err)
5060                 tg3_nvram_unlock(tp);
5061         if (err)
5062                 goto out;
5063
5064         for (i = 0; i < cpu_scratch_size; i += sizeof(u32))
5065                 write_op(tp, cpu_scratch_base + i, 0);
5066         tw32(cpu_base + CPU_STATE, 0xffffffff);
5067         tw32(cpu_base + CPU_MODE, tr32(cpu_base+CPU_MODE)|CPU_MODE_HALT);
5068         for (i = 0; i < (info->text_len / sizeof(u32)); i++)
5069                 write_op(tp, (cpu_scratch_base +
5070                               (info->text_base & 0xffff) +
5071                               (i * sizeof(u32))),
5072                          (info->text_data ?
5073                           info->text_data[i] : 0));
5074         for (i = 0; i < (info->rodata_len / sizeof(u32)); i++)
5075                 write_op(tp, (cpu_scratch_base +
5076                               (info->rodata_base & 0xffff) +
5077                               (i * sizeof(u32))),
5078                          (info->rodata_data ?
5079                           info->rodata_data[i] : 0));
5080         for (i = 0; i < (info->data_len / sizeof(u32)); i++)
5081                 write_op(tp, (cpu_scratch_base +
5082                               (info->data_base & 0xffff) +
5083                               (i * sizeof(u32))),
5084                          (info->data_data ?
5085                           info->data_data[i] : 0));
5086
5087         err = 0;
5088
5089 out:
5090         return err;
5091 }
5092
5093 /* tp->lock is held. */
5094 static int tg3_load_5701_a0_firmware_fix(struct tg3 *tp)
5095 {
5096         struct fw_info info;
5097         int err, i;
5098
5099         info.text_base = TG3_FW_TEXT_ADDR;
5100         info.text_len = TG3_FW_TEXT_LEN;
5101         info.text_data = &tg3FwText[0];
5102         info.rodata_base = TG3_FW_RODATA_ADDR;
5103         info.rodata_len = TG3_FW_RODATA_LEN;
5104         info.rodata_data = &tg3FwRodata[0];
5105         info.data_base = TG3_FW_DATA_ADDR;
5106         info.data_len = TG3_FW_DATA_LEN;
5107         info.data_data = NULL;
5108
5109         err = tg3_load_firmware_cpu(tp, RX_CPU_BASE,
5110                                     RX_CPU_SCRATCH_BASE, RX_CPU_SCRATCH_SIZE,
5111                                     &info);
5112         if (err)
5113                 return err;
5114
5115         err = tg3_load_firmware_cpu(tp, TX_CPU_BASE,
5116                                     TX_CPU_SCRATCH_BASE, TX_CPU_SCRATCH_SIZE,
5117                                     &info);
5118         if (err)
5119                 return err;
5120
5121         /* Now startup only the RX cpu. */
5122         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5123         tw32_f(RX_CPU_BASE + CPU_PC,    TG3_FW_TEXT_ADDR);
5124
5125         for (i = 0; i < 5; i++) {
5126                 if (tr32(RX_CPU_BASE + CPU_PC) == TG3_FW_TEXT_ADDR)
5127                         break;
5128                 tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5129                 tw32(RX_CPU_BASE + CPU_MODE,  CPU_MODE_HALT);
5130                 tw32_f(RX_CPU_BASE + CPU_PC,    TG3_FW_TEXT_ADDR);
5131                 udelay(1000);
5132         }
5133         if (i >= 5) {
5134                 printk(KERN_ERR PFX "tg3_load_firmware fails for %s "
5135                        "to set RX CPU PC, is %08x should be %08x\n",
5136                        tp->dev->name, tr32(RX_CPU_BASE + CPU_PC),
5137                        TG3_FW_TEXT_ADDR);
5138                 return -ENODEV;
5139         }
5140         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5141         tw32_f(RX_CPU_BASE + CPU_MODE,  0x00000000);
5142
5143         return 0;
5144 }
5145
5146 #if TG3_TSO_SUPPORT != 0
5147
5148 #define TG3_TSO_FW_RELEASE_MAJOR        0x1
5149 #define TG3_TSO_FW_RELASE_MINOR         0x6
5150 #define TG3_TSO_FW_RELEASE_FIX          0x0
5151 #define TG3_TSO_FW_START_ADDR           0x08000000
5152 #define TG3_TSO_FW_TEXT_ADDR            0x08000000
5153 #define TG3_TSO_FW_TEXT_LEN             0x1aa0
5154 #define TG3_TSO_FW_RODATA_ADDR          0x08001aa0
5155 #define TG3_TSO_FW_RODATA_LEN           0x60
5156 #define TG3_TSO_FW_DATA_ADDR            0x08001b20
5157 #define TG3_TSO_FW_DATA_LEN             0x30
5158 #define TG3_TSO_FW_SBSS_ADDR            0x08001b50
5159 #define TG3_TSO_FW_SBSS_LEN             0x2c
5160 #define TG3_TSO_FW_BSS_ADDR             0x08001b80
5161 #define TG3_TSO_FW_BSS_LEN              0x894
5162
5163 static u32 tg3TsoFwText[(TG3_TSO_FW_TEXT_LEN / 4) + 1] = {
5164         0x0e000003, 0x00000000, 0x08001b24, 0x00000000, 0x10000003, 0x00000000,
5165         0x0000000d, 0x0000000d, 0x3c1d0800, 0x37bd4000, 0x03a0f021, 0x3c100800,
5166         0x26100000, 0x0e000010, 0x00000000, 0x0000000d, 0x27bdffe0, 0x3c04fefe,
5167         0xafbf0018, 0x0e0005d8, 0x34840002, 0x0e000668, 0x00000000, 0x3c030800,
5168         0x90631b68, 0x24020002, 0x3c040800, 0x24841aac, 0x14620003, 0x24050001,
5169         0x3c040800, 0x24841aa0, 0x24060006, 0x00003821, 0xafa00010, 0x0e00067c,
5170         0xafa00014, 0x8f625c50, 0x34420001, 0xaf625c50, 0x8f625c90, 0x34420001,
5171         0xaf625c90, 0x2402ffff, 0x0e000034, 0xaf625404, 0x8fbf0018, 0x03e00008,
5172         0x27bd0020, 0x00000000, 0x00000000, 0x00000000, 0x27bdffe0, 0xafbf001c,
5173         0xafb20018, 0xafb10014, 0x0e00005b, 0xafb00010, 0x24120002, 0x24110001,
5174         0x8f706820, 0x32020100, 0x10400003, 0x00000000, 0x0e0000bb, 0x00000000,
5175         0x8f706820, 0x32022000, 0x10400004, 0x32020001, 0x0e0001f0, 0x24040001,
5176         0x32020001, 0x10400003, 0x00000000, 0x0e0000a3, 0x00000000, 0x3c020800,
5177         0x90421b98, 0x14520003, 0x00000000, 0x0e0004c0, 0x00000000, 0x0a00003c,
5178         0xaf715028, 0x8fbf001c, 0x8fb20018, 0x8fb10014, 0x8fb00010, 0x03e00008,
5179         0x27bd0020, 0x27bdffe0, 0x3c040800, 0x24841ac0, 0x00002821, 0x00003021,
5180         0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014, 0x3c040800,
5181         0x248423d8, 0xa4800000, 0x3c010800, 0xa0201b98, 0x3c010800, 0xac201b9c,
5182         0x3c010800, 0xac201ba0, 0x3c010800, 0xac201ba4, 0x3c010800, 0xac201bac,
5183         0x3c010800, 0xac201bb8, 0x3c010800, 0xac201bbc, 0x8f624434, 0x3c010800,
5184         0xac221b88, 0x8f624438, 0x3c010800, 0xac221b8c, 0x8f624410, 0xac80f7a8,
5185         0x3c010800, 0xac201b84, 0x3c010800, 0xac2023e0, 0x3c010800, 0xac2023c8,
5186         0x3c010800, 0xac2023cc, 0x3c010800, 0xac202400, 0x3c010800, 0xac221b90,
5187         0x8f620068, 0x24030007, 0x00021702, 0x10430005, 0x00000000, 0x8f620068,
5188         0x00021702, 0x14400004, 0x24020001, 0x3c010800, 0x0a000097, 0xac20240c,
5189         0xac820034, 0x3c040800, 0x24841acc, 0x3c050800, 0x8ca5240c, 0x00003021,
5190         0x00003821, 0xafa00010, 0x0e00067c, 0xafa00014, 0x8fbf0018, 0x03e00008,
5191         0x27bd0020, 0x27bdffe0, 0x3c040800, 0x24841ad8, 0x00002821, 0x00003021,
5192         0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014, 0x0e00005b,
5193         0x00000000, 0x0e0000b4, 0x00002021, 0x8fbf0018, 0x03e00008, 0x27bd0020,
5194         0x24020001, 0x8f636820, 0x00821004, 0x00021027, 0x00621824, 0x03e00008,
5195         0xaf636820, 0x27bdffd0, 0xafbf002c, 0xafb60028, 0xafb50024, 0xafb40020,
5196         0xafb3001c, 0xafb20018, 0xafb10014, 0xafb00010, 0x8f675c5c, 0x3c030800,
5197         0x24631bbc, 0x8c620000, 0x14470005, 0x3c0200ff, 0x3c020800, 0x90421b98,
5198         0x14400119, 0x3c0200ff, 0x3442fff8, 0x00e28824, 0xac670000, 0x00111902,
5199         0x306300ff, 0x30e20003, 0x000211c0, 0x00622825, 0x00a04021, 0x00071602,
5200         0x3c030800, 0x90631b98, 0x3044000f, 0x14600036, 0x00804821, 0x24020001,
5201         0x3c010800, 0xa0221b98, 0x00051100, 0x00821025, 0x3c010800, 0xac201b9c,
5202         0x3c010800, 0xac201ba0, 0x3c010800, 0xac201ba4, 0x3c010800, 0xac201bac,
5203         0x3c010800, 0xac201bb8, 0x3c010800, 0xac201bb0, 0x3c010800, 0xac201bb4,
5204         0x3c010800, 0xa42223d8, 0x9622000c, 0x30437fff, 0x3c010800, 0xa4222410,
5205         0x30428000, 0x3c010800, 0xa4231bc6, 0x10400005, 0x24020001, 0x3c010800,
5206         0xac2223f4, 0x0a000102, 0x2406003e, 0x24060036, 0x3c010800, 0xac2023f4,
5207         0x9622000a, 0x3c030800, 0x94631bc6, 0x3c010800, 0xac2023f0, 0x3c010800,
5208         0xac2023f8, 0x00021302, 0x00021080, 0x00c21021, 0x00621821, 0x3c010800,
5209         0xa42223d0, 0x3c010800, 0x0a000115, 0xa4231b96, 0x9622000c, 0x3c010800,
5210         0xa42223ec, 0x3c040800, 0x24841b9c, 0x8c820000, 0x00021100, 0x3c010800,
5211         0x00220821, 0xac311bc8, 0x8c820000, 0x00021100, 0x3c010800, 0x00220821,
5212         0xac271bcc, 0x8c820000, 0x25030001, 0x306601ff, 0x00021100, 0x3c010800,
5213         0x00220821, 0xac261bd0, 0x8c820000, 0x00021100, 0x3c010800, 0x00220821,
5214         0xac291bd4, 0x96230008, 0x3c020800, 0x8c421bac, 0x00432821, 0x3c010800,
5215         0xac251bac, 0x9622000a, 0x30420004, 0x14400018, 0x00061100, 0x8f630c14,
5216         0x3063000f, 0x2c620002, 0x1440000b, 0x3c02c000, 0x8f630c14, 0x3c020800,
5217         0x8c421b40, 0x3063000f, 0x24420001, 0x3c010800, 0xac221b40, 0x2c620002,
5218         0x1040fff7, 0x3c02c000, 0x00e21825, 0xaf635c5c, 0x8f625c50, 0x30420002,
5219         0x10400014, 0x00000000, 0x0a000147, 0x00000000, 0x3c030800, 0x8c631b80,
5220         0x3c040800, 0x94841b94, 0x01221025, 0x3c010800, 0xa42223da, 0x24020001,
5221         0x3c010800, 0xac221bb8, 0x24630001, 0x0085202a, 0x3c010800, 0x10800003,
5222         0xac231b80, 0x3c010800, 0xa4251b94, 0x3c060800, 0x24c61b9c, 0x8cc20000,
5223         0x24420001, 0xacc20000, 0x28420080, 0x14400005, 0x00000000, 0x0e000656,
5224         0x24040002, 0x0a0001e6, 0x00000000, 0x3c020800, 0x8c421bb8, 0x10400078,
5225         0x24020001, 0x3c050800, 0x90a51b98, 0x14a20072, 0x00000000, 0x3c150800,
5226         0x96b51b96, 0x3c040800, 0x8c841bac, 0x32a3ffff, 0x0083102a, 0x1440006c,
5227         0x00000000, 0x14830003, 0x00000000, 0x3c010800, 0xac2523f0, 0x1060005c,
5228         0x00009021, 0x24d60004, 0x0060a021, 0x24d30014, 0x8ec20000, 0x00028100,
5229         0x3c110800, 0x02308821, 0x0e000625, 0x8e311bc8, 0x00402821, 0x10a00054,
5230         0x00000000, 0x9628000a, 0x31020040, 0x10400005, 0x2407180c, 0x8e22000c,
5231         0x2407188c, 0x00021400, 0xaca20018, 0x3c030800, 0x00701821, 0x8c631bd0,
5232         0x3c020800, 0x00501021, 0x8c421bd4, 0x00031d00, 0x00021400, 0x00621825,
5233         0xaca30014, 0x8ec30004, 0x96220008, 0x00432023, 0x3242ffff, 0x3083ffff,
5234         0x00431021, 0x0282102a, 0x14400002, 0x02b23023, 0x00803021, 0x8e620000,
5235         0x30c4ffff, 0x00441021, 0xae620000, 0x8e220000, 0xaca20000, 0x8e220004,
5236         0x8e63fff4, 0x00431021, 0xaca20004, 0xa4a6000e, 0x8e62fff4, 0x00441021,
5237         0xae62fff4, 0x96230008, 0x0043102a, 0x14400005, 0x02469021, 0x8e62fff0,
5238         0xae60fff4, 0x24420001, 0xae62fff0, 0xaca00008, 0x3242ffff, 0x14540008,
5239         0x24020305, 0x31020080, 0x54400001, 0x34e70010, 0x24020905, 0xa4a2000c,
5240         0x0a0001cb, 0x34e70020, 0xa4a2000c, 0x3c020800, 0x8c4223f0, 0x10400003,
5241         0x3c024b65, 0x0a0001d3, 0x34427654, 0x3c02b49a, 0x344289ab, 0xaca2001c,
5242         0x30e2ffff, 0xaca20010, 0x0e0005a2, 0x00a02021, 0x3242ffff, 0x0054102b,
5243         0x1440ffa9, 0x00000000, 0x24020002, 0x3c010800, 0x0a0001e6, 0xa0221b98,
5244         0x8ec2083c, 0x24420001, 0x0a0001e6, 0xaec2083c, 0x0e0004c0, 0x00000000,
5245         0x8fbf002c, 0x8fb60028, 0x8fb50024, 0x8fb40020, 0x8fb3001c, 0x8fb20018,
5246         0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0030, 0x27bdffd0, 0xafbf0028,
5247         0xafb30024, 0xafb20020, 0xafb1001c, 0xafb00018, 0x8f725c9c, 0x3c0200ff,
5248         0x3442fff8, 0x3c070800, 0x24e71bb4, 0x02428824, 0x9623000e, 0x8ce20000,
5249         0x00431021, 0xace20000, 0x8e220010, 0x30420020, 0x14400011, 0x00809821,
5250         0x0e00063b, 0x02202021, 0x3c02c000, 0x02421825, 0xaf635c9c, 0x8f625c90,
5251         0x30420002, 0x1040011e, 0x00000000, 0xaf635c9c, 0x8f625c90, 0x30420002,
5252         0x10400119, 0x00000000, 0x0a00020d, 0x00000000, 0x8e240008, 0x8e230014,
5253         0x00041402, 0x000231c0, 0x00031502, 0x304201ff, 0x2442ffff, 0x3042007f,
5254         0x00031942, 0x30637800, 0x00021100, 0x24424000, 0x00624821, 0x9522000a,
5255         0x3084ffff, 0x30420008, 0x104000b0, 0x000429c0, 0x3c020800, 0x8c422400,
5256         0x14400024, 0x24c50008, 0x94c20014, 0x3c010800, 0xa42223d0, 0x8cc40010,
5257         0x00041402, 0x3c010800, 0xa42223d2, 0x3c010800, 0xa42423d4, 0x94c2000e,
5258         0x3083ffff, 0x00431023, 0x3c010800, 0xac222408, 0x94c2001a, 0x3c010800,
5259         0xac262400, 0x3c010800, 0xac322404, 0x3c010800, 0xac2223fc, 0x3c02c000,
5260         0x02421825, 0xaf635c9c, 0x8f625c90, 0x30420002, 0x104000e5, 0x00000000,
5261         0xaf635c9c, 0x8f625c90, 0x30420002, 0x104000e0, 0x00000000, 0x0a000246,
5262         0x00000000, 0x94c2000e, 0x3c030800, 0x946323d4, 0x00434023, 0x3103ffff,
5263         0x2c620008, 0x1040001c, 0x00000000, 0x94c20014, 0x24420028, 0x00a22821,
5264         0x00031042, 0x1840000b, 0x00002021, 0x24e60848, 0x00403821, 0x94a30000,
5265         0x8cc20000, 0x24840001, 0x00431021, 0xacc20000, 0x0087102a, 0x1440fff9,
5266         0x24a50002, 0x31020001, 0x1040001f, 0x3c024000, 0x3c040800, 0x248423fc,
5267         0xa0a00001, 0x94a30000, 0x8c820000, 0x00431021, 0x0a000285, 0xac820000,
5268         0x8f626800, 0x3c030010, 0x00431024, 0x10400009, 0x00000000, 0x94c2001a,
5269         0x3c030800, 0x8c6323fc, 0x00431021, 0x3c010800, 0xac2223fc, 0x0a000286,
5270         0x3c024000, 0x94c2001a, 0x94c4001c, 0x3c030800, 0x8c6323fc, 0x00441023,
5271         0x00621821, 0x3c010800, 0xac2323fc, 0x3c024000, 0x02421825, 0xaf635c9c,
5272         0x8f625c90, 0x30420002, 0x1440fffc, 0x00000000, 0x9522000a, 0x30420010,
5273         0x1040009b, 0x00000000, 0x3c030800, 0x946323d4, 0x3c070800, 0x24e72400,
5274         0x8ce40000, 0x8f626800, 0x24630030, 0x00832821, 0x3c030010, 0x00431024,
5275         0x1440000a, 0x00000000, 0x94a20004, 0x3c040800, 0x8c842408, 0x3c030800,
5276         0x8c6323fc, 0x00441023, 0x00621821, 0x3c010800, 0xac2323fc, 0x3c040800,
5277         0x8c8423fc, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402, 0x00822021,
5278         0x00041027, 0xa4a20006, 0x3c030800, 0x8c632404, 0x3c0200ff, 0x3442fff8,
5279         0x00628824, 0x96220008, 0x24050001, 0x24034000, 0x000231c0, 0x00801021,
5280         0xa4c2001a, 0xa4c0001c, 0xace00000, 0x3c010800, 0xac251b60, 0xaf635cb8,
5281         0x8f625cb0, 0x30420002, 0x10400003, 0x00000000, 0x3c010800, 0xac201b60,
5282         0x8e220008, 0xaf625cb8, 0x8f625cb0, 0x30420002, 0x10400003, 0x00000000,
5283         0x3c010800, 0xac201b60, 0x3c020800, 0x8c421b60, 0x1040ffec, 0x00000000,
5284         0x3c040800, 0x0e00063b, 0x8c842404, 0x0a00032a, 0x00000000, 0x3c030800,
5285         0x90631b98, 0x24020002, 0x14620003, 0x3c034b65, 0x0a0002e1, 0x00008021,
5286         0x8e22001c, 0x34637654, 0x10430002, 0x24100002, 0x24100001, 0x00c02021,
5287         0x0e000350, 0x02003021, 0x24020003, 0x3c010800, 0xa0221b98, 0x24020002,
5288         0x1202000a, 0x24020001, 0x3c030800, 0x8c6323f0, 0x10620006, 0x00000000,
5289         0x3c020800, 0x944223d8, 0x00021400, 0x0a00031f, 0xae220014, 0x3c040800,
5290         0x248423da, 0x94820000, 0x00021400, 0xae220014, 0x3c020800, 0x8c421bbc,
5291         0x3c03c000, 0x3c010800, 0xa0201b98, 0x00431025, 0xaf625c5c, 0x8f625c50,
5292         0x30420002, 0x10400009, 0x00000000, 0x2484f7e2, 0x8c820000, 0x00431025,
5293         0xaf625c5c, 0x8f625c50, 0x30420002, 0x1440fffa, 0x00000000, 0x3c020800,
5294         0x24421b84, 0x8c430000, 0x24630001, 0xac430000, 0x8f630c14, 0x3063000f,
5295         0x2c620002, 0x1440000c, 0x3c024000, 0x8f630c14, 0x3c020800, 0x8c421b40,
5296         0x3063000f, 0x24420001, 0x3c010800, 0xac221b40, 0x2c620002, 0x1040fff7,
5297         0x00000000, 0x3c024000, 0x02421825, 0xaf635c9c, 0x8f625c90, 0x30420002,
5298         0x1440fffc, 0x00000000, 0x12600003, 0x00000000, 0x0e0004c0, 0x00000000,
5299         0x8fbf0028, 0x8fb30024, 0x8fb20020, 0x8fb1001c, 0x8fb00018, 0x03e00008,
5300         0x27bd0030, 0x8f634450, 0x3c040800, 0x24841b88, 0x8c820000, 0x00031c02,
5301         0x0043102b, 0x14400007, 0x3c038000, 0x8c840004, 0x8f624450, 0x00021c02,
5302         0x0083102b, 0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024,
5303         0x1440fffd, 0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3c024000,
5304         0x00822025, 0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00000000,
5305         0x03e00008, 0x00000000, 0x27bdffe0, 0x00805821, 0x14c00011, 0x256e0008,
5306         0x3c020800, 0x8c4223f4, 0x10400007, 0x24020016, 0x3c010800, 0xa42223d2,
5307         0x2402002a, 0x3c010800, 0x0a000364, 0xa42223d4, 0x8d670010, 0x00071402,
5308         0x3c010800, 0xa42223d2, 0x3c010800, 0xa42723d4, 0x3c040800, 0x948423d4,
5309         0x3c030800, 0x946323d2, 0x95cf0006, 0x3c020800, 0x944223d0, 0x00832023,
5310         0x01e2c023, 0x3065ffff, 0x24a20028, 0x01c24821, 0x3082ffff, 0x14c0001a,
5311         0x01226021, 0x9582000c, 0x3042003f, 0x3c010800, 0xa42223d6, 0x95820004,
5312         0x95830006, 0x3c010800, 0xac2023e4, 0x3c010800, 0xac2023e8, 0x00021400,
5313         0x00431025, 0x3c010800, 0xac221bc0, 0x95220004, 0x3c010800, 0xa4221bc4,
5314         0x95230002, 0x01e51023, 0x0043102a, 0x10400010, 0x24020001, 0x3c010800,
5315         0x0a000398, 0xac2223f8, 0x3c030800, 0x8c6323e8, 0x3c020800, 0x94421bc4,
5316         0x00431021, 0xa5220004, 0x3c020800, 0x94421bc0, 0xa5820004, 0x3c020800,
5317         0x8c421bc0, 0xa5820006, 0x3c020800, 0x8c4223f0, 0x3c0d0800, 0x8dad23e4,
5318         0x3c0a0800, 0x144000e5, 0x8d4a23e8, 0x3c020800, 0x94421bc4, 0x004a1821,
5319         0x3063ffff, 0x0062182b, 0x24020002, 0x10c2000d, 0x01435023, 0x3c020800,
5320         0x944223d6, 0x30420009, 0x10400008, 0x00000000, 0x9582000c, 0x3042fff6,
5321         0xa582000c, 0x3c020800, 0x944223d6, 0x30420009, 0x01a26823, 0x3c020800,
5322         0x8c4223f8, 0x1040004a, 0x01203821, 0x3c020800, 0x944223d2, 0x00004021,
5323         0xa520000a, 0x01e21023, 0xa5220002, 0x3082ffff, 0x00021042, 0x18400008,
5324         0x00003021, 0x00401821, 0x94e20000, 0x25080001, 0x00c23021, 0x0103102a,
5325         0x1440fffb, 0x24e70002, 0x00061c02, 0x30c2ffff, 0x00623021, 0x00061402,
5326         0x00c23021, 0x00c02821, 0x00061027, 0xa522000a, 0x00003021, 0x2527000c,
5327         0x00004021, 0x94e20000, 0x25080001, 0x00c23021, 0x2d020004, 0x1440fffb,
5328         0x24e70002, 0x95220002, 0x00004021, 0x91230009, 0x00442023, 0x01803821,
5329         0x3082ffff, 0xa4e00010, 0x00621821, 0x00021042, 0x18400010, 0x00c33021,
5330         0x00404821, 0x94e20000, 0x24e70002, 0x00c23021, 0x30e2007f, 0x14400006,
5331         0x25080001, 0x8d630000, 0x3c02007f, 0x3442ff80, 0x00625824, 0x25670008,
5332         0x0109102a, 0x1440fff3, 0x00000000, 0x30820001, 0x10400005, 0x00061c02,
5333         0xa0e00001, 0x94e20000, 0x00c23021, 0x00061c02, 0x30c2ffff, 0x00623021,
5334         0x00061402, 0x00c23021, 0x0a00047d, 0x30c6ffff, 0x24020002, 0x14c20081,
5335         0x00000000, 0x3c020800, 0x8c42240c, 0x14400007, 0x00000000, 0x3c020800,
5336         0x944223d2, 0x95230002, 0x01e21023, 0x10620077, 0x00000000, 0x3c020800,
5337         0x944223d2, 0x01e21023, 0xa5220002, 0x3c020800, 0x8c42240c, 0x1040001a,
5338         0x31e3ffff, 0x8dc70010, 0x3c020800, 0x94421b96, 0x00e04021, 0x00072c02,
5339         0x00aa2021, 0x00431023, 0x00823823, 0x00072402, 0x30e2ffff, 0x00823821,
5340         0x00071027, 0xa522000a, 0x3102ffff, 0x3c040800, 0x948423d4, 0x00453023,
5341         0x00e02821, 0x00641823, 0x006d1821, 0x00c33021, 0x00061c02, 0x30c2ffff,
5342         0x0a00047d, 0x00623021, 0x01203821, 0x00004021, 0x3082ffff, 0x00021042,
5343         0x18400008, 0x00003021, 0x00401821, 0x94e20000, 0x25080001, 0x00c23021,
5344         0x0103102a, 0x1440fffb, 0x24e70002, 0x00061c02, 0x30c2ffff, 0x00623021,
5345         0x00061402, 0x00c23021, 0x00c02821, 0x00061027, 0xa522000a, 0x00003021,
5346         0x2527000c, 0x00004021, 0x94e20000, 0x25080001, 0x00c23021, 0x2d020004,
5347         0x1440fffb, 0x24e70002, 0x95220002, 0x00004021, 0x91230009, 0x00442023,
5348         0x01803821, 0x3082ffff, 0xa4e00010, 0x3c040800, 0x948423d4, 0x00621821,
5349         0x00c33021, 0x00061c02, 0x30c2ffff, 0x00623021, 0x00061c02, 0x3c020800,
5350         0x944223d0, 0x00c34821, 0x00441023, 0x00021fc2, 0x00431021, 0x00021043,
5351         0x18400010, 0x00003021, 0x00402021, 0x94e20000, 0x24e70002, 0x00c23021,
5352         0x30e2007f, 0x14400006, 0x25080001, 0x8d630000, 0x3c02007f, 0x3442ff80,
5353         0x00625824, 0x25670008, 0x0104102a, 0x1440fff3, 0x00000000, 0x3c020800,
5354         0x944223ec, 0x00c23021, 0x3122ffff, 0x00c23021, 0x00061c02, 0x30c2ffff,
5355         0x00623021, 0x00061402, 0x00c23021, 0x00c04021, 0x00061027, 0xa5820010,
5356         0xadc00014, 0x0a00049d, 0xadc00000, 0x8dc70010, 0x00e04021, 0x11400007,
5357         0x00072c02, 0x00aa3021, 0x00061402, 0x30c3ffff, 0x00433021, 0x00061402,
5358         0x00c22821, 0x00051027, 0xa522000a, 0x3c030800, 0x946323d4, 0x3102ffff,
5359         0x01e21021, 0x00433023, 0x00cd3021, 0x00061c02, 0x30c2ffff, 0x00623021,
5360         0x00061402, 0x00c23021, 0x00c04021, 0x00061027, 0xa5820010, 0x3102ffff,
5361         0x00051c00, 0x00431025, 0xadc20010, 0x3c020800, 0x8c4223f4, 0x10400005,
5362         0x2de205eb, 0x14400002, 0x25e2fff2, 0x34028870, 0xa5c20034, 0x3c030800,
5363         0x246323e8, 0x8c620000, 0x24420001, 0xac620000, 0x3c040800, 0x8c8423e4,
5364         0x3c020800, 0x8c421bc0, 0x3303ffff, 0x00832021, 0x00431821, 0x0062102b,
5365         0x3c010800, 0xac2423e4, 0x10400003, 0x2482ffff, 0x3c010800, 0xac2223e4,
5366         0x3c010800, 0xac231bc0, 0x03e00008, 0x27bd0020, 0x27bdffb8, 0x3c050800,
5367         0x24a51b96, 0xafbf0044, 0xafbe0040, 0xafb7003c, 0xafb60038, 0xafb50034,
5368         0xafb40030, 0xafb3002c, 0xafb20028, 0xafb10024, 0xafb00020, 0x94a90000,
5369         0x3c020800, 0x944223d0, 0x3c030800, 0x8c631bb0, 0x3c040800, 0x8c841bac,
5370         0x01221023, 0x0064182a, 0xa7a9001e, 0x106000be, 0xa7a20016, 0x24be0022,
5371         0x97b6001e, 0x24b3001a, 0x24b70016, 0x8fc20000, 0x14400008, 0x00000000,
5372         0x8fc2fff8, 0x97a30016, 0x8fc4fff4, 0x00431021, 0x0082202a, 0x148000b0,
5373         0x00000000, 0x97d50818, 0x32a2ffff, 0x104000a3, 0x00009021, 0x0040a021,
5374         0x00008821, 0x0e000625, 0x00000000, 0x00403021, 0x14c00007, 0x00000000,
5375         0x3c020800, 0x8c4223dc, 0x24420001, 0x3c010800, 0x0a000596, 0xac2223dc,
5376         0x3c100800, 0x02118021, 0x8e101bc8, 0x9608000a, 0x31020040, 0x10400005,
5377         0x2407180c, 0x8e02000c, 0x2407188c, 0x00021400, 0xacc20018, 0x31020080,
5378         0x54400001, 0x34e70010, 0x3c020800, 0x00511021, 0x8c421bd0, 0x3c030800,
5379         0x00711821, 0x8c631bd4, 0x00021500, 0x00031c00, 0x00431025, 0xacc20014,
5380         0x96040008, 0x3242ffff, 0x00821021, 0x0282102a, 0x14400002, 0x02b22823,
5381         0x00802821, 0x8e020000, 0x02459021, 0xacc20000, 0x8e020004, 0x00c02021,
5382         0x26310010, 0xac820004, 0x30e2ffff, 0xac800008, 0xa485000e, 0xac820010,
5383         0x24020305, 0x0e0005a2, 0xa482000c, 0x3242ffff, 0x0054102b, 0x1440ffc5,
5384         0x3242ffff, 0x0a00058e, 0x00000000, 0x8e620000, 0x8e63fffc, 0x0043102a,
5385         0x10400067, 0x00000000, 0x8e62fff0, 0x00028900, 0x3c100800, 0x02118021,
5386         0x0e000625, 0x8e101bc8, 0x00403021, 0x14c00005, 0x00000000, 0x8e62082c,
5387         0x24420001, 0x0a000596, 0xae62082c, 0x9608000a, 0x31020040, 0x10400005,
5388         0x2407180c, 0x8e02000c, 0x2407188c, 0x00021400, 0xacc20018, 0x3c020800,
5389         0x00511021, 0x8c421bd0, 0x3c030800, 0x00711821, 0x8c631bd4, 0x00021500,
5390         0x00031c00, 0x00431025, 0xacc20014, 0x8e63fff4, 0x96020008, 0x00432023,
5391         0x3242ffff, 0x3083ffff, 0x00431021, 0x02c2102a, 0x10400003, 0x00802821,
5392         0x97a9001e, 0x01322823, 0x8e620000, 0x30a4ffff, 0x00441021, 0xae620000,
5393         0xa4c5000e, 0x8e020000, 0xacc20000, 0x8e020004, 0x8e63fff4, 0x00431021,
5394         0xacc20004, 0x8e63fff4, 0x96020008, 0x00641821, 0x0062102a, 0x14400006,
5395         0x02459021, 0x8e62fff0, 0xae60fff4, 0x24420001, 0x0a000571, 0xae62fff0,
5396         0xae63fff4, 0xacc00008, 0x3242ffff, 0x10560003, 0x31020004, 0x10400006,
5397         0x24020305, 0x31020080, 0x54400001, 0x34e70010, 0x34e70020, 0x24020905,
5398         0xa4c2000c, 0x8ee30000, 0x8ee20004, 0x14620007, 0x3c02b49a, 0x8ee20860,
5399         0x54400001, 0x34e70400, 0x3c024b65, 0x0a000588, 0x34427654, 0x344289ab,
5400         0xacc2001c, 0x30e2ffff, 0xacc20010, 0x0e0005a2, 0x00c02021, 0x3242ffff,
5401         0x0056102b, 0x1440ff9b, 0x00000000, 0x8e620000, 0x8e63fffc, 0x0043102a,
5402         0x1440ff48, 0x00000000, 0x8fbf0044, 0x8fbe0040, 0x8fb7003c, 0x8fb60038,
5403         0x8fb50034, 0x8fb40030, 0x8fb3002c, 0x8fb20028, 0x8fb10024, 0x8fb00020,
5404         0x03e00008, 0x27bd0048, 0x27bdffe8, 0xafbf0014, 0xafb00010, 0x8f624450,
5405         0x8f634410, 0x0a0005b1, 0x00808021, 0x8f626820, 0x30422000, 0x10400003,
5406         0x00000000, 0x0e0001f0, 0x00002021, 0x8f624450, 0x8f634410, 0x3042ffff,
5407         0x0043102b, 0x1440fff5, 0x00000000, 0x8f630c14, 0x3063000f, 0x2c620002,
5408         0x1440000b, 0x00000000, 0x8f630c14, 0x3c020800, 0x8c421b40, 0x3063000f,
5409         0x24420001, 0x3c010800, 0xac221b40, 0x2c620002, 0x1040fff7, 0x00000000,
5410         0xaf705c18, 0x8f625c10, 0x30420002, 0x10400009, 0x00000000, 0x8f626820,
5411         0x30422000, 0x1040fff8, 0x00000000, 0x0e0001f0, 0x00002021, 0x0a0005c4,
5412         0x00000000, 0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x00000000,
5413         0x00000000, 0x00000000, 0x27bdffe8, 0x3c1bc000, 0xafbf0014, 0xafb00010,
5414         0xaf60680c, 0x8f626804, 0x34420082, 0xaf626804, 0x8f634000, 0x24020b50,
5415         0x3c010800, 0xac221b54, 0x24020b78, 0x3c010800, 0xac221b64, 0x34630002,
5416         0xaf634000, 0x0e000605, 0x00808021, 0x3c010800, 0xa0221b68, 0x304200ff,
5417         0x24030002, 0x14430005, 0x00000000, 0x3c020800, 0x8c421b54, 0x0a0005f8,
5418         0xac5000c0, 0x3c020800, 0x8c421b54, 0xac5000bc, 0x8f624434, 0x8f634438,
5419         0x8f644410, 0x3c010800, 0xac221b5c, 0x3c010800, 0xac231b6c, 0x3c010800,
5420         0xac241b58, 0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x3c040800,
5421         0x8c870000, 0x3c03aa55, 0x3463aa55, 0x3c06c003, 0xac830000, 0x8cc20000,
5422         0x14430007, 0x24050002, 0x3c0355aa, 0x346355aa, 0xac830000, 0x8cc20000,
5423         0x50430001, 0x24050001, 0x3c020800, 0xac470000, 0x03e00008, 0x00a01021,
5424         0x27bdfff8, 0x18800009, 0x00002821, 0x8f63680c, 0x8f62680c, 0x1043fffe,
5425         0x00000000, 0x24a50001, 0x00a4102a, 0x1440fff9, 0x00000000, 0x03e00008,
5426         0x27bd0008, 0x8f634450, 0x3c020800, 0x8c421b5c, 0x00031c02, 0x0043102b,
5427         0x14400008, 0x3c038000, 0x3c040800, 0x8c841b6c, 0x8f624450, 0x00021c02,
5428         0x0083102b, 0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024,
5429         0x1440fffd, 0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3082ffff,
5430         0x2442e000, 0x2c422001, 0x14400003, 0x3c024000, 0x0a000648, 0x2402ffff,
5431         0x00822025, 0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00001021,
5432         0x03e00008, 0x00000000, 0x8f624450, 0x3c030800, 0x8c631b58, 0x0a000651,
5433         0x3042ffff, 0x8f624450, 0x3042ffff, 0x0043102b, 0x1440fffc, 0x00000000,
5434         0x03e00008, 0x00000000, 0x27bdffe0, 0x00802821, 0x3c040800, 0x24841af0,
5435         0x00003021, 0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014,
5436         0x0a000660, 0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x00000000,
5437         0x00000000, 0x00000000, 0x3c020800, 0x34423000, 0x3c030800, 0x34633000,
5438         0x3c040800, 0x348437ff, 0x3c010800, 0xac221b74, 0x24020040, 0x3c010800,
5439         0xac221b78, 0x3c010800, 0xac201b70, 0xac600000, 0x24630004, 0x0083102b,
5440         0x5040fffd, 0xac600000, 0x03e00008, 0x00000000, 0x00804821, 0x8faa0010,
5441         0x3c020800, 0x8c421b70, 0x3c040800, 0x8c841b78, 0x8fab0014, 0x24430001,
5442         0x0044102b, 0x3c010800, 0xac231b70, 0x14400003, 0x00004021, 0x3c010800,
5443         0xac201b70, 0x3c020800, 0x8c421b70, 0x3c030800, 0x8c631b74, 0x91240000,
5444         0x00021140, 0x00431021, 0x00481021, 0x25080001, 0xa0440000, 0x29020008,
5445         0x1440fff4, 0x25290001, 0x3c020800, 0x8c421b70, 0x3c030800, 0x8c631b74,
5446         0x8f64680c, 0x00021140, 0x00431021, 0xac440008, 0xac45000c, 0xac460010,
5447         0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c, 0x00000000, 0x00000000,
5448 };
5449
5450 static u32 tg3TsoFwRodata[] = {
5451         0x4d61696e, 0x43707542, 0x00000000, 0x4d61696e, 0x43707541, 0x00000000,
5452         0x00000000, 0x00000000, 0x73746b6f, 0x66666c64, 0x496e0000, 0x73746b6f,
5453         0x66662a2a, 0x00000000, 0x53774576, 0x656e7430, 0x00000000, 0x00000000,
5454         0x00000000, 0x00000000, 0x66617461, 0x6c457272, 0x00000000, 0x00000000,
5455         0x00000000,
5456 };
5457
5458 static u32 tg3TsoFwData[] = {
5459         0x00000000, 0x73746b6f, 0x66666c64, 0x5f76312e, 0x362e3000, 0x00000000,
5460         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
5461         0x00000000,
5462 };
5463
5464 /* 5705 needs a special version of the TSO firmware.  */
5465 #define TG3_TSO5_FW_RELEASE_MAJOR       0x1
5466 #define TG3_TSO5_FW_RELASE_MINOR        0x2
5467 #define TG3_TSO5_FW_RELEASE_FIX         0x0
5468 #define TG3_TSO5_FW_START_ADDR          0x00010000
5469 #define TG3_TSO5_FW_TEXT_ADDR           0x00010000
5470 #define TG3_TSO5_FW_TEXT_LEN            0xe90
5471 #define TG3_TSO5_FW_RODATA_ADDR         0x00010e90
5472 #define TG3_TSO5_FW_RODATA_LEN          0x50
5473 #define TG3_TSO5_FW_DATA_ADDR           0x00010f00
5474 #define TG3_TSO5_FW_DATA_LEN            0x20
5475 #define TG3_TSO5_FW_SBSS_ADDR           0x00010f20
5476 #define TG3_TSO5_FW_SBSS_LEN            0x28
5477 #define TG3_TSO5_FW_BSS_ADDR            0x00010f50
5478 #define TG3_TSO5_FW_BSS_LEN             0x88
5479
5480 static u32 tg3Tso5FwText[(TG3_TSO5_FW_TEXT_LEN / 4) + 1] = {
5481         0x0c004003, 0x00000000, 0x00010f04, 0x00000000, 0x10000003, 0x00000000,
5482         0x0000000d, 0x0000000d, 0x3c1d0001, 0x37bde000, 0x03a0f021, 0x3c100001,
5483         0x26100000, 0x0c004010, 0x00000000, 0x0000000d, 0x27bdffe0, 0x3c04fefe,
5484         0xafbf0018, 0x0c0042e8, 0x34840002, 0x0c004364, 0x00000000, 0x3c030001,
5485         0x90630f34, 0x24020002, 0x3c040001, 0x24840e9c, 0x14620003, 0x24050001,
5486         0x3c040001, 0x24840e90, 0x24060002, 0x00003821, 0xafa00010, 0x0c004378,
5487         0xafa00014, 0x0c00402c, 0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020,
5488         0x00000000, 0x00000000, 0x27bdffe0, 0xafbf001c, 0xafb20018, 0xafb10014,
5489         0x0c0042d4, 0xafb00010, 0x3c128000, 0x24110001, 0x8f706810, 0x32020400,
5490         0x10400007, 0x00000000, 0x8f641008, 0x00921024, 0x14400003, 0x00000000,
5491         0x0c004064, 0x00000000, 0x3c020001, 0x90420f56, 0x10510003, 0x32020200,
5492         0x1040fff1, 0x00000000, 0x0c0041b4, 0x00000000, 0x08004034, 0x00000000,
5493         0x8fbf001c, 0x8fb20018, 0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0020,
5494         0x27bdffe0, 0x3c040001, 0x24840eb0, 0x00002821, 0x00003021, 0x00003821,
5495         0xafbf0018, 0xafa00010, 0x0c004378, 0xafa00014, 0x0000d021, 0x24020130,
5496         0xaf625000, 0x3c010001, 0xa4200f50, 0x3c010001, 0xa0200f57, 0x8fbf0018,
5497         0x03e00008, 0x27bd0020, 0x00000000, 0x00000000, 0x3c030001, 0x24630f60,
5498         0x90620000, 0x27bdfff0, 0x14400003, 0x0080c021, 0x08004073, 0x00004821,
5499         0x3c022000, 0x03021024, 0x10400003, 0x24090002, 0x08004073, 0xa0600000,
5500         0x24090001, 0x00181040, 0x30431f80, 0x346f8008, 0x1520004b, 0x25eb0028,
5501         0x3c040001, 0x00832021, 0x8c848010, 0x3c050001, 0x24a50f7a, 0x00041402,
5502         0xa0a20000, 0x3c010001, 0xa0240f7b, 0x3c020001, 0x00431021, 0x94428014,
5503         0x3c010001, 0xa0220f7c, 0x3c0c0001, 0x01836021, 0x8d8c8018, 0x304200ff,
5504         0x24420008, 0x000220c3, 0x24020001, 0x3c010001, 0xa0220f60, 0x0124102b,
5505         0x1040000c, 0x00003821, 0x24a6000e, 0x01602821, 0x8ca20000, 0x8ca30004,
5506         0x24a50008, 0x24e70001, 0xacc20000, 0xacc30004, 0x00e4102b, 0x1440fff8,
5507         0x24c60008, 0x00003821, 0x3c080001, 0x25080f7b, 0x91060000, 0x3c020001,
5508         0x90420f7c, 0x2503000d, 0x00c32821, 0x00461023, 0x00021fc2, 0x00431021,
5509         0x00021043, 0x1840000c, 0x00002021, 0x91020001, 0x00461023, 0x00021fc2,
5510         0x00431021, 0x00021843, 0x94a20000, 0x24e70001, 0x00822021, 0x00e3102a,
5511         0x1440fffb, 0x24a50002, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402,
5512         0x00822021, 0x3c02ffff, 0x01821024, 0x3083ffff, 0x00431025, 0x3c010001,
5513         0x080040fa, 0xac220f80, 0x3c050001, 0x24a50f7c, 0x90a20000, 0x3c0c0001,
5514         0x01836021, 0x8d8c8018, 0x000220c2, 0x1080000e, 0x00003821, 0x01603021,
5515         0x24a5000c, 0x8ca20000, 0x8ca30004, 0x24a50008, 0x24e70001, 0xacc20000,
5516         0xacc30004, 0x00e4102b, 0x1440fff8, 0x24c60008, 0x3c050001, 0x24a50f7c,
5517         0x90a20000, 0x30430007, 0x24020004, 0x10620011, 0x28620005, 0x10400005,
5518         0x24020002, 0x10620008, 0x000710c0, 0x080040fa, 0x00000000, 0x24020006,
5519         0x1062000e, 0x000710c0, 0x080040fa, 0x00000000, 0x00a21821, 0x9463000c,
5520         0x004b1021, 0x080040fa, 0xa4430000, 0x000710c0, 0x00a21821, 0x8c63000c,
5521         0x004b1021, 0x080040fa, 0xac430000, 0x00a21821, 0x8c63000c, 0x004b2021,
5522         0x00a21021, 0xac830000, 0x94420010, 0xa4820004, 0x95e70006, 0x3c020001,
5523         0x90420f7c, 0x3c030001, 0x90630f7a, 0x00e2c823, 0x3c020001, 0x90420f7b,
5524         0x24630028, 0x01e34021, 0x24420028, 0x15200012, 0x01e23021, 0x94c2000c,
5525         0x3c010001, 0xa4220f78, 0x94c20004, 0x94c30006, 0x3c010001, 0xa4200f76,
5526         0x3c010001, 0xa4200f72, 0x00021400, 0x00431025, 0x3c010001, 0xac220f6c,
5527         0x95020004, 0x3c010001, 0x08004124, 0xa4220f70, 0x3c020001, 0x94420f70,
5528         0x3c030001, 0x94630f72, 0x00431021, 0xa5020004, 0x3c020001, 0x94420f6c,
5529         0xa4c20004, 0x3c020001, 0x8c420f6c, 0xa4c20006, 0x3c040001, 0x94840f72,
5530         0x3c020001, 0x94420f70, 0x3c0a0001, 0x954a0f76, 0x00441821, 0x3063ffff,
5531         0x0062182a, 0x24020002, 0x1122000b, 0x00832023, 0x3c030001, 0x94630f78,
5532         0x30620009, 0x10400006, 0x3062fff6, 0xa4c2000c, 0x3c020001, 0x94420f78,
5533         0x30420009, 0x01425023, 0x24020001, 0x1122001b, 0x29220002, 0x50400005,
5534         0x24020002, 0x11200007, 0x31a2ffff, 0x08004197, 0x00000000, 0x1122001d,
5535         0x24020016, 0x08004197, 0x31a2ffff, 0x3c0e0001, 0x95ce0f80, 0x10800005,
5536         0x01806821, 0x01c42021, 0x00041c02, 0x3082ffff, 0x00627021, 0x000e1027,
5537         0xa502000a, 0x3c030001, 0x90630f7b, 0x31a2ffff, 0x00e21021, 0x0800418d,
5538         0x00432023, 0x3c020001, 0x94420f80, 0x00442021, 0x00041c02, 0x3082ffff,
5539         0x00622021, 0x00807021, 0x00041027, 0x08004185, 0xa502000a, 0x3c050001,
5540         0x24a50f7a, 0x90a30000, 0x14620002, 0x24e2fff2, 0xa5e20034, 0x90a20000,
5541         0x00e21023, 0xa5020002, 0x3c030001, 0x94630f80, 0x3c020001, 0x94420f5a,
5542         0x30e5ffff, 0x00641821, 0x00451023, 0x00622023, 0x00041c02, 0x3082ffff,
5543         0x00622021, 0x00041027, 0xa502000a, 0x3c030001, 0x90630f7c, 0x24620001,
5544         0x14a20005, 0x00807021, 0x01631021, 0x90420000, 0x08004185, 0x00026200,
5545         0x24620002, 0x14a20003, 0x306200fe, 0x004b1021, 0x944c0000, 0x3c020001,
5546         0x94420f82, 0x3183ffff, 0x3c040001, 0x90840f7b, 0x00431021, 0x00e21021,
5547         0x00442023, 0x008a2021, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402,
5548         0x00822021, 0x00806821, 0x00041027, 0xa4c20010, 0x31a2ffff, 0x000e1c00,
5549         0x00431025, 0x3c040001, 0x24840f72, 0xade20010, 0x94820000, 0x3c050001,
5550         0x94a50f76, 0x3c030001, 0x8c630f6c, 0x24420001, 0x00b92821, 0xa4820000,
5551         0x3322ffff, 0x00622021, 0x0083182b, 0x3c010001, 0xa4250f76, 0x10600003,
5552         0x24a2ffff, 0x3c010001, 0xa4220f76, 0x3c024000, 0x03021025, 0x3c010001,
5553         0xac240f6c, 0xaf621008, 0x03e00008, 0x27bd0010, 0x3c030001, 0x90630f56,
5554         0x27bdffe8, 0x24020001, 0xafbf0014, 0x10620026, 0xafb00010, 0x8f620cf4,
5555         0x2442ffff, 0x3042007f, 0x00021100, 0x8c434000, 0x3c010001, 0xac230f64,
5556         0x8c434008, 0x24444000, 0x8c5c4004, 0x30620040, 0x14400002, 0x24020088,
5557         0x24020008, 0x3c010001, 0xa4220f68, 0x30620004, 0x10400005, 0x24020001,
5558         0x3c010001, 0xa0220f57, 0x080041d5, 0x00031402, 0x3c010001, 0xa0200f57,
5559         0x00031402, 0x3c010001, 0xa4220f54, 0x9483000c, 0x24020001, 0x3c010001,
5560         0xa4200f50, 0x3c010001, 0xa0220f56, 0x3c010001, 0xa4230f62, 0x24020001,
5561         0x1342001e, 0x00000000, 0x13400005, 0x24020003, 0x13420067, 0x00000000,
5562         0x080042cf, 0x00000000, 0x3c020001, 0x94420f62, 0x241a0001, 0x3c010001,
5563         0xa4200f5e, 0x3c010001, 0xa4200f52, 0x304407ff, 0x00021bc2, 0x00031823,
5564         0x3063003e, 0x34630036, 0x00021242, 0x3042003c, 0x00621821, 0x3c010001,
5565         0xa4240f58, 0x00832021, 0x24630030, 0x3c010001, 0xa4240f5a, 0x3c010001,
5566         0xa4230f5c, 0x3c060001, 0x24c60f52, 0x94c50000, 0x94c30002, 0x3c040001,
5567         0x94840f5a, 0x00651021, 0x0044102a, 0x10400013, 0x3c108000, 0x00a31021,
5568         0xa4c20000, 0x3c02a000, 0xaf620cf4, 0x3c010001, 0xa0200f56, 0x8f641008,
5569         0x00901024, 0x14400003, 0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4,
5570         0x00501024, 0x104000b7, 0x00000000, 0x0800420f, 0x00000000, 0x3c030001,
5571         0x94630f50, 0x00851023, 0xa4c40000, 0x00621821, 0x3042ffff, 0x3c010001,
5572         0xa4230f50, 0xaf620ce8, 0x3c020001, 0x94420f68, 0x34420024, 0xaf620cec,
5573         0x94c30002, 0x3c020001, 0x94420f50, 0x14620012, 0x3c028000, 0x3c108000,
5574         0x3c02a000, 0xaf620cf4, 0x3c010001, 0xa0200f56, 0x8f641008, 0x00901024,
5575         0x14400003, 0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4, 0x00501024,
5576         0x1440fff7, 0x00000000, 0x080042cf, 0x241a0003, 0xaf620cf4, 0x3c108000,
5577         0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064, 0x00000000,
5578         0x8f620cf4, 0x00501024, 0x1440fff7, 0x00000000, 0x080042cf, 0x241a0003,
5579         0x3c070001, 0x24e70f50, 0x94e20000, 0x03821021, 0xaf620ce0, 0x3c020001,
5580         0x8c420f64, 0xaf620ce4, 0x3c050001, 0x94a50f54, 0x94e30000, 0x3c040001,
5581         0x94840f58, 0x3c020001, 0x94420f5e, 0x00a32823, 0x00822023, 0x30a6ffff,
5582         0x3083ffff, 0x00c3102b, 0x14400043, 0x00000000, 0x3c020001, 0x94420f5c,
5583         0x00021400, 0x00621025, 0xaf620ce8, 0x94e20000, 0x3c030001, 0x94630f54,
5584         0x00441021, 0xa4e20000, 0x3042ffff, 0x14430021, 0x3c020008, 0x3c020001,
5585         0x90420f57, 0x10400006, 0x3c03000c, 0x3c020001, 0x94420f68, 0x34630624,
5586         0x0800427c, 0x0000d021, 0x3c020001, 0x94420f68, 0x3c030008, 0x34630624,
5587         0x00431025, 0xaf620cec, 0x3c108000, 0x3c02a000, 0xaf620cf4, 0x3c010001,
5588         0xa0200f56, 0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064,
5589         0x00000000, 0x8f620cf4, 0x00501024, 0x10400015, 0x00000000, 0x08004283,
5590         0x00000000, 0x3c030001, 0x94630f68, 0x34420624, 0x3c108000, 0x00621825,
5591         0x3c028000, 0xaf630cec, 0xaf620cf4, 0x8f641008, 0x00901024, 0x14400003,
5592         0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4, 0x00501024, 0x1440fff7,
5593         0x00000000, 0x3c010001, 0x080042cf, 0xa4200f5e, 0x3c020001, 0x94420f5c,
5594         0x00021400, 0x00c21025, 0xaf620ce8, 0x3c020001, 0x90420f57, 0x10400009,
5595         0x3c03000c, 0x3c020001, 0x94420f68, 0x34630624, 0x0000d021, 0x00431025,
5596         0xaf620cec, 0x080042c1, 0x3c108000, 0x3c020001, 0x94420f68, 0x3c030008,
5597         0x34630604, 0x00431025, 0xaf620cec, 0x3c020001, 0x94420f5e, 0x00451021,
5598         0x3c010001, 0xa4220f5e, 0x3c108000, 0x3c02a000, 0xaf620cf4, 0x3c010001,
5599         0xa0200f56, 0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064,
5600         0x00000000, 0x8f620cf4, 0x00501024, 0x1440fff7, 0x00000000, 0x8fbf0014,
5601         0x8fb00010, 0x03e00008, 0x27bd0018, 0x00000000, 0x27bdffe0, 0x3c040001,
5602         0x24840ec0, 0x00002821, 0x00003021, 0x00003821, 0xafbf0018, 0xafa00010,
5603         0x0c004378, 0xafa00014, 0x0000d021, 0x24020130, 0xaf625000, 0x3c010001,
5604         0xa4200f50, 0x3c010001, 0xa0200f57, 0x8fbf0018, 0x03e00008, 0x27bd0020,
5605         0x27bdffe8, 0x3c1bc000, 0xafbf0014, 0xafb00010, 0xaf60680c, 0x8f626804,
5606         0x34420082, 0xaf626804, 0x8f634000, 0x24020b50, 0x3c010001, 0xac220f20,
5607         0x24020b78, 0x3c010001, 0xac220f30, 0x34630002, 0xaf634000, 0x0c004315,
5608         0x00808021, 0x3c010001, 0xa0220f34, 0x304200ff, 0x24030002, 0x14430005,
5609         0x00000000, 0x3c020001, 0x8c420f20, 0x08004308, 0xac5000c0, 0x3c020001,
5610         0x8c420f20, 0xac5000bc, 0x8f624434, 0x8f634438, 0x8f644410, 0x3c010001,
5611         0xac220f28, 0x3c010001, 0xac230f38, 0x3c010001, 0xac240f24, 0x8fbf0014,
5612         0x8fb00010, 0x03e00008, 0x27bd0018, 0x03e00008, 0x24020001, 0x27bdfff8,
5613         0x18800009, 0x00002821, 0x8f63680c, 0x8f62680c, 0x1043fffe, 0x00000000,
5614         0x24a50001, 0x00a4102a, 0x1440fff9, 0x00000000, 0x03e00008, 0x27bd0008,
5615         0x8f634450, 0x3c020001, 0x8c420f28, 0x00031c02, 0x0043102b, 0x14400008,
5616         0x3c038000, 0x3c040001, 0x8c840f38, 0x8f624450, 0x00021c02, 0x0083102b,
5617         0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024, 0x1440fffd,
5618         0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3082ffff, 0x2442e000,
5619         0x2c422001, 0x14400003, 0x3c024000, 0x08004347, 0x2402ffff, 0x00822025,
5620         0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00001021, 0x03e00008,
5621         0x00000000, 0x8f624450, 0x3c030001, 0x8c630f24, 0x08004350, 0x3042ffff,
5622         0x8f624450, 0x3042ffff, 0x0043102b, 0x1440fffc, 0x00000000, 0x03e00008,
5623         0x00000000, 0x27bdffe0, 0x00802821, 0x3c040001, 0x24840ed0, 0x00003021,
5624         0x00003821, 0xafbf0018, 0xafa00010, 0x0c004378, 0xafa00014, 0x0800435f,
5625         0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x3c020001, 0x3442d600,
5626         0x3c030001, 0x3463d600, 0x3c040001, 0x3484ddff, 0x3c010001, 0xac220f40,
5627         0x24020040, 0x3c010001, 0xac220f44, 0x3c010001, 0xac200f3c, 0xac600000,
5628