3cb7e613ab3023e2d2081724525f3f4455aa71c1
[linux-2.6.git] / drivers / net / sfc / falcon.c
1 /****************************************************************************
2  * Driver for Solarflare Solarstorm network controllers and boards
3  * Copyright 2005-2006 Fen Systems Ltd.
4  * Copyright 2006-2008 Solarflare Communications Inc.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU General Public License version 2 as published
8  * by the Free Software Foundation, incorporated herein by reference.
9  */
10
11 #include <linux/bitops.h>
12 #include <linux/delay.h>
13 #include <linux/pci.h>
14 #include <linux/module.h>
15 #include <linux/seq_file.h>
16 #include <linux/i2c.h>
17 #include <linux/i2c-algo-bit.h>
18 #include <linux/mii.h>
19 #include "net_driver.h"
20 #include "bitfield.h"
21 #include "efx.h"
22 #include "mac.h"
23 #include "spi.h"
24 #include "falcon.h"
25 #include "regs.h"
26 #include "io.h"
27 #include "mdio_10g.h"
28 #include "phy.h"
29 #include "workarounds.h"
30
31 /* Falcon hardware control.
32  * Falcon is the internal codename for the SFC4000 controller that is
33  * present in SFE400X evaluation boards
34  */
35
36 /**
37  * struct falcon_nic_data - Falcon NIC state
38  * @next_buffer_table: First available buffer table id
39  * @pci_dev2: The secondary PCI device if present
40  * @i2c_data: Operations and state for I2C bit-bashing algorithm
41  * @int_error_count: Number of internal errors seen recently
42  * @int_error_expire: Time at which error count will be expired
43  */
44 struct falcon_nic_data {
45         unsigned next_buffer_table;
46         struct pci_dev *pci_dev2;
47         struct i2c_algo_bit_data i2c_data;
48
49         unsigned int_error_count;
50         unsigned long int_error_expire;
51 };
52
53 /**************************************************************************
54  *
55  * Configurable values
56  *
57  **************************************************************************
58  */
59
60 static int disable_dma_stats;
61
62 /* This is set to 16 for a good reason.  In summary, if larger than
63  * 16, the descriptor cache holds more than a default socket
64  * buffer's worth of packets (for UDP we can only have at most one
65  * socket buffer's worth outstanding).  This combined with the fact
66  * that we only get 1 TX event per descriptor cache means the NIC
67  * goes idle.
68  */
69 #define TX_DC_ENTRIES 16
70 #define TX_DC_ENTRIES_ORDER 0
71 #define TX_DC_BASE 0x130000
72
73 #define RX_DC_ENTRIES 64
74 #define RX_DC_ENTRIES_ORDER 2
75 #define RX_DC_BASE 0x100000
76
77 static const unsigned int
78 /* "Large" EEPROM device: Atmel AT25640 or similar
79  * 8 KB, 16-bit address, 32 B write block */
80 large_eeprom_type = ((13 << SPI_DEV_TYPE_SIZE_LBN)
81                      | (2 << SPI_DEV_TYPE_ADDR_LEN_LBN)
82                      | (5 << SPI_DEV_TYPE_BLOCK_SIZE_LBN)),
83 /* Default flash device: Atmel AT25F1024
84  * 128 KB, 24-bit address, 32 KB erase block, 256 B write block */
85 default_flash_type = ((17 << SPI_DEV_TYPE_SIZE_LBN)
86                       | (3 << SPI_DEV_TYPE_ADDR_LEN_LBN)
87                       | (0x52 << SPI_DEV_TYPE_ERASE_CMD_LBN)
88                       | (15 << SPI_DEV_TYPE_ERASE_SIZE_LBN)
89                       | (8 << SPI_DEV_TYPE_BLOCK_SIZE_LBN));
90
91 /* RX FIFO XOFF watermark
92  *
93  * When the amount of the RX FIFO increases used increases past this
94  * watermark send XOFF. Only used if RX flow control is enabled (ethtool -A)
95  * This also has an effect on RX/TX arbitration
96  */
97 static int rx_xoff_thresh_bytes = -1;
98 module_param(rx_xoff_thresh_bytes, int, 0644);
99 MODULE_PARM_DESC(rx_xoff_thresh_bytes, "RX fifo XOFF threshold");
100
101 /* RX FIFO XON watermark
102  *
103  * When the amount of the RX FIFO used decreases below this
104  * watermark send XON. Only used if TX flow control is enabled (ethtool -A)
105  * This also has an effect on RX/TX arbitration
106  */
107 static int rx_xon_thresh_bytes = -1;
108 module_param(rx_xon_thresh_bytes, int, 0644);
109 MODULE_PARM_DESC(rx_xon_thresh_bytes, "RX fifo XON threshold");
110
111 /* If FALCON_MAX_INT_ERRORS internal errors occur within
112  * FALCON_INT_ERROR_EXPIRE seconds, we consider the NIC broken and
113  * disable it.
114  */
115 #define FALCON_INT_ERROR_EXPIRE 3600
116 #define FALCON_MAX_INT_ERRORS 5
117
118 /* We poll for events every FLUSH_INTERVAL ms, and check FLUSH_POLL_COUNT times
119  */
120 #define FALCON_FLUSH_INTERVAL 10
121 #define FALCON_FLUSH_POLL_COUNT 100
122
123 /**************************************************************************
124  *
125  * Falcon constants
126  *
127  **************************************************************************
128  */
129
130 /* DMA address mask */
131 #define FALCON_DMA_MASK DMA_BIT_MASK(46)
132
133 /* TX DMA length mask (13-bit) */
134 #define FALCON_TX_DMA_MASK (4096 - 1)
135
136 /* Size and alignment of special buffers (4KB) */
137 #define FALCON_BUF_SIZE 4096
138
139 /* Dummy SRAM size code */
140 #define SRM_NB_BSZ_ONCHIP_ONLY (-1)
141
142 #define FALCON_IS_DUAL_FUNC(efx)                \
143         (falcon_rev(efx) < FALCON_REV_B0)
144
145 /**************************************************************************
146  *
147  * Falcon hardware access
148  *
149  **************************************************************************/
150
151 static inline void falcon_write_buf_tbl(struct efx_nic *efx, efx_qword_t *value,
152                                         unsigned int index)
153 {
154         efx_sram_writeq(efx, efx->membase + efx->type->buf_tbl_base,
155                         value, index);
156 }
157
158 /* Read the current event from the event queue */
159 static inline efx_qword_t *falcon_event(struct efx_channel *channel,
160                                         unsigned int index)
161 {
162         return (((efx_qword_t *) (channel->eventq.addr)) + index);
163 }
164
165 /* See if an event is present
166  *
167  * We check both the high and low dword of the event for all ones.  We
168  * wrote all ones when we cleared the event, and no valid event can
169  * have all ones in either its high or low dwords.  This approach is
170  * robust against reordering.
171  *
172  * Note that using a single 64-bit comparison is incorrect; even
173  * though the CPU read will be atomic, the DMA write may not be.
174  */
175 static inline int falcon_event_present(efx_qword_t *event)
176 {
177         return (!(EFX_DWORD_IS_ALL_ONES(event->dword[0]) |
178                   EFX_DWORD_IS_ALL_ONES(event->dword[1])));
179 }
180
181 /**************************************************************************
182  *
183  * I2C bus - this is a bit-bashing interface using GPIO pins
184  * Note that it uses the output enables to tristate the outputs
185  * SDA is the data pin and SCL is the clock
186  *
187  **************************************************************************
188  */
189 static void falcon_setsda(void *data, int state)
190 {
191         struct efx_nic *efx = (struct efx_nic *)data;
192         efx_oword_t reg;
193
194         efx_reado(efx, &reg, FR_AB_GPIO_CTL);
195         EFX_SET_OWORD_FIELD(reg, FRF_AB_GPIO3_OEN, !state);
196         efx_writeo(efx, &reg, FR_AB_GPIO_CTL);
197 }
198
199 static void falcon_setscl(void *data, int state)
200 {
201         struct efx_nic *efx = (struct efx_nic *)data;
202         efx_oword_t reg;
203
204         efx_reado(efx, &reg, FR_AB_GPIO_CTL);
205         EFX_SET_OWORD_FIELD(reg, FRF_AB_GPIO0_OEN, !state);
206         efx_writeo(efx, &reg, FR_AB_GPIO_CTL);
207 }
208
209 static int falcon_getsda(void *data)
210 {
211         struct efx_nic *efx = (struct efx_nic *)data;
212         efx_oword_t reg;
213
214         efx_reado(efx, &reg, FR_AB_GPIO_CTL);
215         return EFX_OWORD_FIELD(reg, FRF_AB_GPIO3_IN);
216 }
217
218 static int falcon_getscl(void *data)
219 {
220         struct efx_nic *efx = (struct efx_nic *)data;
221         efx_oword_t reg;
222
223         efx_reado(efx, &reg, FR_AB_GPIO_CTL);
224         return EFX_OWORD_FIELD(reg, FRF_AB_GPIO0_IN);
225 }
226
227 static struct i2c_algo_bit_data falcon_i2c_bit_operations = {
228         .setsda         = falcon_setsda,
229         .setscl         = falcon_setscl,
230         .getsda         = falcon_getsda,
231         .getscl         = falcon_getscl,
232         .udelay         = 5,
233         /* Wait up to 50 ms for slave to let us pull SCL high */
234         .timeout        = DIV_ROUND_UP(HZ, 20),
235 };
236
237 /**************************************************************************
238  *
239  * Falcon special buffer handling
240  * Special buffers are used for event queues and the TX and RX
241  * descriptor rings.
242  *
243  *************************************************************************/
244
245 /*
246  * Initialise a Falcon special buffer
247  *
248  * This will define a buffer (previously allocated via
249  * falcon_alloc_special_buffer()) in Falcon's buffer table, allowing
250  * it to be used for event queues, descriptor rings etc.
251  */
252 static void
253 falcon_init_special_buffer(struct efx_nic *efx,
254                            struct efx_special_buffer *buffer)
255 {
256         efx_qword_t buf_desc;
257         int index;
258         dma_addr_t dma_addr;
259         int i;
260
261         EFX_BUG_ON_PARANOID(!buffer->addr);
262
263         /* Write buffer descriptors to NIC */
264         for (i = 0; i < buffer->entries; i++) {
265                 index = buffer->index + i;
266                 dma_addr = buffer->dma_addr + (i * 4096);
267                 EFX_LOG(efx, "mapping special buffer %d at %llx\n",
268                         index, (unsigned long long)dma_addr);
269                 EFX_POPULATE_QWORD_3(buf_desc,
270                                      FRF_AZ_BUF_ADR_REGION, 0,
271                                      FRF_AZ_BUF_ADR_FBUF, dma_addr >> 12,
272                                      FRF_AZ_BUF_OWNER_ID_FBUF, 0);
273                 falcon_write_buf_tbl(efx, &buf_desc, index);
274         }
275 }
276
277 /* Unmaps a buffer from Falcon and clears the buffer table entries */
278 static void
279 falcon_fini_special_buffer(struct efx_nic *efx,
280                            struct efx_special_buffer *buffer)
281 {
282         efx_oword_t buf_tbl_upd;
283         unsigned int start = buffer->index;
284         unsigned int end = (buffer->index + buffer->entries - 1);
285
286         if (!buffer->entries)
287                 return;
288
289         EFX_LOG(efx, "unmapping special buffers %d-%d\n",
290                 buffer->index, buffer->index + buffer->entries - 1);
291
292         EFX_POPULATE_OWORD_4(buf_tbl_upd,
293                              FRF_AZ_BUF_UPD_CMD, 0,
294                              FRF_AZ_BUF_CLR_CMD, 1,
295                              FRF_AZ_BUF_CLR_END_ID, end,
296                              FRF_AZ_BUF_CLR_START_ID, start);
297         efx_writeo(efx, &buf_tbl_upd, FR_AZ_BUF_TBL_UPD);
298 }
299
300 /*
301  * Allocate a new Falcon special buffer
302  *
303  * This allocates memory for a new buffer, clears it and allocates a
304  * new buffer ID range.  It does not write into Falcon's buffer table.
305  *
306  * This call will allocate 4KB buffers, since Falcon can't use 8KB
307  * buffers for event queues and descriptor rings.
308  */
309 static int falcon_alloc_special_buffer(struct efx_nic *efx,
310                                        struct efx_special_buffer *buffer,
311                                        unsigned int len)
312 {
313         struct falcon_nic_data *nic_data = efx->nic_data;
314
315         len = ALIGN(len, FALCON_BUF_SIZE);
316
317         buffer->addr = pci_alloc_consistent(efx->pci_dev, len,
318                                             &buffer->dma_addr);
319         if (!buffer->addr)
320                 return -ENOMEM;
321         buffer->len = len;
322         buffer->entries = len / FALCON_BUF_SIZE;
323         BUG_ON(buffer->dma_addr & (FALCON_BUF_SIZE - 1));
324
325         /* All zeros is a potentially valid event so memset to 0xff */
326         memset(buffer->addr, 0xff, len);
327
328         /* Select new buffer ID */
329         buffer->index = nic_data->next_buffer_table;
330         nic_data->next_buffer_table += buffer->entries;
331
332         EFX_LOG(efx, "allocating special buffers %d-%d at %llx+%x "
333                 "(virt %p phys %llx)\n", buffer->index,
334                 buffer->index + buffer->entries - 1,
335                 (u64)buffer->dma_addr, len,
336                 buffer->addr, (u64)virt_to_phys(buffer->addr));
337
338         return 0;
339 }
340
341 static void falcon_free_special_buffer(struct efx_nic *efx,
342                                        struct efx_special_buffer *buffer)
343 {
344         if (!buffer->addr)
345                 return;
346
347         EFX_LOG(efx, "deallocating special buffers %d-%d at %llx+%x "
348                 "(virt %p phys %llx)\n", buffer->index,
349                 buffer->index + buffer->entries - 1,
350                 (u64)buffer->dma_addr, buffer->len,
351                 buffer->addr, (u64)virt_to_phys(buffer->addr));
352
353         pci_free_consistent(efx->pci_dev, buffer->len, buffer->addr,
354                             buffer->dma_addr);
355         buffer->addr = NULL;
356         buffer->entries = 0;
357 }
358
359 /**************************************************************************
360  *
361  * Falcon generic buffer handling
362  * These buffers are used for interrupt status and MAC stats
363  *
364  **************************************************************************/
365
366 static int falcon_alloc_buffer(struct efx_nic *efx,
367                                struct efx_buffer *buffer, unsigned int len)
368 {
369         buffer->addr = pci_alloc_consistent(efx->pci_dev, len,
370                                             &buffer->dma_addr);
371         if (!buffer->addr)
372                 return -ENOMEM;
373         buffer->len = len;
374         memset(buffer->addr, 0, len);
375         return 0;
376 }
377
378 static void falcon_free_buffer(struct efx_nic *efx, struct efx_buffer *buffer)
379 {
380         if (buffer->addr) {
381                 pci_free_consistent(efx->pci_dev, buffer->len,
382                                     buffer->addr, buffer->dma_addr);
383                 buffer->addr = NULL;
384         }
385 }
386
387 /**************************************************************************
388  *
389  * Falcon TX path
390  *
391  **************************************************************************/
392
393 /* Returns a pointer to the specified transmit descriptor in the TX
394  * descriptor queue belonging to the specified channel.
395  */
396 static inline efx_qword_t *falcon_tx_desc(struct efx_tx_queue *tx_queue,
397                                                unsigned int index)
398 {
399         return (((efx_qword_t *) (tx_queue->txd.addr)) + index);
400 }
401
402 /* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */
403 static inline void falcon_notify_tx_desc(struct efx_tx_queue *tx_queue)
404 {
405         unsigned write_ptr;
406         efx_dword_t reg;
407
408         write_ptr = tx_queue->write_count & EFX_TXQ_MASK;
409         EFX_POPULATE_DWORD_1(reg, FRF_AZ_TX_DESC_WPTR_DWORD, write_ptr);
410         efx_writed_page(tx_queue->efx, &reg,
411                         FR_AZ_TX_DESC_UPD_DWORD_P0, tx_queue->queue);
412 }
413
414
415 /* For each entry inserted into the software descriptor ring, create a
416  * descriptor in the hardware TX descriptor ring (in host memory), and
417  * write a doorbell.
418  */
419 void falcon_push_buffers(struct efx_tx_queue *tx_queue)
420 {
421
422         struct efx_tx_buffer *buffer;
423         efx_qword_t *txd;
424         unsigned write_ptr;
425
426         BUG_ON(tx_queue->write_count == tx_queue->insert_count);
427
428         do {
429                 write_ptr = tx_queue->write_count & EFX_TXQ_MASK;
430                 buffer = &tx_queue->buffer[write_ptr];
431                 txd = falcon_tx_desc(tx_queue, write_ptr);
432                 ++tx_queue->write_count;
433
434                 /* Create TX descriptor ring entry */
435                 EFX_POPULATE_QWORD_4(*txd,
436                                      FSF_AZ_TX_KER_CONT, buffer->continuation,
437                                      FSF_AZ_TX_KER_BYTE_COUNT, buffer->len,
438                                      FSF_AZ_TX_KER_BUF_REGION, 0,
439                                      FSF_AZ_TX_KER_BUF_ADDR, buffer->dma_addr);
440         } while (tx_queue->write_count != tx_queue->insert_count);
441
442         wmb(); /* Ensure descriptors are written before they are fetched */
443         falcon_notify_tx_desc(tx_queue);
444 }
445
446 /* Allocate hardware resources for a TX queue */
447 int falcon_probe_tx(struct efx_tx_queue *tx_queue)
448 {
449         struct efx_nic *efx = tx_queue->efx;
450         BUILD_BUG_ON(EFX_TXQ_SIZE < 512 || EFX_TXQ_SIZE > 4096 ||
451                      EFX_TXQ_SIZE & EFX_TXQ_MASK);
452         return falcon_alloc_special_buffer(efx, &tx_queue->txd,
453                                            EFX_TXQ_SIZE * sizeof(efx_qword_t));
454 }
455
456 void falcon_init_tx(struct efx_tx_queue *tx_queue)
457 {
458         efx_oword_t tx_desc_ptr;
459         struct efx_nic *efx = tx_queue->efx;
460
461         tx_queue->flushed = false;
462
463         /* Pin TX descriptor ring */
464         falcon_init_special_buffer(efx, &tx_queue->txd);
465
466         /* Push TX descriptor ring to card */
467         EFX_POPULATE_OWORD_10(tx_desc_ptr,
468                               FRF_AZ_TX_DESCQ_EN, 1,
469                               FRF_AZ_TX_ISCSI_DDIG_EN, 0,
470                               FRF_AZ_TX_ISCSI_HDIG_EN, 0,
471                               FRF_AZ_TX_DESCQ_BUF_BASE_ID, tx_queue->txd.index,
472                               FRF_AZ_TX_DESCQ_EVQ_ID,
473                               tx_queue->channel->channel,
474                               FRF_AZ_TX_DESCQ_OWNER_ID, 0,
475                               FRF_AZ_TX_DESCQ_LABEL, tx_queue->queue,
476                               FRF_AZ_TX_DESCQ_SIZE,
477                               __ffs(tx_queue->txd.entries),
478                               FRF_AZ_TX_DESCQ_TYPE, 0,
479                               FRF_BZ_TX_NON_IP_DROP_DIS, 1);
480
481         if (falcon_rev(efx) >= FALCON_REV_B0) {
482                 int csum = tx_queue->queue == EFX_TX_QUEUE_OFFLOAD_CSUM;
483                 EFX_SET_OWORD_FIELD(tx_desc_ptr, FRF_BZ_TX_IP_CHKSM_DIS, !csum);
484                 EFX_SET_OWORD_FIELD(tx_desc_ptr, FRF_BZ_TX_TCP_CHKSM_DIS,
485                                     !csum);
486         }
487
488         efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base,
489                          tx_queue->queue);
490
491         if (falcon_rev(efx) < FALCON_REV_B0) {
492                 efx_oword_t reg;
493
494                 /* Only 128 bits in this register */
495                 BUILD_BUG_ON(EFX_TX_QUEUE_COUNT >= 128);
496
497                 efx_reado(efx, &reg, FR_AA_TX_CHKSM_CFG);
498                 if (tx_queue->queue == EFX_TX_QUEUE_OFFLOAD_CSUM)
499                         clear_bit_le(tx_queue->queue, (void *)&reg);
500                 else
501                         set_bit_le(tx_queue->queue, (void *)&reg);
502                 efx_writeo(efx, &reg, FR_AA_TX_CHKSM_CFG);
503         }
504 }
505
506 static void falcon_flush_tx_queue(struct efx_tx_queue *tx_queue)
507 {
508         struct efx_nic *efx = tx_queue->efx;
509         efx_oword_t tx_flush_descq;
510
511         /* Post a flush command */
512         EFX_POPULATE_OWORD_2(tx_flush_descq,
513                              FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
514                              FRF_AZ_TX_FLUSH_DESCQ, tx_queue->queue);
515         efx_writeo(efx, &tx_flush_descq, FR_AZ_TX_FLUSH_DESCQ);
516 }
517
518 void falcon_fini_tx(struct efx_tx_queue *tx_queue)
519 {
520         struct efx_nic *efx = tx_queue->efx;
521         efx_oword_t tx_desc_ptr;
522
523         /* The queue should have been flushed */
524         WARN_ON(!tx_queue->flushed);
525
526         /* Remove TX descriptor ring from card */
527         EFX_ZERO_OWORD(tx_desc_ptr);
528         efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base,
529                          tx_queue->queue);
530
531         /* Unpin TX descriptor ring */
532         falcon_fini_special_buffer(efx, &tx_queue->txd);
533 }
534
535 /* Free buffers backing TX queue */
536 void falcon_remove_tx(struct efx_tx_queue *tx_queue)
537 {
538         falcon_free_special_buffer(tx_queue->efx, &tx_queue->txd);
539 }
540
541 /**************************************************************************
542  *
543  * Falcon RX path
544  *
545  **************************************************************************/
546
547 /* Returns a pointer to the specified descriptor in the RX descriptor queue */
548 static inline efx_qword_t *falcon_rx_desc(struct efx_rx_queue *rx_queue,
549                                                unsigned int index)
550 {
551         return (((efx_qword_t *) (rx_queue->rxd.addr)) + index);
552 }
553
554 /* This creates an entry in the RX descriptor queue */
555 static inline void falcon_build_rx_desc(struct efx_rx_queue *rx_queue,
556                                         unsigned index)
557 {
558         struct efx_rx_buffer *rx_buf;
559         efx_qword_t *rxd;
560
561         rxd = falcon_rx_desc(rx_queue, index);
562         rx_buf = efx_rx_buffer(rx_queue, index);
563         EFX_POPULATE_QWORD_3(*rxd,
564                              FSF_AZ_RX_KER_BUF_SIZE,
565                              rx_buf->len -
566                              rx_queue->efx->type->rx_buffer_padding,
567                              FSF_AZ_RX_KER_BUF_REGION, 0,
568                              FSF_AZ_RX_KER_BUF_ADDR, rx_buf->dma_addr);
569 }
570
571 /* This writes to the RX_DESC_WPTR register for the specified receive
572  * descriptor ring.
573  */
574 void falcon_notify_rx_desc(struct efx_rx_queue *rx_queue)
575 {
576         efx_dword_t reg;
577         unsigned write_ptr;
578
579         while (rx_queue->notified_count != rx_queue->added_count) {
580                 falcon_build_rx_desc(rx_queue,
581                                      rx_queue->notified_count &
582                                      EFX_RXQ_MASK);
583                 ++rx_queue->notified_count;
584         }
585
586         wmb();
587         write_ptr = rx_queue->added_count & EFX_RXQ_MASK;
588         EFX_POPULATE_DWORD_1(reg, FRF_AZ_RX_DESC_WPTR_DWORD, write_ptr);
589         efx_writed_page(rx_queue->efx, &reg,
590                         FR_AZ_RX_DESC_UPD_DWORD_P0, rx_queue->queue);
591 }
592
593 int falcon_probe_rx(struct efx_rx_queue *rx_queue)
594 {
595         struct efx_nic *efx = rx_queue->efx;
596         BUILD_BUG_ON(EFX_RXQ_SIZE < 512 || EFX_RXQ_SIZE > 4096 ||
597                      EFX_RXQ_SIZE & EFX_RXQ_MASK);
598         return falcon_alloc_special_buffer(efx, &rx_queue->rxd,
599                                            EFX_RXQ_SIZE * sizeof(efx_qword_t));
600 }
601
602 void falcon_init_rx(struct efx_rx_queue *rx_queue)
603 {
604         efx_oword_t rx_desc_ptr;
605         struct efx_nic *efx = rx_queue->efx;
606         bool is_b0 = falcon_rev(efx) >= FALCON_REV_B0;
607         bool iscsi_digest_en = is_b0;
608
609         EFX_LOG(efx, "RX queue %d ring in special buffers %d-%d\n",
610                 rx_queue->queue, rx_queue->rxd.index,
611                 rx_queue->rxd.index + rx_queue->rxd.entries - 1);
612
613         rx_queue->flushed = false;
614
615         /* Pin RX descriptor ring */
616         falcon_init_special_buffer(efx, &rx_queue->rxd);
617
618         /* Push RX descriptor ring to card */
619         EFX_POPULATE_OWORD_10(rx_desc_ptr,
620                               FRF_AZ_RX_ISCSI_DDIG_EN, iscsi_digest_en,
621                               FRF_AZ_RX_ISCSI_HDIG_EN, iscsi_digest_en,
622                               FRF_AZ_RX_DESCQ_BUF_BASE_ID, rx_queue->rxd.index,
623                               FRF_AZ_RX_DESCQ_EVQ_ID,
624                               rx_queue->channel->channel,
625                               FRF_AZ_RX_DESCQ_OWNER_ID, 0,
626                               FRF_AZ_RX_DESCQ_LABEL, rx_queue->queue,
627                               FRF_AZ_RX_DESCQ_SIZE,
628                               __ffs(rx_queue->rxd.entries),
629                               FRF_AZ_RX_DESCQ_TYPE, 0 /* kernel queue */ ,
630                               /* For >=B0 this is scatter so disable */
631                               FRF_AZ_RX_DESCQ_JUMBO, !is_b0,
632                               FRF_AZ_RX_DESCQ_EN, 1);
633         efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
634                          rx_queue->queue);
635 }
636
637 static void falcon_flush_rx_queue(struct efx_rx_queue *rx_queue)
638 {
639         struct efx_nic *efx = rx_queue->efx;
640         efx_oword_t rx_flush_descq;
641
642         /* Post a flush command */
643         EFX_POPULATE_OWORD_2(rx_flush_descq,
644                              FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
645                              FRF_AZ_RX_FLUSH_DESCQ, rx_queue->queue);
646         efx_writeo(efx, &rx_flush_descq, FR_AZ_RX_FLUSH_DESCQ);
647 }
648
649 void falcon_fini_rx(struct efx_rx_queue *rx_queue)
650 {
651         efx_oword_t rx_desc_ptr;
652         struct efx_nic *efx = rx_queue->efx;
653
654         /* The queue should already have been flushed */
655         WARN_ON(!rx_queue->flushed);
656
657         /* Remove RX descriptor ring from card */
658         EFX_ZERO_OWORD(rx_desc_ptr);
659         efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
660                          rx_queue->queue);
661
662         /* Unpin RX descriptor ring */
663         falcon_fini_special_buffer(efx, &rx_queue->rxd);
664 }
665
666 /* Free buffers backing RX queue */
667 void falcon_remove_rx(struct efx_rx_queue *rx_queue)
668 {
669         falcon_free_special_buffer(rx_queue->efx, &rx_queue->rxd);
670 }
671
672 /**************************************************************************
673  *
674  * Falcon event queue processing
675  * Event queues are processed by per-channel tasklets.
676  *
677  **************************************************************************/
678
679 /* Update a channel's event queue's read pointer (RPTR) register
680  *
681  * This writes the EVQ_RPTR_REG register for the specified channel's
682  * event queue.
683  *
684  * Note that EVQ_RPTR_REG contains the index of the "last read" event,
685  * whereas channel->eventq_read_ptr contains the index of the "next to
686  * read" event.
687  */
688 void falcon_eventq_read_ack(struct efx_channel *channel)
689 {
690         efx_dword_t reg;
691         struct efx_nic *efx = channel->efx;
692
693         EFX_POPULATE_DWORD_1(reg, FRF_AZ_EVQ_RPTR, channel->eventq_read_ptr);
694         efx_writed_table(efx, &reg, efx->type->evq_rptr_tbl_base,
695                             channel->channel);
696 }
697
698 /* Use HW to insert a SW defined event */
699 void falcon_generate_event(struct efx_channel *channel, efx_qword_t *event)
700 {
701         efx_oword_t drv_ev_reg;
702
703         BUILD_BUG_ON(FRF_AZ_DRV_EV_DATA_LBN != 0 ||
704                      FRF_AZ_DRV_EV_DATA_WIDTH != 64);
705         drv_ev_reg.u32[0] = event->u32[0];
706         drv_ev_reg.u32[1] = event->u32[1];
707         drv_ev_reg.u32[2] = 0;
708         drv_ev_reg.u32[3] = 0;
709         EFX_SET_OWORD_FIELD(drv_ev_reg, FRF_AZ_DRV_EV_QID, channel->channel);
710         efx_writeo(channel->efx, &drv_ev_reg, FR_AZ_DRV_EV);
711 }
712
713 /* Handle a transmit completion event
714  *
715  * Falcon batches TX completion events; the message we receive is of
716  * the form "complete all TX events up to this index".
717  */
718 static void falcon_handle_tx_event(struct efx_channel *channel,
719                                    efx_qword_t *event)
720 {
721         unsigned int tx_ev_desc_ptr;
722         unsigned int tx_ev_q_label;
723         struct efx_tx_queue *tx_queue;
724         struct efx_nic *efx = channel->efx;
725
726         if (likely(EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_COMP))) {
727                 /* Transmit completion */
728                 tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_DESC_PTR);
729                 tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
730                 tx_queue = &efx->tx_queue[tx_ev_q_label];
731                 channel->irq_mod_score +=
732                         (tx_ev_desc_ptr - tx_queue->read_count) &
733                         EFX_TXQ_MASK;
734                 efx_xmit_done(tx_queue, tx_ev_desc_ptr);
735         } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_WQ_FF_FULL)) {
736                 /* Rewrite the FIFO write pointer */
737                 tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
738                 tx_queue = &efx->tx_queue[tx_ev_q_label];
739
740                 if (efx_dev_registered(efx))
741                         netif_tx_lock(efx->net_dev);
742                 falcon_notify_tx_desc(tx_queue);
743                 if (efx_dev_registered(efx))
744                         netif_tx_unlock(efx->net_dev);
745         } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_PKT_ERR) &&
746                    EFX_WORKAROUND_10727(efx)) {
747                 efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH);
748         } else {
749                 EFX_ERR(efx, "channel %d unexpected TX event "
750                         EFX_QWORD_FMT"\n", channel->channel,
751                         EFX_QWORD_VAL(*event));
752         }
753 }
754
755 /* Detect errors included in the rx_evt_pkt_ok bit. */
756 static void falcon_handle_rx_not_ok(struct efx_rx_queue *rx_queue,
757                                     const efx_qword_t *event,
758                                     bool *rx_ev_pkt_ok,
759                                     bool *discard)
760 {
761         struct efx_nic *efx = rx_queue->efx;
762         bool rx_ev_buf_owner_id_err, rx_ev_ip_hdr_chksum_err;
763         bool rx_ev_tcp_udp_chksum_err, rx_ev_eth_crc_err;
764         bool rx_ev_frm_trunc, rx_ev_drib_nib, rx_ev_tobe_disc;
765         bool rx_ev_other_err, rx_ev_pause_frm;
766         bool rx_ev_ip_frag_err, rx_ev_hdr_type, rx_ev_mcast_pkt;
767         unsigned rx_ev_pkt_type;
768
769         rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
770         rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
771         rx_ev_tobe_disc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_TOBE_DISC);
772         rx_ev_pkt_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_TYPE);
773         rx_ev_buf_owner_id_err = EFX_QWORD_FIELD(*event,
774                                                  FSF_AZ_RX_EV_BUF_OWNER_ID_ERR);
775         rx_ev_ip_frag_err = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_IP_FRAG_ERR);
776         rx_ev_ip_hdr_chksum_err = EFX_QWORD_FIELD(*event,
777                                                   FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR);
778         rx_ev_tcp_udp_chksum_err = EFX_QWORD_FIELD(*event,
779                                                    FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR);
780         rx_ev_eth_crc_err = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_ETH_CRC_ERR);
781         rx_ev_frm_trunc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_FRM_TRUNC);
782         rx_ev_drib_nib = ((falcon_rev(efx) >= FALCON_REV_B0) ?
783                           0 : EFX_QWORD_FIELD(*event, FSF_AA_RX_EV_DRIB_NIB));
784         rx_ev_pause_frm = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PAUSE_FRM_ERR);
785
786         /* Every error apart from tobe_disc and pause_frm */
787         rx_ev_other_err = (rx_ev_drib_nib | rx_ev_tcp_udp_chksum_err |
788                            rx_ev_buf_owner_id_err | rx_ev_eth_crc_err |
789                            rx_ev_frm_trunc | rx_ev_ip_hdr_chksum_err);
790
791         /* Count errors that are not in MAC stats.  Ignore expected
792          * checksum errors during self-test. */
793         if (rx_ev_frm_trunc)
794                 ++rx_queue->channel->n_rx_frm_trunc;
795         else if (rx_ev_tobe_disc)
796                 ++rx_queue->channel->n_rx_tobe_disc;
797         else if (!efx->loopback_selftest) {
798                 if (rx_ev_ip_hdr_chksum_err)
799                         ++rx_queue->channel->n_rx_ip_hdr_chksum_err;
800                 else if (rx_ev_tcp_udp_chksum_err)
801                         ++rx_queue->channel->n_rx_tcp_udp_chksum_err;
802         }
803         if (rx_ev_ip_frag_err)
804                 ++rx_queue->channel->n_rx_ip_frag_err;
805
806         /* The frame must be discarded if any of these are true. */
807         *discard = (rx_ev_eth_crc_err | rx_ev_frm_trunc | rx_ev_drib_nib |
808                     rx_ev_tobe_disc | rx_ev_pause_frm);
809
810         /* TOBE_DISC is expected on unicast mismatches; don't print out an
811          * error message.  FRM_TRUNC indicates RXDP dropped the packet due
812          * to a FIFO overflow.
813          */
814 #ifdef EFX_ENABLE_DEBUG
815         if (rx_ev_other_err) {
816                 EFX_INFO_RL(efx, " RX queue %d unexpected RX event "
817                             EFX_QWORD_FMT "%s%s%s%s%s%s%s%s\n",
818                             rx_queue->queue, EFX_QWORD_VAL(*event),
819                             rx_ev_buf_owner_id_err ? " [OWNER_ID_ERR]" : "",
820                             rx_ev_ip_hdr_chksum_err ?
821                             " [IP_HDR_CHKSUM_ERR]" : "",
822                             rx_ev_tcp_udp_chksum_err ?
823                             " [TCP_UDP_CHKSUM_ERR]" : "",
824                             rx_ev_eth_crc_err ? " [ETH_CRC_ERR]" : "",
825                             rx_ev_frm_trunc ? " [FRM_TRUNC]" : "",
826                             rx_ev_drib_nib ? " [DRIB_NIB]" : "",
827                             rx_ev_tobe_disc ? " [TOBE_DISC]" : "",
828                             rx_ev_pause_frm ? " [PAUSE]" : "");
829         }
830 #endif
831 }
832
833 /* Handle receive events that are not in-order. */
834 static void falcon_handle_rx_bad_index(struct efx_rx_queue *rx_queue,
835                                        unsigned index)
836 {
837         struct efx_nic *efx = rx_queue->efx;
838         unsigned expected, dropped;
839
840         expected = rx_queue->removed_count & EFX_RXQ_MASK;
841         dropped = (index - expected) & EFX_RXQ_MASK;
842         EFX_INFO(efx, "dropped %d events (index=%d expected=%d)\n",
843                 dropped, index, expected);
844
845         efx_schedule_reset(efx, EFX_WORKAROUND_5676(efx) ?
846                            RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE);
847 }
848
849 /* Handle a packet received event
850  *
851  * Falcon silicon gives a "discard" flag if it's a unicast packet with the
852  * wrong destination address
853  * Also "is multicast" and "matches multicast filter" flags can be used to
854  * discard non-matching multicast packets.
855  */
856 static void falcon_handle_rx_event(struct efx_channel *channel,
857                                    const efx_qword_t *event)
858 {
859         unsigned int rx_ev_desc_ptr, rx_ev_byte_cnt;
860         unsigned int rx_ev_hdr_type, rx_ev_mcast_pkt;
861         unsigned expected_ptr;
862         bool rx_ev_pkt_ok, discard = false, checksummed;
863         struct efx_rx_queue *rx_queue;
864         struct efx_nic *efx = channel->efx;
865
866         /* Basic packet information */
867         rx_ev_byte_cnt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_BYTE_CNT);
868         rx_ev_pkt_ok = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_OK);
869         rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
870         WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_JUMBO_CONT));
871         WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_SOP) != 1);
872         WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_Q_LABEL) !=
873                 channel->channel);
874
875         rx_queue = &efx->rx_queue[channel->channel];
876
877         rx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_DESC_PTR);
878         expected_ptr = rx_queue->removed_count & EFX_RXQ_MASK;
879         if (unlikely(rx_ev_desc_ptr != expected_ptr))
880                 falcon_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr);
881
882         if (likely(rx_ev_pkt_ok)) {
883                 /* If packet is marked as OK and packet type is TCP/IPv4 or
884                  * UDP/IPv4, then we can rely on the hardware checksum.
885                  */
886                 checksummed =
887                         rx_ev_hdr_type == FSE_AB_RX_EV_HDR_TYPE_IPV4_TCP ||
888                         rx_ev_hdr_type == FSE_AB_RX_EV_HDR_TYPE_IPV4_UDP;
889         } else {
890                 falcon_handle_rx_not_ok(rx_queue, event, &rx_ev_pkt_ok,
891                                         &discard);
892                 checksummed = false;
893         }
894
895         /* Detect multicast packets that didn't match the filter */
896         rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
897         if (rx_ev_mcast_pkt) {
898                 unsigned int rx_ev_mcast_hash_match =
899                         EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_HASH_MATCH);
900
901                 if (unlikely(!rx_ev_mcast_hash_match))
902                         discard = true;
903         }
904
905         channel->irq_mod_score += 2;
906
907         /* Handle received packet */
908         efx_rx_packet(rx_queue, rx_ev_desc_ptr, rx_ev_byte_cnt,
909                       checksummed, discard);
910 }
911
912 /* Global events are basically PHY events */
913 static void falcon_handle_global_event(struct efx_channel *channel,
914                                        efx_qword_t *event)
915 {
916         struct efx_nic *efx = channel->efx;
917         bool handled = false;
918
919         if (EFX_QWORD_FIELD(*event, FSF_AB_GLB_EV_G_PHY0_INTR) ||
920             EFX_QWORD_FIELD(*event, FSF_AB_GLB_EV_XG_PHY0_INTR) ||
921             EFX_QWORD_FIELD(*event, FSF_AB_GLB_EV_XFP_PHY0_INTR)) {
922                 efx->phy_op->clear_interrupt(efx);
923                 queue_work(efx->workqueue, &efx->phy_work);
924                 handled = true;
925         }
926
927         if ((falcon_rev(efx) >= FALCON_REV_B0) &&
928             EFX_QWORD_FIELD(*event, FSF_BB_GLB_EV_XG_MGT_INTR)) {
929                 queue_work(efx->workqueue, &efx->mac_work);
930                 handled = true;
931         }
932
933         if (falcon_rev(efx) <= FALCON_REV_A1 ?
934             EFX_QWORD_FIELD(*event, FSF_AA_GLB_EV_RX_RECOVERY) :
935             EFX_QWORD_FIELD(*event, FSF_BB_GLB_EV_RX_RECOVERY)) {
936                 EFX_ERR(efx, "channel %d seen global RX_RESET "
937                         "event. Resetting.\n", channel->channel);
938
939                 atomic_inc(&efx->rx_reset);
940                 efx_schedule_reset(efx, EFX_WORKAROUND_6555(efx) ?
941                                    RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE);
942                 handled = true;
943         }
944
945         if (!handled)
946                 EFX_ERR(efx, "channel %d unknown global event "
947                         EFX_QWORD_FMT "\n", channel->channel,
948                         EFX_QWORD_VAL(*event));
949 }
950
951 static void falcon_handle_driver_event(struct efx_channel *channel,
952                                        efx_qword_t *event)
953 {
954         struct efx_nic *efx = channel->efx;
955         unsigned int ev_sub_code;
956         unsigned int ev_sub_data;
957
958         ev_sub_code = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBCODE);
959         ev_sub_data = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA);
960
961         switch (ev_sub_code) {
962         case FSE_AZ_TX_DESCQ_FLS_DONE_EV:
963                 EFX_TRACE(efx, "channel %d TXQ %d flushed\n",
964                           channel->channel, ev_sub_data);
965                 break;
966         case FSE_AZ_RX_DESCQ_FLS_DONE_EV:
967                 EFX_TRACE(efx, "channel %d RXQ %d flushed\n",
968                           channel->channel, ev_sub_data);
969                 break;
970         case FSE_AZ_EVQ_INIT_DONE_EV:
971                 EFX_LOG(efx, "channel %d EVQ %d initialised\n",
972                         channel->channel, ev_sub_data);
973                 break;
974         case FSE_AZ_SRM_UPD_DONE_EV:
975                 EFX_TRACE(efx, "channel %d SRAM update done\n",
976                           channel->channel);
977                 break;
978         case FSE_AZ_WAKE_UP_EV:
979                 EFX_TRACE(efx, "channel %d RXQ %d wakeup event\n",
980                           channel->channel, ev_sub_data);
981                 break;
982         case FSE_AZ_TIMER_EV:
983                 EFX_TRACE(efx, "channel %d RX queue %d timer expired\n",
984                           channel->channel, ev_sub_data);
985                 break;
986         case FSE_AA_RX_RECOVER_EV:
987                 EFX_ERR(efx, "channel %d seen DRIVER RX_RESET event. "
988                         "Resetting.\n", channel->channel);
989                 atomic_inc(&efx->rx_reset);
990                 efx_schedule_reset(efx,
991                                    EFX_WORKAROUND_6555(efx) ?
992                                    RESET_TYPE_RX_RECOVERY :
993                                    RESET_TYPE_DISABLE);
994                 break;
995         case FSE_BZ_RX_DSC_ERROR_EV:
996                 EFX_ERR(efx, "RX DMA Q %d reports descriptor fetch error."
997                         " RX Q %d is disabled.\n", ev_sub_data, ev_sub_data);
998                 efx_schedule_reset(efx, RESET_TYPE_RX_DESC_FETCH);
999                 break;
1000         case FSE_BZ_TX_DSC_ERROR_EV:
1001                 EFX_ERR(efx, "TX DMA Q %d reports descriptor fetch error."
1002                         " TX Q %d is disabled.\n", ev_sub_data, ev_sub_data);
1003                 efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH);
1004                 break;
1005         default:
1006                 EFX_TRACE(efx, "channel %d unknown driver event code %d "
1007                           "data %04x\n", channel->channel, ev_sub_code,
1008                           ev_sub_data);
1009                 break;
1010         }
1011 }
1012
1013 int falcon_process_eventq(struct efx_channel *channel, int rx_quota)
1014 {
1015         unsigned int read_ptr;
1016         efx_qword_t event, *p_event;
1017         int ev_code;
1018         int rx_packets = 0;
1019
1020         read_ptr = channel->eventq_read_ptr;
1021
1022         do {
1023                 p_event = falcon_event(channel, read_ptr);
1024                 event = *p_event;
1025
1026                 if (!falcon_event_present(&event))
1027                         /* End of events */
1028                         break;
1029
1030                 EFX_TRACE(channel->efx, "channel %d event is "EFX_QWORD_FMT"\n",
1031                           channel->channel, EFX_QWORD_VAL(event));
1032
1033                 /* Clear this event by marking it all ones */
1034                 EFX_SET_QWORD(*p_event);
1035
1036                 ev_code = EFX_QWORD_FIELD(event, FSF_AZ_EV_CODE);
1037
1038                 switch (ev_code) {
1039                 case FSE_AZ_EV_CODE_RX_EV:
1040                         falcon_handle_rx_event(channel, &event);
1041                         ++rx_packets;
1042                         break;
1043                 case FSE_AZ_EV_CODE_TX_EV:
1044                         falcon_handle_tx_event(channel, &event);
1045                         break;
1046                 case FSE_AZ_EV_CODE_DRV_GEN_EV:
1047                         channel->eventq_magic = EFX_QWORD_FIELD(
1048                                 event, FSF_AZ_DRV_GEN_EV_MAGIC);
1049                         EFX_LOG(channel->efx, "channel %d received generated "
1050                                 "event "EFX_QWORD_FMT"\n", channel->channel,
1051                                 EFX_QWORD_VAL(event));
1052                         break;
1053                 case FSE_AZ_EV_CODE_GLOBAL_EV:
1054                         falcon_handle_global_event(channel, &event);
1055                         break;
1056                 case FSE_AZ_EV_CODE_DRIVER_EV:
1057                         falcon_handle_driver_event(channel, &event);
1058                         break;
1059                 default:
1060                         EFX_ERR(channel->efx, "channel %d unknown event type %d"
1061                                 " (data " EFX_QWORD_FMT ")\n", channel->channel,
1062                                 ev_code, EFX_QWORD_VAL(event));
1063                 }
1064
1065                 /* Increment read pointer */
1066                 read_ptr = (read_ptr + 1) & EFX_EVQ_MASK;
1067
1068         } while (rx_packets < rx_quota);
1069
1070         channel->eventq_read_ptr = read_ptr;
1071         return rx_packets;
1072 }
1073
1074 void falcon_set_int_moderation(struct efx_channel *channel)
1075 {
1076         efx_dword_t timer_cmd;
1077         struct efx_nic *efx = channel->efx;
1078
1079         /* Set timer register */
1080         if (channel->irq_moderation) {
1081                 /* Round to resolution supported by hardware.  The value we
1082                  * program is based at 0.  So actual interrupt moderation
1083                  * achieved is ((x + 1) * res).
1084                  */
1085                 channel->irq_moderation -= (channel->irq_moderation %
1086                                             FALCON_IRQ_MOD_RESOLUTION);
1087                 if (channel->irq_moderation < FALCON_IRQ_MOD_RESOLUTION)
1088                         channel->irq_moderation = FALCON_IRQ_MOD_RESOLUTION;
1089                 EFX_POPULATE_DWORD_2(timer_cmd,
1090                                      FRF_AB_TC_TIMER_MODE,
1091                                      FFE_BB_TIMER_MODE_INT_HLDOFF,
1092                                      FRF_AB_TC_TIMER_VAL,
1093                                      channel->irq_moderation /
1094                                      FALCON_IRQ_MOD_RESOLUTION - 1);
1095         } else {
1096                 EFX_POPULATE_DWORD_2(timer_cmd,
1097                                      FRF_AB_TC_TIMER_MODE,
1098                                      FFE_BB_TIMER_MODE_DIS,
1099                                      FRF_AB_TC_TIMER_VAL, 0);
1100         }
1101         BUILD_BUG_ON(FR_AA_TIMER_COMMAND_KER != FR_BZ_TIMER_COMMAND_P0);
1102         efx_writed_page_locked(efx, &timer_cmd, FR_BZ_TIMER_COMMAND_P0,
1103                                channel->channel);
1104
1105 }
1106
1107 /* Allocate buffer table entries for event queue */
1108 int falcon_probe_eventq(struct efx_channel *channel)
1109 {
1110         struct efx_nic *efx = channel->efx;
1111         BUILD_BUG_ON(EFX_EVQ_SIZE < 512 || EFX_EVQ_SIZE > 32768 ||
1112                      EFX_EVQ_SIZE & EFX_EVQ_MASK);
1113         return falcon_alloc_special_buffer(efx, &channel->eventq,
1114                                            EFX_EVQ_SIZE * sizeof(efx_qword_t));
1115 }
1116
1117 void falcon_init_eventq(struct efx_channel *channel)
1118 {
1119         efx_oword_t evq_ptr;
1120         struct efx_nic *efx = channel->efx;
1121
1122         EFX_LOG(efx, "channel %d event queue in special buffers %d-%d\n",
1123                 channel->channel, channel->eventq.index,
1124                 channel->eventq.index + channel->eventq.entries - 1);
1125
1126         /* Pin event queue buffer */
1127         falcon_init_special_buffer(efx, &channel->eventq);
1128
1129         /* Fill event queue with all ones (i.e. empty events) */
1130         memset(channel->eventq.addr, 0xff, channel->eventq.len);
1131
1132         /* Push event queue to card */
1133         EFX_POPULATE_OWORD_3(evq_ptr,
1134                              FRF_AZ_EVQ_EN, 1,
1135                              FRF_AZ_EVQ_SIZE, __ffs(channel->eventq.entries),
1136                              FRF_AZ_EVQ_BUF_BASE_ID, channel->eventq.index);
1137         efx_writeo_table(efx, &evq_ptr, efx->type->evq_ptr_tbl_base,
1138                          channel->channel);
1139
1140         falcon_set_int_moderation(channel);
1141 }
1142
1143 void falcon_fini_eventq(struct efx_channel *channel)
1144 {
1145         efx_oword_t eventq_ptr;
1146         struct efx_nic *efx = channel->efx;
1147
1148         /* Remove event queue from card */
1149         EFX_ZERO_OWORD(eventq_ptr);
1150         efx_writeo_table(efx, &eventq_ptr, efx->type->evq_ptr_tbl_base,
1151                          channel->channel);
1152
1153         /* Unpin event queue */
1154         falcon_fini_special_buffer(efx, &channel->eventq);
1155 }
1156
1157 /* Free buffers backing event queue */
1158 void falcon_remove_eventq(struct efx_channel *channel)
1159 {
1160         falcon_free_special_buffer(channel->efx, &channel->eventq);
1161 }
1162
1163
1164 /* Generates a test event on the event queue.  A subsequent call to
1165  * process_eventq() should pick up the event and place the value of
1166  * "magic" into channel->eventq_magic;
1167  */
1168 void falcon_generate_test_event(struct efx_channel *channel, unsigned int magic)
1169 {
1170         efx_qword_t test_event;
1171
1172         EFX_POPULATE_QWORD_2(test_event, FSF_AZ_EV_CODE,
1173                              FSE_AZ_EV_CODE_DRV_GEN_EV,
1174                              FSF_AZ_DRV_GEN_EV_MAGIC, magic);
1175         falcon_generate_event(channel, &test_event);
1176 }
1177
1178 void falcon_sim_phy_event(struct efx_nic *efx)
1179 {
1180         efx_qword_t phy_event;
1181
1182         EFX_POPULATE_QWORD_1(phy_event, FSF_AZ_EV_CODE,
1183                              FSE_AZ_EV_CODE_GLOBAL_EV);
1184         if (EFX_IS10G(efx))
1185                 EFX_SET_QWORD_FIELD(phy_event, FSF_AB_GLB_EV_XG_PHY0_INTR, 1);
1186         else
1187                 EFX_SET_QWORD_FIELD(phy_event, FSF_AB_GLB_EV_G_PHY0_INTR, 1);
1188
1189         falcon_generate_event(&efx->channel[0], &phy_event);
1190 }
1191
1192 /**************************************************************************
1193  *
1194  * Flush handling
1195  *
1196  **************************************************************************/
1197
1198
1199 static void falcon_poll_flush_events(struct efx_nic *efx)
1200 {
1201         struct efx_channel *channel = &efx->channel[0];
1202         struct efx_tx_queue *tx_queue;
1203         struct efx_rx_queue *rx_queue;
1204         unsigned int read_ptr = channel->eventq_read_ptr;
1205         unsigned int end_ptr = (read_ptr - 1) & EFX_EVQ_MASK;
1206
1207         do {
1208                 efx_qword_t *event = falcon_event(channel, read_ptr);
1209                 int ev_code, ev_sub_code, ev_queue;
1210                 bool ev_failed;
1211
1212                 if (!falcon_event_present(event))
1213                         break;
1214
1215                 ev_code = EFX_QWORD_FIELD(*event, FSF_AZ_EV_CODE);
1216                 ev_sub_code = EFX_QWORD_FIELD(*event,
1217                                               FSF_AZ_DRIVER_EV_SUBCODE);
1218                 if (ev_code == FSE_AZ_EV_CODE_DRIVER_EV &&
1219                     ev_sub_code == FSE_AZ_TX_DESCQ_FLS_DONE_EV) {
1220                         ev_queue = EFX_QWORD_FIELD(*event,
1221                                                    FSF_AZ_DRIVER_EV_SUBDATA);
1222                         if (ev_queue < EFX_TX_QUEUE_COUNT) {
1223                                 tx_queue = efx->tx_queue + ev_queue;
1224                                 tx_queue->flushed = true;
1225                         }
1226                 } else if (ev_code == FSE_AZ_EV_CODE_DRIVER_EV &&
1227                            ev_sub_code == FSE_AZ_RX_DESCQ_FLS_DONE_EV) {
1228                         ev_queue = EFX_QWORD_FIELD(
1229                                 *event, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
1230                         ev_failed = EFX_QWORD_FIELD(
1231                                 *event, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
1232                         if (ev_queue < efx->n_rx_queues) {
1233                                 rx_queue = efx->rx_queue + ev_queue;
1234
1235                                 /* retry the rx flush */
1236                                 if (ev_failed)
1237                                         falcon_flush_rx_queue(rx_queue);
1238                                 else
1239                                         rx_queue->flushed = true;
1240                         }
1241                 }
1242
1243                 read_ptr = (read_ptr + 1) & EFX_EVQ_MASK;
1244         } while (read_ptr != end_ptr);
1245 }
1246
1247 /* Handle tx and rx flushes at the same time, since they run in
1248  * parallel in the hardware and there's no reason for us to
1249  * serialise them */
1250 int falcon_flush_queues(struct efx_nic *efx)
1251 {
1252         struct efx_rx_queue *rx_queue;
1253         struct efx_tx_queue *tx_queue;
1254         int i;
1255         bool outstanding;
1256
1257         /* Issue flush requests */
1258         efx_for_each_tx_queue(tx_queue, efx) {
1259                 tx_queue->flushed = false;
1260                 falcon_flush_tx_queue(tx_queue);
1261         }
1262         efx_for_each_rx_queue(rx_queue, efx) {
1263                 rx_queue->flushed = false;
1264                 falcon_flush_rx_queue(rx_queue);
1265         }
1266
1267         /* Poll the evq looking for flush completions. Since we're not pushing
1268          * any more rx or tx descriptors at this point, we're in no danger of
1269          * overflowing the evq whilst we wait */
1270         for (i = 0; i < FALCON_FLUSH_POLL_COUNT; ++i) {
1271                 msleep(FALCON_FLUSH_INTERVAL);
1272                 falcon_poll_flush_events(efx);
1273
1274                 /* Check if every queue has been succesfully flushed */
1275                 outstanding = false;
1276                 efx_for_each_tx_queue(tx_queue, efx)
1277                         outstanding |= !tx_queue->flushed;
1278                 efx_for_each_rx_queue(rx_queue, efx)
1279                         outstanding |= !rx_queue->flushed;
1280                 if (!outstanding)
1281                         return 0;
1282         }
1283
1284         /* Mark the queues as all flushed. We're going to return failure
1285          * leading to a reset, or fake up success anyway. "flushed" now
1286          * indicates that we tried to flush. */
1287         efx_for_each_tx_queue(tx_queue, efx) {
1288                 if (!tx_queue->flushed)
1289                         EFX_ERR(efx, "tx queue %d flush command timed out\n",
1290                                 tx_queue->queue);
1291                 tx_queue->flushed = true;
1292         }
1293         efx_for_each_rx_queue(rx_queue, efx) {
1294                 if (!rx_queue->flushed)
1295                         EFX_ERR(efx, "rx queue %d flush command timed out\n",
1296                                 rx_queue->queue);
1297                 rx_queue->flushed = true;
1298         }
1299
1300         if (EFX_WORKAROUND_7803(efx))
1301                 return 0;
1302
1303         return -ETIMEDOUT;
1304 }
1305
1306 /**************************************************************************
1307  *
1308  * Falcon hardware interrupts
1309  * The hardware interrupt handler does very little work; all the event
1310  * queue processing is carried out by per-channel tasklets.
1311  *
1312  **************************************************************************/
1313
1314 /* Enable/disable/generate Falcon interrupts */
1315 static inline void falcon_interrupts(struct efx_nic *efx, int enabled,
1316                                      int force)
1317 {
1318         efx_oword_t int_en_reg_ker;
1319
1320         EFX_POPULATE_OWORD_2(int_en_reg_ker,
1321                              FRF_AZ_KER_INT_KER, force,
1322                              FRF_AZ_DRV_INT_EN_KER, enabled);
1323         efx_writeo(efx, &int_en_reg_ker, FR_AZ_INT_EN_KER);
1324 }
1325
1326 void falcon_enable_interrupts(struct efx_nic *efx)
1327 {
1328         efx_oword_t int_adr_reg_ker;
1329         struct efx_channel *channel;
1330
1331         EFX_ZERO_OWORD(*((efx_oword_t *) efx->irq_status.addr));
1332         wmb(); /* Ensure interrupt vector is clear before interrupts enabled */
1333
1334         /* Program address */
1335         EFX_POPULATE_OWORD_2(int_adr_reg_ker,
1336                              FRF_AZ_NORM_INT_VEC_DIS_KER,
1337                              EFX_INT_MODE_USE_MSI(efx),
1338                              FRF_AZ_INT_ADR_KER, efx->irq_status.dma_addr);
1339         efx_writeo(efx, &int_adr_reg_ker, FR_AZ_INT_ADR_KER);
1340
1341         /* Enable interrupts */
1342         falcon_interrupts(efx, 1, 0);
1343
1344         /* Force processing of all the channels to get the EVQ RPTRs up to
1345            date */
1346         efx_for_each_channel(channel, efx)
1347                 efx_schedule_channel(channel);
1348 }
1349
1350 void falcon_disable_interrupts(struct efx_nic *efx)
1351 {
1352         /* Disable interrupts */
1353         falcon_interrupts(efx, 0, 0);
1354 }
1355
1356 /* Generate a Falcon test interrupt
1357  * Interrupt must already have been enabled, otherwise nasty things
1358  * may happen.
1359  */
1360 void falcon_generate_interrupt(struct efx_nic *efx)
1361 {
1362         falcon_interrupts(efx, 1, 1);
1363 }
1364
1365 /* Acknowledge a legacy interrupt from Falcon
1366  *
1367  * This acknowledges a legacy (not MSI) interrupt via INT_ACK_KER_REG.
1368  *
1369  * Due to SFC bug 3706 (silicon revision <=A1) reads can be duplicated in the
1370  * BIU. Interrupt acknowledge is read sensitive so must write instead
1371  * (then read to ensure the BIU collector is flushed)
1372  *
1373  * NB most hardware supports MSI interrupts
1374  */
1375 static inline void falcon_irq_ack_a1(struct efx_nic *efx)
1376 {
1377         efx_dword_t reg;
1378
1379         EFX_POPULATE_DWORD_1(reg, FRF_AA_INT_ACK_KER_FIELD, 0xb7eb7e);
1380         efx_writed(efx, &reg, FR_AA_INT_ACK_KER);
1381         efx_readd(efx, &reg, FR_AA_WORK_AROUND_BROKEN_PCI_READS);
1382 }
1383
1384 /* Process a fatal interrupt
1385  * Disable bus mastering ASAP and schedule a reset
1386  */
1387 static irqreturn_t falcon_fatal_interrupt(struct efx_nic *efx)
1388 {
1389         struct falcon_nic_data *nic_data = efx->nic_data;
1390         efx_oword_t *int_ker = efx->irq_status.addr;
1391         efx_oword_t fatal_intr;
1392         int error, mem_perr;
1393
1394         efx_reado(efx, &fatal_intr, FR_AZ_FATAL_INTR_KER);
1395         error = EFX_OWORD_FIELD(fatal_intr, FRF_AZ_FATAL_INTR);
1396
1397         EFX_ERR(efx, "SYSTEM ERROR " EFX_OWORD_FMT " status "
1398                 EFX_OWORD_FMT ": %s\n", EFX_OWORD_VAL(*int_ker),
1399                 EFX_OWORD_VAL(fatal_intr),
1400                 error ? "disabling bus mastering" : "no recognised error");
1401         if (error == 0)
1402                 goto out;
1403
1404         /* If this is a memory parity error dump which blocks are offending */
1405         mem_perr = EFX_OWORD_FIELD(fatal_intr, FRF_AZ_MEM_PERR_INT_KER);
1406         if (mem_perr) {
1407                 efx_oword_t reg;
1408                 efx_reado(efx, &reg, FR_AZ_MEM_STAT);
1409                 EFX_ERR(efx, "SYSTEM ERROR: memory parity error "
1410                         EFX_OWORD_FMT "\n", EFX_OWORD_VAL(reg));
1411         }
1412
1413         /* Disable both devices */
1414         pci_clear_master(efx->pci_dev);
1415         if (FALCON_IS_DUAL_FUNC(efx))
1416                 pci_clear_master(nic_data->pci_dev2);
1417         falcon_disable_interrupts(efx);
1418
1419         /* Count errors and reset or disable the NIC accordingly */
1420         if (nic_data->int_error_count == 0 ||
1421             time_after(jiffies, nic_data->int_error_expire)) {
1422                 nic_data->int_error_count = 0;
1423                 nic_data->int_error_expire =
1424                         jiffies + FALCON_INT_ERROR_EXPIRE * HZ;
1425         }
1426         if (++nic_data->int_error_count < FALCON_MAX_INT_ERRORS) {
1427                 EFX_ERR(efx, "SYSTEM ERROR - reset scheduled\n");
1428                 efx_schedule_reset(efx, RESET_TYPE_INT_ERROR);
1429         } else {
1430                 EFX_ERR(efx, "SYSTEM ERROR - max number of errors seen."
1431                         "NIC will be disabled\n");
1432                 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
1433         }
1434 out:
1435         return IRQ_HANDLED;
1436 }
1437
1438 /* Handle a legacy interrupt from Falcon
1439  * Acknowledges the interrupt and schedule event queue processing.
1440  */
1441 static irqreturn_t falcon_legacy_interrupt_b0(int irq, void *dev_id)
1442 {
1443         struct efx_nic *efx = dev_id;
1444         efx_oword_t *int_ker = efx->irq_status.addr;
1445         irqreturn_t result = IRQ_NONE;
1446         struct efx_channel *channel;
1447         efx_dword_t reg;
1448         u32 queues;
1449         int syserr;
1450
1451         /* Read the ISR which also ACKs the interrupts */
1452         efx_readd(efx, &reg, FR_BZ_INT_ISR0);
1453         queues = EFX_EXTRACT_DWORD(reg, 0, 31);
1454
1455         /* Check to see if we have a serious error condition */
1456         syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
1457         if (unlikely(syserr))
1458                 return falcon_fatal_interrupt(efx);
1459
1460         /* Schedule processing of any interrupting queues */
1461         efx_for_each_channel(channel, efx) {
1462                 if ((queues & 1) ||
1463                     falcon_event_present(
1464                             falcon_event(channel, channel->eventq_read_ptr))) {
1465                         efx_schedule_channel(channel);
1466                         result = IRQ_HANDLED;
1467                 }
1468                 queues >>= 1;
1469         }
1470
1471         if (result == IRQ_HANDLED) {
1472                 efx->last_irq_cpu = raw_smp_processor_id();
1473                 EFX_TRACE(efx, "IRQ %d on CPU %d status " EFX_DWORD_FMT "\n",
1474                           irq, raw_smp_processor_id(), EFX_DWORD_VAL(reg));
1475         }
1476
1477         return result;
1478 }
1479
1480
1481 static irqreturn_t falcon_legacy_interrupt_a1(int irq, void *dev_id)
1482 {
1483         struct efx_nic *efx = dev_id;
1484         efx_oword_t *int_ker = efx->irq_status.addr;
1485         struct efx_channel *channel;
1486         int syserr;
1487         int queues;
1488
1489         /* Check to see if this is our interrupt.  If it isn't, we
1490          * exit without having touched the hardware.
1491          */
1492         if (unlikely(EFX_OWORD_IS_ZERO(*int_ker))) {
1493                 EFX_TRACE(efx, "IRQ %d on CPU %d not for me\n", irq,
1494                           raw_smp_processor_id());
1495                 return IRQ_NONE;
1496         }
1497         efx->last_irq_cpu = raw_smp_processor_id();
1498         EFX_TRACE(efx, "IRQ %d on CPU %d status " EFX_OWORD_FMT "\n",
1499                   irq, raw_smp_processor_id(), EFX_OWORD_VAL(*int_ker));
1500
1501         /* Check to see if we have a serious error condition */
1502         syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
1503         if (unlikely(syserr))
1504                 return falcon_fatal_interrupt(efx);
1505
1506         /* Determine interrupting queues, clear interrupt status
1507          * register and acknowledge the device interrupt.
1508          */
1509         BUILD_BUG_ON(INT_EVQS_WIDTH > EFX_MAX_CHANNELS);
1510         queues = EFX_OWORD_FIELD(*int_ker, INT_EVQS);
1511         EFX_ZERO_OWORD(*int_ker);
1512         wmb(); /* Ensure the vector is cleared before interrupt ack */
1513         falcon_irq_ack_a1(efx);
1514
1515         /* Schedule processing of any interrupting queues */
1516         channel = &efx->channel[0];
1517         while (queues) {
1518                 if (queues & 0x01)
1519                         efx_schedule_channel(channel);
1520                 channel++;
1521                 queues >>= 1;
1522         }
1523
1524         return IRQ_HANDLED;
1525 }
1526
1527 /* Handle an MSI interrupt from Falcon
1528  *
1529  * Handle an MSI hardware interrupt.  This routine schedules event
1530  * queue processing.  No interrupt acknowledgement cycle is necessary.
1531  * Also, we never need to check that the interrupt is for us, since
1532  * MSI interrupts cannot be shared.
1533  */
1534 static irqreturn_t falcon_msi_interrupt(int irq, void *dev_id)
1535 {
1536         struct efx_channel *channel = dev_id;
1537         struct efx_nic *efx = channel->efx;
1538         efx_oword_t *int_ker = efx->irq_status.addr;
1539         int syserr;
1540
1541         efx->last_irq_cpu = raw_smp_processor_id();
1542         EFX_TRACE(efx, "IRQ %d on CPU %d status " EFX_OWORD_FMT "\n",
1543                   irq, raw_smp_processor_id(), EFX_OWORD_VAL(*int_ker));
1544
1545         /* Check to see if we have a serious error condition */
1546         syserr = EFX_OWORD_FIELD(*int_ker, FATAL_INT);
1547         if (unlikely(syserr))
1548                 return falcon_fatal_interrupt(efx);
1549
1550         /* Schedule processing of the channel */
1551         efx_schedule_channel(channel);
1552
1553         return IRQ_HANDLED;
1554 }
1555
1556
1557 /* Setup RSS indirection table.
1558  * This maps from the hash value of the packet to RXQ
1559  */
1560 static void falcon_setup_rss_indir_table(struct efx_nic *efx)
1561 {
1562         int i = 0;
1563         unsigned long offset;
1564         efx_dword_t dword;
1565
1566         if (falcon_rev(efx) < FALCON_REV_B0)
1567                 return;
1568
1569         for (offset = FR_BZ_RX_INDIRECTION_TBL;
1570              offset < FR_BZ_RX_INDIRECTION_TBL + 0x800;
1571              offset += 0x10) {
1572                 EFX_POPULATE_DWORD_1(dword, FRF_BZ_IT_QUEUE,
1573                                      i % efx->n_rx_queues);
1574                 efx_writed(efx, &dword, offset);
1575                 i++;
1576         }
1577 }
1578
1579 /* Hook interrupt handler(s)
1580  * Try MSI and then legacy interrupts.
1581  */
1582 int falcon_init_interrupt(struct efx_nic *efx)
1583 {
1584         struct efx_channel *channel;
1585         int rc;
1586
1587         if (!EFX_INT_MODE_USE_MSI(efx)) {
1588                 irq_handler_t handler;
1589                 if (falcon_rev(efx) >= FALCON_REV_B0)
1590                         handler = falcon_legacy_interrupt_b0;
1591                 else
1592                         handler = falcon_legacy_interrupt_a1;
1593
1594                 rc = request_irq(efx->legacy_irq, handler, IRQF_SHARED,
1595                                  efx->name, efx);
1596                 if (rc) {
1597                         EFX_ERR(efx, "failed to hook legacy IRQ %d\n",
1598                                 efx->pci_dev->irq);
1599                         goto fail1;
1600                 }
1601                 return 0;
1602         }
1603
1604         /* Hook MSI or MSI-X interrupt */
1605         efx_for_each_channel(channel, efx) {
1606                 rc = request_irq(channel->irq, falcon_msi_interrupt,
1607                                  IRQF_PROBE_SHARED, /* Not shared */
1608                                  channel->name, channel);
1609                 if (rc) {
1610                         EFX_ERR(efx, "failed to hook IRQ %d\n", channel->irq);
1611                         goto fail2;
1612                 }
1613         }
1614
1615         return 0;
1616
1617  fail2:
1618         efx_for_each_channel(channel, efx)
1619                 free_irq(channel->irq, channel);
1620  fail1:
1621         return rc;
1622 }
1623
1624 void falcon_fini_interrupt(struct efx_nic *efx)
1625 {
1626         struct efx_channel *channel;
1627         efx_oword_t reg;
1628
1629         /* Disable MSI/MSI-X interrupts */
1630         efx_for_each_channel(channel, efx) {
1631                 if (channel->irq)
1632                         free_irq(channel->irq, channel);
1633         }
1634
1635         /* ACK legacy interrupt */
1636         if (falcon_rev(efx) >= FALCON_REV_B0)
1637                 efx_reado(efx, &reg, FR_BZ_INT_ISR0);
1638         else
1639                 falcon_irq_ack_a1(efx);
1640
1641         /* Disable legacy interrupt */
1642         if (efx->legacy_irq)
1643                 free_irq(efx->legacy_irq, efx);
1644 }
1645
1646 /**************************************************************************
1647  *
1648  * EEPROM/flash
1649  *
1650  **************************************************************************
1651  */
1652
1653 #define FALCON_SPI_MAX_LEN sizeof(efx_oword_t)
1654
1655 static int falcon_spi_poll(struct efx_nic *efx)
1656 {
1657         efx_oword_t reg;
1658         efx_reado(efx, &reg, FR_AB_EE_SPI_HCMD);
1659         return EFX_OWORD_FIELD(reg, FRF_AB_EE_SPI_HCMD_CMD_EN) ? -EBUSY : 0;
1660 }
1661
1662 /* Wait for SPI command completion */
1663 static int falcon_spi_wait(struct efx_nic *efx)
1664 {
1665         /* Most commands will finish quickly, so we start polling at
1666          * very short intervals.  Sometimes the command may have to
1667          * wait for VPD or expansion ROM access outside of our
1668          * control, so we allow up to 100 ms. */
1669         unsigned long timeout = jiffies + 1 + DIV_ROUND_UP(HZ, 10);
1670         int i;
1671
1672         for (i = 0; i < 10; i++) {
1673                 if (!falcon_spi_poll(efx))
1674                         return 0;
1675                 udelay(10);
1676         }
1677
1678         for (;;) {
1679                 if (!falcon_spi_poll(efx))
1680                         return 0;
1681                 if (time_after_eq(jiffies, timeout)) {
1682                         EFX_ERR(efx, "timed out waiting for SPI\n");
1683                         return -ETIMEDOUT;
1684                 }
1685                 schedule_timeout_uninterruptible(1);
1686         }
1687 }
1688
1689 int falcon_spi_cmd(const struct efx_spi_device *spi,
1690                    unsigned int command, int address,
1691                    const void *in, void *out, size_t len)
1692 {
1693         struct efx_nic *efx = spi->efx;
1694         bool addressed = (address >= 0);
1695         bool reading = (out != NULL);
1696         efx_oword_t reg;
1697         int rc;
1698
1699         /* Input validation */
1700         if (len > FALCON_SPI_MAX_LEN)
1701                 return -EINVAL;
1702         BUG_ON(!mutex_is_locked(&efx->spi_lock));
1703
1704         /* Check that previous command is not still running */
1705         rc = falcon_spi_poll(efx);
1706         if (rc)
1707                 return rc;
1708
1709         /* Program address register, if we have an address */
1710         if (addressed) {
1711                 EFX_POPULATE_OWORD_1(reg, FRF_AB_EE_SPI_HADR_ADR, address);
1712                 efx_writeo(efx, &reg, FR_AB_EE_SPI_HADR);
1713         }
1714
1715         /* Program data register, if we have data */
1716         if (in != NULL) {
1717                 memcpy(&reg, in, len);
1718                 efx_writeo(efx, &reg, FR_AB_EE_SPI_HDATA);
1719         }
1720
1721         /* Issue read/write command */
1722         EFX_POPULATE_OWORD_7(reg,
1723                              FRF_AB_EE_SPI_HCMD_CMD_EN, 1,
1724                              FRF_AB_EE_SPI_HCMD_SF_SEL, spi->device_id,
1725                              FRF_AB_EE_SPI_HCMD_DABCNT, len,
1726                              FRF_AB_EE_SPI_HCMD_READ, reading,
1727                              FRF_AB_EE_SPI_HCMD_DUBCNT, 0,
1728                              FRF_AB_EE_SPI_HCMD_ADBCNT,
1729                              (addressed ? spi->addr_len : 0),
1730                              FRF_AB_EE_SPI_HCMD_ENC, command);
1731         efx_writeo(efx, &reg, FR_AB_EE_SPI_HCMD);
1732
1733         /* Wait for read/write to complete */
1734         rc = falcon_spi_wait(efx);
1735         if (rc)
1736                 return rc;
1737
1738         /* Read data */
1739         if (out != NULL) {
1740                 efx_reado(efx, &reg, FR_AB_EE_SPI_HDATA);
1741                 memcpy(out, &reg, len);
1742         }
1743
1744         return 0;
1745 }
1746
1747 static size_t
1748 falcon_spi_write_limit(const struct efx_spi_device *spi, size_t start)
1749 {
1750         return min(FALCON_SPI_MAX_LEN,
1751                    (spi->block_size - (start & (spi->block_size - 1))));
1752 }
1753
1754 static inline u8
1755 efx_spi_munge_command(const struct efx_spi_device *spi,
1756                       const u8 command, const unsigned int address)
1757 {
1758         return command | (((address >> 8) & spi->munge_address) << 3);
1759 }
1760
1761 /* Wait up to 10 ms for buffered write completion */
1762 int falcon_spi_wait_write(const struct efx_spi_device *spi)
1763 {
1764         struct efx_nic *efx = spi->efx;
1765         unsigned long timeout = jiffies + 1 + DIV_ROUND_UP(HZ, 100);
1766         u8 status;
1767         int rc;
1768
1769         for (;;) {
1770                 rc = falcon_spi_cmd(spi, SPI_RDSR, -1, NULL,
1771                                     &status, sizeof(status));
1772                 if (rc)
1773                         return rc;
1774                 if (!(status & SPI_STATUS_NRDY))
1775                         return 0;
1776                 if (time_after_eq(jiffies, timeout)) {
1777                         EFX_ERR(efx, "SPI write timeout on device %d"
1778                                 " last status=0x%02x\n",
1779                                 spi->device_id, status);
1780                         return -ETIMEDOUT;
1781                 }
1782                 schedule_timeout_uninterruptible(1);
1783         }
1784 }
1785
1786 int falcon_spi_read(const struct efx_spi_device *spi, loff_t start,
1787                     size_t len, size_t *retlen, u8 *buffer)
1788 {
1789         size_t block_len, pos = 0;
1790         unsigned int command;
1791         int rc = 0;
1792
1793         while (pos < len) {
1794                 block_len = min(len - pos, FALCON_SPI_MAX_LEN);
1795
1796                 command = efx_spi_munge_command(spi, SPI_READ, start + pos);
1797                 rc = falcon_spi_cmd(spi, command, start + pos, NULL,
1798                                     buffer + pos, block_len);
1799                 if (rc)
1800                         break;
1801                 pos += block_len;
1802
1803                 /* Avoid locking up the system */
1804                 cond_resched();
1805                 if (signal_pending(current)) {
1806                         rc = -EINTR;
1807                         break;
1808                 }
1809         }
1810
1811         if (retlen)
1812                 *retlen = pos;
1813         return rc;
1814 }
1815
1816 int falcon_spi_write(const struct efx_spi_device *spi, loff_t start,
1817                      size_t len, size_t *retlen, const u8 *buffer)
1818 {
1819         u8 verify_buffer[FALCON_SPI_MAX_LEN];
1820         size_t block_len, pos = 0;
1821         unsigned int command;
1822         int rc = 0;
1823
1824         while (pos < len) {
1825                 rc = falcon_spi_cmd(spi, SPI_WREN, -1, NULL, NULL, 0);
1826                 if (rc)
1827                         break;
1828
1829                 block_len = min(len - pos,
1830                                 falcon_spi_write_limit(spi, start + pos));
1831                 command = efx_spi_munge_command(spi, SPI_WRITE, start + pos);
1832                 rc = falcon_spi_cmd(spi, command, start + pos,
1833                                     buffer + pos, NULL, block_len);
1834                 if (rc)
1835                         break;
1836
1837                 rc = falcon_spi_wait_write(spi);
1838                 if (rc)
1839                         break;
1840
1841                 command = efx_spi_munge_command(spi, SPI_READ, start + pos);
1842                 rc = falcon_spi_cmd(spi, command, start + pos,
1843                                     NULL, verify_buffer, block_len);
1844                 if (memcmp(verify_buffer, buffer + pos, block_len)) {
1845                         rc = -EIO;
1846                         break;
1847                 }
1848
1849                 pos += block_len;
1850
1851                 /* Avoid locking up the system */
1852                 cond_resched();
1853                 if (signal_pending(current)) {
1854                         rc = -EINTR;
1855                         break;
1856                 }
1857         }
1858
1859         if (retlen)
1860                 *retlen = pos;
1861         return rc;
1862 }
1863
1864 /**************************************************************************
1865  *
1866  * MAC wrapper
1867  *
1868  **************************************************************************
1869  */
1870
1871 static int falcon_reset_macs(struct efx_nic *efx)
1872 {
1873         efx_oword_t reg;
1874         int count;
1875
1876         if (falcon_rev(efx) < FALCON_REV_B0) {
1877                 /* It's not safe to use GLB_CTL_REG to reset the
1878                  * macs, so instead use the internal MAC resets
1879                  */
1880                 if (!EFX_IS10G(efx)) {
1881                         EFX_POPULATE_OWORD_1(reg, FRF_AB_GM_SW_RST, 1);
1882                         efx_writeo(efx, &reg, FR_AB_GM_CFG1);
1883                         udelay(1000);
1884
1885                         EFX_POPULATE_OWORD_1(reg, FRF_AB_GM_SW_RST, 0);
1886                         efx_writeo(efx, &reg, FR_AB_GM_CFG1);
1887                         udelay(1000);
1888                         return 0;
1889                 } else {
1890                         EFX_POPULATE_OWORD_1(reg, FRF_AB_XM_CORE_RST, 1);
1891                         efx_writeo(efx, &reg, FR_AB_XM_GLB_CFG);
1892
1893                         for (count = 0; count < 10000; count++) {
1894                                 efx_reado(efx, &reg, FR_AB_XM_GLB_CFG);
1895                                 if (EFX_OWORD_FIELD(reg, FRF_AB_XM_CORE_RST) ==
1896                                     0)
1897                                         return 0;
1898                                 udelay(10);
1899                         }
1900
1901                         EFX_ERR(efx, "timed out waiting for XMAC core reset\n");
1902                         return -ETIMEDOUT;
1903                 }
1904         }
1905
1906         /* MAC stats will fail whilst the TX fifo is draining. Serialise
1907          * the drain sequence with the statistics fetch */
1908         efx_stats_disable(efx);
1909
1910         efx_reado(efx, &reg, FR_AB_MAC_CTRL);
1911         EFX_SET_OWORD_FIELD(reg, FRF_BB_TXFIFO_DRAIN_EN, 1);
1912         efx_writeo(efx, &reg, FR_AB_MAC_CTRL);
1913
1914         efx_reado(efx, &reg, FR_AB_GLB_CTL);
1915         EFX_SET_OWORD_FIELD(reg, FRF_AB_RST_XGTX, 1);
1916         EFX_SET_OWORD_FIELD(reg, FRF_AB_RST_XGRX, 1);
1917         EFX_SET_OWORD_FIELD(reg, FRF_AB_RST_EM, 1);
1918         efx_writeo(efx, &reg, FR_AB_GLB_CTL);
1919
1920         count = 0;
1921         while (1) {
1922                 efx_reado(efx, &reg, FR_AB_GLB_CTL);
1923                 if (!EFX_OWORD_FIELD(reg, FRF_AB_RST_XGTX) &&
1924                     !EFX_OWORD_FIELD(reg, FRF_AB_RST_XGRX) &&
1925                     !EFX_OWORD_FIELD(reg, FRF_AB_RST_EM)) {
1926                         EFX_LOG(efx, "Completed MAC reset after %d loops\n",
1927                                 count);
1928                         break;
1929                 }
1930                 if (count > 20) {
1931                         EFX_ERR(efx, "MAC reset failed\n");
1932                         break;
1933                 }
1934                 count++;
1935                 udelay(10);
1936         }
1937
1938         efx_stats_enable(efx);
1939
1940         /* If we've reset the EM block and the link is up, then
1941          * we'll have to kick the XAUI link so the PHY can recover */
1942         if (efx->link_up && EFX_IS10G(efx) && EFX_WORKAROUND_5147(efx))
1943                 falcon_reset_xaui(efx);
1944
1945         return 0;
1946 }
1947
1948 void falcon_drain_tx_fifo(struct efx_nic *efx)
1949 {
1950         efx_oword_t reg;
1951
1952         if ((falcon_rev(efx) < FALCON_REV_B0) ||
1953             (efx->loopback_mode != LOOPBACK_NONE))
1954                 return;
1955
1956         efx_reado(efx, &reg, FR_AB_MAC_CTRL);
1957         /* There is no point in draining more than once */
1958         if (EFX_OWORD_FIELD(reg, FRF_BB_TXFIFO_DRAIN_EN))
1959                 return;
1960
1961         falcon_reset_macs(efx);
1962 }
1963
1964 void falcon_deconfigure_mac_wrapper(struct efx_nic *efx)
1965 {
1966         efx_oword_t reg;
1967
1968         if (falcon_rev(efx) < FALCON_REV_B0)
1969                 return;
1970
1971         /* Isolate the MAC -> RX */
1972         efx_reado(efx, &reg, FR_AZ_RX_CFG);
1973         EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_INGR_EN, 0);
1974         efx_writeo(efx, &reg, FR_AZ_RX_CFG);
1975
1976         if (!efx->link_up)
1977                 falcon_drain_tx_fifo(efx);
1978 }
1979
1980 void falcon_reconfigure_mac_wrapper(struct efx_nic *efx)
1981 {
1982         efx_oword_t reg;
1983         int link_speed;
1984         bool tx_fc;
1985
1986         switch (efx->link_speed) {
1987         case 10000: link_speed = 3; break;
1988         case 1000:  link_speed = 2; break;
1989         case 100:   link_speed = 1; break;
1990         default:    link_speed = 0; break;
1991         }
1992         /* MAC_LINK_STATUS controls MAC backpressure but doesn't work
1993          * as advertised.  Disable to ensure packets are not
1994          * indefinitely held and TX queue can be flushed at any point
1995          * while the link is down. */
1996         EFX_POPULATE_OWORD_5(reg,
1997                              FRF_AB_MAC_XOFF_VAL, 0xffff /* max pause time */,
1998                              FRF_AB_MAC_BCAD_ACPT, 1,
1999                              FRF_AB_MAC_UC_PROM, efx->promiscuous,
2000                              FRF_AB_MAC_LINK_STATUS, 1, /* always set */
2001                              FRF_AB_MAC_SPEED, link_speed);
2002         /* On B0, MAC backpressure can be disabled and packets get
2003          * discarded. */
2004         if (falcon_rev(efx) >= FALCON_REV_B0) {
2005                 EFX_SET_OWORD_FIELD(reg, FRF_BB_TXFIFO_DRAIN_EN,
2006                                     !efx->link_up);
2007         }
2008
2009         efx_writeo(efx, &reg, FR_AB_MAC_CTRL);
2010
2011         /* Restore the multicast hash registers. */
2012         falcon_set_multicast_hash(efx);
2013
2014         /* Transmission of pause frames when RX crosses the threshold is
2015          * covered by RX_XOFF_MAC_EN and XM_TX_CFG_REG:XM_FCNTL.
2016          * Action on receipt of pause frames is controller by XM_DIS_FCNTL */
2017         tx_fc = !!(efx->link_fc & EFX_FC_TX);
2018         efx_reado(efx, &reg, FR_AZ_RX_CFG);
2019         EFX_SET_OWORD_FIELD(reg, FRF_AZ_RX_XOFF_MAC_EN, tx_fc);
2020
2021         /* Unisolate the MAC -> RX */
2022         if (falcon_rev(efx) >= FALCON_REV_B0)
2023                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_INGR_EN, 1);
2024         efx_writeo(efx, &reg, FR_AZ_RX_CFG);
2025 }
2026
2027 int falcon_dma_stats(struct efx_nic *efx, unsigned int done_offset)
2028 {
2029         efx_oword_t reg;
2030         u32 *dma_done;
2031         int i;
2032
2033         if (disable_dma_stats)
2034                 return 0;
2035
2036         /* Statistics fetch will fail if the MAC is in TX drain */
2037         if (falcon_rev(efx) >= FALCON_REV_B0) {
2038                 efx_oword_t temp;
2039                 efx_reado(efx, &temp, FR_AB_MAC_CTRL);
2040                 if (EFX_OWORD_FIELD(temp, FRF_BB_TXFIFO_DRAIN_EN))
2041                         return 0;
2042         }
2043
2044         dma_done = (efx->stats_buffer.addr + done_offset);
2045         *dma_done = FALCON_STATS_NOT_DONE;
2046         wmb(); /* ensure done flag is clear */
2047
2048         /* Initiate DMA transfer of stats */
2049         EFX_POPULATE_OWORD_2(reg,
2050                              FRF_AB_MAC_STAT_DMA_CMD, 1,
2051                              FRF_AB_MAC_STAT_DMA_ADR,
2052                              efx->stats_buffer.dma_addr);
2053         efx_writeo(efx, &reg, FR_AB_MAC_STAT_DMA);
2054
2055         /* Wait for transfer to complete */
2056         for (i = 0; i < 400; i++) {
2057                 if (*(volatile u32 *)dma_done == FALCON_STATS_DONE) {
2058                         rmb(); /* Ensure the stats are valid. */
2059                         return 0;
2060                 }
2061                 udelay(10);
2062         }
2063
2064         EFX_ERR(efx, "timed out waiting for statistics\n");
2065         return -ETIMEDOUT;
2066 }
2067
2068 /**************************************************************************
2069  *
2070  * PHY access via GMII
2071  *
2072  **************************************************************************
2073  */
2074
2075 /* Wait for GMII access to complete */
2076 static int falcon_gmii_wait(struct efx_nic *efx)
2077 {
2078         efx_dword_t md_stat;
2079         int count;
2080
2081         /* wait upto 50ms - taken max from datasheet */
2082         for (count = 0; count < 5000; count++) {
2083                 efx_readd(efx, &md_stat, FR_AB_MD_STAT);
2084                 if (EFX_DWORD_FIELD(md_stat, FRF_AB_MD_BSY) == 0) {
2085                         if (EFX_DWORD_FIELD(md_stat, FRF_AB_MD_LNFL) != 0 ||
2086                             EFX_DWORD_FIELD(md_stat, FRF_AB_MD_BSERR) != 0) {
2087                                 EFX_ERR(efx, "error from GMII access "
2088                                         EFX_DWORD_FMT"\n",
2089                                         EFX_DWORD_VAL(md_stat));
2090                                 return -EIO;
2091                         }
2092                         return 0;
2093                 }
2094                 udelay(10);
2095         }
2096         EFX_ERR(efx, "timed out waiting for GMII\n");
2097         return -ETIMEDOUT;
2098 }
2099
2100 /* Write an MDIO register of a PHY connected to Falcon. */
2101 static int falcon_mdio_write(struct net_device *net_dev,
2102                              int prtad, int devad, u16 addr, u16 value)
2103 {
2104         struct efx_nic *efx = netdev_priv(net_dev);
2105         efx_oword_t reg;
2106         int rc;
2107
2108         EFX_REGDUMP(efx, "writing MDIO %d register %d.%d with 0x%04x\n",
2109                     prtad, devad, addr, value);
2110
2111         spin_lock_bh(&efx->phy_lock);
2112
2113         /* Check MDIO not currently being accessed */
2114         rc = falcon_gmii_wait(efx);
2115         if (rc)
2116                 goto out;
2117
2118         /* Write the address/ID register */
2119         EFX_POPULATE_OWORD_1(reg, FRF_AB_MD_PHY_ADR, addr);
2120         efx_writeo(efx, &reg, FR_AB_MD_PHY_ADR);
2121
2122         EFX_POPULATE_OWORD_2(reg, FRF_AB_MD_PRT_ADR, prtad,
2123                              FRF_AB_MD_DEV_ADR, devad);
2124         efx_writeo(efx, &reg, FR_AB_MD_ID);
2125
2126         /* Write data */
2127         EFX_POPULATE_OWORD_1(reg, FRF_AB_MD_TXD, value);
2128         efx_writeo(efx, &reg, FR_AB_MD_TXD);
2129
2130         EFX_POPULATE_OWORD_2(reg,
2131                              FRF_AB_MD_WRC, 1,
2132                              FRF_AB_MD_GC, 0);
2133         efx_writeo(efx, &reg, FR_AB_MD_CS);
2134
2135         /* Wait for data to be written */
2136         rc = falcon_gmii_wait(efx);
2137         if (rc) {
2138                 /* Abort the write operation */
2139                 EFX_POPULATE_OWORD_2(reg,
2140                                      FRF_AB_MD_WRC, 0,
2141                                      FRF_AB_MD_GC, 1);
2142                 efx_writeo(efx, &reg, FR_AB_MD_CS);
2143                 udelay(10);
2144         }
2145
2146  out:
2147         spin_unlock_bh(&efx->phy_lock);
2148         return rc;
2149 }
2150
2151 /* Read an MDIO register of a PHY connected to Falcon. */
2152 static int falcon_mdio_read(struct net_device *net_dev,
2153                             int prtad, int devad, u16 addr)
2154 {
2155         struct efx_nic *efx = netdev_priv(net_dev);
2156         efx_oword_t reg;
2157         int rc;
2158
2159         spin_lock_bh(&efx->phy_lock);
2160
2161         /* Check MDIO not currently being accessed */
2162         rc = falcon_gmii_wait(efx);
2163         if (rc)
2164                 goto out;
2165
2166         EFX_POPULATE_OWORD_1(reg, FRF_AB_MD_PHY_ADR, addr);
2167         efx_writeo(efx, &reg, FR_AB_MD_PHY_ADR);
2168
2169         EFX_POPULATE_OWORD_2(reg, FRF_AB_MD_PRT_ADR, prtad,
2170                              FRF_AB_MD_DEV_ADR, devad);
2171         efx_writeo(efx, &reg, FR_AB_MD_ID);
2172
2173         /* Request data to be read */
2174         EFX_POPULATE_OWORD_2(reg, FRF_AB_MD_RDC, 1, FRF_AB_MD_GC, 0);
2175         efx_writeo(efx, &reg, FR_AB_MD_CS);
2176
2177         /* Wait for data to become available */
2178         rc = falcon_gmii_wait(efx);
2179         if (rc == 0) {
2180                 efx_reado(efx, &reg, FR_AB_MD_RXD);
2181                 rc = EFX_OWORD_FIELD(reg, FRF_AB_MD_RXD);
2182                 EFX_REGDUMP(efx, "read from MDIO %d register %d.%d, got %04x\n",
2183                             prtad, devad, addr, rc);
2184         } else {
2185                 /* Abort the read operation */
2186                 EFX_POPULATE_OWORD_2(reg,
2187                                      FRF_AB_MD_RIC, 0,
2188                                      FRF_AB_MD_GC, 1);
2189                 efx_writeo(efx, &reg, FR_AB_MD_CS);
2190
2191                 EFX_LOG(efx, "read from MDIO %d register %d.%d, got error %d\n",
2192                         prtad, devad, addr, rc);
2193         }
2194
2195  out:
2196         spin_unlock_bh(&efx->phy_lock);
2197         return rc;
2198 }
2199
2200 static int falcon_probe_phy(struct efx_nic *efx)
2201 {
2202         switch (efx->phy_type) {
2203         case PHY_TYPE_SFX7101:
2204                 efx->phy_op = &falcon_sfx7101_phy_ops;
2205                 break;
2206         case PHY_TYPE_SFT9001A:
2207         case PHY_TYPE_SFT9001B:
2208                 efx->phy_op = &falcon_sft9001_phy_ops;
2209                 break;
2210         case PHY_TYPE_QT2022C2:
2211         case PHY_TYPE_QT2025C:
2212                 efx->phy_op = &falcon_xfp_phy_ops;
2213                 break;
2214         default:
2215                 EFX_ERR(efx, "Unknown PHY type %d\n",
2216                         efx->phy_type);
2217                 return -1;
2218         }
2219
2220         if (efx->phy_op->macs & EFX_XMAC)
2221                 efx->loopback_modes |= ((1 << LOOPBACK_XGMII) |
2222                                         (1 << LOOPBACK_XGXS) |
2223                                         (1 << LOOPBACK_XAUI));
2224         if (efx->phy_op->macs & EFX_GMAC)
2225                 efx->loopback_modes |= (1 << LOOPBACK_GMAC);
2226         efx->loopback_modes |= efx->phy_op->loopbacks;
2227
2228         return 0;
2229 }
2230
2231 int falcon_switch_mac(struct efx_nic *efx)
2232 {
2233         struct efx_mac_operations *old_mac_op = efx->mac_op;
2234         efx_oword_t nic_stat;
2235         unsigned strap_val;
2236         int rc = 0;
2237
2238         /* Don't try to fetch MAC stats while we're switching MACs */
2239         efx_stats_disable(efx);
2240
2241         /* Internal loopbacks override the phy speed setting */
2242         if (efx->loopback_mode == LOOPBACK_GMAC) {
2243                 efx->link_speed = 1000;
2244                 efx->link_fd = true;
2245         } else if (LOOPBACK_INTERNAL(efx)) {
2246                 efx->link_speed = 10000;
2247                 efx->link_fd = true;
2248         }
2249
2250         WARN_ON(!mutex_is_locked(&efx->mac_lock));
2251         efx->mac_op = (EFX_IS10G(efx) ?
2252                        &falcon_xmac_operations : &falcon_gmac_operations);
2253
2254         /* Always push the NIC_STAT_REG setting even if the mac hasn't
2255          * changed, because this function is run post online reset */
2256         efx_reado(efx, &nic_stat, FR_AB_NIC_STAT);
2257         strap_val = EFX_IS10G(efx) ? 5 : 3;
2258         if (falcon_rev(efx) >= FALCON_REV_B0) {
2259                 EFX_SET_OWORD_FIELD(nic_stat, FRF_BB_EE_STRAP_EN, 1);
2260                 EFX_SET_OWORD_FIELD(nic_stat, FRF_BB_EE_STRAP, strap_val);
2261                 efx_writeo(efx, &nic_stat, FR_AB_NIC_STAT);
2262         } else {
2263                 /* Falcon A1 does not support 1G/10G speed switching
2264                  * and must not be used with a PHY that does. */
2265                 BUG_ON(EFX_OWORD_FIELD(nic_stat, FRF_AB_STRAP_PINS) !=
2266                        strap_val);
2267         }
2268
2269         if (old_mac_op == efx->mac_op)
2270                 goto out;
2271
2272         EFX_LOG(efx, "selected %cMAC\n", EFX_IS10G(efx) ? 'X' : 'G');
2273         /* Not all macs support a mac-level link state */
2274         efx->mac_up = true;
2275
2276         rc = falcon_reset_macs(efx);
2277 out:
2278         efx_stats_enable(efx);
2279         return rc;
2280 }
2281
2282 /* This call is responsible for hooking in the MAC and PHY operations */
2283 int falcon_probe_port(struct efx_nic *efx)
2284 {
2285         int rc;
2286
2287         /* Hook in PHY operations table */
2288         rc = falcon_probe_phy(efx);
2289         if (rc)
2290                 return rc;
2291
2292         /* Set up MDIO structure for PHY */
2293         efx->mdio.mmds = efx->phy_op->mmds;
2294         efx->mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22;
2295         efx->mdio.mdio_read = falcon_mdio_read;
2296         efx->mdio.mdio_write = falcon_mdio_write;
2297
2298         /* Hardware flow ctrl. FalconA RX FIFO too small for pause generation */
2299         if (falcon_rev(efx) >= FALCON_REV_B0)
2300                 efx->wanted_fc = EFX_FC_RX | EFX_FC_TX;
2301         else
2302                 efx->wanted_fc = EFX_FC_RX;
2303
2304         /* Allocate buffer for stats */
2305         rc = falcon_alloc_buffer(efx, &efx->stats_buffer,
2306                                  FALCON_MAC_STATS_SIZE);
2307         if (rc)
2308                 return rc;
2309         EFX_LOG(efx, "stats buffer at %llx (virt %p phys %llx)\n",
2310                 (u64)efx->stats_buffer.dma_addr,
2311                 efx->stats_buffer.addr,
2312                 (u64)virt_to_phys(efx->stats_buffer.addr));
2313
2314         return 0;
2315 }
2316
2317 void falcon_remove_port(struct efx_nic *efx)
2318 {
2319         falcon_free_buffer(efx, &efx->stats_buffer);
2320 }
2321
2322 /**************************************************************************
2323  *
2324  * Multicast filtering
2325  *
2326  **************************************************************************
2327  */
2328
2329 void falcon_set_multicast_hash(struct efx_nic *efx)
2330 {
2331         union efx_multicast_hash *mc_hash = &efx->multicast_hash;
2332
2333         /* Broadcast packets go through the multicast hash filter.
2334          * ether_crc_le() of the broadcast address is 0xbe2612ff
2335          * so we always add bit 0xff to the mask.
2336          */
2337         set_bit_le(0xff, mc_hash->byte);
2338
2339         efx_writeo(efx, &mc_hash->oword[0], FR_AB_MAC_MC_HASH_REG0);
2340         efx_writeo(efx, &mc_hash->oword[1], FR_AB_MAC_MC_HASH_REG1);
2341 }
2342
2343
2344 /**************************************************************************
2345  *
2346  * Falcon test code
2347  *
2348  **************************************************************************/
2349
2350 int falcon_read_nvram(struct efx_nic *efx, struct falcon_nvconfig *nvconfig_out)
2351 {
2352         struct falcon_nvconfig *nvconfig;
2353         struct efx_spi_device *spi;
2354         void *region;
2355         int rc, magic_num, struct_ver;
2356         __le16 *word, *limit;
2357         u32 csum;
2358
2359         spi = efx->spi_flash ? efx->spi_flash : efx->spi_eeprom;
2360         if (!spi)
2361                 return -EINVAL;
2362
2363         region = kmalloc(FALCON_NVCONFIG_END, GFP_KERNEL);
2364         if (!region)
2365                 return -ENOMEM;
2366         nvconfig = region + FALCON_NVCONFIG_OFFSET;
2367
2368         mutex_lock(&efx->spi_lock);
2369         rc = falcon_spi_read(spi, 0, FALCON_NVCONFIG_END, NULL, region);
2370         mutex_unlock(&efx->spi_lock);
2371         if (rc) {
2372                 EFX_ERR(efx, "Failed to read %s\n",
2373                         efx->spi_flash ? "flash" : "EEPROM");
2374                 rc = -EIO;
2375                 goto out;
2376         }
2377
2378         magic_num = le16_to_cpu(nvconfig->board_magic_num);
2379         struct_ver = le16_to_cpu(nvconfig->board_struct_ver);
2380
2381         rc = -EINVAL;
2382         if (magic_num != FALCON_NVCONFIG_BOARD_MAGIC_NUM) {
2383                 EFX_ERR(efx, "NVRAM bad magic 0x%x\n", magic_num);
2384                 goto out;
2385         }
2386         if (struct_ver < 2) {
2387                 EFX_ERR(efx, "NVRAM has ancient version 0x%x\n", struct_ver);
2388                 goto out;
2389         } else if (struct_ver < 4) {
2390                 word = &nvconfig->board_magic_num;
2391                 limit = (__le16 *) (nvconfig + 1);
2392         } else {
2393                 word = region;
2394                 limit = region + FALCON_NVCONFIG_END;
2395         }
2396         for (csum = 0; word < limit; ++word)
2397                 csum += le16_to_cpu(*word);
2398
2399         if (~csum & 0xffff) {
2400                 EFX_ERR(efx, "NVRAM has incorrect checksum\n");
2401                 goto out;
2402         }
2403
2404         rc = 0;
2405         if (nvconfig_out)
2406                 memcpy(nvconfig_out, nvconfig, sizeof(*nvconfig));
2407
2408  out:
2409         kfree(region);
2410         return rc;
2411 }
2412
2413 /* Registers tested in the falcon register test */
2414 static struct {
2415         unsigned address;
2416         efx_oword_t mask;
2417 } efx_test_registers[] = {
2418         { FR_AZ_ADR_REGION,
2419           EFX_OWORD32(0x0001FFFF, 0x0001FFFF, 0x0001FFFF, 0x0001FFFF) },
2420         { FR_AZ_RX_CFG,
2421           EFX_OWORD32(0xFFFFFFFE, 0x00017FFF, 0x00000000, 0x00000000) },
2422         { FR_AZ_TX_CFG,
2423           EFX_OWORD32(0x7FFF0037, 0x00000000, 0x00000000, 0x00000000) },
2424         { FR_AZ_TX_RESERVED,
2425           EFX_OWORD32(0xFFFEFE80, 0x1FFFFFFF, 0x020000FE, 0x007FFFFF) },
2426         { FR_AB_MAC_CTRL,
2427           EFX_OWORD32(0xFFFF0000, 0x00000000, 0x00000000, 0x00000000) },
2428         { FR_AZ_SRM_TX_DC_CFG,
2429           EFX_OWORD32(0x001FFFFF, 0x00000000, 0x00000000, 0x00000000) },
2430         { FR_AZ_RX_DC_CFG,
2431           EFX_OWORD32(0x0000000F, 0x00000000, 0x00000000, 0x00000000) },
2432         { FR_AZ_RX_DC_PF_WM,
2433           EFX_OWORD32(0x000003FF, 0x00000000, 0x00000000, 0x00000000) },
2434         { FR_BZ_DP_CTRL,
2435           EFX_OWORD32(0x00000FFF, 0x00000000, 0x00000000, 0x00000000) },
2436         { FR_AB_GM_CFG2,
2437           EFX_OWORD32(0x00007337, 0x00000000, 0x00000000, 0x00000000) },
2438         { FR_AB_GMF_CFG0,
2439           EFX_OWORD32(0x00001F1F, 0x00000000, 0x00000000, 0x00000000) },
2440         { FR_AB_XM_GLB_CFG,
2441           EFX_OWORD32(0x00000C68, 0x00000000, 0x00000000, 0x00000000) },
2442         { FR_AB_XM_TX_CFG,
2443           EFX_OWORD32(0x00080164, 0x00000000, 0x00000000, 0x00000000) },
2444         { FR_AB_XM_RX_CFG,
2445           EFX_OWORD32(0x07100A0C, 0x00000000, 0x00000000, 0x00000000) },
2446         { FR_AB_XM_RX_PARAM,
2447           EFX_OWORD32(0x00001FF8, 0x00000000, 0x00000000, 0x00000000) },
2448         { FR_AB_XM_FC,
2449           EFX_OWORD32(0xFFFF0001, 0x00000000, 0x00000000, 0x00000000) },
2450         { FR_AB_XM_ADR_LO,
2451           EFX_OWORD32(0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000) },
2452         { FR_AB_XX_SD_CTL,
2453           EFX_OWORD32(0x0003FF0F, 0x00000000, 0x00000000, 0x00000000) },
2454 };
2455
2456 static bool efx_masked_compare_oword(const efx_oword_t *a, const efx_oword_t *b,
2457                                      const efx_oword_t *mask)
2458 {
2459         return ((a->u64[0] ^ b->u64[0]) & mask->u64[0]) ||
2460                 ((a->u64[1] ^ b->u64[1]) & mask->u64[1]);
2461 }
2462
2463 int falcon_test_registers(struct efx_nic *efx)
2464 {
2465         unsigned address = 0, i, j;
2466         efx_oword_t mask, imask, original, reg, buf;
2467
2468         /* Falcon should be in loopback to isolate the XMAC from the PHY */
2469         WARN_ON(!LOOPBACK_INTERNAL(efx));
2470
2471         for (i = 0; i < ARRAY_SIZE(efx_test_registers); ++i) {
2472                 address = efx_test_registers[i].address;
2473                 mask = imask = efx_test_registers[i].mask;
2474                 EFX_INVERT_OWORD(imask);
2475
2476                 efx_reado(efx, &original, address);
2477
2478                 /* bit sweep on and off */
2479                 for (j = 0; j < 128; j++) {
2480                         if (!EFX_EXTRACT_OWORD32(mask, j, j))
2481                                 continue;
2482
2483                         /* Test this testable bit can be set in isolation */
2484                         EFX_AND_OWORD(reg, original, mask);
2485                         EFX_SET_OWORD32(reg, j, j, 1);
2486
2487                         efx_writeo(efx, &reg, address);
2488                         efx_reado(efx, &buf, address);
2489
2490                         if (efx_masked_compare_oword(&reg, &buf, &mask))
2491                                 goto fail;
2492
2493                         /* Test this testable bit can be cleared in isolation */
2494                         EFX_OR_OWORD(reg, original, mask);
2495                         EFX_SET_OWORD32(reg, j, j, 0);
2496
2497                         efx_writeo(efx, &reg, address);
2498                         efx_reado(efx, &buf, address);
2499
2500                         if (efx_masked_compare_oword(&reg, &buf, &mask))
2501                                 goto fail;
2502                 }
2503
2504                 efx_writeo(efx, &original, address);
2505         }
2506
2507         return 0;
2508
2509 fail:
2510         EFX_ERR(efx, "wrote "EFX_OWORD_FMT" read "EFX_OWORD_FMT
2511                 " at address 0x%x mask "EFX_OWORD_FMT"\n", EFX_OWORD_VAL(reg),
2512                 EFX_OWORD_VAL(buf), address, EFX_OWORD_VAL(mask));
2513         return -EIO;
2514 }
2515
2516 /**************************************************************************
2517  *
2518  * Device reset
2519  *
2520  **************************************************************************
2521  */
2522
2523 /* Resets NIC to known state.  This routine must be called in process
2524  * context and is allowed to sleep. */
2525 int falcon_reset_hw(struct efx_nic *efx, enum reset_type method)
2526 {
2527         struct falcon_nic_data *nic_data = efx->nic_data;
2528         efx_oword_t glb_ctl_reg_ker;
2529         int rc;
2530
2531         EFX_LOG(efx, "performing hardware reset (%d)\n", method);
2532
2533         /* Initiate device reset */
2534         if (method == RESET_TYPE_WORLD) {
2535                 rc = pci_save_state(efx->pci_dev);
2536                 if (rc) {
2537                         EFX_ERR(efx, "failed to backup PCI state of primary "
2538                                 "function prior to hardware reset\n");
2539                         goto fail1;
2540                 }
2541                 if (FALCON_IS_DUAL_FUNC(efx)) {
2542                         rc = pci_save_state(nic_data->pci_dev2);
2543                         if (rc) {
2544                                 EFX_ERR(efx, "failed to backup PCI state of "
2545                                         "secondary function prior to "
2546                                         "hardware reset\n");
2547                                 goto fail2;
2548                         }
2549                 }
2550
2551                 EFX_POPULATE_OWORD_2(glb_ctl_reg_ker,
2552                                      FRF_AB_EXT_PHY_RST_DUR,
2553                                      FFE_AB_EXT_PHY_RST_DUR_10240US,
2554                                      FRF_AB_SWRST, 1);
2555         } else {
2556                 EFX_POPULATE_OWORD_7(glb_ctl_reg_ker,
2557                                      /* exclude PHY from "invisible" reset */
2558                                      FRF_AB_EXT_PHY_RST_CTL,
2559                                      method == RESET_TYPE_INVISIBLE,
2560                                      /* exclude EEPROM/flash and PCIe */
2561                                      FRF_AB_PCIE_CORE_RST_CTL, 1,
2562                                      FRF_AB_PCIE_NSTKY_RST_CTL, 1,
2563                                      FRF_AB_PCIE_SD_RST_CTL, 1,
2564                                      FRF_AB_EE_RST_CTL, 1,
2565                                      FRF_AB_EXT_PHY_RST_DUR,
2566                                      FFE_AB_EXT_PHY_RST_DUR_10240US,
2567                                      FRF_AB_SWRST, 1);
2568         }
2569         efx_writeo(efx, &glb_ctl_reg_ker, FR_AB_GLB_CTL);
2570
2571         EFX_LOG(efx, "waiting for hardware reset\n");
2572         schedule_timeout_uninterruptible(HZ / 20);
2573
2574         /* Restore PCI configuration if needed */
2575         if (method == RESET_TYPE_WORLD) {
2576                 if (FALCON_IS_DUAL_FUNC(efx)) {
2577                         rc = pci_restore_state(nic_data->pci_dev2);
2578                         if (rc) {
2579                                 EFX_ERR(efx, "failed to restore PCI config for "
2580                                         "the secondary function\n");
2581                                 goto fail3;
2582                         }
2583                 }
2584                 rc = pci_restore_state(efx->pci_dev);
2585                 if (rc) {
2586                         EFX_ERR(efx, "failed to restore PCI config for the "
2587                                 "primary function\n");
2588                         goto fail4;
2589                 }
2590                 EFX_LOG(efx, "successfully restored PCI config\n");
2591         }
2592
2593         /* Assert that reset complete */
2594         efx_reado(efx, &glb_ctl_reg_ker, FR_AB_GLB_CTL);
2595         if (EFX_OWORD_FIELD(glb_ctl_reg_ker, FRF_AB_SWRST) != 0) {
2596                 rc = -ETIMEDOUT;
2597                 EFX_ERR(efx, "timed out waiting for hardware reset\n");
2598                 goto fail5;
2599         }
2600         EFX_LOG(efx, "hardware reset complete\n");
2601
2602         return 0;
2603
2604         /* pci_save_state() and pci_restore_state() MUST be called in pairs */
2605 fail2:
2606 fail3:
2607         pci_restore_state(efx->pci_dev);
2608 fail1:
2609 fail4:
2610 fail5:
2611         return rc;
2612 }
2613
2614 /* Zeroes out the SRAM contents.  This routine must be called in
2615  * process context and is allowed to sleep.
2616  */
2617 static int falcon_reset_sram(struct efx_nic *efx)
2618 {
2619         efx_oword_t srm_cfg_reg_ker, gpio_cfg_reg_ker;
2620         int count;
2621
2622         /* Set the SRAM wake/sleep GPIO appropriately. */
2623         efx_reado(efx, &gpio_cfg_reg_ker, FR_AB_GPIO_CTL);
2624         EFX_SET_OWORD_FIELD(gpio_cfg_reg_ker, FRF_AB_GPIO1_OEN, 1);
2625         EFX_SET_OWORD_FIELD(gpio_cfg_reg_ker, FRF_AB_GPIO1_OUT, 1);
2626         efx_writeo(efx, &gpio_cfg_reg_ker, FR_AB_GPIO_CTL);
2627
2628         /* Initiate SRAM reset */
2629         EFX_POPULATE_OWORD_2(srm_cfg_reg_ker,
2630                              FRF_AZ_SRM_INIT_EN, 1,
2631                              FRF_AZ_SRM_NB_SZ, 0);
2632         efx_writeo(efx, &srm_cfg_reg_ker, FR_AZ_SRM_CFG);
2633
2634         /* Wait for SRAM reset to complete */
2635         count = 0;
2636         do {
2637                 EFX_LOG(efx, "waiting for SRAM reset (attempt %d)...\n", count);
2638
2639                 /* SRAM reset is slow; expect around 16ms */
2640                 schedule_timeout_uninterruptible(HZ / 50);
2641
2642                 /* Check for reset complete */
2643                 efx_reado(efx, &srm_cfg_reg_ker, FR_AZ_SRM_CFG);
2644                 if (!EFX_OWORD_FIELD(srm_cfg_reg_ker, FRF_AZ_SRM_INIT_EN)) {
2645                         EFX_LOG(efx, "SRAM reset complete\n");
2646
2647                         return 0;
2648                 }
2649         } while (++count < 20); /* wait upto 0.4 sec */
2650
2651         EFX_ERR(efx, "timed out waiting for SRAM reset\n");
2652         return -ETIMEDOUT;
2653 }
2654
2655 static int falcon_spi_device_init(struct efx_nic *efx,
2656                                   struct efx_spi_device **spi_device_ret,
2657                                   unsigned int device_id, u32 device_type)
2658 {
2659         struct efx_spi_device *spi_device;
2660
2661         if (device_type != 0) {
2662                 spi_device = kzalloc(sizeof(*spi_device), GFP_KERNEL);
2663                 if (!spi_device)
2664                         return -ENOMEM;
2665                 spi_device->device_id = device_id;
2666                 spi_device->size =
2667                         1 << SPI_DEV_TYPE_FIELD(device_type, SPI_DEV_TYPE_SIZE);
2668                 spi_device->addr_len =
2669                         SPI_DEV_TYPE_FIELD(device_type, SPI_DEV_TYPE_ADDR_LEN);
2670                 spi_device->munge_address = (spi_device->size == 1 << 9 &&
2671                                              spi_device->addr_len == 1);
2672                 spi_device->erase_command =
2673                         SPI_DEV_TYPE_FIELD(device_type, SPI_DEV_TYPE_ERASE_CMD);
2674                 spi_device->erase_size =
2675                         1 << SPI_DEV_TYPE_FIELD(device_type,
2676                                                 SPI_DEV_TYPE_ERASE_SIZE);
2677                 spi_device->block_size =
2678                         1 << SPI_DEV_TYPE_FIELD(device_type,
2679                                                 SPI_DEV_TYPE_BLOCK_SIZE);
2680
2681                 spi_device->efx = efx;
2682         } else {
2683                 spi_device = NULL;
2684         }
2685
2686         kfree(*spi_device_ret);
2687         *spi_device_ret = spi_device;
2688         return 0;
2689 }
2690
2691
2692 static void falcon_remove_spi_devices(struct efx_nic *efx)
2693 {
2694         kfree(efx->spi_eeprom);
2695         efx->spi_eeprom = NULL;
2696         kfree(efx->spi_flash);
2697         efx->spi_flash = NULL;
2698 }
2699
2700 /* Extract non-volatile configuration */
2701 static int falcon_probe_nvconfig(struct efx_nic *efx)
2702 {
2703         struct falcon_nvconfig *nvconfig;
2704         int board_rev;
2705         int rc;
2706
2707         nvconfig = kmalloc(sizeof(*nvconfig), GFP_KERNEL);
2708         if (!nvconfig)
2709                 return -ENOMEM;
2710
2711         rc = falcon_read_nvram(efx, nvconfig);
2712         if (rc == -EINVAL) {
2713                 EFX_ERR(efx, "NVRAM is invalid therefore using defaults\n");
2714                 efx->phy_type = PHY_TYPE_NONE;
2715                 efx->mdio.prtad = MDIO_PRTAD_NONE;
2716                 board_rev = 0;
2717                 rc = 0;
2718         } else if (rc) {
2719                 goto fail1;
2720         } else {
2721                 struct falcon_nvconfig_board_v2 *v2 = &nvconfig->board_v2;
2722                 struct falcon_nvconfig_board_v3 *v3 = &nvconfig->board_v3;
2723
2724                 efx->phy_type = v2->port0_phy_type;
2725                 efx->mdio.prtad = v2->port0_phy_addr;
2726                 board_rev = le16_to_cpu(v2->board_revision);
2727
2728                 if (le16_to_cpu(nvconfig->board_struct_ver) >= 3) {
2729                         rc = falcon_spi_device_init(
2730                                 efx, &efx->spi_flash, FFE_AB_SPI_DEVICE_FLASH,
2731                                 le32_to_cpu(v3->spi_device_type
2732                                             [FFE_AB_SPI_DEVICE_FLASH]));
2733                         if (rc)
2734                                 goto fail2;
2735                         rc = falcon_spi_device_init(
2736                                 efx, &efx->spi_eeprom, FFE_AB_SPI_DEVICE_EEPROM,
2737                                 le32_to_cpu(v3->spi_device_type
2738                                             [FFE_AB_SPI_DEVICE_EEPROM]));
2739                         if (rc)
2740                                 goto fail2;
2741                 }
2742         }
2743
2744         /* Read the MAC addresses */
2745         memcpy(efx->mac_address, nvconfig->mac_address[0], ETH_ALEN);
2746
2747         EFX_LOG(efx, "PHY is %d phy_id %d\n", efx->phy_type, efx->mdio.prtad);
2748
2749         falcon_probe_board(efx, board_rev);
2750
2751         kfree(nvconfig);
2752         return 0;
2753
2754  fail2:
2755         falcon_remove_spi_devices(efx);
2756  fail1:
2757         kfree(nvconfig);
2758         return rc;
2759 }
2760
2761 /* Probe the NIC variant (revision, ASIC vs FPGA, function count, port
2762  * count, port speed).  Set workaround and feature flags accordingly.
2763  */
2764 static int falcon_probe_nic_variant(struct efx_nic *efx)
2765 {
2766         efx_oword_t altera_build;
2767         efx_oword_t nic_stat;
2768
2769         efx_reado(efx, &altera_build, FR_AZ_ALTERA_BUILD);
2770         if (EFX_OWORD_FIELD(altera_build, FRF_AZ_ALTERA_BUILD_VER)) {
2771                 EFX_ERR(efx, "Falcon FPGA not supported\n");
2772                 return -ENODEV;
2773         }
2774
2775         efx_reado(efx, &nic_stat, FR_AB_NIC_STAT);
2776
2777         switch (falcon_rev(efx)) {
2778         case FALCON_REV_A0:
2779         case 0xff:
2780                 EFX_ERR(efx, "Falcon rev A0 not supported\n");
2781                 return -ENODEV;
2782
2783         case FALCON_REV_A1:
2784                 if (EFX_OWORD_FIELD(nic_stat, FRF_AA_STRAP_PCIE) == 0) {
2785                         EFX_ERR(efx, "Falcon rev A1 PCI-X not supported\n");
2786                         return -ENODEV;
2787                 }
2788                 break;
2789
2790         case FALCON_REV_B0:
2791                 break;
2792
2793         default:
2794                 EFX_ERR(efx, "Unknown Falcon rev %d\n", falcon_rev(efx));
2795                 return -ENODEV;
2796         }
2797
2798         /* Initial assumed speed */
2799         efx->link_speed = EFX_OWORD_FIELD(nic_stat, FRF_AB_STRAP_10G) ? 10000 : 1000;
2800
2801         return 0;
2802 }
2803
2804 /* Probe all SPI devices on the NIC */
2805 static void falcon_probe_spi_devices(struct efx_nic *efx)
2806 {
2807         efx_oword_t nic_stat, gpio_ctl, ee_vpd_cfg;
2808         int boot_dev;
2809
2810         efx_reado(efx, &gpio_ctl, FR_AB_GPIO_CTL);
2811         efx_reado(efx, &nic_stat, FR_AB_NIC_STAT);
2812         efx_reado(efx, &ee_vpd_cfg, FR_AB_EE_VPD_CFG0);
2813
2814         if (EFX_OWORD_FIELD(gpio_ctl, FRF_AB_GPIO3_PWRUP_VALUE)) {
2815                 boot_dev = (EFX_OWORD_FIELD(nic_stat, FRF_AB_SF_PRST) ?
2816                             FFE_AB_SPI_DEVICE_FLASH : FFE_AB_SPI_DEVICE_EEPROM);
2817                 EFX_LOG(efx, "Booted from %s\n",
2818                         boot_dev == FFE_AB_SPI_DEVICE_FLASH ? "flash" : "EEPROM");
2819         } else {
2820                 /* Disable VPD and set clock dividers to safe
2821                  * values for initial programming. */
2822                 boot_dev = -1;
2823                 EFX_LOG(efx, "Booted from internal ASIC settings;"
2824                         " setting SPI config\n");
2825                 EFX_POPULATE_OWORD_3(ee_vpd_cfg, FRF_AB_EE_VPD_EN, 0,
2826                                      /* 125 MHz / 7 ~= 20 MHz */
2827                                      FRF_AB_EE_SF_CLOCK_DIV, 7,
2828                                      /* 125 MHz / 63 ~= 2 MHz */
2829                                      FRF_AB_EE_EE_CLOCK_DIV, 63);
2830                 efx_writeo(efx, &ee_vpd_cfg, FR_AB_EE_VPD_CFG0);
2831         }
2832
2833         if (boot_dev == FFE_AB_SPI_DEVICE_FLASH)
2834                 falcon_spi_device_init(efx, &efx->spi_flash,
2835                                        FFE_AB_SPI_DEVICE_FLASH,
2836                                        default_flash_type);
2837         if (boot_dev == FFE_AB_SPI_DEVICE_EEPROM)
2838                 falcon_spi_device_init(efx, &efx->spi_eeprom,
2839                                        FFE_AB_SPI_DEVICE_EEPROM,
2840                                        large_eeprom_type);
2841 }
2842
2843 int falcon_probe_nic(struct efx_nic *efx)
2844 {
2845         struct falcon_nic_data *nic_data;
2846         int rc;
2847
2848         /* Allocate storage for hardware specific data */
2849         nic_data = kzalloc(sizeof(*nic_data), GFP_KERNEL);
2850         if (!nic_data)
2851                 return -ENOMEM;
2852         efx->nic_data = nic_data;
2853
2854         /* Determine number of ports etc. */
2855         rc = falcon_probe_nic_variant(efx);
2856         if (rc)
2857                 goto fail1;
2858
2859         /* Probe secondary function if expected */
2860         if (FALCON_IS_DUAL_FUNC(efx)) {
2861                 struct pci_dev *dev = pci_dev_get(efx->pci_dev);
2862
2863                 while ((dev = pci_get_device(EFX_VENDID_SFC, FALCON_A_S_DEVID,
2864                                              dev))) {
2865                         if (dev->bus == efx->pci_dev->bus &&
2866                             dev->devfn == efx->pci_dev->devfn + 1) {
2867                                 nic_data->pci_dev2 = dev;
2868                                 break;
2869                         }
2870                 }
2871                 if (!nic_data->pci_dev2) {
2872                         EFX_ERR(efx, "failed to find secondary function\n");
2873                         rc = -ENODEV;
2874                         goto fail2;
2875                 }
2876         }
2877
2878         /* Now we can reset the NIC */
2879         rc = falcon_reset_hw(efx, RESET_TYPE_ALL);
2880         if (rc) {
2881                 EFX_ERR(efx, "failed to reset NIC\n");
2882                 goto fail3;
2883         }
2884
2885         /* Allocate memory for INT_KER */
2886         rc = falcon_alloc_buffer(efx, &efx->irq_status, sizeof(efx_oword_t));
2887         if (rc)
2888                 goto fail4;
2889         BUG_ON(efx->irq_status.dma_addr & 0x0f);
2890
2891         EFX_LOG(efx, "INT_KER at %llx (virt %p phys %llx)\n",
2892                 (u64)efx->irq_status.dma_addr,
2893                 efx->irq_status.addr, (u64)virt_to_phys(efx->irq_status.addr));
2894
2895         falcon_probe_spi_devices(efx);
2896
2897         /* Read in the non-volatile configuration */
2898         rc = falcon_probe_nvconfig(efx);
2899         if (rc)
2900                 goto fail5;
2901
2902         /* Initialise I2C adapter */
2903         efx->i2c_adap.owner = THIS_MODULE;
2904         nic_data->i2c_data = falcon_i2c_bit_operations;
2905         nic_data->i2c_data.data = efx;
2906         efx->i2c_adap.algo_data = &nic_data->i2c_data;
2907         efx->i2c_adap.dev.parent = &efx->pci_dev->dev;
2908         strlcpy(efx->i2c_adap.name, "SFC4000 GPIO", sizeof(efx->i2c_adap.name));
2909         rc = i2c_bit_add_bus(&efx->i2c_adap);
2910         if (rc)
2911                 goto fail5;
2912
2913         return 0;
2914
2915  fail5:
2916         falcon_remove_spi_devices(efx);
2917         falcon_free_buffer(efx, &efx->irq_status);
2918  fail4:
2919  fail3:
2920         if (nic_data->pci_dev2) {
2921                 pci_dev_put(nic_data->pci_dev2);
2922                 nic_data->pci_dev2 = NULL;
2923         }
2924  fail2:
2925  fail1:
2926         kfree(efx->nic_data);
2927         return rc;
2928 }
2929
2930 static void falcon_init_rx_cfg(struct efx_nic *efx)
2931 {
2932         /* Prior to Siena the RX DMA engine will split each frame at
2933          * intervals of RX_USR_BUF_SIZE (32-byte units). We set it to
2934          * be so large that that never happens. */
2935         const unsigned huge_buf_size = (3 * 4096) >> 5;
2936         /* RX control FIFO thresholds (32 entries) */
2937         const unsigned ctrl_xon_thr = 20;
2938         const unsigned ctrl_xoff_thr = 25;
2939         /* RX data FIFO thresholds (256-byte units; size varies) */
2940         int data_xon_thr = rx_xon_thresh_bytes >> 8;
2941         int data_xoff_thr = rx_xoff_thresh_bytes >> 8;
2942         efx_oword_t reg;
2943
2944         efx_reado(efx, &reg, FR_AZ_RX_CFG);
2945         if (falcon_rev(efx) <= FALCON_REV_A1) {
2946                 /* Data FIFO size is 5.5K */
2947                 if (data_xon_thr < 0)
2948                         data_xon_thr = 512 >> 8;
2949                 if (data_xoff_thr < 0)
2950                         data_xoff_thr = 2048 >> 8;
2951                 EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_DESC_PUSH_EN, 0);
2952                 EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_USR_BUF_SIZE,
2953                                     huge_buf_size);
2954                 EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XON_MAC_TH, data_xon_thr);
2955                 EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XOFF_MAC_TH, data_xoff_thr);
2956                 EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XON_TX_TH, ctrl_xon_thr);
2957                 EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XOFF_TX_TH, ctrl_xoff_thr);
2958         } else {
2959                 /* Data FIFO size is 80K; register fields moved */
2960                 if (data_xon_thr < 0)
2961                         data_xon_thr = 27648 >> 8; /* ~3*max MTU */
2962                 if (data_xoff_thr < 0)
2963                         data_xoff_thr = 54272 >> 8; /* ~80Kb - 3*max MTU */
2964                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_DESC_PUSH_EN, 0);
2965                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_USR_BUF_SIZE,
2966                                     huge_buf_size);
2967                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_XON_MAC_TH, data_xon_thr);
2968                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_XOFF_MAC_TH, data_xoff_thr);
2969                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_XON_TX_TH, ctrl_xon_thr);
2970                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_XOFF_TX_TH, ctrl_xoff_thr);
2971                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_INGR_EN, 1);
2972         }
2973         efx_writeo(efx, &reg, FR_AZ_RX_CFG);
2974 }
2975
2976 /* This call performs hardware-specific global initialisation, such as
2977  * defining the descriptor cache sizes and number of RSS channels.
2978  * It does not set up any buffers, descriptor rings or event queues.
2979  */
2980 int falcon_init_nic(struct efx_nic *efx)
2981 {
2982         efx_oword_t temp;
2983         int rc;
2984
2985         /* Use on-chip SRAM */
2986         efx_reado(efx, &temp, FR_AB_NIC_STAT);
2987         EFX_SET_OWORD_FIELD(temp, FRF_AB_ONCHIP_SRAM, 1);
2988         efx_writeo(efx, &temp, FR_AB_NIC_STAT);
2989
2990         /* Set the source of the GMAC clock */
2991         if (falcon_rev(efx) == FALCON_REV_B0) {
2992                 efx_reado(efx, &temp, FR_AB_GPIO_CTL);
2993                 EFX_SET_OWORD_FIELD(temp, FRF_AB_USE_NIC_CLK, true);
2994                 efx_writeo(efx, &temp, FR_AB_GPIO_CTL);
2995         }
2996
2997         rc = falcon_reset_sram(efx);
2998         if (rc)
2999                 return rc;
3000
3001         /* Set positions of descriptor caches in SRAM. */
3002         EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_TX_DC_BASE_ADR, TX_DC_BASE / 8);
3003         efx_writeo(efx, &temp, FR_AZ_SRM_TX_DC_CFG);
3004         EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_RX_DC_BASE_ADR, RX_DC_BASE / 8);
3005         efx_writeo(efx, &temp, FR_AZ_SRM_RX_DC_CFG);
3006
3007         /* Set TX descriptor cache size. */
3008         BUILD_BUG_ON(TX_DC_ENTRIES != (16 << TX_DC_ENTRIES_ORDER));
3009         EFX_POPULATE_OWORD_1(temp, FRF_AZ_TX_DC_SIZE, TX_DC_ENTRIES_ORDER);
3010         efx_writeo(efx, &temp, FR_AZ_TX_DC_CFG);
3011
3012         /* Set RX descriptor cache size.  Set low watermark to size-8, as
3013          * this allows most efficient prefetching.
3014          */
3015         BUILD_BUG_ON(RX_DC_ENTRIES != (16 << RX_DC_ENTRIES_ORDER));
3016         EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_SIZE, RX_DC_ENTRIES_ORDER);
3017         efx_writeo(efx, &temp, FR_AZ_RX_DC_CFG);
3018         EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_PF_LWM, RX_DC_ENTRIES - 8);
3019         efx_writeo(efx, &temp, FR_AZ_RX_DC_PF_WM);
3020
3021         /* Clear the parity enables on the TX data fifos as
3022          * they produce false parity errors because of timing issues
3023          */
3024         if (EFX_WORKAROUND_5129(efx)) {
3025                 efx_reado(efx, &temp, FR_AZ_CSR_SPARE);
3026                 EFX_SET_OWORD_FIELD(temp, FRF_AB_MEM_PERR_EN_TX_DATA, 0);
3027                 efx_writeo(efx, &temp, FR_AZ_CSR_SPARE);
3028         }
3029
3030         /* Enable all the genuinely fatal interrupts.  (They are still
3031          * masked by the overall interrupt mask, controlled by
3032          * falcon_interrupts()).
3033          *
3034          * Note: All other fatal interrupts are enabled
3035          */
3036         EFX_POPULATE_OWORD_3(temp,
3037                              FRF_AZ_ILL_ADR_INT_KER_EN, 1,
3038                              FRF_AZ_RBUF_OWN_INT_KER_EN, 1,
3039                              FRF_AZ_TBUF_OWN_INT_KER_EN, 1);
3040         EFX_INVERT_OWORD(temp);
3041         efx_writeo(efx, &temp, FR_AZ_FATAL_INTR_KER);
3042
3043         if (EFX_WORKAROUND_7244(efx)) {
3044                 efx_reado(efx, &temp, FR_BZ_RX_FILTER_CTL);
3045                 EFX_SET_OWORD_FIELD(temp, FRF_BZ_UDP_FULL_SRCH_LIMIT, 8);
3046                 EFX_SET_OWORD_FIELD(temp, FRF_BZ_UDP_WILD_SRCH_LIMIT, 8);
3047                 EFX_SET_OWORD_FIELD(temp, FRF_BZ_TCP_FULL_SRCH_LIMIT, 8);
3048                 EFX_SET_OWORD_FIELD(temp, FRF_BZ_TCP_WILD_SRCH_LIMIT, 8);
3049                 efx_writeo(efx, &temp, FR_BZ_RX_FILTER_CTL);
3050         }
3051
3052         falcon_setup_rss_indir_table(efx);
3053
3054         /* XXX This is documented only for Falcon A0/A1 */
3055         /* Setup RX.  Wait for descriptor is broken and must
3056          * be disabled.  RXDP recovery shouldn't be needed, but is.
3057          */
3058         efx_reado(efx, &temp, FR_AA_RX_SELF_RST);
3059         EFX_SET_OWORD_FIELD(temp, FRF_AA_RX_NODESC_WAIT_DIS, 1);
3060         EFX_SET_OWORD_FIELD(temp, FRF_AA_RX_SELF_RST_EN, 1);
3061         if (EFX_WORKAROUND_5583(efx))
3062                 EFX_SET_OWORD_FIELD(temp, FRF_AA_RX_ISCSI_DIS, 1);
3063         efx_writeo(efx, &temp, FR_AA_RX_SELF_RST);
3064
3065         /* Disable the ugly timer-based TX DMA backoff and allow TX DMA to be
3066          * controlled by the RX FIFO fill level. Set arbitration to one pkt/Q.
3067          */
3068         efx_reado(efx, &temp, FR_AZ_TX_RESERVED);
3069         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER, 0xfe);
3070         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER_EN, 1);
3071         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
3072         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PUSH_EN, 0);
3073         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_DIS_NON_IP_EV, 1);
3074         /* Enable SW_EV to inherit in char driver - assume harmless here */
3075         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_SOFT_EVT_EN, 1);
3076         /* Prefetch threshold 2 => fetch when descriptor cache half empty */
3077         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_THRESHOLD, 2);
3078         /* Squash TX of packets of 16 bytes or less */
3079         if (falcon_rev(efx) >= FALCON_REV_B0 && EFX_WORKAROUND_9141(efx))
3080                 EFX_SET_OWORD_FIELD(temp, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
3081         efx_writeo(efx, &temp, FR_AZ_TX_RESERVED);
3082
3083         /* Do not enable TX_NO_EOP_DISC_EN, since it limits packets to 16
3084          * descriptors (which is bad).
3085          */
3086         efx_reado(efx, &temp, FR_AZ_TX_CFG);
3087         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
3088         efx_writeo(efx, &temp, FR_AZ_TX_CFG);
3089
3090         falcon_init_rx_cfg(efx);
3091
3092         /* Set destination of both TX and RX Flush events */
3093         if (falcon_rev(efx) >= FALCON_REV_B0) {
3094                 EFX_POPULATE_OWORD_1(temp, FRF_BZ_FLS_EVQ_ID, 0);
3095                 efx_writeo(efx, &temp, FR_BZ_DP_CTRL);
3096         }
3097
3098         return 0;
3099 }
3100
3101 void falcon_remove_nic(struct efx_nic *efx)
3102 {
3103         struct falcon_nic_data *nic_data = efx->nic_data;
3104         int rc;
3105
3106         /* Remove I2C adapter and clear it in preparation for a retry */
3107         rc = i2c_del_adapter(&efx->i2c_adap);
3108         BUG_ON(rc);
3109         memset(&efx->i2c_adap, 0, sizeof(efx->i2c_adap));
3110
3111         falcon_remove_spi_devices(efx);
3112         falcon_free_buffer(efx, &efx->irq_status);
3113
3114         falcon_reset_hw(efx, RESET_TYPE_ALL);
3115
3116         /* Release the second function after the reset */
3117         if (nic_data->pci_dev2) {
3118                 pci_dev_put(nic_data->pci_dev2);
3119                 nic_data->pci_dev2 = NULL;
3120         }
3121
3122         /* Tear down the private nic state */
3123         kfree(efx->nic_data);
3124         efx->nic_data = NULL;
3125 }
3126
3127 void falcon_update_nic_stats(struct efx_nic *efx)
3128 {
3129         efx_oword_t cnt;
3130
3131         efx_reado(efx, &cnt, FR_AZ_RX_NODESC_DROP);
3132         efx->n_rx_nodesc_drop_cnt +=
3133                 EFX_OWORD_FIELD(cnt, FRF_AB_RX_NODESC_DROP_CNT);
3134 }
3135
3136 /**************************************************************************
3137  *
3138  * Revision-dependent attributes used by efx.c
3139  *
3140  **************************************************************************
3141  */
3142
3143 struct efx_nic_type falcon_a_nic_type = {
3144         .mem_bar = 2,
3145         .mem_map_size = 0x20000,
3146         .txd_ptr_tbl_base = FR_AA_TX_DESC_PTR_TBL_KER,
3147         .rxd_ptr_tbl_base = FR_AA_RX_DESC_PTR_TBL_KER,
3148         .buf_tbl_base = FR_AA_BUF_FULL_TBL_KER,
3149         .evq_ptr_tbl_base = FR_AA_EVQ_PTR_TBL_KER,
3150         .evq_rptr_tbl_base = FR_AA_EVQ_RPTR_KER,
3151         .max_dma_mask = FALCON_DMA_MASK,
3152         .tx_dma_mask = FALCON_TX_DMA_MASK,
3153         .bug5391_mask = 0xf,
3154         .rx_buffer_padding = 0x24,
3155         .max_interrupt_mode = EFX_INT_MODE_MSI,
3156         .phys_addr_channels = 4,
3157 };
3158
3159 struct efx_nic_type falcon_b_nic_type = {
3160         .mem_bar = 2,
3161         /* Map everything up to and including the RSS indirection
3162          * table.  Don't map MSI-X table, MSI-X PBA since Linux
3163          * requires that they not be mapped.  */
3164         .mem_map_size = (FR_BZ_RX_INDIRECTION_TBL +
3165                          FR_BZ_RX_INDIRECTION_TBL_STEP *
3166                          FR_BZ_RX_INDIRECTION_TBL_ROWS),
3167         .txd_ptr_tbl_base = FR_BZ_TX_DESC_PTR_TBL,
3168         .rxd_ptr_tbl_base = FR_BZ_RX_DESC_PTR_TBL,
3169         .buf_tbl_base = FR_BZ_BUF_FULL_TBL,
3170         .evq_ptr_tbl_base = FR_BZ_EVQ_PTR_TBL,
3171         .evq_rptr_tbl_base = FR_BZ_EVQ_RPTR,
3172         .max_dma_mask = FALCON_DMA_MASK,
3173         .tx_dma_mask = FALCON_TX_DMA_MASK,
3174         .bug5391_mask = 0,
3175         .rx_buffer_padding = 0,
3176         .max_interrupt_mode = EFX_INT_MODE_MSIX,
3177         .phys_addr_channels = 32, /* Hardware limit is 64, but the legacy
3178                                    * interrupt handler only supports 32
3179                                    * channels */
3180 };
3181