Merge branch 'pci/stephen-const' into next
[linux-3.10.git] / drivers / rapidio / devices / tsi721.c
1 /*
2  * RapidIO mport driver for Tsi721 PCIExpress-to-SRIO bridge
3  *
4  * Copyright 2011 Integrated Device Technology, Inc.
5  * Alexandre Bounine <alexandre.bounine@idt.com>
6  * Chul Kim <chul.kim@idt.com>
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms of the GNU General Public License as published by the Free
10  * Software Foundation; either version 2 of the License, or (at your option)
11  * any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along with
19  * this program; if not, write to the Free Software Foundation, Inc., 59
20  * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
21  */
22
23 #include <linux/io.h>
24 #include <linux/errno.h>
25 #include <linux/init.h>
26 #include <linux/ioport.h>
27 #include <linux/kernel.h>
28 #include <linux/module.h>
29 #include <linux/pci.h>
30 #include <linux/rio.h>
31 #include <linux/rio_drv.h>
32 #include <linux/dma-mapping.h>
33 #include <linux/interrupt.h>
34 #include <linux/kfifo.h>
35 #include <linux/delay.h>
36
37 #include "tsi721.h"
38
39 #define DEBUG_PW        /* Inbound Port-Write debugging */
40
41 static void tsi721_omsg_handler(struct tsi721_device *priv, int ch);
42 static void tsi721_imsg_handler(struct tsi721_device *priv, int ch);
43
44 /**
45  * tsi721_lcread - read from local SREP config space
46  * @mport: RapidIO master port info
47  * @index: ID of RapdiIO interface
48  * @offset: Offset into configuration space
49  * @len: Length (in bytes) of the maintenance transaction
50  * @data: Value to be read into
51  *
52  * Generates a local SREP space read. Returns %0 on
53  * success or %-EINVAL on failure.
54  */
55 static int tsi721_lcread(struct rio_mport *mport, int index, u32 offset,
56                          int len, u32 *data)
57 {
58         struct tsi721_device *priv = mport->priv;
59
60         if (len != sizeof(u32))
61                 return -EINVAL; /* only 32-bit access is supported */
62
63         *data = ioread32(priv->regs + offset);
64
65         return 0;
66 }
67
68 /**
69  * tsi721_lcwrite - write into local SREP config space
70  * @mport: RapidIO master port info
71  * @index: ID of RapdiIO interface
72  * @offset: Offset into configuration space
73  * @len: Length (in bytes) of the maintenance transaction
74  * @data: Value to be written
75  *
76  * Generates a local write into SREP configuration space. Returns %0 on
77  * success or %-EINVAL on failure.
78  */
79 static int tsi721_lcwrite(struct rio_mport *mport, int index, u32 offset,
80                           int len, u32 data)
81 {
82         struct tsi721_device *priv = mport->priv;
83
84         if (len != sizeof(u32))
85                 return -EINVAL; /* only 32-bit access is supported */
86
87         iowrite32(data, priv->regs + offset);
88
89         return 0;
90 }
91
92 /**
93  * tsi721_maint_dma - Helper function to generate RapidIO maintenance
94  *                    transactions using designated Tsi721 DMA channel.
95  * @priv: pointer to tsi721 private data
96  * @sys_size: RapdiIO transport system size
97  * @destid: Destination ID of transaction
98  * @hopcount: Number of hops to target device
99  * @offset: Offset into configuration space
100  * @len: Length (in bytes) of the maintenance transaction
101  * @data: Location to be read from or write into
102  * @do_wr: Operation flag (1 == MAINT_WR)
103  *
104  * Generates a RapidIO maintenance transaction (Read or Write).
105  * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
106  */
107 static int tsi721_maint_dma(struct tsi721_device *priv, u32 sys_size,
108                         u16 destid, u8 hopcount, u32 offset, int len,
109                         u32 *data, int do_wr)
110 {
111         void __iomem *regs = priv->regs + TSI721_DMAC_BASE(priv->mdma.ch_id);
112         struct tsi721_dma_desc *bd_ptr;
113         u32 rd_count, swr_ptr, ch_stat;
114         int i, err = 0;
115         u32 op = do_wr ? MAINT_WR : MAINT_RD;
116
117         if (offset > (RIO_MAINT_SPACE_SZ - len) || (len != sizeof(u32)))
118                 return -EINVAL;
119
120         bd_ptr = priv->mdma.bd_base;
121
122         rd_count = ioread32(regs + TSI721_DMAC_DRDCNT);
123
124         /* Initialize DMA descriptor */
125         bd_ptr[0].type_id = cpu_to_le32((DTYPE2 << 29) | (op << 19) | destid);
126         bd_ptr[0].bcount = cpu_to_le32((sys_size << 26) | 0x04);
127         bd_ptr[0].raddr_lo = cpu_to_le32((hopcount << 24) | offset);
128         bd_ptr[0].raddr_hi = 0;
129         if (do_wr)
130                 bd_ptr[0].data[0] = cpu_to_be32p(data);
131         else
132                 bd_ptr[0].data[0] = 0xffffffff;
133
134         mb();
135
136         /* Start DMA operation */
137         iowrite32(rd_count + 2, regs + TSI721_DMAC_DWRCNT);
138         ioread32(regs + TSI721_DMAC_DWRCNT);
139         i = 0;
140
141         /* Wait until DMA transfer is finished */
142         while ((ch_stat = ioread32(regs + TSI721_DMAC_STS))
143                                                         & TSI721_DMAC_STS_RUN) {
144                 udelay(1);
145                 if (++i >= 5000000) {
146                         dev_dbg(&priv->pdev->dev,
147                                 "%s : DMA[%d] read timeout ch_status=%x\n",
148                                 __func__, priv->mdma.ch_id, ch_stat);
149                         if (!do_wr)
150                                 *data = 0xffffffff;
151                         err = -EIO;
152                         goto err_out;
153                 }
154         }
155
156         if (ch_stat & TSI721_DMAC_STS_ABORT) {
157                 /* If DMA operation aborted due to error,
158                  * reinitialize DMA channel
159                  */
160                 dev_dbg(&priv->pdev->dev, "%s : DMA ABORT ch_stat=%x\n",
161                         __func__, ch_stat);
162                 dev_dbg(&priv->pdev->dev, "OP=%d : destid=%x hc=%x off=%x\n",
163                         do_wr ? MAINT_WR : MAINT_RD, destid, hopcount, offset);
164                 iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT);
165                 iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
166                 udelay(10);
167                 iowrite32(0, regs + TSI721_DMAC_DWRCNT);
168                 udelay(1);
169                 if (!do_wr)
170                         *data = 0xffffffff;
171                 err = -EIO;
172                 goto err_out;
173         }
174
175         if (!do_wr)
176                 *data = be32_to_cpu(bd_ptr[0].data[0]);
177
178         /*
179          * Update descriptor status FIFO RD pointer.
180          * NOTE: Skipping check and clear FIFO entries because we are waiting
181          * for transfer to be completed.
182          */
183         swr_ptr = ioread32(regs + TSI721_DMAC_DSWP);
184         iowrite32(swr_ptr, regs + TSI721_DMAC_DSRP);
185 err_out:
186
187         return err;
188 }
189
190 /**
191  * tsi721_cread_dma - Generate a RapidIO maintenance read transaction
192  *                    using Tsi721 BDMA engine.
193  * @mport: RapidIO master port control structure
194  * @index: ID of RapdiIO interface
195  * @destid: Destination ID of transaction
196  * @hopcount: Number of hops to target device
197  * @offset: Offset into configuration space
198  * @len: Length (in bytes) of the maintenance transaction
199  * @val: Location to be read into
200  *
201  * Generates a RapidIO maintenance read transaction.
202  * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
203  */
204 static int tsi721_cread_dma(struct rio_mport *mport, int index, u16 destid,
205                         u8 hopcount, u32 offset, int len, u32 *data)
206 {
207         struct tsi721_device *priv = mport->priv;
208
209         return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,
210                                 offset, len, data, 0);
211 }
212
213 /**
214  * tsi721_cwrite_dma - Generate a RapidIO maintenance write transaction
215  *                     using Tsi721 BDMA engine
216  * @mport: RapidIO master port control structure
217  * @index: ID of RapdiIO interface
218  * @destid: Destination ID of transaction
219  * @hopcount: Number of hops to target device
220  * @offset: Offset into configuration space
221  * @len: Length (in bytes) of the maintenance transaction
222  * @val: Value to be written
223  *
224  * Generates a RapidIO maintenance write transaction.
225  * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
226  */
227 static int tsi721_cwrite_dma(struct rio_mport *mport, int index, u16 destid,
228                          u8 hopcount, u32 offset, int len, u32 data)
229 {
230         struct tsi721_device *priv = mport->priv;
231         u32 temp = data;
232
233         return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,
234                                 offset, len, &temp, 1);
235 }
236
237 /**
238  * tsi721_pw_handler - Tsi721 inbound port-write interrupt handler
239  * @mport: RapidIO master port structure
240  *
241  * Handles inbound port-write interrupts. Copies PW message from an internal
242  * buffer into PW message FIFO and schedules deferred routine to process
243  * queued messages.
244  */
245 static int
246 tsi721_pw_handler(struct rio_mport *mport)
247 {
248         struct tsi721_device *priv = mport->priv;
249         u32 pw_stat;
250         u32 pw_buf[TSI721_RIO_PW_MSG_SIZE/sizeof(u32)];
251
252
253         pw_stat = ioread32(priv->regs + TSI721_RIO_PW_RX_STAT);
254
255         if (pw_stat & TSI721_RIO_PW_RX_STAT_PW_VAL) {
256                 pw_buf[0] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(0));
257                 pw_buf[1] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(1));
258                 pw_buf[2] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(2));
259                 pw_buf[3] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(3));
260
261                 /* Queue PW message (if there is room in FIFO),
262                  * otherwise discard it.
263                  */
264                 spin_lock(&priv->pw_fifo_lock);
265                 if (kfifo_avail(&priv->pw_fifo) >= TSI721_RIO_PW_MSG_SIZE)
266                         kfifo_in(&priv->pw_fifo, pw_buf,
267                                                 TSI721_RIO_PW_MSG_SIZE);
268                 else
269                         priv->pw_discard_count++;
270                 spin_unlock(&priv->pw_fifo_lock);
271         }
272
273         /* Clear pending PW interrupts */
274         iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL,
275                   priv->regs + TSI721_RIO_PW_RX_STAT);
276
277         schedule_work(&priv->pw_work);
278
279         return 0;
280 }
281
282 static void tsi721_pw_dpc(struct work_struct *work)
283 {
284         struct tsi721_device *priv = container_of(work, struct tsi721_device,
285                                                     pw_work);
286         u32 msg_buffer[RIO_PW_MSG_SIZE/sizeof(u32)]; /* Use full size PW message
287                                                         buffer for RIO layer */
288
289         /*
290          * Process port-write messages
291          */
292         while (kfifo_out_spinlocked(&priv->pw_fifo, (unsigned char *)msg_buffer,
293                          TSI721_RIO_PW_MSG_SIZE, &priv->pw_fifo_lock)) {
294                 /* Process one message */
295 #ifdef DEBUG_PW
296                 {
297                 u32 i;
298                 pr_debug("%s : Port-Write Message:", __func__);
299                 for (i = 0; i < RIO_PW_MSG_SIZE/sizeof(u32); ) {
300                         pr_debug("0x%02x: %08x %08x %08x %08x", i*4,
301                                 msg_buffer[i], msg_buffer[i + 1],
302                                 msg_buffer[i + 2], msg_buffer[i + 3]);
303                         i += 4;
304                 }
305                 pr_debug("\n");
306                 }
307 #endif
308                 /* Pass the port-write message to RIO core for processing */
309                 rio_inb_pwrite_handler((union rio_pw_msg *)msg_buffer);
310         }
311 }
312
313 /**
314  * tsi721_pw_enable - enable/disable port-write interface init
315  * @mport: Master port implementing the port write unit
316  * @enable:    1=enable; 0=disable port-write message handling
317  */
318 static int tsi721_pw_enable(struct rio_mport *mport, int enable)
319 {
320         struct tsi721_device *priv = mport->priv;
321         u32 rval;
322
323         rval = ioread32(priv->regs + TSI721_RIO_EM_INT_ENABLE);
324
325         if (enable)
326                 rval |= TSI721_RIO_EM_INT_ENABLE_PW_RX;
327         else
328                 rval &= ~TSI721_RIO_EM_INT_ENABLE_PW_RX;
329
330         /* Clear pending PW interrupts */
331         iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL,
332                   priv->regs + TSI721_RIO_PW_RX_STAT);
333         /* Update enable bits */
334         iowrite32(rval, priv->regs + TSI721_RIO_EM_INT_ENABLE);
335
336         return 0;
337 }
338
339 /**
340  * tsi721_dsend - Send a RapidIO doorbell
341  * @mport: RapidIO master port info
342  * @index: ID of RapidIO interface
343  * @destid: Destination ID of target device
344  * @data: 16-bit info field of RapidIO doorbell
345  *
346  * Sends a RapidIO doorbell message. Always returns %0.
347  */
348 static int tsi721_dsend(struct rio_mport *mport, int index,
349                         u16 destid, u16 data)
350 {
351         struct tsi721_device *priv = mport->priv;
352         u32 offset;
353
354         offset = (((mport->sys_size) ? RIO_TT_CODE_16 : RIO_TT_CODE_8) << 18) |
355                  (destid << 2);
356
357         dev_dbg(&priv->pdev->dev,
358                 "Send Doorbell 0x%04x to destID 0x%x\n", data, destid);
359         iowrite16be(data, priv->odb_base + offset);
360
361         return 0;
362 }
363
364 /**
365  * tsi721_dbell_handler - Tsi721 doorbell interrupt handler
366  * @mport: RapidIO master port structure
367  *
368  * Handles inbound doorbell interrupts. Copies doorbell entry from an internal
369  * buffer into DB message FIFO and schedules deferred  routine to process
370  * queued DBs.
371  */
372 static int
373 tsi721_dbell_handler(struct rio_mport *mport)
374 {
375         struct tsi721_device *priv = mport->priv;
376         u32 regval;
377
378         /* Disable IDB interrupts */
379         regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
380         regval &= ~TSI721_SR_CHINT_IDBQRCV;
381         iowrite32(regval,
382                 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
383
384         schedule_work(&priv->idb_work);
385
386         return 0;
387 }
388
389 static void tsi721_db_dpc(struct work_struct *work)
390 {
391         struct tsi721_device *priv = container_of(work, struct tsi721_device,
392                                                     idb_work);
393         struct rio_mport *mport;
394         struct rio_dbell *dbell;
395         int found = 0;
396         u32 wr_ptr, rd_ptr;
397         u64 *idb_entry;
398         u32 regval;
399         union {
400                 u64 msg;
401                 u8  bytes[8];
402         } idb;
403
404         /*
405          * Process queued inbound doorbells
406          */
407         mport = priv->mport;
408
409         wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
410         rd_ptr = ioread32(priv->regs + TSI721_IDQ_RP(IDB_QUEUE)) % IDB_QSIZE;
411
412         while (wr_ptr != rd_ptr) {
413                 idb_entry = (u64 *)(priv->idb_base +
414                                         (TSI721_IDB_ENTRY_SIZE * rd_ptr));
415                 rd_ptr++;
416                 rd_ptr %= IDB_QSIZE;
417                 idb.msg = *idb_entry;
418                 *idb_entry = 0;
419
420                 /* Process one doorbell */
421                 list_for_each_entry(dbell, &mport->dbells, node) {
422                         if ((dbell->res->start <= DBELL_INF(idb.bytes)) &&
423                             (dbell->res->end >= DBELL_INF(idb.bytes))) {
424                                 found = 1;
425                                 break;
426                         }
427                 }
428
429                 if (found) {
430                         dbell->dinb(mport, dbell->dev_id, DBELL_SID(idb.bytes),
431                                     DBELL_TID(idb.bytes), DBELL_INF(idb.bytes));
432                 } else {
433                         dev_dbg(&priv->pdev->dev,
434                                 "spurious inb doorbell, sid %2.2x tid %2.2x"
435                                 " info %4.4x\n", DBELL_SID(idb.bytes),
436                                 DBELL_TID(idb.bytes), DBELL_INF(idb.bytes));
437                 }
438         }
439
440         iowrite32(rd_ptr & (IDB_QSIZE - 1),
441                 priv->regs + TSI721_IDQ_RP(IDB_QUEUE));
442
443         /* Re-enable IDB interrupts */
444         regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
445         regval |= TSI721_SR_CHINT_IDBQRCV;
446         iowrite32(regval,
447                 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
448 }
449
450 /**
451  * tsi721_irqhandler - Tsi721 interrupt handler
452  * @irq: Linux interrupt number
453  * @ptr: Pointer to interrupt-specific data (mport structure)
454  *
455  * Handles Tsi721 interrupts signaled using MSI and INTA. Checks reported
456  * interrupt events and calls an event-specific handler(s).
457  */
458 static irqreturn_t tsi721_irqhandler(int irq, void *ptr)
459 {
460         struct rio_mport *mport = (struct rio_mport *)ptr;
461         struct tsi721_device *priv = mport->priv;
462         u32 dev_int;
463         u32 dev_ch_int;
464         u32 intval;
465         u32 ch_inte;
466
467         dev_int = ioread32(priv->regs + TSI721_DEV_INT);
468         if (!dev_int)
469                 return IRQ_NONE;
470
471         dev_ch_int = ioread32(priv->regs + TSI721_DEV_CHAN_INT);
472
473         if (dev_int & TSI721_DEV_INT_SR2PC_CH) {
474                 /* Service SR2PC Channel interrupts */
475                 if (dev_ch_int & TSI721_INT_SR2PC_CHAN(IDB_QUEUE)) {
476                         /* Service Inbound Doorbell interrupt */
477                         intval = ioread32(priv->regs +
478                                                 TSI721_SR_CHINT(IDB_QUEUE));
479                         if (intval & TSI721_SR_CHINT_IDBQRCV)
480                                 tsi721_dbell_handler(mport);
481                         else
482                                 dev_info(&priv->pdev->dev,
483                                         "Unsupported SR_CH_INT %x\n", intval);
484
485                         /* Clear interrupts */
486                         iowrite32(intval,
487                                 priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
488                         ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
489                 }
490         }
491
492         if (dev_int & TSI721_DEV_INT_SMSG_CH) {
493                 int ch;
494
495                 /*
496                  * Service channel interrupts from Messaging Engine
497                  */
498
499                 if (dev_ch_int & TSI721_INT_IMSG_CHAN_M) { /* Inbound Msg */
500                         /* Disable signaled OB MSG Channel interrupts */
501                         ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
502                         ch_inte &= ~(dev_ch_int & TSI721_INT_IMSG_CHAN_M);
503                         iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
504
505                         /*
506                          * Process Inbound Message interrupt for each MBOX
507                          */
508                         for (ch = 4; ch < RIO_MAX_MBOX + 4; ch++) {
509                                 if (!(dev_ch_int & TSI721_INT_IMSG_CHAN(ch)))
510                                         continue;
511                                 tsi721_imsg_handler(priv, ch);
512                         }
513                 }
514
515                 if (dev_ch_int & TSI721_INT_OMSG_CHAN_M) { /* Outbound Msg */
516                         /* Disable signaled OB MSG Channel interrupts */
517                         ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
518                         ch_inte &= ~(dev_ch_int & TSI721_INT_OMSG_CHAN_M);
519                         iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
520
521                         /*
522                          * Process Outbound Message interrupts for each MBOX
523                          */
524
525                         for (ch = 0; ch < RIO_MAX_MBOX; ch++) {
526                                 if (!(dev_ch_int & TSI721_INT_OMSG_CHAN(ch)))
527                                         continue;
528                                 tsi721_omsg_handler(priv, ch);
529                         }
530                 }
531         }
532
533         if (dev_int & TSI721_DEV_INT_SRIO) {
534                 /* Service SRIO MAC interrupts */
535                 intval = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);
536                 if (intval & TSI721_RIO_EM_INT_STAT_PW_RX)
537                         tsi721_pw_handler(mport);
538         }
539
540 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
541         if (dev_int & TSI721_DEV_INT_BDMA_CH) {
542                 int ch;
543
544                 if (dev_ch_int & TSI721_INT_BDMA_CHAN_M) {
545                         dev_dbg(&priv->pdev->dev,
546                                 "IRQ from DMA channel 0x%08x\n", dev_ch_int);
547
548                         for (ch = 0; ch < TSI721_DMA_MAXCH; ch++) {
549                                 if (!(dev_ch_int & TSI721_INT_BDMA_CHAN(ch)))
550                                         continue;
551                                 tsi721_bdma_handler(&priv->bdma[ch]);
552                         }
553                 }
554         }
555 #endif
556         return IRQ_HANDLED;
557 }
558
559 static void tsi721_interrupts_init(struct tsi721_device *priv)
560 {
561         u32 intr;
562
563         /* Enable IDB interrupts */
564         iowrite32(TSI721_SR_CHINT_ALL,
565                 priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
566         iowrite32(TSI721_SR_CHINT_IDBQRCV,
567                 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
568
569         /* Enable SRIO MAC interrupts */
570         iowrite32(TSI721_RIO_EM_DEV_INT_EN_INT,
571                 priv->regs + TSI721_RIO_EM_DEV_INT_EN);
572
573         /* Enable interrupts from channels in use */
574 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
575         intr = TSI721_INT_SR2PC_CHAN(IDB_QUEUE) |
576                 (TSI721_INT_BDMA_CHAN_M &
577                  ~TSI721_INT_BDMA_CHAN(TSI721_DMACH_MAINT));
578 #else
579         intr = TSI721_INT_SR2PC_CHAN(IDB_QUEUE);
580 #endif
581         iowrite32(intr, priv->regs + TSI721_DEV_CHAN_INTE);
582
583         if (priv->flags & TSI721_USING_MSIX)
584                 intr = TSI721_DEV_INT_SRIO;
585         else
586                 intr = TSI721_DEV_INT_SR2PC_CH | TSI721_DEV_INT_SRIO |
587                         TSI721_DEV_INT_SMSG_CH | TSI721_DEV_INT_BDMA_CH;
588
589         iowrite32(intr, priv->regs + TSI721_DEV_INTE);
590         ioread32(priv->regs + TSI721_DEV_INTE);
591 }
592
593 #ifdef CONFIG_PCI_MSI
594 /**
595  * tsi721_omsg_msix - MSI-X interrupt handler for outbound messaging
596  * @irq: Linux interrupt number
597  * @ptr: Pointer to interrupt-specific data (mport structure)
598  *
599  * Handles outbound messaging interrupts signaled using MSI-X.
600  */
601 static irqreturn_t tsi721_omsg_msix(int irq, void *ptr)
602 {
603         struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;
604         int mbox;
605
606         mbox = (irq - priv->msix[TSI721_VECT_OMB0_DONE].vector) % RIO_MAX_MBOX;
607         tsi721_omsg_handler(priv, mbox);
608         return IRQ_HANDLED;
609 }
610
611 /**
612  * tsi721_imsg_msix - MSI-X interrupt handler for inbound messaging
613  * @irq: Linux interrupt number
614  * @ptr: Pointer to interrupt-specific data (mport structure)
615  *
616  * Handles inbound messaging interrupts signaled using MSI-X.
617  */
618 static irqreturn_t tsi721_imsg_msix(int irq, void *ptr)
619 {
620         struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;
621         int mbox;
622
623         mbox = (irq - priv->msix[TSI721_VECT_IMB0_RCV].vector) % RIO_MAX_MBOX;
624         tsi721_imsg_handler(priv, mbox + 4);
625         return IRQ_HANDLED;
626 }
627
628 /**
629  * tsi721_srio_msix - Tsi721 MSI-X SRIO MAC interrupt handler
630  * @irq: Linux interrupt number
631  * @ptr: Pointer to interrupt-specific data (mport structure)
632  *
633  * Handles Tsi721 interrupts from SRIO MAC.
634  */
635 static irqreturn_t tsi721_srio_msix(int irq, void *ptr)
636 {
637         struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;
638         u32 srio_int;
639
640         /* Service SRIO MAC interrupts */
641         srio_int = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);
642         if (srio_int & TSI721_RIO_EM_INT_STAT_PW_RX)
643                 tsi721_pw_handler((struct rio_mport *)ptr);
644
645         return IRQ_HANDLED;
646 }
647
648 /**
649  * tsi721_sr2pc_ch_msix - Tsi721 MSI-X SR2PC Channel interrupt handler
650  * @irq: Linux interrupt number
651  * @ptr: Pointer to interrupt-specific data (mport structure)
652  *
653  * Handles Tsi721 interrupts from SR2PC Channel.
654  * NOTE: At this moment services only one SR2PC channel associated with inbound
655  * doorbells.
656  */
657 static irqreturn_t tsi721_sr2pc_ch_msix(int irq, void *ptr)
658 {
659         struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;
660         u32 sr_ch_int;
661
662         /* Service Inbound DB interrupt from SR2PC channel */
663         sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
664         if (sr_ch_int & TSI721_SR_CHINT_IDBQRCV)
665                 tsi721_dbell_handler((struct rio_mport *)ptr);
666
667         /* Clear interrupts */
668         iowrite32(sr_ch_int, priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
669         /* Read back to ensure that interrupt was cleared */
670         sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
671
672         return IRQ_HANDLED;
673 }
674
675 /**
676  * tsi721_request_msix - register interrupt service for MSI-X mode.
677  * @mport: RapidIO master port structure
678  *
679  * Registers MSI-X interrupt service routines for interrupts that are active
680  * immediately after mport initialization. Messaging interrupt service routines
681  * should be registered during corresponding open requests.
682  */
683 static int tsi721_request_msix(struct rio_mport *mport)
684 {
685         struct tsi721_device *priv = mport->priv;
686         int err = 0;
687
688         err = request_irq(priv->msix[TSI721_VECT_IDB].vector,
689                         tsi721_sr2pc_ch_msix, 0,
690                         priv->msix[TSI721_VECT_IDB].irq_name, (void *)mport);
691         if (err)
692                 goto out;
693
694         err = request_irq(priv->msix[TSI721_VECT_PWRX].vector,
695                         tsi721_srio_msix, 0,
696                         priv->msix[TSI721_VECT_PWRX].irq_name, (void *)mport);
697         if (err)
698                 free_irq(
699                         priv->msix[TSI721_VECT_IDB].vector,
700                         (void *)mport);
701 out:
702         return err;
703 }
704
705 /**
706  * tsi721_enable_msix - Attempts to enable MSI-X support for Tsi721.
707  * @priv: pointer to tsi721 private data
708  *
709  * Configures MSI-X support for Tsi721. Supports only an exact number
710  * of requested vectors.
711  */
712 static int tsi721_enable_msix(struct tsi721_device *priv)
713 {
714         struct msix_entry entries[TSI721_VECT_MAX];
715         int err;
716         int i;
717
718         entries[TSI721_VECT_IDB].entry = TSI721_MSIX_SR2PC_IDBQ_RCV(IDB_QUEUE);
719         entries[TSI721_VECT_PWRX].entry = TSI721_MSIX_SRIO_MAC_INT;
720
721         /*
722          * Initialize MSI-X entries for Messaging Engine:
723          * this driver supports four RIO mailboxes (inbound and outbound)
724          * NOTE: Inbound message MBOX 0...4 use IB channels 4...7. Therefore
725          * offset +4 is added to IB MBOX number.
726          */
727         for (i = 0; i < RIO_MAX_MBOX; i++) {
728                 entries[TSI721_VECT_IMB0_RCV + i].entry =
729                                         TSI721_MSIX_IMSG_DQ_RCV(i + 4);
730                 entries[TSI721_VECT_IMB0_INT + i].entry =
731                                         TSI721_MSIX_IMSG_INT(i + 4);
732                 entries[TSI721_VECT_OMB0_DONE + i].entry =
733                                         TSI721_MSIX_OMSG_DONE(i);
734                 entries[TSI721_VECT_OMB0_INT + i].entry =
735                                         TSI721_MSIX_OMSG_INT(i);
736         }
737
738 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
739         /*
740          * Initialize MSI-X entries for Block DMA Engine:
741          * this driver supports XXX DMA channels
742          * (one is reserved for SRIO maintenance transactions)
743          */
744         for (i = 0; i < TSI721_DMA_CHNUM; i++) {
745                 entries[TSI721_VECT_DMA0_DONE + i].entry =
746                                         TSI721_MSIX_DMACH_DONE(i);
747                 entries[TSI721_VECT_DMA0_INT + i].entry =
748                                         TSI721_MSIX_DMACH_INT(i);
749         }
750 #endif /* CONFIG_RAPIDIO_DMA_ENGINE */
751
752         err = pci_enable_msix(priv->pdev, entries, ARRAY_SIZE(entries));
753         if (err) {
754                 if (err > 0)
755                         dev_info(&priv->pdev->dev,
756                                  "Only %d MSI-X vectors available, "
757                                  "not using MSI-X\n", err);
758                 else
759                         dev_err(&priv->pdev->dev,
760                                 "Failed to enable MSI-X (err=%d)\n", err);
761                 return err;
762         }
763
764         /*
765          * Copy MSI-X vector information into tsi721 private structure
766          */
767         priv->msix[TSI721_VECT_IDB].vector = entries[TSI721_VECT_IDB].vector;
768         snprintf(priv->msix[TSI721_VECT_IDB].irq_name, IRQ_DEVICE_NAME_MAX,
769                  DRV_NAME "-idb@pci:%s", pci_name(priv->pdev));
770         priv->msix[TSI721_VECT_PWRX].vector = entries[TSI721_VECT_PWRX].vector;
771         snprintf(priv->msix[TSI721_VECT_PWRX].irq_name, IRQ_DEVICE_NAME_MAX,
772                  DRV_NAME "-pwrx@pci:%s", pci_name(priv->pdev));
773
774         for (i = 0; i < RIO_MAX_MBOX; i++) {
775                 priv->msix[TSI721_VECT_IMB0_RCV + i].vector =
776                                 entries[TSI721_VECT_IMB0_RCV + i].vector;
777                 snprintf(priv->msix[TSI721_VECT_IMB0_RCV + i].irq_name,
778                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbr%d@pci:%s",
779                          i, pci_name(priv->pdev));
780
781                 priv->msix[TSI721_VECT_IMB0_INT + i].vector =
782                                 entries[TSI721_VECT_IMB0_INT + i].vector;
783                 snprintf(priv->msix[TSI721_VECT_IMB0_INT + i].irq_name,
784                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbi%d@pci:%s",
785                          i, pci_name(priv->pdev));
786
787                 priv->msix[TSI721_VECT_OMB0_DONE + i].vector =
788                                 entries[TSI721_VECT_OMB0_DONE + i].vector;
789                 snprintf(priv->msix[TSI721_VECT_OMB0_DONE + i].irq_name,
790                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombd%d@pci:%s",
791                          i, pci_name(priv->pdev));
792
793                 priv->msix[TSI721_VECT_OMB0_INT + i].vector =
794                                 entries[TSI721_VECT_OMB0_INT + i].vector;
795                 snprintf(priv->msix[TSI721_VECT_OMB0_INT + i].irq_name,
796                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombi%d@pci:%s",
797                          i, pci_name(priv->pdev));
798         }
799
800 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
801         for (i = 0; i < TSI721_DMA_CHNUM; i++) {
802                 priv->msix[TSI721_VECT_DMA0_DONE + i].vector =
803                                 entries[TSI721_VECT_DMA0_DONE + i].vector;
804                 snprintf(priv->msix[TSI721_VECT_DMA0_DONE + i].irq_name,
805                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-dmad%d@pci:%s",
806                          i, pci_name(priv->pdev));
807
808                 priv->msix[TSI721_VECT_DMA0_INT + i].vector =
809                                 entries[TSI721_VECT_DMA0_INT + i].vector;
810                 snprintf(priv->msix[TSI721_VECT_DMA0_INT + i].irq_name,
811                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-dmai%d@pci:%s",
812                          i, pci_name(priv->pdev));
813         }
814 #endif /* CONFIG_RAPIDIO_DMA_ENGINE */
815
816         return 0;
817 }
818 #endif /* CONFIG_PCI_MSI */
819
820 static int tsi721_request_irq(struct rio_mport *mport)
821 {
822         struct tsi721_device *priv = mport->priv;
823         int err;
824
825 #ifdef CONFIG_PCI_MSI
826         if (priv->flags & TSI721_USING_MSIX)
827                 err = tsi721_request_msix(mport);
828         else
829 #endif
830                 err = request_irq(priv->pdev->irq, tsi721_irqhandler,
831                           (priv->flags & TSI721_USING_MSI) ? 0 : IRQF_SHARED,
832                           DRV_NAME, (void *)mport);
833
834         if (err)
835                 dev_err(&priv->pdev->dev,
836                         "Unable to allocate interrupt, Error: %d\n", err);
837
838         return err;
839 }
840
841 /**
842  * tsi721_init_pc2sr_mapping - initializes outbound (PCIe->SRIO)
843  * translation regions.
844  * @priv: pointer to tsi721 private data
845  *
846  * Disables SREP translation regions.
847  */
848 static void tsi721_init_pc2sr_mapping(struct tsi721_device *priv)
849 {
850         int i;
851
852         /* Disable all PC2SR translation windows */
853         for (i = 0; i < TSI721_OBWIN_NUM; i++)
854                 iowrite32(0, priv->regs + TSI721_OBWINLB(i));
855 }
856
857 /**
858  * tsi721_init_sr2pc_mapping - initializes inbound (SRIO->PCIe)
859  * translation regions.
860  * @priv: pointer to tsi721 private data
861  *
862  * Disables inbound windows.
863  */
864 static void tsi721_init_sr2pc_mapping(struct tsi721_device *priv)
865 {
866         int i;
867
868         /* Disable all SR2PC inbound windows */
869         for (i = 0; i < TSI721_IBWIN_NUM; i++)
870                 iowrite32(0, priv->regs + TSI721_IBWINLB(i));
871 }
872
873 /**
874  * tsi721_port_write_init - Inbound port write interface init
875  * @priv: pointer to tsi721 private data
876  *
877  * Initializes inbound port write handler.
878  * Returns %0 on success or %-ENOMEM on failure.
879  */
880 static int tsi721_port_write_init(struct tsi721_device *priv)
881 {
882         priv->pw_discard_count = 0;
883         INIT_WORK(&priv->pw_work, tsi721_pw_dpc);
884         spin_lock_init(&priv->pw_fifo_lock);
885         if (kfifo_alloc(&priv->pw_fifo,
886                         TSI721_RIO_PW_MSG_SIZE * 32, GFP_KERNEL)) {
887                 dev_err(&priv->pdev->dev, "PW FIFO allocation failed\n");
888                 return -ENOMEM;
889         }
890
891         /* Use reliable port-write capture mode */
892         iowrite32(TSI721_RIO_PW_CTL_PWC_REL, priv->regs + TSI721_RIO_PW_CTL);
893         return 0;
894 }
895
896 static int tsi721_doorbell_init(struct tsi721_device *priv)
897 {
898         /* Outbound Doorbells do not require any setup.
899          * Tsi721 uses dedicated PCI BAR1 to generate doorbells.
900          * That BAR1 was mapped during the probe routine.
901          */
902
903         /* Initialize Inbound Doorbell processing DPC and queue */
904         priv->db_discard_count = 0;
905         INIT_WORK(&priv->idb_work, tsi721_db_dpc);
906
907         /* Allocate buffer for inbound doorbells queue */
908         priv->idb_base = dma_zalloc_coherent(&priv->pdev->dev,
909                                 IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,
910                                 &priv->idb_dma, GFP_KERNEL);
911         if (!priv->idb_base)
912                 return -ENOMEM;
913
914         dev_dbg(&priv->pdev->dev, "Allocated IDB buffer @ %p (phys = %llx)\n",
915                 priv->idb_base, (unsigned long long)priv->idb_dma);
916
917         iowrite32(TSI721_IDQ_SIZE_VAL(IDB_QSIZE),
918                 priv->regs + TSI721_IDQ_SIZE(IDB_QUEUE));
919         iowrite32(((u64)priv->idb_dma >> 32),
920                 priv->regs + TSI721_IDQ_BASEU(IDB_QUEUE));
921         iowrite32(((u64)priv->idb_dma & TSI721_IDQ_BASEL_ADDR),
922                 priv->regs + TSI721_IDQ_BASEL(IDB_QUEUE));
923         /* Enable accepting all inbound doorbells */
924         iowrite32(0, priv->regs + TSI721_IDQ_MASK(IDB_QUEUE));
925
926         iowrite32(TSI721_IDQ_INIT, priv->regs + TSI721_IDQ_CTL(IDB_QUEUE));
927
928         iowrite32(0, priv->regs + TSI721_IDQ_RP(IDB_QUEUE));
929
930         return 0;
931 }
932
933 static void tsi721_doorbell_free(struct tsi721_device *priv)
934 {
935         if (priv->idb_base == NULL)
936                 return;
937
938         /* Free buffer allocated for inbound doorbell queue */
939         dma_free_coherent(&priv->pdev->dev, IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,
940                           priv->idb_base, priv->idb_dma);
941         priv->idb_base = NULL;
942 }
943
944 /**
945  * tsi721_bdma_maint_init - Initialize maintenance request BDMA channel.
946  * @priv: pointer to tsi721 private data
947  *
948  * Initialize BDMA channel allocated for RapidIO maintenance read/write
949  * request generation
950  * Returns %0 on success or %-ENOMEM on failure.
951  */
952 static int tsi721_bdma_maint_init(struct tsi721_device *priv)
953 {
954         struct tsi721_dma_desc *bd_ptr;
955         u64             *sts_ptr;
956         dma_addr_t      bd_phys, sts_phys;
957         int             sts_size;
958         int             bd_num = 2;
959         void __iomem    *regs;
960
961         dev_dbg(&priv->pdev->dev,
962                 "Init Block DMA Engine for Maintenance requests, CH%d\n",
963                 TSI721_DMACH_MAINT);
964
965         /*
966          * Initialize DMA channel for maintenance requests
967          */
968
969         priv->mdma.ch_id = TSI721_DMACH_MAINT;
970         regs = priv->regs + TSI721_DMAC_BASE(TSI721_DMACH_MAINT);
971
972         /* Allocate space for DMA descriptors */
973         bd_ptr = dma_zalloc_coherent(&priv->pdev->dev,
974                                         bd_num * sizeof(struct tsi721_dma_desc),
975                                         &bd_phys, GFP_KERNEL);
976         if (!bd_ptr)
977                 return -ENOMEM;
978
979         priv->mdma.bd_num = bd_num;
980         priv->mdma.bd_phys = bd_phys;
981         priv->mdma.bd_base = bd_ptr;
982
983         dev_dbg(&priv->pdev->dev, "DMA descriptors @ %p (phys = %llx)\n",
984                 bd_ptr, (unsigned long long)bd_phys);
985
986         /* Allocate space for descriptor status FIFO */
987         sts_size = (bd_num >= TSI721_DMA_MINSTSSZ) ?
988                                         bd_num : TSI721_DMA_MINSTSSZ;
989         sts_size = roundup_pow_of_two(sts_size);
990         sts_ptr = dma_zalloc_coherent(&priv->pdev->dev,
991                                      sts_size * sizeof(struct tsi721_dma_sts),
992                                      &sts_phys, GFP_KERNEL);
993         if (!sts_ptr) {
994                 /* Free space allocated for DMA descriptors */
995                 dma_free_coherent(&priv->pdev->dev,
996                                   bd_num * sizeof(struct tsi721_dma_desc),
997                                   bd_ptr, bd_phys);
998                 priv->mdma.bd_base = NULL;
999                 return -ENOMEM;
1000         }
1001
1002         priv->mdma.sts_phys = sts_phys;
1003         priv->mdma.sts_base = sts_ptr;
1004         priv->mdma.sts_size = sts_size;
1005
1006         dev_dbg(&priv->pdev->dev,
1007                 "desc status FIFO @ %p (phys = %llx) size=0x%x\n",
1008                 sts_ptr, (unsigned long long)sts_phys, sts_size);
1009
1010         /* Initialize DMA descriptors ring */
1011         bd_ptr[bd_num - 1].type_id = cpu_to_le32(DTYPE3 << 29);
1012         bd_ptr[bd_num - 1].next_lo = cpu_to_le32((u64)bd_phys &
1013                                                  TSI721_DMAC_DPTRL_MASK);
1014         bd_ptr[bd_num - 1].next_hi = cpu_to_le32((u64)bd_phys >> 32);
1015
1016         /* Setup DMA descriptor pointers */
1017         iowrite32(((u64)bd_phys >> 32), regs + TSI721_DMAC_DPTRH);
1018         iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK),
1019                 regs + TSI721_DMAC_DPTRL);
1020
1021         /* Setup descriptor status FIFO */
1022         iowrite32(((u64)sts_phys >> 32), regs + TSI721_DMAC_DSBH);
1023         iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK),
1024                 regs + TSI721_DMAC_DSBL);
1025         iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size),
1026                 regs + TSI721_DMAC_DSSZ);
1027
1028         /* Clear interrupt bits */
1029         iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT);
1030
1031         ioread32(regs + TSI721_DMAC_INT);
1032
1033         /* Toggle DMA channel initialization */
1034         iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
1035         ioread32(regs + TSI721_DMAC_CTL);
1036         udelay(10);
1037
1038         return 0;
1039 }
1040
1041 static int tsi721_bdma_maint_free(struct tsi721_device *priv)
1042 {
1043         u32 ch_stat;
1044         struct tsi721_bdma_maint *mdma = &priv->mdma;
1045         void __iomem *regs = priv->regs + TSI721_DMAC_BASE(mdma->ch_id);
1046
1047         if (mdma->bd_base == NULL)
1048                 return 0;
1049
1050         /* Check if DMA channel still running */
1051         ch_stat = ioread32(regs + TSI721_DMAC_STS);
1052         if (ch_stat & TSI721_DMAC_STS_RUN)
1053                 return -EFAULT;
1054
1055         /* Put DMA channel into init state */
1056         iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
1057
1058         /* Free space allocated for DMA descriptors */
1059         dma_free_coherent(&priv->pdev->dev,
1060                 mdma->bd_num * sizeof(struct tsi721_dma_desc),
1061                 mdma->bd_base, mdma->bd_phys);
1062         mdma->bd_base = NULL;
1063
1064         /* Free space allocated for status FIFO */
1065         dma_free_coherent(&priv->pdev->dev,
1066                 mdma->sts_size * sizeof(struct tsi721_dma_sts),
1067                 mdma->sts_base, mdma->sts_phys);
1068         mdma->sts_base = NULL;
1069         return 0;
1070 }
1071
1072 /* Enable Inbound Messaging Interrupts */
1073 static void
1074 tsi721_imsg_interrupt_enable(struct tsi721_device *priv, int ch,
1075                                   u32 inte_mask)
1076 {
1077         u32 rval;
1078
1079         if (!inte_mask)
1080                 return;
1081
1082         /* Clear pending Inbound Messaging interrupts */
1083         iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));
1084
1085         /* Enable Inbound Messaging interrupts */
1086         rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));
1087         iowrite32(rval | inte_mask, priv->regs + TSI721_IBDMAC_INTE(ch));
1088
1089         if (priv->flags & TSI721_USING_MSIX)
1090                 return; /* Finished if we are in MSI-X mode */
1091
1092         /*
1093          * For MSI and INTA interrupt signalling we need to enable next levels
1094          */
1095
1096         /* Enable Device Channel Interrupt */
1097         rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1098         iowrite32(rval | TSI721_INT_IMSG_CHAN(ch),
1099                   priv->regs + TSI721_DEV_CHAN_INTE);
1100 }
1101
1102 /* Disable Inbound Messaging Interrupts */
1103 static void
1104 tsi721_imsg_interrupt_disable(struct tsi721_device *priv, int ch,
1105                                    u32 inte_mask)
1106 {
1107         u32 rval;
1108
1109         if (!inte_mask)
1110                 return;
1111
1112         /* Clear pending Inbound Messaging interrupts */
1113         iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));
1114
1115         /* Disable Inbound Messaging interrupts */
1116         rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));
1117         rval &= ~inte_mask;
1118         iowrite32(rval, priv->regs + TSI721_IBDMAC_INTE(ch));
1119
1120         if (priv->flags & TSI721_USING_MSIX)
1121                 return; /* Finished if we are in MSI-X mode */
1122
1123         /*
1124          * For MSI and INTA interrupt signalling we need to disable next levels
1125          */
1126
1127         /* Disable Device Channel Interrupt */
1128         rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1129         rval &= ~TSI721_INT_IMSG_CHAN(ch);
1130         iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);
1131 }
1132
1133 /* Enable Outbound Messaging interrupts */
1134 static void
1135 tsi721_omsg_interrupt_enable(struct tsi721_device *priv, int ch,
1136                                   u32 inte_mask)
1137 {
1138         u32 rval;
1139
1140         if (!inte_mask)
1141                 return;
1142
1143         /* Clear pending Outbound Messaging interrupts */
1144         iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));
1145
1146         /* Enable Outbound Messaging channel interrupts */
1147         rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));
1148         iowrite32(rval | inte_mask, priv->regs + TSI721_OBDMAC_INTE(ch));
1149
1150         if (priv->flags & TSI721_USING_MSIX)
1151                 return; /* Finished if we are in MSI-X mode */
1152
1153         /*
1154          * For MSI and INTA interrupt signalling we need to enable next levels
1155          */
1156
1157         /* Enable Device Channel Interrupt */
1158         rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1159         iowrite32(rval | TSI721_INT_OMSG_CHAN(ch),
1160                   priv->regs + TSI721_DEV_CHAN_INTE);
1161 }
1162
1163 /* Disable Outbound Messaging interrupts */
1164 static void
1165 tsi721_omsg_interrupt_disable(struct tsi721_device *priv, int ch,
1166                                    u32 inte_mask)
1167 {
1168         u32 rval;
1169
1170         if (!inte_mask)
1171                 return;
1172
1173         /* Clear pending Outbound Messaging interrupts */
1174         iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));
1175
1176         /* Disable Outbound Messaging interrupts */
1177         rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));
1178         rval &= ~inte_mask;
1179         iowrite32(rval, priv->regs + TSI721_OBDMAC_INTE(ch));
1180
1181         if (priv->flags & TSI721_USING_MSIX)
1182                 return; /* Finished if we are in MSI-X mode */
1183
1184         /*
1185          * For MSI and INTA interrupt signalling we need to disable next levels
1186          */
1187
1188         /* Disable Device Channel Interrupt */
1189         rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1190         rval &= ~TSI721_INT_OMSG_CHAN(ch);
1191         iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);
1192 }
1193
1194 /**
1195  * tsi721_add_outb_message - Add message to the Tsi721 outbound message queue
1196  * @mport: Master port with outbound message queue
1197  * @rdev: Target of outbound message
1198  * @mbox: Outbound mailbox
1199  * @buffer: Message to add to outbound queue
1200  * @len: Length of message
1201  */
1202 static int
1203 tsi721_add_outb_message(struct rio_mport *mport, struct rio_dev *rdev, int mbox,
1204                         void *buffer, size_t len)
1205 {
1206         struct tsi721_device *priv = mport->priv;
1207         struct tsi721_omsg_desc *desc;
1208         u32 tx_slot;
1209
1210         if (!priv->omsg_init[mbox] ||
1211             len > TSI721_MSG_MAX_SIZE || len < 8)
1212                 return -EINVAL;
1213
1214         tx_slot = priv->omsg_ring[mbox].tx_slot;
1215
1216         /* Copy copy message into transfer buffer */
1217         memcpy(priv->omsg_ring[mbox].omq_base[tx_slot], buffer, len);
1218
1219         if (len & 0x7)
1220                 len += 8;
1221
1222         /* Build descriptor associated with buffer */
1223         desc = priv->omsg_ring[mbox].omd_base;
1224         desc[tx_slot].type_id = cpu_to_le32((DTYPE4 << 29) | rdev->destid);
1225         if (tx_slot % 4 == 0)
1226                 desc[tx_slot].type_id |= cpu_to_le32(TSI721_OMD_IOF);
1227
1228         desc[tx_slot].msg_info =
1229                 cpu_to_le32((mport->sys_size << 26) | (mbox << 22) |
1230                             (0xe << 12) | (len & 0xff8));
1231         desc[tx_slot].bufptr_lo =
1232                 cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] &
1233                             0xffffffff);
1234         desc[tx_slot].bufptr_hi =
1235                 cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] >> 32);
1236
1237         priv->omsg_ring[mbox].wr_count++;
1238
1239         /* Go to next descriptor */
1240         if (++priv->omsg_ring[mbox].tx_slot == priv->omsg_ring[mbox].size) {
1241                 priv->omsg_ring[mbox].tx_slot = 0;
1242                 /* Move through the ring link descriptor at the end */
1243                 priv->omsg_ring[mbox].wr_count++;
1244         }
1245
1246         mb();
1247
1248         /* Set new write count value */
1249         iowrite32(priv->omsg_ring[mbox].wr_count,
1250                 priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1251         ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1252
1253         return 0;
1254 }
1255
1256 /**
1257  * tsi721_omsg_handler - Outbound Message Interrupt Handler
1258  * @priv: pointer to tsi721 private data
1259  * @ch:   number of OB MSG channel to service
1260  *
1261  * Services channel interrupts from outbound messaging engine.
1262  */
1263 static void tsi721_omsg_handler(struct tsi721_device *priv, int ch)
1264 {
1265         u32 omsg_int;
1266
1267         spin_lock(&priv->omsg_ring[ch].lock);
1268
1269         omsg_int = ioread32(priv->regs + TSI721_OBDMAC_INT(ch));
1270
1271         if (omsg_int & TSI721_OBDMAC_INT_ST_FULL)
1272                 dev_info(&priv->pdev->dev,
1273                         "OB MBOX%d: Status FIFO is full\n", ch);
1274
1275         if (omsg_int & (TSI721_OBDMAC_INT_DONE | TSI721_OBDMAC_INT_IOF_DONE)) {
1276                 u32 srd_ptr;
1277                 u64 *sts_ptr, last_ptr = 0, prev_ptr = 0;
1278                 int i, j;
1279                 u32 tx_slot;
1280
1281                 /*
1282                  * Find last successfully processed descriptor
1283                  */
1284
1285                 /* Check and clear descriptor status FIFO entries */
1286                 srd_ptr = priv->omsg_ring[ch].sts_rdptr;
1287                 sts_ptr = priv->omsg_ring[ch].sts_base;
1288                 j = srd_ptr * 8;
1289                 while (sts_ptr[j]) {
1290                         for (i = 0; i < 8 && sts_ptr[j]; i++, j++) {
1291                                 prev_ptr = last_ptr;
1292                                 last_ptr = le64_to_cpu(sts_ptr[j]);
1293                                 sts_ptr[j] = 0;
1294                         }
1295
1296                         ++srd_ptr;
1297                         srd_ptr %= priv->omsg_ring[ch].sts_size;
1298                         j = srd_ptr * 8;
1299                 }
1300
1301                 if (last_ptr == 0)
1302                         goto no_sts_update;
1303
1304                 priv->omsg_ring[ch].sts_rdptr = srd_ptr;
1305                 iowrite32(srd_ptr, priv->regs + TSI721_OBDMAC_DSRP(ch));
1306
1307                 if (!priv->mport->outb_msg[ch].mcback)
1308                         goto no_sts_update;
1309
1310                 /* Inform upper layer about transfer completion */
1311
1312                 tx_slot = (last_ptr - (u64)priv->omsg_ring[ch].omd_phys)/
1313                                                 sizeof(struct tsi721_omsg_desc);
1314
1315                 /*
1316                  * Check if this is a Link Descriptor (LD).
1317                  * If yes, ignore LD and use descriptor processed
1318                  * before LD.
1319                  */
1320                 if (tx_slot == priv->omsg_ring[ch].size) {
1321                         if (prev_ptr)
1322                                 tx_slot = (prev_ptr -
1323                                         (u64)priv->omsg_ring[ch].omd_phys)/
1324                                                 sizeof(struct tsi721_omsg_desc);
1325                         else
1326                                 goto no_sts_update;
1327                 }
1328
1329                 /* Move slot index to the next message to be sent */
1330                 ++tx_slot;
1331                 if (tx_slot == priv->omsg_ring[ch].size)
1332                         tx_slot = 0;
1333                 BUG_ON(tx_slot >= priv->omsg_ring[ch].size);
1334                 priv->mport->outb_msg[ch].mcback(priv->mport,
1335                                 priv->omsg_ring[ch].dev_id, ch,
1336                                 tx_slot);
1337         }
1338
1339 no_sts_update:
1340
1341         if (omsg_int & TSI721_OBDMAC_INT_ERROR) {
1342                 /*
1343                 * Outbound message operation aborted due to error,
1344                 * reinitialize OB MSG channel
1345                 */
1346
1347                 dev_dbg(&priv->pdev->dev, "OB MSG ABORT ch_stat=%x\n",
1348                         ioread32(priv->regs + TSI721_OBDMAC_STS(ch)));
1349
1350                 iowrite32(TSI721_OBDMAC_INT_ERROR,
1351                                 priv->regs + TSI721_OBDMAC_INT(ch));
1352                 iowrite32(TSI721_OBDMAC_CTL_INIT,
1353                                 priv->regs + TSI721_OBDMAC_CTL(ch));
1354                 ioread32(priv->regs + TSI721_OBDMAC_CTL(ch));
1355
1356                 /* Inform upper level to clear all pending tx slots */
1357                 if (priv->mport->outb_msg[ch].mcback)
1358                         priv->mport->outb_msg[ch].mcback(priv->mport,
1359                                         priv->omsg_ring[ch].dev_id, ch,
1360                                         priv->omsg_ring[ch].tx_slot);
1361                 /* Synch tx_slot tracking */
1362                 iowrite32(priv->omsg_ring[ch].tx_slot,
1363                         priv->regs + TSI721_OBDMAC_DRDCNT(ch));
1364                 ioread32(priv->regs + TSI721_OBDMAC_DRDCNT(ch));
1365                 priv->omsg_ring[ch].wr_count = priv->omsg_ring[ch].tx_slot;
1366                 priv->omsg_ring[ch].sts_rdptr = 0;
1367         }
1368
1369         /* Clear channel interrupts */
1370         iowrite32(omsg_int, priv->regs + TSI721_OBDMAC_INT(ch));
1371
1372         if (!(priv->flags & TSI721_USING_MSIX)) {
1373                 u32 ch_inte;
1374
1375                 /* Re-enable channel interrupts */
1376                 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1377                 ch_inte |= TSI721_INT_OMSG_CHAN(ch);
1378                 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
1379         }
1380
1381         spin_unlock(&priv->omsg_ring[ch].lock);
1382 }
1383
1384 /**
1385  * tsi721_open_outb_mbox - Initialize Tsi721 outbound mailbox
1386  * @mport: Master port implementing Outbound Messaging Engine
1387  * @dev_id: Device specific pointer to pass on event
1388  * @mbox: Mailbox to open
1389  * @entries: Number of entries in the outbound mailbox ring
1390  */
1391 static int tsi721_open_outb_mbox(struct rio_mport *mport, void *dev_id,
1392                                  int mbox, int entries)
1393 {
1394         struct tsi721_device *priv = mport->priv;
1395         struct tsi721_omsg_desc *bd_ptr;
1396         int i, rc = 0;
1397
1398         if ((entries < TSI721_OMSGD_MIN_RING_SIZE) ||
1399             (entries > (TSI721_OMSGD_RING_SIZE)) ||
1400             (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {
1401                 rc = -EINVAL;
1402                 goto out;
1403         }
1404
1405         priv->omsg_ring[mbox].dev_id = dev_id;
1406         priv->omsg_ring[mbox].size = entries;
1407         priv->omsg_ring[mbox].sts_rdptr = 0;
1408         spin_lock_init(&priv->omsg_ring[mbox].lock);
1409
1410         /* Outbound Msg Buffer allocation based on
1411            the number of maximum descriptor entries */
1412         for (i = 0; i < entries; i++) {
1413                 priv->omsg_ring[mbox].omq_base[i] =
1414                         dma_alloc_coherent(
1415                                 &priv->pdev->dev, TSI721_MSG_BUFFER_SIZE,
1416                                 &priv->omsg_ring[mbox].omq_phys[i],
1417                                 GFP_KERNEL);
1418                 if (priv->omsg_ring[mbox].omq_base[i] == NULL) {
1419                         dev_dbg(&priv->pdev->dev,
1420                                 "Unable to allocate OB MSG data buffer for"
1421                                 " MBOX%d\n", mbox);
1422                         rc = -ENOMEM;
1423                         goto out_buf;
1424                 }
1425         }
1426
1427         /* Outbound message descriptor allocation */
1428         priv->omsg_ring[mbox].omd_base = dma_alloc_coherent(
1429                                 &priv->pdev->dev,
1430                                 (entries + 1) * sizeof(struct tsi721_omsg_desc),
1431                                 &priv->omsg_ring[mbox].omd_phys, GFP_KERNEL);
1432         if (priv->omsg_ring[mbox].omd_base == NULL) {
1433                 dev_dbg(&priv->pdev->dev,
1434                         "Unable to allocate OB MSG descriptor memory "
1435                         "for MBOX%d\n", mbox);
1436                 rc = -ENOMEM;
1437                 goto out_buf;
1438         }
1439
1440         priv->omsg_ring[mbox].tx_slot = 0;
1441
1442         /* Outbound message descriptor status FIFO allocation */
1443         priv->omsg_ring[mbox].sts_size = roundup_pow_of_two(entries + 1);
1444         priv->omsg_ring[mbox].sts_base = dma_zalloc_coherent(&priv->pdev->dev,
1445                         priv->omsg_ring[mbox].sts_size *
1446                                                 sizeof(struct tsi721_dma_sts),
1447                         &priv->omsg_ring[mbox].sts_phys, GFP_KERNEL);
1448         if (priv->omsg_ring[mbox].sts_base == NULL) {
1449                 dev_dbg(&priv->pdev->dev,
1450                         "Unable to allocate OB MSG descriptor status FIFO "
1451                         "for MBOX%d\n", mbox);
1452                 rc = -ENOMEM;
1453                 goto out_desc;
1454         }
1455
1456         /*
1457          * Configure Outbound Messaging Engine
1458          */
1459
1460         /* Setup Outbound Message descriptor pointer */
1461         iowrite32(((u64)priv->omsg_ring[mbox].omd_phys >> 32),
1462                         priv->regs + TSI721_OBDMAC_DPTRH(mbox));
1463         iowrite32(((u64)priv->omsg_ring[mbox].omd_phys &
1464                                         TSI721_OBDMAC_DPTRL_MASK),
1465                         priv->regs + TSI721_OBDMAC_DPTRL(mbox));
1466
1467         /* Setup Outbound Message descriptor status FIFO */
1468         iowrite32(((u64)priv->omsg_ring[mbox].sts_phys >> 32),
1469                         priv->regs + TSI721_OBDMAC_DSBH(mbox));
1470         iowrite32(((u64)priv->omsg_ring[mbox].sts_phys &
1471                                         TSI721_OBDMAC_DSBL_MASK),
1472                         priv->regs + TSI721_OBDMAC_DSBL(mbox));
1473         iowrite32(TSI721_DMAC_DSSZ_SIZE(priv->omsg_ring[mbox].sts_size),
1474                 priv->regs + (u32)TSI721_OBDMAC_DSSZ(mbox));
1475
1476         /* Enable interrupts */
1477
1478 #ifdef CONFIG_PCI_MSI
1479         if (priv->flags & TSI721_USING_MSIX) {
1480                 /* Request interrupt service if we are in MSI-X mode */
1481                 rc = request_irq(
1482                         priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,
1483                         tsi721_omsg_msix, 0,
1484                         priv->msix[TSI721_VECT_OMB0_DONE + mbox].irq_name,
1485                         (void *)mport);
1486
1487                 if (rc) {
1488                         dev_dbg(&priv->pdev->dev,
1489                                 "Unable to allocate MSI-X interrupt for "
1490                                 "OBOX%d-DONE\n", mbox);
1491                         goto out_stat;
1492                 }
1493
1494                 rc = request_irq(priv->msix[TSI721_VECT_OMB0_INT + mbox].vector,
1495                         tsi721_omsg_msix, 0,
1496                         priv->msix[TSI721_VECT_OMB0_INT + mbox].irq_name,
1497                         (void *)mport);
1498
1499                 if (rc) {
1500                         dev_dbg(&priv->pdev->dev,
1501                                 "Unable to allocate MSI-X interrupt for "
1502                                 "MBOX%d-INT\n", mbox);
1503                         free_irq(
1504                                 priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,
1505                                 (void *)mport);
1506                         goto out_stat;
1507                 }
1508         }
1509 #endif /* CONFIG_PCI_MSI */
1510
1511         tsi721_omsg_interrupt_enable(priv, mbox, TSI721_OBDMAC_INT_ALL);
1512
1513         /* Initialize Outbound Message descriptors ring */
1514         bd_ptr = priv->omsg_ring[mbox].omd_base;
1515         bd_ptr[entries].type_id = cpu_to_le32(DTYPE5 << 29);
1516         bd_ptr[entries].msg_info = 0;
1517         bd_ptr[entries].next_lo =
1518                 cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys &
1519                 TSI721_OBDMAC_DPTRL_MASK);
1520         bd_ptr[entries].next_hi =
1521                 cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys >> 32);
1522         priv->omsg_ring[mbox].wr_count = 0;
1523         mb();
1524
1525         /* Initialize Outbound Message engine */
1526         iowrite32(TSI721_OBDMAC_CTL_INIT, priv->regs + TSI721_OBDMAC_CTL(mbox));
1527         ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1528         udelay(10);
1529
1530         priv->omsg_init[mbox] = 1;
1531
1532         return 0;
1533
1534 #ifdef CONFIG_PCI_MSI
1535 out_stat:
1536         dma_free_coherent(&priv->pdev->dev,
1537                 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
1538                 priv->omsg_ring[mbox].sts_base,
1539                 priv->omsg_ring[mbox].sts_phys);
1540
1541         priv->omsg_ring[mbox].sts_base = NULL;
1542 #endif /* CONFIG_PCI_MSI */
1543
1544 out_desc:
1545         dma_free_coherent(&priv->pdev->dev,
1546                 (entries + 1) * sizeof(struct tsi721_omsg_desc),
1547                 priv->omsg_ring[mbox].omd_base,
1548                 priv->omsg_ring[mbox].omd_phys);
1549
1550         priv->omsg_ring[mbox].omd_base = NULL;
1551
1552 out_buf:
1553         for (i = 0; i < priv->omsg_ring[mbox].size; i++) {
1554                 if (priv->omsg_ring[mbox].omq_base[i]) {
1555                         dma_free_coherent(&priv->pdev->dev,
1556                                 TSI721_MSG_BUFFER_SIZE,
1557                                 priv->omsg_ring[mbox].omq_base[i],
1558                                 priv->omsg_ring[mbox].omq_phys[i]);
1559
1560                         priv->omsg_ring[mbox].omq_base[i] = NULL;
1561                 }
1562         }
1563
1564 out:
1565         return rc;
1566 }
1567
1568 /**
1569  * tsi721_close_outb_mbox - Close Tsi721 outbound mailbox
1570  * @mport: Master port implementing the outbound message unit
1571  * @mbox: Mailbox to close
1572  */
1573 static void tsi721_close_outb_mbox(struct rio_mport *mport, int mbox)
1574 {
1575         struct tsi721_device *priv = mport->priv;
1576         u32 i;
1577
1578         if (!priv->omsg_init[mbox])
1579                 return;
1580         priv->omsg_init[mbox] = 0;
1581
1582         /* Disable Interrupts */
1583
1584         tsi721_omsg_interrupt_disable(priv, mbox, TSI721_OBDMAC_INT_ALL);
1585
1586 #ifdef CONFIG_PCI_MSI
1587         if (priv->flags & TSI721_USING_MSIX) {
1588                 free_irq(priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,
1589                          (void *)mport);
1590                 free_irq(priv->msix[TSI721_VECT_OMB0_INT + mbox].vector,
1591                          (void *)mport);
1592         }
1593 #endif /* CONFIG_PCI_MSI */
1594
1595         /* Free OMSG Descriptor Status FIFO */
1596         dma_free_coherent(&priv->pdev->dev,
1597                 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
1598                 priv->omsg_ring[mbox].sts_base,
1599                 priv->omsg_ring[mbox].sts_phys);
1600
1601         priv->omsg_ring[mbox].sts_base = NULL;
1602
1603         /* Free OMSG descriptors */
1604         dma_free_coherent(&priv->pdev->dev,
1605                 (priv->omsg_ring[mbox].size + 1) *
1606                         sizeof(struct tsi721_omsg_desc),
1607                 priv->omsg_ring[mbox].omd_base,
1608                 priv->omsg_ring[mbox].omd_phys);
1609
1610         priv->omsg_ring[mbox].omd_base = NULL;
1611
1612         /* Free message buffers */
1613         for (i = 0; i < priv->omsg_ring[mbox].size; i++) {
1614                 if (priv->omsg_ring[mbox].omq_base[i]) {
1615                         dma_free_coherent(&priv->pdev->dev,
1616                                 TSI721_MSG_BUFFER_SIZE,
1617                                 priv->omsg_ring[mbox].omq_base[i],
1618                                 priv->omsg_ring[mbox].omq_phys[i]);
1619
1620                         priv->omsg_ring[mbox].omq_base[i] = NULL;
1621                 }
1622         }
1623 }
1624
1625 /**
1626  * tsi721_imsg_handler - Inbound Message Interrupt Handler
1627  * @priv: pointer to tsi721 private data
1628  * @ch: inbound message channel number to service
1629  *
1630  * Services channel interrupts from inbound messaging engine.
1631  */
1632 static void tsi721_imsg_handler(struct tsi721_device *priv, int ch)
1633 {
1634         u32 mbox = ch - 4;
1635         u32 imsg_int;
1636
1637         spin_lock(&priv->imsg_ring[mbox].lock);
1638
1639         imsg_int = ioread32(priv->regs + TSI721_IBDMAC_INT(ch));
1640
1641         if (imsg_int & TSI721_IBDMAC_INT_SRTO)
1642                 dev_info(&priv->pdev->dev, "IB MBOX%d SRIO timeout\n",
1643                         mbox);
1644
1645         if (imsg_int & TSI721_IBDMAC_INT_PC_ERROR)
1646                 dev_info(&priv->pdev->dev, "IB MBOX%d PCIe error\n",
1647                         mbox);
1648
1649         if (imsg_int & TSI721_IBDMAC_INT_FQ_LOW)
1650                 dev_info(&priv->pdev->dev,
1651                         "IB MBOX%d IB free queue low\n", mbox);
1652
1653         /* Clear IB channel interrupts */
1654         iowrite32(imsg_int, priv->regs + TSI721_IBDMAC_INT(ch));
1655
1656         /* If an IB Msg is received notify the upper layer */
1657         if (imsg_int & TSI721_IBDMAC_INT_DQ_RCV &&
1658                 priv->mport->inb_msg[mbox].mcback)
1659                 priv->mport->inb_msg[mbox].mcback(priv->mport,
1660                                 priv->imsg_ring[mbox].dev_id, mbox, -1);
1661
1662         if (!(priv->flags & TSI721_USING_MSIX)) {
1663                 u32 ch_inte;
1664
1665                 /* Re-enable channel interrupts */
1666                 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1667                 ch_inte |= TSI721_INT_IMSG_CHAN(ch);
1668                 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
1669         }
1670
1671         spin_unlock(&priv->imsg_ring[mbox].lock);
1672 }
1673
1674 /**
1675  * tsi721_open_inb_mbox - Initialize Tsi721 inbound mailbox
1676  * @mport: Master port implementing the Inbound Messaging Engine
1677  * @dev_id: Device specific pointer to pass on event
1678  * @mbox: Mailbox to open
1679  * @entries: Number of entries in the inbound mailbox ring
1680  */
1681 static int tsi721_open_inb_mbox(struct rio_mport *mport, void *dev_id,
1682                                 int mbox, int entries)
1683 {
1684         struct tsi721_device *priv = mport->priv;
1685         int ch = mbox + 4;
1686         int i;
1687         u64 *free_ptr;
1688         int rc = 0;
1689
1690         if ((entries < TSI721_IMSGD_MIN_RING_SIZE) ||
1691             (entries > TSI721_IMSGD_RING_SIZE) ||
1692             (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {
1693                 rc = -EINVAL;
1694                 goto out;
1695         }
1696
1697         /* Initialize IB Messaging Ring */
1698         priv->imsg_ring[mbox].dev_id = dev_id;
1699         priv->imsg_ring[mbox].size = entries;
1700         priv->imsg_ring[mbox].rx_slot = 0;
1701         priv->imsg_ring[mbox].desc_rdptr = 0;
1702         priv->imsg_ring[mbox].fq_wrptr = 0;
1703         for (i = 0; i < priv->imsg_ring[mbox].size; i++)
1704                 priv->imsg_ring[mbox].imq_base[i] = NULL;
1705         spin_lock_init(&priv->imsg_ring[mbox].lock);
1706
1707         /* Allocate buffers for incoming messages */
1708         priv->imsg_ring[mbox].buf_base =
1709                 dma_alloc_coherent(&priv->pdev->dev,
1710                                    entries * TSI721_MSG_BUFFER_SIZE,
1711                                    &priv->imsg_ring[mbox].buf_phys,
1712                                    GFP_KERNEL);
1713
1714         if (priv->imsg_ring[mbox].buf_base == NULL) {
1715                 dev_err(&priv->pdev->dev,
1716                         "Failed to allocate buffers for IB MBOX%d\n", mbox);
1717                 rc = -ENOMEM;
1718                 goto out;
1719         }
1720
1721         /* Allocate memory for circular free list */
1722         priv->imsg_ring[mbox].imfq_base =
1723                 dma_alloc_coherent(&priv->pdev->dev,
1724                                    entries * 8,
1725                                    &priv->imsg_ring[mbox].imfq_phys,
1726                                    GFP_KERNEL);
1727
1728         if (priv->imsg_ring[mbox].imfq_base == NULL) {
1729                 dev_err(&priv->pdev->dev,
1730                         "Failed to allocate free queue for IB MBOX%d\n", mbox);
1731                 rc = -ENOMEM;
1732                 goto out_buf;
1733         }
1734
1735         /* Allocate memory for Inbound message descriptors */
1736         priv->imsg_ring[mbox].imd_base =
1737                 dma_alloc_coherent(&priv->pdev->dev,
1738                                    entries * sizeof(struct tsi721_imsg_desc),
1739                                    &priv->imsg_ring[mbox].imd_phys, GFP_KERNEL);
1740
1741         if (priv->imsg_ring[mbox].imd_base == NULL) {
1742                 dev_err(&priv->pdev->dev,
1743                         "Failed to allocate descriptor memory for IB MBOX%d\n",
1744                         mbox);
1745                 rc = -ENOMEM;
1746                 goto out_dma;
1747         }
1748
1749         /* Fill free buffer pointer list */
1750         free_ptr = priv->imsg_ring[mbox].imfq_base;
1751         for (i = 0; i < entries; i++)
1752                 free_ptr[i] = cpu_to_le64(
1753                                 (u64)(priv->imsg_ring[mbox].buf_phys) +
1754                                 i * 0x1000);
1755
1756         mb();
1757
1758         /*
1759          * For mapping of inbound SRIO Messages into appropriate queues we need
1760          * to set Inbound Device ID register in the messaging engine. We do it
1761          * once when first inbound mailbox is requested.
1762          */
1763         if (!(priv->flags & TSI721_IMSGID_SET)) {
1764                 iowrite32((u32)priv->mport->host_deviceid,
1765                         priv->regs + TSI721_IB_DEVID);
1766                 priv->flags |= TSI721_IMSGID_SET;
1767         }
1768
1769         /*
1770          * Configure Inbound Messaging channel (ch = mbox + 4)
1771          */
1772
1773         /* Setup Inbound Message free queue */
1774         iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys >> 32),
1775                 priv->regs + TSI721_IBDMAC_FQBH(ch));
1776         iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys &
1777                         TSI721_IBDMAC_FQBL_MASK),
1778                 priv->regs+TSI721_IBDMAC_FQBL(ch));
1779         iowrite32(TSI721_DMAC_DSSZ_SIZE(entries),
1780                 priv->regs + TSI721_IBDMAC_FQSZ(ch));
1781
1782         /* Setup Inbound Message descriptor queue */
1783         iowrite32(((u64)priv->imsg_ring[mbox].imd_phys >> 32),
1784                 priv->regs + TSI721_IBDMAC_DQBH(ch));
1785         iowrite32(((u32)priv->imsg_ring[mbox].imd_phys &
1786                    (u32)TSI721_IBDMAC_DQBL_MASK),
1787                 priv->regs+TSI721_IBDMAC_DQBL(ch));
1788         iowrite32(TSI721_DMAC_DSSZ_SIZE(entries),
1789                 priv->regs + TSI721_IBDMAC_DQSZ(ch));
1790
1791         /* Enable interrupts */
1792
1793 #ifdef CONFIG_PCI_MSI
1794         if (priv->flags & TSI721_USING_MSIX) {
1795                 /* Request interrupt service if we are in MSI-X mode */
1796                 rc = request_irq(priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
1797                         tsi721_imsg_msix, 0,
1798                         priv->msix[TSI721_VECT_IMB0_RCV + mbox].irq_name,
1799                         (void *)mport);
1800
1801                 if (rc) {
1802                         dev_dbg(&priv->pdev->dev,
1803                                 "Unable to allocate MSI-X interrupt for "
1804                                 "IBOX%d-DONE\n", mbox);
1805                         goto out_desc;
1806                 }
1807
1808                 rc = request_irq(priv->msix[TSI721_VECT_IMB0_INT + mbox].vector,
1809                         tsi721_imsg_msix, 0,
1810                         priv->msix[TSI721_VECT_IMB0_INT + mbox].irq_name,
1811                         (void *)mport);
1812
1813                 if (rc) {
1814                         dev_dbg(&priv->pdev->dev,
1815                                 "Unable to allocate MSI-X interrupt for "
1816                                 "IBOX%d-INT\n", mbox);
1817                         free_irq(
1818                                 priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
1819                                 (void *)mport);
1820                         goto out_desc;
1821                 }
1822         }
1823 #endif /* CONFIG_PCI_MSI */
1824
1825         tsi721_imsg_interrupt_enable(priv, ch, TSI721_IBDMAC_INT_ALL);
1826
1827         /* Initialize Inbound Message Engine */
1828         iowrite32(TSI721_IBDMAC_CTL_INIT, priv->regs + TSI721_IBDMAC_CTL(ch));
1829         ioread32(priv->regs + TSI721_IBDMAC_CTL(ch));
1830         udelay(10);
1831         priv->imsg_ring[mbox].fq_wrptr = entries - 1;
1832         iowrite32(entries - 1, priv->regs + TSI721_IBDMAC_FQWP(ch));
1833
1834         priv->imsg_init[mbox] = 1;
1835         return 0;
1836
1837 #ifdef CONFIG_PCI_MSI
1838 out_desc:
1839         dma_free_coherent(&priv->pdev->dev,
1840                 priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),
1841                 priv->imsg_ring[mbox].imd_base,
1842                 priv->imsg_ring[mbox].imd_phys);
1843
1844         priv->imsg_ring[mbox].imd_base = NULL;
1845 #endif /* CONFIG_PCI_MSI */
1846
1847 out_dma:
1848         dma_free_coherent(&priv->pdev->dev,
1849                 priv->imsg_ring[mbox].size * 8,
1850                 priv->imsg_ring[mbox].imfq_base,
1851                 priv->imsg_ring[mbox].imfq_phys);
1852
1853         priv->imsg_ring[mbox].imfq_base = NULL;
1854
1855 out_buf:
1856         dma_free_coherent(&priv->pdev->dev,
1857                 priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,
1858                 priv->imsg_ring[mbox].buf_base,
1859                 priv->imsg_ring[mbox].buf_phys);
1860
1861         priv->imsg_ring[mbox].buf_base = NULL;
1862
1863 out:
1864         return rc;
1865 }
1866
1867 /**
1868  * tsi721_close_inb_mbox - Shut down Tsi721 inbound mailbox
1869  * @mport: Master port implementing the Inbound Messaging Engine
1870  * @mbox: Mailbox to close
1871  */
1872 static void tsi721_close_inb_mbox(struct rio_mport *mport, int mbox)
1873 {
1874         struct tsi721_device *priv = mport->priv;
1875         u32 rx_slot;
1876         int ch = mbox + 4;
1877
1878         if (!priv->imsg_init[mbox]) /* mbox isn't initialized yet */
1879                 return;
1880         priv->imsg_init[mbox] = 0;
1881
1882         /* Disable Inbound Messaging Engine */
1883
1884         /* Disable Interrupts */
1885         tsi721_imsg_interrupt_disable(priv, ch, TSI721_OBDMAC_INT_MASK);
1886
1887 #ifdef CONFIG_PCI_MSI
1888         if (priv->flags & TSI721_USING_MSIX) {
1889                 free_irq(priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
1890                                 (void *)mport);
1891                 free_irq(priv->msix[TSI721_VECT_IMB0_INT + mbox].vector,
1892                                 (void *)mport);
1893         }
1894 #endif /* CONFIG_PCI_MSI */
1895
1896         /* Clear Inbound Buffer Queue */
1897         for (rx_slot = 0; rx_slot < priv->imsg_ring[mbox].size; rx_slot++)
1898                 priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;
1899
1900         /* Free memory allocated for message buffers */
1901         dma_free_coherent(&priv->pdev->dev,
1902                 priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,
1903                 priv->imsg_ring[mbox].buf_base,
1904                 priv->imsg_ring[mbox].buf_phys);
1905
1906         priv->imsg_ring[mbox].buf_base = NULL;
1907
1908         /* Free memory allocated for free pointr list */
1909         dma_free_coherent(&priv->pdev->dev,
1910                 priv->imsg_ring[mbox].size * 8,
1911                 priv->imsg_ring[mbox].imfq_base,
1912                 priv->imsg_ring[mbox].imfq_phys);
1913
1914         priv->imsg_ring[mbox].imfq_base = NULL;
1915
1916         /* Free memory allocated for RX descriptors */
1917         dma_free_coherent(&priv->pdev->dev,
1918                 priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),
1919                 priv->imsg_ring[mbox].imd_base,
1920                 priv->imsg_ring[mbox].imd_phys);
1921
1922         priv->imsg_ring[mbox].imd_base = NULL;
1923 }
1924
1925 /**
1926  * tsi721_add_inb_buffer - Add buffer to the Tsi721 inbound message queue
1927  * @mport: Master port implementing the Inbound Messaging Engine
1928  * @mbox: Inbound mailbox number
1929  * @buf: Buffer to add to inbound queue
1930  */
1931 static int tsi721_add_inb_buffer(struct rio_mport *mport, int mbox, void *buf)
1932 {
1933         struct tsi721_device *priv = mport->priv;
1934         u32 rx_slot;
1935         int rc = 0;
1936
1937         rx_slot = priv->imsg_ring[mbox].rx_slot;
1938         if (priv->imsg_ring[mbox].imq_base[rx_slot]) {
1939                 dev_err(&priv->pdev->dev,
1940                         "Error adding inbound buffer %d, buffer exists\n",
1941                         rx_slot);
1942                 rc = -EINVAL;
1943                 goto out;
1944         }
1945
1946         priv->imsg_ring[mbox].imq_base[rx_slot] = buf;
1947
1948         if (++priv->imsg_ring[mbox].rx_slot == priv->imsg_ring[mbox].size)
1949                 priv->imsg_ring[mbox].rx_slot = 0;
1950
1951 out:
1952         return rc;
1953 }
1954
1955 /**
1956  * tsi721_get_inb_message - Fetch inbound message from the Tsi721 MSG Queue
1957  * @mport: Master port implementing the Inbound Messaging Engine
1958  * @mbox: Inbound mailbox number
1959  *
1960  * Returns pointer to the message on success or NULL on failure.
1961  */
1962 static void *tsi721_get_inb_message(struct rio_mport *mport, int mbox)
1963 {
1964         struct tsi721_device *priv = mport->priv;
1965         struct tsi721_imsg_desc *desc;
1966         u32 rx_slot;
1967         void *rx_virt = NULL;
1968         u64 rx_phys;
1969         void *buf = NULL;
1970         u64 *free_ptr;
1971         int ch = mbox + 4;
1972         int msg_size;
1973
1974         if (!priv->imsg_init[mbox])
1975                 return NULL;
1976
1977         desc = priv->imsg_ring[mbox].imd_base;
1978         desc += priv->imsg_ring[mbox].desc_rdptr;
1979
1980         if (!(le32_to_cpu(desc->msg_info) & TSI721_IMD_HO))
1981                 goto out;
1982
1983         rx_slot = priv->imsg_ring[mbox].rx_slot;
1984         while (priv->imsg_ring[mbox].imq_base[rx_slot] == NULL) {
1985                 if (++rx_slot == priv->imsg_ring[mbox].size)
1986                         rx_slot = 0;
1987         }
1988
1989         rx_phys = ((u64)le32_to_cpu(desc->bufptr_hi) << 32) |
1990                         le32_to_cpu(desc->bufptr_lo);
1991
1992         rx_virt = priv->imsg_ring[mbox].buf_base +
1993                   (rx_phys - (u64)priv->imsg_ring[mbox].buf_phys);
1994
1995         buf = priv->imsg_ring[mbox].imq_base[rx_slot];
1996         msg_size = le32_to_cpu(desc->msg_info) & TSI721_IMD_BCOUNT;
1997         if (msg_size == 0)
1998                 msg_size = RIO_MAX_MSG_SIZE;
1999
2000         memcpy(buf, rx_virt, msg_size);
2001         priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;
2002
2003         desc->msg_info &= cpu_to_le32(~TSI721_IMD_HO);
2004         if (++priv->imsg_ring[mbox].desc_rdptr == priv->imsg_ring[mbox].size)
2005                 priv->imsg_ring[mbox].desc_rdptr = 0;
2006
2007         iowrite32(priv->imsg_ring[mbox].desc_rdptr,
2008                 priv->regs + TSI721_IBDMAC_DQRP(ch));
2009
2010         /* Return free buffer into the pointer list */
2011         free_ptr = priv->imsg_ring[mbox].imfq_base;
2012         free_ptr[priv->imsg_ring[mbox].fq_wrptr] = cpu_to_le64(rx_phys);
2013
2014         if (++priv->imsg_ring[mbox].fq_wrptr == priv->imsg_ring[mbox].size)
2015                 priv->imsg_ring[mbox].fq_wrptr = 0;
2016
2017         iowrite32(priv->imsg_ring[mbox].fq_wrptr,
2018                 priv->regs + TSI721_IBDMAC_FQWP(ch));
2019 out:
2020         return buf;
2021 }
2022
2023 /**
2024  * tsi721_messages_init - Initialization of Messaging Engine
2025  * @priv: pointer to tsi721 private data
2026  *
2027  * Configures Tsi721 messaging engine.
2028  */
2029 static int tsi721_messages_init(struct tsi721_device *priv)
2030 {
2031         int     ch;
2032
2033         iowrite32(0, priv->regs + TSI721_SMSG_ECC_LOG);
2034         iowrite32(0, priv->regs + TSI721_RETRY_GEN_CNT);
2035         iowrite32(0, priv->regs + TSI721_RETRY_RX_CNT);
2036
2037         /* Set SRIO Message Request/Response Timeout */
2038         iowrite32(TSI721_RQRPTO_VAL, priv->regs + TSI721_RQRPTO);
2039
2040         /* Initialize Inbound Messaging Engine Registers */
2041         for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++) {
2042                 /* Clear interrupt bits */
2043                 iowrite32(TSI721_IBDMAC_INT_MASK,
2044                         priv->regs + TSI721_IBDMAC_INT(ch));
2045                 /* Clear Status */
2046                 iowrite32(0, priv->regs + TSI721_IBDMAC_STS(ch));
2047
2048                 iowrite32(TSI721_SMSG_ECC_COR_LOG_MASK,
2049                                 priv->regs + TSI721_SMSG_ECC_COR_LOG(ch));
2050                 iowrite32(TSI721_SMSG_ECC_NCOR_MASK,
2051                                 priv->regs + TSI721_SMSG_ECC_NCOR(ch));
2052         }
2053
2054         return 0;
2055 }
2056
2057 /**
2058  * tsi721_disable_ints - disables all device interrupts
2059  * @priv: pointer to tsi721 private data
2060  */
2061 static void tsi721_disable_ints(struct tsi721_device *priv)
2062 {
2063         int ch;
2064
2065         /* Disable all device level interrupts */
2066         iowrite32(0, priv->regs + TSI721_DEV_INTE);
2067
2068         /* Disable all Device Channel interrupts */
2069         iowrite32(0, priv->regs + TSI721_DEV_CHAN_INTE);
2070
2071         /* Disable all Inbound Msg Channel interrupts */
2072         for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++)
2073                 iowrite32(0, priv->regs + TSI721_IBDMAC_INTE(ch));
2074
2075         /* Disable all Outbound Msg Channel interrupts */
2076         for (ch = 0; ch < TSI721_OMSG_CHNUM; ch++)
2077                 iowrite32(0, priv->regs + TSI721_OBDMAC_INTE(ch));
2078
2079         /* Disable all general messaging interrupts */
2080         iowrite32(0, priv->regs + TSI721_SMSG_INTE);
2081
2082         /* Disable all BDMA Channel interrupts */
2083         for (ch = 0; ch < TSI721_DMA_MAXCH; ch++)
2084                 iowrite32(0,
2085                         priv->regs + TSI721_DMAC_BASE(ch) + TSI721_DMAC_INTE);
2086
2087         /* Disable all general BDMA interrupts */
2088         iowrite32(0, priv->regs + TSI721_BDMA_INTE);
2089
2090         /* Disable all SRIO Channel interrupts */
2091         for (ch = 0; ch < TSI721_SRIO_MAXCH; ch++)
2092                 iowrite32(0, priv->regs + TSI721_SR_CHINTE(ch));
2093
2094         /* Disable all general SR2PC interrupts */
2095         iowrite32(0, priv->regs + TSI721_SR2PC_GEN_INTE);
2096
2097         /* Disable all PC2SR interrupts */
2098         iowrite32(0, priv->regs + TSI721_PC2SR_INTE);
2099
2100         /* Disable all I2C interrupts */
2101         iowrite32(0, priv->regs + TSI721_I2C_INT_ENABLE);
2102
2103         /* Disable SRIO MAC interrupts */
2104         iowrite32(0, priv->regs + TSI721_RIO_EM_INT_ENABLE);
2105         iowrite32(0, priv->regs + TSI721_RIO_EM_DEV_INT_EN);
2106 }
2107
2108 /**
2109  * tsi721_setup_mport - Setup Tsi721 as RapidIO subsystem master port
2110  * @priv: pointer to tsi721 private data
2111  *
2112  * Configures Tsi721 as RapidIO master port.
2113  */
2114 static int __devinit tsi721_setup_mport(struct tsi721_device *priv)
2115 {
2116         struct pci_dev *pdev = priv->pdev;
2117         int err = 0;
2118         struct rio_ops *ops;
2119
2120         struct rio_mport *mport;
2121
2122         ops = kzalloc(sizeof(struct rio_ops), GFP_KERNEL);
2123         if (!ops) {
2124                 dev_dbg(&pdev->dev, "Unable to allocate memory for rio_ops\n");
2125                 return -ENOMEM;
2126         }
2127
2128         ops->lcread = tsi721_lcread;
2129         ops->lcwrite = tsi721_lcwrite;
2130         ops->cread = tsi721_cread_dma;
2131         ops->cwrite = tsi721_cwrite_dma;
2132         ops->dsend = tsi721_dsend;
2133         ops->open_inb_mbox = tsi721_open_inb_mbox;
2134         ops->close_inb_mbox = tsi721_close_inb_mbox;
2135         ops->open_outb_mbox = tsi721_open_outb_mbox;
2136         ops->close_outb_mbox = tsi721_close_outb_mbox;
2137         ops->add_outb_message = tsi721_add_outb_message;
2138         ops->add_inb_buffer = tsi721_add_inb_buffer;
2139         ops->get_inb_message = tsi721_get_inb_message;
2140
2141         mport = kzalloc(sizeof(struct rio_mport), GFP_KERNEL);
2142         if (!mport) {
2143                 kfree(ops);
2144                 dev_dbg(&pdev->dev, "Unable to allocate memory for mport\n");
2145                 return -ENOMEM;
2146         }
2147
2148         mport->ops = ops;
2149         mport->index = 0;
2150         mport->sys_size = 0; /* small system */
2151         mport->phy_type = RIO_PHY_SERIAL;
2152         mport->priv = (void *)priv;
2153         mport->phys_efptr = 0x100;
2154         priv->mport = mport;
2155
2156         INIT_LIST_HEAD(&mport->dbells);
2157
2158         rio_init_dbell_res(&mport->riores[RIO_DOORBELL_RESOURCE], 0, 0xffff);
2159         rio_init_mbox_res(&mport->riores[RIO_INB_MBOX_RESOURCE], 0, 3);
2160         rio_init_mbox_res(&mport->riores[RIO_OUTB_MBOX_RESOURCE], 0, 3);
2161         strcpy(mport->name, "Tsi721 mport");
2162
2163         /* Hook up interrupt handler */
2164
2165 #ifdef CONFIG_PCI_MSI
2166         if (!tsi721_enable_msix(priv))
2167                 priv->flags |= TSI721_USING_MSIX;
2168         else if (!pci_enable_msi(pdev))
2169                 priv->flags |= TSI721_USING_MSI;
2170         else
2171                 dev_info(&pdev->dev,
2172                          "MSI/MSI-X is not available. Using legacy INTx.\n");
2173 #endif /* CONFIG_PCI_MSI */
2174
2175         err = tsi721_request_irq(mport);
2176
2177         if (!err) {
2178                 tsi721_interrupts_init(priv);
2179                 ops->pwenable = tsi721_pw_enable;
2180         } else {
2181                 dev_err(&pdev->dev, "Unable to get assigned PCI IRQ "
2182                         "vector %02X err=0x%x\n", pdev->irq, err);
2183                 goto err_exit;
2184         }
2185
2186 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
2187         tsi721_register_dma(priv);
2188 #endif
2189         /* Enable SRIO link */
2190         iowrite32(ioread32(priv->regs + TSI721_DEVCTL) |
2191                   TSI721_DEVCTL_SRBOOT_CMPL,
2192                   priv->regs + TSI721_DEVCTL);
2193
2194         rio_register_mport(mport);
2195
2196         if (mport->host_deviceid >= 0)
2197                 iowrite32(RIO_PORT_GEN_HOST | RIO_PORT_GEN_MASTER |
2198                           RIO_PORT_GEN_DISCOVERED,
2199                           priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));
2200         else
2201                 iowrite32(0, priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));
2202
2203         return 0;
2204
2205 err_exit:
2206         kfree(mport);
2207         kfree(ops);
2208         return err;
2209 }
2210
2211 static int __devinit tsi721_probe(struct pci_dev *pdev,
2212                                   const struct pci_device_id *id)
2213 {
2214         struct tsi721_device *priv;
2215         int i;
2216         int err;
2217
2218         priv = kzalloc(sizeof(struct tsi721_device), GFP_KERNEL);
2219         if (priv == NULL) {
2220                 dev_err(&pdev->dev, "Failed to allocate memory for device\n");
2221                 err = -ENOMEM;
2222                 goto err_exit;
2223         }
2224
2225         err = pci_enable_device(pdev);
2226         if (err) {
2227                 dev_err(&pdev->dev, "Failed to enable PCI device\n");
2228                 goto err_clean;
2229         }
2230
2231         priv->pdev = pdev;
2232
2233 #ifdef DEBUG
2234         for (i = 0; i <= PCI_STD_RESOURCE_END; i++) {
2235                 dev_dbg(&pdev->dev, "res[%d] @ 0x%llx (0x%lx, 0x%lx)\n",
2236                         i, (unsigned long long)pci_resource_start(pdev, i),
2237                         (unsigned long)pci_resource_len(pdev, i),
2238                         pci_resource_flags(pdev, i));
2239         }
2240 #endif
2241         /*
2242          * Verify BAR configuration
2243          */
2244
2245         /* BAR_0 (registers) must be 512KB+ in 32-bit address space */
2246         if (!(pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM) ||
2247             pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM_64 ||
2248             pci_resource_len(pdev, BAR_0) < TSI721_REG_SPACE_SIZE) {
2249                 dev_err(&pdev->dev,
2250                         "Missing or misconfigured CSR BAR0, aborting.\n");
2251                 err = -ENODEV;
2252                 goto err_disable_pdev;
2253         }
2254
2255         /* BAR_1 (outbound doorbells) must be 16MB+ in 32-bit address space */
2256         if (!(pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM) ||
2257             pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM_64 ||
2258             pci_resource_len(pdev, BAR_1) < TSI721_DB_WIN_SIZE) {
2259                 dev_err(&pdev->dev,
2260                         "Missing or misconfigured Doorbell BAR1, aborting.\n");
2261                 err = -ENODEV;
2262                 goto err_disable_pdev;
2263         }
2264
2265         /*
2266          * BAR_2 and BAR_4 (outbound translation) must be in 64-bit PCIe address
2267          * space.
2268          * NOTE: BAR_2 and BAR_4 are not used by this version of driver.
2269          * It may be a good idea to keep them disabled using HW configuration
2270          * to save PCI memory space.
2271          */
2272         if ((pci_resource_flags(pdev, BAR_2) & IORESOURCE_MEM) &&
2273             (pci_resource_flags(pdev, BAR_2) & IORESOURCE_MEM_64)) {
2274                 dev_info(&pdev->dev, "Outbound BAR2 is not used but enabled.\n");
2275         }
2276
2277         if ((pci_resource_flags(pdev, BAR_4) & IORESOURCE_MEM) &&
2278             (pci_resource_flags(pdev, BAR_4) & IORESOURCE_MEM_64)) {
2279                 dev_info(&pdev->dev, "Outbound BAR4 is not used but enabled.\n");
2280         }
2281
2282         err = pci_request_regions(pdev, DRV_NAME);
2283         if (err) {
2284                 dev_err(&pdev->dev, "Cannot obtain PCI resources, "
2285                         "aborting.\n");
2286                 goto err_disable_pdev;
2287         }
2288
2289         pci_set_master(pdev);
2290
2291         priv->regs = pci_ioremap_bar(pdev, BAR_0);
2292         if (!priv->regs) {
2293                 dev_err(&pdev->dev,
2294                         "Unable to map device registers space, aborting\n");
2295                 err = -ENOMEM;
2296                 goto err_free_res;
2297         }
2298
2299         priv->odb_base = pci_ioremap_bar(pdev, BAR_1);
2300         if (!priv->odb_base) {
2301                 dev_err(&pdev->dev,
2302                         "Unable to map outbound doorbells space, aborting\n");
2303                 err = -ENOMEM;
2304                 goto err_unmap_bars;
2305         }
2306
2307         /* Configure DMA attributes. */
2308         if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
2309                 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) {
2310                         dev_info(&pdev->dev, "Unable to set DMA mask\n");
2311                         goto err_unmap_bars;
2312                 }
2313
2314                 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)))
2315                         dev_info(&pdev->dev, "Unable to set consistent DMA mask\n");
2316         } else {
2317                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2318                 if (err)
2319                         dev_info(&pdev->dev, "Unable to set consistent DMA mask\n");
2320         }
2321
2322         BUG_ON(!pci_is_pcie(pdev));
2323
2324         /* Clear "no snoop" and "relaxed ordering" bits, use default MRRS. */
2325         pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL,
2326                 PCI_EXP_DEVCTL_READRQ | PCI_EXP_DEVCTL_RELAX_EN |
2327                 PCI_EXP_DEVCTL_NOSNOOP_EN,
2328                 0x2 << MAX_READ_REQUEST_SZ_SHIFT);
2329
2330         /* Adjust PCIe completion timeout. */
2331         pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL2, 0xf, 0x2);
2332
2333         /*
2334          * FIXUP: correct offsets of MSI-X tables in the MSI-X Capability Block
2335          */
2336         pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0x01);
2337         pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXTBL,
2338                                                 TSI721_MSIXTBL_OFFSET);
2339         pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXPBA,
2340                                                 TSI721_MSIXPBA_OFFSET);
2341         pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0);
2342         /* End of FIXUP */
2343
2344         tsi721_disable_ints(priv);
2345
2346         tsi721_init_pc2sr_mapping(priv);
2347         tsi721_init_sr2pc_mapping(priv);
2348
2349         if (tsi721_bdma_maint_init(priv)) {
2350                 dev_err(&pdev->dev, "BDMA initialization failed, aborting\n");
2351                 err = -ENOMEM;
2352                 goto err_unmap_bars;
2353         }
2354
2355         err = tsi721_doorbell_init(priv);
2356         if (err)
2357                 goto err_free_bdma;
2358
2359         tsi721_port_write_init(priv);
2360
2361         err = tsi721_messages_init(priv);
2362         if (err)
2363                 goto err_free_consistent;
2364
2365         err = tsi721_setup_mport(priv);
2366         if (err)
2367                 goto err_free_consistent;
2368
2369         return 0;
2370
2371 err_free_consistent:
2372         tsi721_doorbell_free(priv);
2373 err_free_bdma:
2374         tsi721_bdma_maint_free(priv);
2375 err_unmap_bars:
2376         if (priv->regs)
2377                 iounmap(priv->regs);
2378         if (priv->odb_base)
2379                 iounmap(priv->odb_base);
2380 err_free_res:
2381         pci_release_regions(pdev);
2382         pci_clear_master(pdev);
2383 err_disable_pdev:
2384         pci_disable_device(pdev);
2385 err_clean:
2386         kfree(priv);
2387 err_exit:
2388         return err;
2389 }
2390
2391 static DEFINE_PCI_DEVICE_TABLE(tsi721_pci_tbl) = {
2392         { PCI_DEVICE(PCI_VENDOR_ID_IDT, PCI_DEVICE_ID_TSI721) },
2393         { 0, }  /* terminate list */
2394 };
2395
2396 MODULE_DEVICE_TABLE(pci, tsi721_pci_tbl);
2397
2398 static struct pci_driver tsi721_driver = {
2399         .name           = "tsi721",
2400         .id_table       = tsi721_pci_tbl,
2401         .probe          = tsi721_probe,
2402 };
2403
2404 static int __init tsi721_init(void)
2405 {
2406         return pci_register_driver(&tsi721_driver);
2407 }
2408
2409 static void __exit tsi721_exit(void)
2410 {
2411         pci_unregister_driver(&tsi721_driver);
2412 }
2413
2414 device_initcall(tsi721_init);