Revert "Merge commit 'main-jb-2012.08.03-B4' into t114-0806"
[linux-2.6.git] / drivers / spi / spi-tegra.c
1 /*
2  * Driver for Nvidia TEGRA spi controller.
3  *
4  * Copyright (C) 2010 Google, Inc.
5  *
6  * Author:
7  *     Erik Gilling <konkers@android.com>
8  *
9  * Copyright (C) 2010-2011 NVIDIA Corporation
10  *
11  * This software is licensed under the terms of the GNU General Public
12  * License version 2, as published by the Free Software Foundation, and
13  * may be copied, distributed, and modified under those terms.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18  * GNU General Public License for more details.
19  *
20  */
21
22 /*#define DEBUG           1*/
23 /*#define VERBOSE_DEBUG   1*/
24
25 #include <linux/kernel.h>
26 #include <linux/module.h>
27 #include <linux/init.h>
28 #include <linux/err.h>
29 #include <linux/platform_device.h>
30 #include <linux/io.h>
31 #include <linux/dma-mapping.h>
32 #include <linux/dmapool.h>
33 #include <linux/clk.h>
34 #include <linux/interrupt.h>
35 #include <linux/delay.h>
36 #include <linux/completion.h>
37 #include <linux/kthread.h>
38 #include <linux/pm_runtime.h>
39
40 #include <linux/spi/spi.h>
41 #include <linux/spi-tegra.h>
42
43 #include <mach/dma.h>
44 #include <mach/clk.h>
45
46 #define SPI_PM_RUNTIME_ENABLE 0
47
48 #define SLINK_COMMAND           0x000
49 #define   SLINK_BIT_LENGTH(x)           (((x) & 0x1f) << 0)
50 #define   SLINK_WORD_SIZE(x)            (((x) & 0x1f) << 5)
51 #define   SLINK_BOTH_EN                 (1 << 10)
52 #define   SLINK_CS_SW                   (1 << 11)
53 #define   SLINK_CS_VALUE                (1 << 12)
54 #define   SLINK_CS_POLARITY             (1 << 13)
55 #define   SLINK_IDLE_SDA_DRIVE_LOW      (0 << 16)
56 #define   SLINK_IDLE_SDA_DRIVE_HIGH     (1 << 16)
57 #define   SLINK_IDLE_SDA_PULL_LOW       (2 << 16)
58 #define   SLINK_IDLE_SDA_PULL_HIGH      (3 << 16)
59 #define   SLINK_IDLE_SDA_MASK           (3 << 16)
60 #define   SLINK_CS_POLARITY1            (1 << 20)
61 #define   SLINK_CK_SDA                  (1 << 21)
62 #define   SLINK_CS_POLARITY2            (1 << 22)
63 #define   SLINK_CS_POLARITY3            (1 << 23)
64 #define   SLINK_IDLE_SCLK_DRIVE_LOW     (0 << 24)
65 #define   SLINK_IDLE_SCLK_DRIVE_HIGH    (1 << 24)
66 #define   SLINK_IDLE_SCLK_PULL_LOW      (2 << 24)
67 #define   SLINK_IDLE_SCLK_PULL_HIGH     (3 << 24)
68 #define   SLINK_IDLE_SCLK_MASK          (3 << 24)
69 #define   SLINK_M_S                     (1 << 28)
70 #define   SLINK_WAIT                    (1 << 29)
71 #define   SLINK_GO                      (1 << 30)
72 #define   SLINK_ENB                     (1 << 31)
73
74 #define SLINK_COMMAND2          0x004
75 #define   SLINK_LSBFE                   (1 << 0)
76 #define   SLINK_SSOE                    (1 << 1)
77 #define   SLINK_SPIE                    (1 << 4)
78 #define   SLINK_BIDIROE                 (1 << 6)
79 #define   SLINK_MODFEN                  (1 << 7)
80 #define   SLINK_INT_SIZE(x)             (((x) & 0x1f) << 8)
81 #define   SLINK_CS_ACTIVE_BETWEEN       (1 << 17)
82 #define   SLINK_SS_EN_CS(x)             (((x) & 0x3) << 18)
83 #define   SLINK_SS_SETUP(x)             (((x) & 0x3) << 20)
84 #define   SLINK_FIFO_REFILLS_0          (0 << 22)
85 #define   SLINK_FIFO_REFILLS_1          (1 << 22)
86 #define   SLINK_FIFO_REFILLS_2          (2 << 22)
87 #define   SLINK_FIFO_REFILLS_3          (3 << 22)
88 #define   SLINK_FIFO_REFILLS_MASK       (3 << 22)
89 #define   SLINK_WAIT_PACK_INT(x)        (((x) & 0x7) << 26)
90 #define   SLINK_SPC0                    (1 << 29)
91 #define   SLINK_TXEN                    (1 << 30)
92 #define   SLINK_RXEN                    (1 << 31)
93
94 #define SLINK_STATUS            0x008
95 #define   SLINK_COUNT(val)              (((val) >> 0) & 0x1f)
96 #define   SLINK_WORD(val)               (((val) >> 5) & 0x1f)
97 #define   SLINK_BLK_CNT(val)            (((val) >> 0) & 0xffff)
98 #define   SLINK_MODF                    (1 << 16)
99 #define   SLINK_RX_UNF                  (1 << 18)
100 #define   SLINK_TX_OVF                  (1 << 19)
101 #define   SLINK_TX_FULL                 (1 << 20)
102 #define   SLINK_TX_EMPTY                (1 << 21)
103 #define   SLINK_RX_FULL                 (1 << 22)
104 #define   SLINK_RX_EMPTY                (1 << 23)
105 #define   SLINK_TX_UNF                  (1 << 24)
106 #define   SLINK_RX_OVF                  (1 << 25)
107 #define   SLINK_TX_FLUSH                (1 << 26)
108 #define   SLINK_RX_FLUSH                (1 << 27)
109 #define   SLINK_SCLK                    (1 << 28)
110 #define   SLINK_ERR                     (1 << 29)
111 #define   SLINK_RDY                     (1 << 30)
112 #define   SLINK_BSY                     (1 << 31)
113
114 #define SLINK_MAS_DATA          0x010
115 #define SLINK_SLAVE_DATA        0x014
116
117 #define SLINK_DMA_CTL           0x018
118 #define   SLINK_DMA_BLOCK_SIZE(x)       (((x) & 0xffff) << 0)
119 #define   SLINK_TX_TRIG_1               (0 << 16)
120 #define   SLINK_TX_TRIG_4               (1 << 16)
121 #define   SLINK_TX_TRIG_8               (2 << 16)
122 #define   SLINK_TX_TRIG_16              (3 << 16)
123 #define   SLINK_TX_TRIG_MASK            (3 << 16)
124 #define   SLINK_RX_TRIG_1               (0 << 18)
125 #define   SLINK_RX_TRIG_4               (1 << 18)
126 #define   SLINK_RX_TRIG_8               (2 << 18)
127 #define   SLINK_RX_TRIG_16              (3 << 18)
128 #define   SLINK_RX_TRIG_MASK            (3 << 18)
129 #define   SLINK_PACKED                  (1 << 20)
130 #define   SLINK_PACK_SIZE_4             (0 << 21)
131 #define   SLINK_PACK_SIZE_8             (1 << 21)
132 #define   SLINK_PACK_SIZE_16            (2 << 21)
133 #define   SLINK_PACK_SIZE_32            (3 << 21)
134 #define   SLINK_PACK_SIZE_MASK          (3 << 21)
135 #define   SLINK_IE_TXC                  (1 << 26)
136 #define   SLINK_IE_RXC                  (1 << 27)
137 #define   SLINK_DMA_EN                  (1 << 31)
138
139 #define SLINK_STATUS2           0x01c
140 #define   SLINK_TX_FIFO_EMPTY_COUNT(val)        (((val) & 0x3f) >> 0)
141 #define   SLINK_RX_FIFO_FULL_COUNT(val)         (((val) & 0x3f0000) >> 16)
142 #define   SLINK_SS_HOLD_TIME(val)               (((val) & 0xF) << 6)
143
144 #define SLINK_TX_FIFO           0x100
145 #define SLINK_RX_FIFO           0x180
146
147 #define DATA_DIR_TX             (1 << 0)
148 #define DATA_DIR_RX             (1 << 1)
149
150 #define SLINK_DMA_TIMEOUT (msecs_to_jiffies(1000))
151
152
153 static const unsigned long spi_tegra_req_sels[] = {
154         TEGRA_DMA_REQ_SEL_SL2B1,
155         TEGRA_DMA_REQ_SEL_SL2B2,
156         TEGRA_DMA_REQ_SEL_SL2B3,
157         TEGRA_DMA_REQ_SEL_SL2B4,
158 #ifndef CONFIG_ARCH_TEGRA_2x_SOC
159         TEGRA_DMA_REQ_SEL_SL2B5,
160         TEGRA_DMA_REQ_SEL_SL2B6,
161 #endif
162
163 };
164
165 #define DEFAULT_SPI_DMA_BUF_LEN         (16*1024)
166 #define TX_FIFO_EMPTY_COUNT_MAX         SLINK_TX_FIFO_EMPTY_COUNT(0x20)
167 #define RX_FIFO_FULL_COUNT_ZERO         SLINK_RX_FIFO_FULL_COUNT(0)
168
169 #define SLINK_STATUS2_RESET \
170         (TX_FIFO_EMPTY_COUNT_MAX | \
171         RX_FIFO_FULL_COUNT_ZERO << 16)
172
173 #define MAX_CHIP_SELECT         4
174 #define SLINK_FIFO_DEPTH        32
175
176 struct spi_tegra_data {
177         struct spi_master       *master;
178         struct platform_device  *pdev;
179         spinlock_t              lock;
180         spinlock_t              reg_lock;
181         char                    port_name[32];
182
183         struct clk              *clk;
184         struct clk              *sclk;
185         void __iomem            *base;
186         phys_addr_t             phys;
187         unsigned                irq;
188
189         u32                     cur_speed;
190
191         struct list_head        queue;
192         struct spi_transfer     *cur;
193         struct spi_device       *cur_spi;
194         unsigned                cur_pos;
195         unsigned                cur_len;
196         unsigned                words_per_32bit;
197         unsigned                bytes_per_word;
198         unsigned                curr_dma_words;
199
200         unsigned                cur_direction;
201
202         bool                    is_dma_allowed;
203
204         struct tegra_dma_req    rx_dma_req;
205         struct tegra_dma_channel *rx_dma;
206         u32                     *rx_buf;
207         dma_addr_t              rx_buf_phys;
208         unsigned                cur_rx_pos;
209
210         struct tegra_dma_req    tx_dma_req;
211         struct tegra_dma_channel *tx_dma;
212         u32                     *tx_buf;
213         dma_addr_t              tx_buf_phys;
214         unsigned                cur_tx_pos;
215
216         unsigned                dma_buf_size;
217         unsigned                max_buf_size;
218         bool                    is_curr_dma_xfer;
219
220         bool                    is_clkon_always;
221         int                     clk_state;
222         bool                    is_suspended;
223
224         bool                    is_hw_based_cs;
225
226         struct completion       rx_dma_complete;
227         struct completion       tx_dma_complete;
228         bool                    is_transfer_in_progress;
229
230         u32                     rx_complete;
231         u32                     tx_complete;
232         u32                     tx_status;
233         u32                     rx_status;
234         u32                     status_reg;
235         bool                    is_packed;
236         unsigned long           packed_size;
237
238         u32                     command_reg;
239         u32                     command2_reg;
240         u32                     dma_control_reg;
241         u32                     def_command_reg;
242         u32                     def_command2_reg;
243
244         struct spi_clk_parent   *parent_clk_list;
245         int                     parent_clk_count;
246         unsigned long           max_rate;
247         unsigned long           max_parent_rate;
248         int                     min_div;
249         struct workqueue_struct *spi_workqueue;
250         struct work_struct spi_transfer_work;
251 };
252
253 static int tegra_spi_runtime_idle(struct device *dev);
254 static int tegra_spi_runtime_resume(struct device *dev);
255
256 static inline unsigned long spi_tegra_readl(struct spi_tegra_data *tspi,
257                     unsigned long reg)
258 {
259         unsigned long flags;
260         unsigned long val;
261
262         spin_lock_irqsave(&tspi->reg_lock, flags);
263         if (tspi->clk_state < 1)
264                 BUG();
265         val = readl(tspi->base + reg);
266         spin_unlock_irqrestore(&tspi->reg_lock, flags);
267         return val;
268 }
269
270 static inline void spi_tegra_writel(struct spi_tegra_data *tspi,
271                     unsigned long val, unsigned long reg)
272 {
273         unsigned long flags;
274
275         spin_lock_irqsave(&tspi->reg_lock, flags);
276         if (tspi->clk_state < 1)
277                 BUG();
278         writel(val, tspi->base + reg);
279
280         /* Synchronize write by reading back the register */
281         readl(tspi->base + SLINK_MAS_DATA);
282         spin_unlock_irqrestore(&tspi->reg_lock, flags);
283 }
284
285 static int tegra_spi_clk_disable(struct spi_tegra_data *tspi)
286 {
287         unsigned long flags;
288
289         /* Flush all write which are in PPSB queue by reading back */
290         spi_tegra_readl(tspi, SLINK_MAS_DATA);
291
292         spin_lock_irqsave(&tspi->reg_lock, flags);
293         tspi->clk_state--;
294         spin_unlock_irqrestore(&tspi->reg_lock, flags);
295         clk_disable(tspi->clk);
296         clk_disable(tspi->sclk);
297         return 0;
298 }
299
300 static int tegra_spi_clk_enable(struct spi_tegra_data *tspi)
301 {
302         unsigned long flags;
303
304         clk_enable(tspi->sclk);
305         clk_enable(tspi->clk);
306         spin_lock_irqsave(&tspi->reg_lock, flags);
307         tspi->clk_state++;
308         spin_unlock_irqrestore(&tspi->reg_lock, flags);
309         return 0;
310 }
311
312 #if SPI_PM_RUNTIME_ENABLE
313 #define spi_pm_runtime_get_sync(dev) pm_runtime_get_sync(dev)
314 #define spi_pm_runtime_put_sync(dev) pm_runtime_put_sync(dev)
315 #define spi_pm_runtime_enable(dev) pm_runtime_enable(dev)
316 #define spi_pm_runtime_disable(dev) pm_runtime_disable(dev)
317 #define spi_pm_runtime_enabled(dev) pm_runtime_enabled(dev)
318 #define spi_pm_runtime_status_suspended(dev) pm_runtime_status_suspended(dev)
319 #else
320 #define spi_pm_runtime_get_sync(dev) tegra_spi_runtime_resume(dev)
321 #define spi_pm_runtime_put_sync(dev) tegra_spi_runtime_idle(dev)
322 #define spi_pm_runtime_enable(dev) do { } while(0)
323 #define spi_pm_runtime_disable(dev) do { } while(0)
324 #define spi_pm_runtime_enabled(dev) true
325 #define spi_pm_runtime_status_suspended(dev) true
326 #endif
327
328 static void cancel_dma(struct tegra_dma_channel *dma_chan,
329         struct tegra_dma_req *req)
330 {
331         tegra_dma_cancel(dma_chan);
332         if (req->status == -TEGRA_DMA_REQ_ERROR_ABORTED)
333                 req->complete(req);
334 }
335
336 static void spi_tegra_clear_status(struct spi_tegra_data *tspi)
337 {
338         unsigned long val;
339         unsigned long val_write = 0;
340
341         val = spi_tegra_readl(tspi, SLINK_STATUS);
342
343         val_write = SLINK_RDY;
344         if (val & SLINK_TX_OVF)
345                 val_write |= SLINK_TX_OVF;
346         if (val & SLINK_RX_OVF)
347                 val_write |= SLINK_RX_OVF;
348         if (val & SLINK_RX_UNF)
349                 val_write |= SLINK_RX_UNF;
350         if (val & SLINK_TX_UNF)
351                 val_write |= SLINK_TX_UNF;
352
353         spi_tegra_writel(tspi, val_write, SLINK_STATUS);
354 }
355
356 static unsigned long spi_tegra_get_packed_size(struct spi_tegra_data *tspi,
357                                   struct spi_transfer *t)
358 {
359         unsigned long val;
360
361         switch (tspi->bytes_per_word) {
362         case 0:
363                 val = SLINK_PACK_SIZE_4;
364                 break;
365         case 1:
366                 val = SLINK_PACK_SIZE_8;
367                 break;
368         case 2:
369                 val = SLINK_PACK_SIZE_16;
370                 break;
371         case 4:
372                 val = SLINK_PACK_SIZE_32;
373                 break;
374         default:
375                 val = 0;
376         }
377         return val;
378 }
379
380 static unsigned spi_tegra_calculate_curr_xfer_param(
381         struct spi_device *spi, struct spi_tegra_data *tspi,
382         struct spi_transfer *t)
383 {
384         unsigned remain_len = t->len - tspi->cur_pos;
385         unsigned max_word;
386         unsigned bits_per_word ;
387         unsigned max_len;
388         unsigned total_fifo_words;
389
390         bits_per_word = t->bits_per_word ? t->bits_per_word :
391                                                 spi->bits_per_word;
392         tspi->bytes_per_word = (bits_per_word - 1) / 8 + 1;
393
394         if (bits_per_word == 8 || bits_per_word == 16) {
395                 tspi->is_packed = 1;
396                 tspi->words_per_32bit = 32/bits_per_word;
397         } else {
398                 tspi->is_packed = 0;
399                 tspi->words_per_32bit = 1;
400         }
401         tspi->packed_size = spi_tegra_get_packed_size(tspi, t);
402
403         if (tspi->is_packed) {
404                 max_len = min(remain_len, tspi->max_buf_size);
405                 tspi->curr_dma_words = max_len/tspi->bytes_per_word;
406                 total_fifo_words = max_len/4;
407         } else {
408                 max_word = (remain_len - 1) / tspi->bytes_per_word + 1;
409                 max_word = min(max_word, tspi->max_buf_size/4);
410                 tspi->curr_dma_words = max_word;
411                 total_fifo_words = max_word;
412         }
413         return total_fifo_words;
414 }
415
416 static unsigned spi_tegra_fill_tx_fifo_from_client_txbuf(
417         struct spi_tegra_data *tspi, struct spi_transfer *t)
418 {
419         unsigned nbytes;
420         unsigned tx_empty_count;
421         unsigned long fifo_status;
422         u8 *tx_buf = (u8 *)t->tx_buf + tspi->cur_tx_pos;
423         unsigned max_n_32bit;
424         unsigned i, count;
425         unsigned long x;
426         unsigned int written_words;
427
428         fifo_status = spi_tegra_readl(tspi, SLINK_STATUS2);
429         tx_empty_count = SLINK_TX_FIFO_EMPTY_COUNT(fifo_status);
430
431         if (tspi->is_packed) {
432                 nbytes = tspi->curr_dma_words * tspi->bytes_per_word;
433                 max_n_32bit = (min(nbytes,  tx_empty_count*4) - 1)/4 + 1;
434                 for (count = 0; count < max_n_32bit; ++count) {
435                         x = 0;
436                         for (i = 0; (i < 4) && nbytes; i++, nbytes--)
437                                 x |= (*tx_buf++) << (i*8);
438                         spi_tegra_writel(tspi, x, SLINK_TX_FIFO);
439                 }
440                 written_words =  min(max_n_32bit * tspi->words_per_32bit,
441                                         tspi->curr_dma_words);
442         } else {
443                 max_n_32bit = min(tspi->curr_dma_words,  tx_empty_count);
444                 nbytes = max_n_32bit * tspi->bytes_per_word;
445                 for (count = 0; count < max_n_32bit; ++count) {
446                         x = 0;
447                         for (i = 0; nbytes && (i < tspi->bytes_per_word);
448                                                         ++i, nbytes--)
449                                 x |= ((*tx_buf++) << i*8);
450                         spi_tegra_writel(tspi, x, SLINK_TX_FIFO);
451                 }
452                 written_words = max_n_32bit;
453         }
454         tspi->cur_tx_pos += written_words * tspi->bytes_per_word;
455         return written_words;
456 }
457
458 static unsigned int spi_tegra_read_rx_fifo_to_client_rxbuf(
459                 struct spi_tegra_data *tspi, struct spi_transfer *t)
460 {
461         unsigned rx_full_count;
462         unsigned long fifo_status;
463         u8 *rx_buf = (u8 *)t->rx_buf + tspi->cur_rx_pos;
464         unsigned i, count;
465         unsigned long x;
466         unsigned int read_words = 0;
467         unsigned len;
468
469         fifo_status = spi_tegra_readl(tspi, SLINK_STATUS2);
470         rx_full_count = SLINK_RX_FIFO_FULL_COUNT(fifo_status);
471         dev_dbg(&tspi->pdev->dev, "Rx fifo count %d\n", rx_full_count);
472         if (tspi->is_packed) {
473                 len = tspi->curr_dma_words * tspi->bytes_per_word;
474                 for (count = 0; count < rx_full_count; ++count) {
475                         x = spi_tegra_readl(tspi, SLINK_RX_FIFO);
476                         for (i = 0; len && (i < 4); ++i, len--)
477                                 *rx_buf++ = (x >> i*8) & 0xFF;
478                 }
479                 tspi->cur_rx_pos += tspi->curr_dma_words * tspi->bytes_per_word;
480                 read_words += tspi->curr_dma_words;
481         } else {
482                 unsigned int rx_mask, bits_per_word;
483
484                 bits_per_word = t->bits_per_word ? t->bits_per_word :
485                                                 tspi->cur_spi->bits_per_word;
486                 rx_mask = (1 << bits_per_word) - 1;
487                 for (count = 0; count < rx_full_count; ++count) {
488                         x = spi_tegra_readl(tspi, SLINK_RX_FIFO);
489                         x &= rx_mask;
490                         for (i = 0; (i < tspi->bytes_per_word); ++i)
491                                 *rx_buf++ = (x >> (i*8)) & 0xFF;
492                 }
493                 tspi->cur_rx_pos += rx_full_count * tspi->bytes_per_word;
494                 read_words += rx_full_count;
495         }
496         return read_words;
497 }
498
499 static void spi_tegra_copy_client_txbuf_to_spi_txbuf(
500                 struct spi_tegra_data *tspi, struct spi_transfer *t)
501 {
502         unsigned len;
503
504         /* Make the dma buffer to read by cpu */
505         dma_sync_single_for_cpu(&tspi->pdev->dev, tspi->tx_buf_phys,
506                                 tspi->dma_buf_size, DMA_TO_DEVICE);
507
508         if (tspi->is_packed) {
509                 len = tspi->curr_dma_words * tspi->bytes_per_word;
510                 memcpy(tspi->tx_buf, t->tx_buf + tspi->cur_pos, len);
511         } else {
512                 unsigned int i;
513                 unsigned int count;
514                 u8 *tx_buf = (u8 *)t->tx_buf + tspi->cur_tx_pos;
515                 unsigned consume = tspi->curr_dma_words * tspi->bytes_per_word;
516                 unsigned int x;
517
518                 for (count = 0; count < tspi->curr_dma_words; ++count) {
519                         x = 0;
520                         for (i = 0; consume && (i < tspi->bytes_per_word);
521                                                         ++i, consume--)
522                                 x |= ((*tx_buf++) << i*8);
523                         tspi->tx_buf[count] = x;
524                 }
525         }
526         tspi->cur_tx_pos += tspi->curr_dma_words * tspi->bytes_per_word;
527
528         /* Make the dma buffer to read by dma */
529         dma_sync_single_for_device(&tspi->pdev->dev, tspi->tx_buf_phys,
530                                 tspi->dma_buf_size, DMA_TO_DEVICE);
531 }
532
533 static void spi_tegra_copy_spi_rxbuf_to_client_rxbuf(
534                 struct spi_tegra_data *tspi, struct spi_transfer *t)
535 {
536         unsigned len;
537
538         /* Make the dma buffer to read by cpu */
539         dma_sync_single_for_cpu(&tspi->pdev->dev, tspi->rx_buf_phys,
540                 tspi->dma_buf_size, DMA_FROM_DEVICE);
541
542         if (tspi->is_packed) {
543                 len = tspi->curr_dma_words * tspi->bytes_per_word;
544                 memcpy(t->rx_buf + tspi->cur_rx_pos, tspi->rx_buf, len);
545         } else {
546                 unsigned int i;
547                 unsigned int count;
548                 unsigned char *rx_buf = t->rx_buf + tspi->cur_rx_pos;
549                 unsigned int x;
550                 unsigned int rx_mask, bits_per_word;
551
552                 bits_per_word = t->bits_per_word ? t->bits_per_word :
553                                                 tspi->cur_spi->bits_per_word;
554                 rx_mask = (1 << bits_per_word) - 1;
555                 for (count = 0; count < tspi->curr_dma_words; ++count) {
556                         x = tspi->rx_buf[count];
557                         x &= rx_mask;
558                         for (i = 0; (i < tspi->bytes_per_word); ++i)
559                                 *rx_buf++ = (x >> (i*8)) & 0xFF;
560                 }
561         }
562         tspi->cur_rx_pos += tspi->curr_dma_words * tspi->bytes_per_word;
563
564         /* Make the dma buffer to read by dma */
565         dma_sync_single_for_device(&tspi->pdev->dev, tspi->rx_buf_phys,
566                 tspi->dma_buf_size, DMA_FROM_DEVICE);
567 }
568
569 static int spi_tegra_start_dma_based_transfer(
570                 struct spi_tegra_data *tspi, struct spi_transfer *t)
571 {
572         unsigned long val;
573         unsigned long test_val;
574         unsigned int len;
575         int ret = 0;
576
577         INIT_COMPLETION(tspi->rx_dma_complete);
578         INIT_COMPLETION(tspi->tx_dma_complete);
579
580         /* Make sure that Rx and Tx fifo are empty */
581         test_val = spi_tegra_readl(tspi, SLINK_STATUS);
582         if (((test_val >> 20) & 0xF) != 0xA)
583                 dev_err(&tspi->pdev->dev,
584                         "The Rx and Tx fifo are not empty status 0x%08lx\n",
585                                 test_val);
586
587         val = SLINK_DMA_BLOCK_SIZE(tspi->curr_dma_words - 1);
588         val |= tspi->packed_size;
589         if (tspi->is_packed)
590                 len = DIV_ROUND_UP(tspi->curr_dma_words * tspi->bytes_per_word,
591                                         4) * 4;
592         else
593                 len = tspi->curr_dma_words * 4;
594
595         if (len & 0xF)
596                 val |= SLINK_TX_TRIG_1 | SLINK_RX_TRIG_1;
597         else if (((len) >> 4) & 0x1)
598                 val |= SLINK_TX_TRIG_4 | SLINK_RX_TRIG_4;
599         else
600                 val |= SLINK_TX_TRIG_8 | SLINK_RX_TRIG_8;
601
602         if (tspi->cur_direction & DATA_DIR_TX)
603                 val |= SLINK_IE_TXC;
604
605         if (tspi->cur_direction & DATA_DIR_RX)
606                 val |= SLINK_IE_RXC;
607
608         spi_tegra_writel(tspi, val, SLINK_DMA_CTL);
609         tspi->dma_control_reg = val;
610
611         if (tspi->cur_direction & DATA_DIR_TX) {
612                 spi_tegra_copy_client_txbuf_to_spi_txbuf(tspi, t);
613                 wmb();
614                 tspi->tx_dma_req.size = len;
615                 ret = tegra_dma_enqueue_req(tspi->tx_dma, &tspi->tx_dma_req);
616                 if (ret < 0) {
617                         dev_err(&tspi->pdev->dev,
618                                 "Error in starting tx dma error = %d\n", ret);
619                         return ret;
620                 }
621
622                 /* Wait for tx fifo to be fill before starting slink */
623                 test_val = spi_tegra_readl(tspi, SLINK_STATUS);
624                 while (!(test_val & SLINK_TX_FULL))
625                         test_val = spi_tegra_readl(tspi, SLINK_STATUS);
626         }
627
628         if (tspi->cur_direction & DATA_DIR_RX) {
629                 /* Make the dma buffer to read by dma */
630                 dma_sync_single_for_device(&tspi->pdev->dev, tspi->rx_buf_phys,
631                                 tspi->dma_buf_size, DMA_FROM_DEVICE);
632
633                 tspi->rx_dma_req.size = len;
634                 ret = tegra_dma_enqueue_req(tspi->rx_dma, &tspi->rx_dma_req);
635                 if (ret < 0) {
636                         dev_err(&tspi->pdev->dev,
637                                 "Error in starting rx dma error = %d\n", ret);
638                         if (tspi->cur_direction & DATA_DIR_TX)
639                                 cancel_dma(tspi->tx_dma, &tspi->tx_dma_req);
640                         return ret;
641                 }
642         }
643         tspi->is_curr_dma_xfer = true;
644         if (tspi->is_packed) {
645                 val |= SLINK_PACKED;
646                 spi_tegra_writel(tspi, val, SLINK_DMA_CTL);
647                 udelay(1);
648                 wmb();
649         }
650         tspi->dma_control_reg = val;
651
652         val |= SLINK_DMA_EN;
653         spi_tegra_writel(tspi, val, SLINK_DMA_CTL);
654         return ret;
655 }
656
657 static int spi_tegra_start_cpu_based_transfer(
658                 struct spi_tegra_data *tspi, struct spi_transfer *t)
659 {
660         unsigned long val;
661         unsigned curr_words;
662
663         val = tspi->packed_size;
664         if (tspi->cur_direction & DATA_DIR_TX)
665                 val |= SLINK_IE_TXC;
666
667         if (tspi->cur_direction & DATA_DIR_RX)
668                 val |= SLINK_IE_RXC;
669
670         spi_tegra_writel(tspi, val, SLINK_DMA_CTL);
671         tspi->dma_control_reg = val;
672
673         if (tspi->cur_direction & DATA_DIR_TX)
674                 curr_words = spi_tegra_fill_tx_fifo_from_client_txbuf(tspi, t);
675         else
676                 curr_words = tspi->curr_dma_words;
677         val |= SLINK_DMA_BLOCK_SIZE(curr_words - 1);
678         spi_tegra_writel(tspi, val, SLINK_DMA_CTL);
679         tspi->dma_control_reg = val;
680
681         tspi->is_curr_dma_xfer = false;
682         if (tspi->is_packed) {
683                 val |= SLINK_PACKED;
684                 spi_tegra_writel(tspi, val, SLINK_DMA_CTL);
685                 udelay(1);
686                 wmb();
687         }
688         tspi->dma_control_reg = val;
689         val |= SLINK_DMA_EN;
690         spi_tegra_writel(tspi, val, SLINK_DMA_CTL);
691         return 0;
692 }
693
694 static void set_best_clk_source(struct spi_tegra_data *tspi,
695                 unsigned long speed)
696 {
697         long new_rate;
698         unsigned long err_rate;
699         int rate = speed * 4;
700         unsigned int fin_err = speed * 4;
701         int final_index = -1;
702         int count;
703         int ret;
704         struct clk *pclk;
705         unsigned long prate, crate, nrate;
706         unsigned long cdiv;
707
708         if (!tspi->parent_clk_count || !tspi->parent_clk_list)
709                 return;
710
711         /* make sure divisor is more than min_div */
712         pclk = clk_get_parent(tspi->clk);
713         prate = clk_get_rate(pclk);
714         crate = clk_get_rate(tspi->clk);
715         cdiv = DIV_ROUND_UP(prate, crate);
716         if (cdiv < tspi->min_div) {
717                 nrate = DIV_ROUND_UP(prate, tspi->min_div);
718                 clk_set_rate(tspi->clk, nrate);
719         }
720
721         for (count = 0; count < tspi->parent_clk_count; ++count) {
722                 if (!tspi->parent_clk_list[count].parent_clk)
723                         continue;
724                 ret = clk_set_parent(tspi->clk,
725                         tspi->parent_clk_list[count].parent_clk);
726                 if (ret < 0) {
727                         dev_warn(&tspi->pdev->dev,
728                                 "Error in setting parent clk src %s\n",
729                                 tspi->parent_clk_list[count].name);
730                         continue;
731                 }
732
733                 new_rate = clk_round_rate(tspi->clk, rate);
734                 if (new_rate < 0)
735                         continue;
736
737                 err_rate = abs(new_rate - rate);
738                 if (err_rate < fin_err) {
739                         final_index = count;
740                         fin_err = err_rate;
741                 }
742         }
743
744         if (final_index >= 0) {
745                 dev_info(&tspi->pdev->dev, "Setting clk_src %s\n",
746                                 tspi->parent_clk_list[final_index].name);
747                 clk_set_parent(tspi->clk,
748                         tspi->parent_clk_list[final_index].parent_clk);
749         }
750 }
751
752 static void spi_tegra_start_transfer(struct spi_device *spi,
753                     struct spi_transfer *t, bool is_first_of_msg,
754                     bool is_single_xfer)
755 {
756         struct spi_tegra_data *tspi = spi_master_get_devdata(spi->master);
757         u32 speed;
758         u8 bits_per_word;
759         unsigned total_fifo_words;
760         int ret;
761         struct tegra_spi_device_controller_data *cdata = spi->controller_data;
762         unsigned long command;
763         unsigned long command2;
764 #ifndef CONFIG_ARCH_TEGRA_2x_SOC
765         unsigned long status2;
766 #endif
767         int cs_setup_count;
768         int cs_hold_count;
769
770         unsigned int cs_pol_bit[] = {
771                         SLINK_CS_POLARITY,
772                         SLINK_CS_POLARITY1,
773                         SLINK_CS_POLARITY2,
774                         SLINK_CS_POLARITY3,
775         };
776
777         bits_per_word = t->bits_per_word ? t->bits_per_word :
778                                         spi->bits_per_word;
779
780         speed = t->speed_hz ? t->speed_hz : spi->max_speed_hz;
781         if (speed != tspi->cur_speed) {
782                 set_best_clk_source(tspi, speed);
783                 clk_set_rate(tspi->clk, speed * 4);
784                 tspi->cur_speed = speed;
785         }
786
787         tspi->cur = t;
788         tspi->cur_spi = spi;
789         tspi->cur_pos = 0;
790         tspi->cur_rx_pos = 0;
791         tspi->cur_tx_pos = 0;
792         tspi->rx_complete = 0;
793         tspi->tx_complete = 0;
794         total_fifo_words = spi_tegra_calculate_curr_xfer_param(spi, tspi, t);
795
796         command2 = tspi->def_command2_reg;
797         if (is_first_of_msg) {
798                 if ((ret = spi_pm_runtime_get_sync(&tspi->pdev->dev)) < 0) {
799                         dev_err(&tspi->pdev->dev,
800                                 "%s: spi_pm_runtime_get_sync() returns %d\n",
801                                 __func__, ret);
802                         return;
803                 }
804
805                 spi_tegra_clear_status(tspi);
806
807                 command = tspi->def_command_reg;
808                 command |= SLINK_BIT_LENGTH(bits_per_word - 1);
809
810                 /* possibly use the hw based chip select */
811                 tspi->is_hw_based_cs = false;
812                 if (cdata && cdata->is_hw_based_cs && is_single_xfer) {
813                         if ((tspi->curr_dma_words * tspi->bytes_per_word) ==
814                                                 (t->len - tspi->cur_pos)) {
815                                 cs_setup_count = cdata->cs_setup_clk_count >> 1;
816                                 if (cs_setup_count > 3)
817                                         cs_setup_count = 3;
818                                 cs_hold_count = cdata->cs_hold_clk_count;
819                                 if (cs_hold_count > 0xF)
820                                         cs_hold_count = 0xF;
821                                 tspi->is_hw_based_cs = true;
822
823                                 command &= ~SLINK_CS_SW;
824                                 command2 &= ~SLINK_SS_SETUP(3);
825                                 command2 |= SLINK_SS_SETUP(cs_setup_count);
826 #ifndef CONFIG_ARCH_TEGRA_2x_SOC
827                                 status2 = spi_tegra_readl(tspi, SLINK_STATUS2);
828                                 status2 &= ~SLINK_SS_HOLD_TIME(0xF);
829                                 status2 |= SLINK_SS_HOLD_TIME(cs_hold_count);
830                                 spi_tegra_writel(tspi, status2, SLINK_STATUS2);
831 #endif
832                         }
833                 }
834                 if (!tspi->is_hw_based_cs) {
835                         command |= SLINK_CS_SW;
836                         command ^= cs_pol_bit[spi->chip_select];
837                 }
838
839                 command &= ~SLINK_IDLE_SCLK_MASK & ~SLINK_CK_SDA;
840                 if (spi->mode & SPI_CPHA)
841                         command |= SLINK_CK_SDA;
842
843                 if (spi->mode & SPI_CPOL)
844                         command |= SLINK_IDLE_SCLK_DRIVE_HIGH;
845                 else
846                         command |= SLINK_IDLE_SCLK_DRIVE_LOW;
847         } else {
848                 command = tspi->command_reg;
849                 command &= ~SLINK_BIT_LENGTH(~0);
850                 command |= SLINK_BIT_LENGTH(bits_per_word - 1);
851         }
852
853         spi_tegra_writel(tspi, command, SLINK_COMMAND);
854         tspi->command_reg = command;
855
856         dev_dbg(&tspi->pdev->dev, "The def 0x%x and written 0x%lx\n",
857                                 tspi->def_command_reg, command);
858
859         command2 &= ~(SLINK_SS_EN_CS(~0) | SLINK_RXEN | SLINK_TXEN);
860         tspi->cur_direction = 0;
861         if (t->rx_buf) {
862                 command2 |= SLINK_RXEN;
863                 tspi->cur_direction |= DATA_DIR_RX;
864         }
865         if (t->tx_buf) {
866                 command2 |= SLINK_TXEN;
867                 tspi->cur_direction |= DATA_DIR_TX;
868         }
869         command2 |= SLINK_SS_EN_CS(spi->chip_select);
870         spi_tegra_writel(tspi, command2, SLINK_COMMAND2);
871         tspi->command2_reg = command2;
872
873         if (total_fifo_words > SLINK_FIFO_DEPTH)
874                 ret = spi_tegra_start_dma_based_transfer(tspi, t);
875         else
876                 ret = spi_tegra_start_cpu_based_transfer(tspi, t);
877         WARN_ON(ret < 0);
878 }
879
880 static int spi_tegra_setup(struct spi_device *spi)
881 {
882         struct spi_tegra_data *tspi = spi_master_get_devdata(spi->master);
883         unsigned long cs_bit;
884         unsigned long val;
885         unsigned long flags;
886
887         dev_dbg(&spi->dev, "setup %d bpw, %scpol, %scpha, %dHz\n",
888                 spi->bits_per_word,
889                 spi->mode & SPI_CPOL ? "" : "~",
890                 spi->mode & SPI_CPHA ? "" : "~",
891                 spi->max_speed_hz);
892
893         BUG_ON(spi->chip_select >= MAX_CHIP_SELECT);
894         switch (spi->chip_select) {
895         case 0:
896                 cs_bit = SLINK_CS_POLARITY;
897                 break;
898
899         case 1:
900                 cs_bit = SLINK_CS_POLARITY1;
901                 break;
902
903         case 2:
904                 cs_bit = SLINK_CS_POLARITY2;
905                 break;
906
907         case 3:
908                 cs_bit = SLINK_CS_POLARITY3;
909                 break;
910
911         default:
912                 return -EINVAL;
913         }
914
915         spi_pm_runtime_get_sync(&tspi->pdev->dev);
916
917         spin_lock_irqsave(&tspi->lock, flags);
918         val = tspi->def_command_reg;
919         if (spi->mode & SPI_CS_HIGH)
920                 val |= cs_bit;
921         else
922                 val &= ~cs_bit;
923         tspi->def_command_reg = val;
924         spi_tegra_writel(tspi, tspi->def_command_reg, SLINK_COMMAND);
925         spin_unlock_irqrestore(&tspi->lock, flags);
926
927         spi_pm_runtime_put_sync(&tspi->pdev->dev);
928         return 0;
929 }
930
931 static void tegra_spi_transfer_work(struct work_struct *work)
932 {
933         struct spi_tegra_data *tspi;
934         struct spi_device *spi;
935         struct spi_message *m;
936         struct spi_transfer *t;
937         int single_xfer = 0;
938         unsigned long flags;
939
940         tspi = container_of(work, struct spi_tegra_data, spi_transfer_work);
941
942         spin_lock_irqsave(&tspi->lock, flags);
943
944         if (tspi->is_transfer_in_progress || tspi->is_suspended) {
945                 spin_unlock_irqrestore(&tspi->lock, flags);
946                 return;
947         }
948         if (list_empty(&tspi->queue)) {
949                 spin_unlock_irqrestore(&tspi->lock, flags);
950                 return;
951         }
952
953         m = list_first_entry(&tspi->queue, struct spi_message, queue);
954         spi = m->state;
955         single_xfer = list_is_singular(&m->transfers);
956         m->actual_length = 0;
957         m->status = 0;
958         t = list_first_entry(&m->transfers, struct spi_transfer, transfer_list);
959         tspi->is_transfer_in_progress = true;
960
961         spin_unlock_irqrestore(&tspi->lock, flags);
962         spi_tegra_start_transfer(spi, t, true, single_xfer);
963 }
964
965 static int spi_tegra_transfer(struct spi_device *spi, struct spi_message *m)
966 {
967         struct spi_tegra_data *tspi = spi_master_get_devdata(spi->master);
968         struct spi_transfer *t;
969         unsigned long flags;
970         int was_empty;
971         int bytes_per_word;
972
973         if (list_empty(&m->transfers) || !m->complete)
974                 return -EINVAL;
975
976         list_for_each_entry(t, &m->transfers, transfer_list) {
977                 if (t->bits_per_word < 0 || t->bits_per_word > 32)
978                         return -EINVAL;
979
980                 if (t->len == 0)
981                         return -EINVAL;
982
983                 /* Check that the all words are available */
984                 if (t->bits_per_word)
985                         bytes_per_word = (t->bits_per_word + 7)/8;
986                 else
987                         bytes_per_word = (spi->bits_per_word + 7)/8;
988
989                 if (t->len % bytes_per_word != 0)
990                         return -EINVAL;
991
992                 if (!t->rx_buf && !t->tx_buf)
993                         return -EINVAL;
994         }
995
996         spin_lock_irqsave(&tspi->lock, flags);
997
998         if (WARN_ON(tspi->is_suspended)) {
999                 spin_unlock_irqrestore(&tspi->lock, flags);
1000                 return -EBUSY;
1001         }
1002
1003         m->state = spi;
1004         was_empty = list_empty(&tspi->queue);
1005         list_add_tail(&m->queue, &tspi->queue);
1006         if (was_empty)
1007                 queue_work(tspi->spi_workqueue, &tspi->spi_transfer_work);
1008
1009         spin_unlock_irqrestore(&tspi->lock, flags);
1010         return 0;
1011 }
1012
1013 static void spi_tegra_curr_transfer_complete(struct spi_tegra_data *tspi,
1014         unsigned err, unsigned cur_xfer_size, unsigned long *irq_flags)
1015 {
1016         struct spi_message *m;
1017         struct spi_device *spi;
1018         struct spi_transfer *t;
1019         int single_xfer = 0;
1020
1021         /* Check if CS need to be toggele here */
1022         if (tspi->cur && tspi->cur->cs_change &&
1023                                 tspi->cur->delay_usecs) {
1024                 udelay(tspi->cur->delay_usecs);
1025         }
1026
1027         m = list_first_entry(&tspi->queue, struct spi_message, queue);
1028         if (err)
1029                 m->status = -EIO;
1030         spi = m->state;
1031
1032         m->actual_length += cur_xfer_size;
1033
1034         if (!list_is_last(&tspi->cur->transfer_list, &m->transfers)) {
1035                 tspi->cur = list_first_entry(&tspi->cur->transfer_list,
1036                         struct spi_transfer, transfer_list);
1037                 spin_unlock_irqrestore(&tspi->lock, *irq_flags);
1038                 spi_tegra_start_transfer(spi, tspi->cur, false, 0);
1039                 spin_lock_irqsave(&tspi->lock, *irq_flags);
1040         } else {
1041                 list_del(&m->queue);
1042                 m->complete(m->context);
1043                 if (!list_empty(&tspi->queue)) {
1044                         if (tspi->is_suspended) {
1045                                 spi_tegra_writel(tspi, tspi->def_command_reg,
1046                                                 SLINK_COMMAND);
1047                                 spi_tegra_writel(tspi, tspi->def_command2_reg,
1048                                                 SLINK_COMMAND2);
1049                                 tspi->is_transfer_in_progress = false;
1050                                 return;
1051                         }
1052                         m = list_first_entry(&tspi->queue, struct spi_message,
1053                                 queue);
1054                         spi = m->state;
1055                         single_xfer = list_is_singular(&m->transfers);
1056                         m->actual_length = 0;
1057                         m->status = 0;
1058
1059                         t = list_first_entry(&m->transfers, struct spi_transfer,
1060                                                 transfer_list);
1061                         spin_unlock_irqrestore(&tspi->lock, *irq_flags);
1062                         spi_tegra_start_transfer(spi, t, true, single_xfer);
1063                         spin_lock_irqsave(&tspi->lock, *irq_flags);
1064                 } else {
1065                         spi_tegra_writel(tspi, tspi->def_command_reg,
1066                                                                 SLINK_COMMAND);
1067                         spi_tegra_writel(tspi, tspi->def_command2_reg,
1068                                                                 SLINK_COMMAND2);
1069                         /* Provide delay to stablize the signal state */
1070                         spin_unlock_irqrestore(&tspi->lock, *irq_flags);
1071                         udelay(10);
1072                         spi_pm_runtime_put_sync(&tspi->pdev->dev);
1073                         spin_lock_irqsave(&tspi->lock, *irq_flags);
1074                         tspi->is_transfer_in_progress = false;
1075                         /* Check if any new request has come between
1076                          * clock disable */
1077                         queue_work(tspi->spi_workqueue,
1078                                         &tspi->spi_transfer_work);
1079                 }
1080         }
1081         return;
1082 }
1083
1084 static void tegra_spi_tx_dma_complete(struct tegra_dma_req *req)
1085 {
1086         struct spi_tegra_data *tspi = req->dev;
1087         complete(&tspi->tx_dma_complete);
1088 }
1089
1090 static void tegra_spi_rx_dma_complete(struct tegra_dma_req *req)
1091 {
1092         struct spi_tegra_data *tspi = req->dev;
1093         complete(&tspi->rx_dma_complete);
1094 }
1095
1096 static void handle_cpu_based_xfer(void *context_data)
1097 {
1098         struct spi_tegra_data *tspi = context_data;
1099         struct spi_transfer *t = tspi->cur;
1100         unsigned long flags;
1101
1102         spin_lock_irqsave(&tspi->lock, flags);
1103         if (tspi->tx_status ||  tspi->rx_status ||
1104                                 (tspi->status_reg & SLINK_BSY)) {
1105                 dev_err(&tspi->pdev->dev, "%s ERROR bit set 0x%x\n",
1106                                          __func__, tspi->status_reg);
1107                 dev_err(&tspi->pdev->dev, "%s 0x%08x:0x%08x:0x%08x\n",
1108                                 __func__, tspi->command_reg, tspi->command2_reg,
1109                                 tspi->dma_control_reg);
1110                 tegra_periph_reset_assert(tspi->clk);
1111                 udelay(2);
1112                 tegra_periph_reset_deassert(tspi->clk);
1113                 WARN_ON(1);
1114                 spi_tegra_curr_transfer_complete(tspi,
1115                         tspi->tx_status ||  tspi->rx_status, t->len, &flags);
1116                 goto exit;
1117         }
1118
1119         dev_vdbg(&tspi->pdev->dev, "Current direction %x\n",
1120                                         tspi->cur_direction);
1121         if (tspi->cur_direction & DATA_DIR_RX)
1122                 spi_tegra_read_rx_fifo_to_client_rxbuf(tspi, t);
1123
1124         if (tspi->cur_direction & DATA_DIR_TX)
1125                 tspi->cur_pos = tspi->cur_tx_pos;
1126         else if (tspi->cur_direction & DATA_DIR_RX)
1127                 tspi->cur_pos = tspi->cur_rx_pos;
1128         else
1129                 WARN_ON(1);
1130
1131         dev_vdbg(&tspi->pdev->dev,
1132                 "current position %d and length of the transfer %d\n",
1133                         tspi->cur_pos, t->len);
1134         if (tspi->cur_pos == t->len) {
1135                 spi_tegra_curr_transfer_complete(tspi,
1136                         tspi->tx_status || tspi->rx_status, t->len, &flags);
1137                 goto exit;
1138         }
1139
1140         spi_tegra_calculate_curr_xfer_param(tspi->cur_spi, tspi, t);
1141         spi_tegra_start_cpu_based_transfer(tspi, t);
1142 exit:
1143         spin_unlock_irqrestore(&tspi->lock, flags);
1144         return;
1145 }
1146
1147 static irqreturn_t spi_tegra_isr_thread(int irq, void *context_data)
1148 {
1149         struct spi_tegra_data *tspi = context_data;
1150         struct spi_transfer *t = tspi->cur;
1151         long wait_status;
1152         int err = 0;
1153         unsigned total_fifo_words;
1154         unsigned long flags;
1155
1156         if (!tspi->is_curr_dma_xfer) {
1157                 handle_cpu_based_xfer(context_data);
1158                 return IRQ_HANDLED;
1159         }
1160
1161         /* Abort dmas if any error */
1162         if (tspi->cur_direction & DATA_DIR_TX) {
1163                 if (tspi->tx_status) {
1164                         cancel_dma(tspi->tx_dma, &tspi->tx_dma_req);
1165                         err += 1;
1166                 } else {
1167                         wait_status = wait_for_completion_interruptible_timeout(
1168                                 &tspi->tx_dma_complete, SLINK_DMA_TIMEOUT);
1169                         if (wait_status <= 0) {
1170                                 cancel_dma(tspi->tx_dma, &tspi->tx_dma_req);
1171                                 dev_err(&tspi->pdev->dev,
1172                                         "Error in Dma Tx transfer\n");
1173                                 err += 1;
1174                         }
1175                 }
1176         }
1177
1178         if (tspi->cur_direction & DATA_DIR_RX) {
1179                 if (tspi->rx_status) {
1180                         cancel_dma(tspi->rx_dma, &tspi->rx_dma_req);
1181                         err += 2;
1182                 } else {
1183                         wait_status = wait_for_completion_interruptible_timeout(
1184                                 &tspi->rx_dma_complete, SLINK_DMA_TIMEOUT);
1185                         if (wait_status <= 0) {
1186                                 cancel_dma(tspi->rx_dma, &tspi->rx_dma_req);
1187                                 dev_err(&tspi->pdev->dev,
1188                                         "Error in Dma Rx transfer\n");
1189                                 err += 2;
1190                         }
1191                 }
1192         }
1193
1194         spin_lock_irqsave(&tspi->lock, flags);
1195         if (err) {
1196                 dev_err(&tspi->pdev->dev, "%s ERROR bit set 0x%x\n",
1197                                          __func__, tspi->status_reg);
1198                 dev_err(&tspi->pdev->dev, "%s 0x%08x:0x%08x:0x%08x\n",
1199                                 __func__, tspi->command_reg, tspi->command2_reg,
1200                                 tspi->dma_control_reg);
1201                 tegra_periph_reset_assert(tspi->clk);
1202                 udelay(2);
1203                 tegra_periph_reset_deassert(tspi->clk);
1204                 WARN_ON(1);
1205                 spi_tegra_curr_transfer_complete(tspi, err, t->len, &flags);
1206                 spin_unlock_irqrestore(&tspi->lock, flags);
1207                 return IRQ_HANDLED;
1208         }
1209
1210         if (tspi->cur_direction & DATA_DIR_RX)
1211                 spi_tegra_copy_spi_rxbuf_to_client_rxbuf(tspi, t);
1212
1213         if (tspi->cur_direction & DATA_DIR_TX)
1214                 tspi->cur_pos = tspi->cur_tx_pos;
1215         else if (tspi->cur_direction & DATA_DIR_RX)
1216                 tspi->cur_pos = tspi->cur_rx_pos;
1217         else
1218                 WARN_ON(1);
1219
1220         if (tspi->cur_pos == t->len) {
1221                 spi_tegra_curr_transfer_complete(tspi,
1222                         tspi->tx_status || tspi->rx_status, t->len, &flags);
1223                 spin_unlock_irqrestore(&tspi->lock, flags);
1224                 return IRQ_HANDLED;
1225         }
1226
1227         /* Continue transfer in current message */
1228         total_fifo_words = spi_tegra_calculate_curr_xfer_param(tspi->cur_spi,
1229                                                         tspi, t);
1230         if (total_fifo_words > SLINK_FIFO_DEPTH)
1231                 err = spi_tegra_start_dma_based_transfer(tspi, t);
1232         else
1233                 err = spi_tegra_start_cpu_based_transfer(tspi, t);
1234
1235         spin_unlock_irqrestore(&tspi->lock, flags);
1236         WARN_ON(err < 0);
1237         return IRQ_HANDLED;
1238 }
1239
1240 static irqreturn_t spi_tegra_isr(int irq, void *context_data)
1241 {
1242         struct spi_tegra_data *tspi = context_data;
1243
1244         tspi->status_reg = spi_tegra_readl(tspi, SLINK_STATUS);
1245         if (tspi->cur_direction & DATA_DIR_TX)
1246                 tspi->tx_status = tspi->status_reg &
1247                                         (SLINK_TX_OVF | SLINK_TX_UNF);
1248
1249         if (tspi->cur_direction & DATA_DIR_RX)
1250                 tspi->rx_status = tspi->status_reg &
1251                                         (SLINK_RX_OVF | SLINK_RX_UNF);
1252         spi_tegra_clear_status(tspi);
1253
1254
1255         return IRQ_WAKE_THREAD;
1256 }
1257
1258 static void spi_tegra_deinit_dma_param(struct spi_tegra_data *tspi,
1259         bool dma_to_memory)
1260 {
1261         struct tegra_dma_channel *tdc;
1262         u32 *dma_buf;
1263         dma_addr_t dma_phys;
1264
1265         if (dma_to_memory) {
1266                 dma_buf = tspi->rx_buf;
1267                 tdc = tspi->rx_dma;
1268                 dma_phys = tspi->rx_buf_phys;
1269                 tspi->rx_dma = NULL;
1270                 tspi->rx_buf = NULL;
1271         } else {
1272                 dma_buf = tspi->tx_buf;
1273                 tdc = tspi->tx_dma;
1274                 dma_phys = tspi->tx_buf_phys;
1275                 tspi->tx_buf = NULL;
1276                 tspi->tx_dma = NULL;
1277         }
1278
1279         dma_free_coherent(&tspi->pdev->dev, tspi->dma_buf_size,
1280                         dma_buf, dma_phys);
1281         tegra_dma_free_channel(tdc);
1282 }
1283
1284 static int __devinit spi_tegra_init_dma_param(struct spi_tegra_data *tspi,
1285                         bool dma_to_memory)
1286 {
1287         struct tegra_dma_req *dma_req;
1288         struct tegra_dma_channel *tdc;
1289         u32 *dma_buf;
1290         dma_addr_t dma_phys;
1291
1292         tdc = tegra_dma_allocate_channel(TEGRA_DMA_MODE_ONESHOT, "spi_%s_%d",
1293                         (dma_to_memory) ? "rx" : "tx", tspi->pdev->id);
1294         if (!tdc) {
1295                 dev_err(&tspi->pdev->dev, "can not allocate rx dma channel\n");
1296                 return -ENODEV;
1297         }
1298
1299         dma_buf = dma_alloc_coherent(&tspi->pdev->dev, tspi->dma_buf_size,
1300                                 &dma_phys, GFP_KERNEL);
1301         if (!dma_buf) {
1302                 dev_err(&tspi->pdev->dev, "can not allocate rx bounce buffer");
1303                 tegra_dma_free_channel(tdc);
1304                 return -ENOMEM;
1305         }
1306
1307         dma_req = (dma_to_memory) ? &tspi->rx_dma_req : &tspi->tx_dma_req;
1308         memset(dma_req, 0, sizeof(*dma_req));
1309
1310         dma_req->req_sel = spi_tegra_req_sels[tspi->pdev->id];
1311         dma_req->dev = tspi;
1312         dma_req->dest_bus_width = 32;
1313         dma_req->source_bus_width = 32;
1314         dma_req->to_memory = (dma_to_memory) ? 1 : 0;
1315         dma_req->virt_addr = dma_buf;
1316         dma_req->dest_wrap = 0;
1317         dma_req->source_wrap = 0;
1318
1319         if (dma_to_memory) {
1320                 dma_req->complete = tegra_spi_rx_dma_complete;
1321                 dma_req->dest_addr = dma_phys;
1322                 dma_req->source_addr = tspi->phys + SLINK_RX_FIFO;
1323                 dma_req->source_wrap = 4;
1324                 tspi->rx_buf_phys = dma_phys;
1325                 tspi->rx_buf = dma_buf;
1326                 tspi->rx_dma = tdc;
1327         } else {
1328                 dma_req->complete = tegra_spi_tx_dma_complete;
1329                 dma_req->dest_addr = tspi->phys + SLINK_TX_FIFO;
1330                 dma_req->source_addr = dma_phys;
1331                 dma_req->dest_wrap = 4;
1332                 tspi->tx_buf = dma_buf;
1333                 tspi->tx_buf_phys = dma_phys;
1334                 tspi->tx_dma = tdc;
1335         }
1336         return 0;
1337 }
1338
1339 static int __devinit spi_tegra_probe(struct platform_device *pdev)
1340 {
1341         struct spi_master       *master;
1342         struct spi_tegra_data   *tspi;
1343         struct resource         *r;
1344         struct tegra_spi_platform_data *pdata = pdev->dev.platform_data;
1345         int ret, spi_irq;
1346         int i;
1347         char spi_wq_name[20];
1348
1349         master = spi_alloc_master(&pdev->dev, sizeof *tspi);
1350         if (master == NULL) {
1351                 dev_err(&pdev->dev, "master allocation failed\n");
1352                 return -ENOMEM;
1353         }
1354
1355         /* the spi->mode bits understood by this driver: */
1356         master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
1357
1358         if (pdev->id != -1)
1359                 master->bus_num = pdev->id;
1360
1361         master->setup = spi_tegra_setup;
1362         master->transfer = spi_tegra_transfer;
1363         master->num_chipselect = MAX_CHIP_SELECT;
1364
1365         dev_set_drvdata(&pdev->dev, master);
1366         tspi = spi_master_get_devdata(master);
1367         tspi->master = master;
1368         tspi->pdev = pdev;
1369         tspi->is_transfer_in_progress = false;
1370         tspi->is_suspended = false;
1371         spin_lock_init(&tspi->lock);
1372         spin_lock_init(&tspi->reg_lock);
1373
1374         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1375         if (!r) {
1376                 dev_err(&pdev->dev, "No IO memory resource\n");
1377                 ret = -ENODEV;
1378                 goto exit_free_master;
1379         }
1380         tspi->phys = r->start;
1381         tspi->base = devm_request_and_ioremap(&pdev->dev, r);
1382         if (!tspi->base) {
1383                 dev_err(&pdev->dev,
1384                         "Cannot request memregion/iomap dma address\n");
1385                 ret = -EADDRNOTAVAIL;
1386                 goto exit_free_master;
1387         }
1388
1389         spi_irq = platform_get_irq(pdev, 0);
1390         if (unlikely(spi_irq < 0)) {
1391                 dev_err(&pdev->dev, "can't find irq resource\n");
1392                 ret = -ENXIO;
1393                 goto exit_free_master;
1394         }
1395         tspi->irq = spi_irq;
1396
1397         sprintf(tspi->port_name, "tegra_spi_%d", pdev->id);
1398         ret = devm_request_threaded_irq(&pdev->dev, tspi->irq,
1399                         spi_tegra_isr, spi_tegra_isr_thread, IRQF_ONESHOT,
1400                         tspi->port_name, tspi);
1401         if (ret < 0) {
1402                 dev_err(&pdev->dev, "Failed to register ISR for IRQ %d\n",
1403                                         tspi->irq);
1404                 goto exit_free_master;
1405         }
1406
1407         tspi->clk = devm_clk_get(&pdev->dev, "spi");
1408         if (IS_ERR(tspi->clk)) {
1409                 dev_err(&pdev->dev, "can not get clock\n");
1410                 ret = PTR_ERR(tspi->clk);
1411                 goto exit_free_master;
1412         }
1413
1414         tspi->sclk = devm_clk_get(&pdev->dev, "sclk");
1415         if (IS_ERR(tspi->sclk)) {
1416                 dev_err(&pdev->dev, "can not get sclock\n");
1417                 ret = PTR_ERR(tspi->sclk);
1418                 goto exit_free_master;
1419         }
1420
1421         INIT_LIST_HEAD(&tspi->queue);
1422
1423         if (pdata) {
1424                 tspi->is_clkon_always = pdata->is_clkon_always;
1425                 tspi->is_dma_allowed = pdata->is_dma_based;
1426                 tspi->dma_buf_size = (pdata->max_dma_buffer) ?
1427                                 pdata->max_dma_buffer : DEFAULT_SPI_DMA_BUF_LEN;
1428                 tspi->parent_clk_count = pdata->parent_clk_count;
1429                 tspi->parent_clk_list = pdata->parent_clk_list;
1430                 tspi->max_rate = pdata->max_rate;
1431         } else {
1432                 tspi->is_clkon_always = false;
1433                 tspi->is_dma_allowed = true;
1434                 tspi->dma_buf_size = DEFAULT_SPI_DMA_BUF_LEN;
1435                 tspi->parent_clk_count = 0;
1436                 tspi->parent_clk_list = NULL;
1437                 tspi->max_rate = 0;
1438         }
1439
1440         tspi->max_parent_rate = 0;
1441         tspi->min_div = 0;
1442
1443         if (tspi->parent_clk_count) {
1444                 tspi->max_parent_rate = tspi->parent_clk_list[0].fixed_clk_rate;
1445                 for (i = 1; i < tspi->parent_clk_count; ++i) {
1446                         tspi->max_parent_rate = max(tspi->max_parent_rate,
1447                                 tspi->parent_clk_list[i].fixed_clk_rate);
1448                 }
1449                 if (tspi->max_rate)
1450                         tspi->min_div = DIV_ROUND_UP(tspi->max_parent_rate,
1451                                                 tspi->max_rate);
1452         }
1453         tspi->max_buf_size = SLINK_FIFO_DEPTH << 2;
1454
1455         if (!tspi->is_dma_allowed)
1456                 goto skip_dma_alloc;
1457
1458         init_completion(&tspi->tx_dma_complete);
1459         init_completion(&tspi->rx_dma_complete);
1460
1461         ret = spi_tegra_init_dma_param(tspi, true);
1462         if (ret < 0) {
1463                 dev_err(&pdev->dev, "Error in rx dma init\n");
1464                 goto exit_free_master;
1465         }
1466
1467         ret = spi_tegra_init_dma_param(tspi, false);
1468         if (ret < 0) {
1469                 dev_err(&pdev->dev, "Error in tx dma init\n");
1470                 goto exit_rx_dma_free;
1471         }
1472
1473         tspi->max_buf_size = tspi->dma_buf_size;
1474         tspi->def_command_reg  = SLINK_CS_SW | SLINK_M_S;
1475         tspi->def_command2_reg = SLINK_CS_ACTIVE_BETWEEN;
1476
1477 skip_dma_alloc:
1478         spi_pm_runtime_enable(&pdev->dev);
1479         if (!spi_pm_runtime_enabled(&pdev->dev)) {
1480                 ret = tegra_spi_runtime_resume(&pdev->dev);
1481                 if (ret) {
1482                         dev_err(&pdev->dev, "runtime resume failed %d", ret);
1483                         goto exit_pm_disable;
1484                 }
1485         }
1486
1487         /* Enable clock if it is require to be enable always */
1488         if (tspi->is_clkon_always)
1489                 spi_pm_runtime_get_sync(&pdev->dev);
1490
1491         master->dev.of_node = pdev->dev.of_node;
1492         ret = spi_register_master(master);
1493         if (ret < 0) {
1494                 dev_err(&pdev->dev, "can not register to master err %d\n", ret);
1495                 goto exit_pm_suspend;
1496         }
1497
1498         /* create the workqueue for the kbc path */
1499         snprintf(spi_wq_name, sizeof(spi_wq_name), "spi_tegra-%d", pdev->id);
1500         tspi->spi_workqueue = create_singlethread_workqueue(spi_wq_name);
1501         if (!tspi->spi_workqueue) {
1502                 dev_err(&pdev->dev, "Failed to create work queue\n");
1503                 ret = -ENODEV;
1504                 goto exit_master_unregister;
1505         }
1506
1507         INIT_WORK(&tspi->spi_transfer_work, tegra_spi_transfer_work);
1508
1509         return ret;
1510
1511 exit_master_unregister:
1512         spi_unregister_master(master);
1513
1514         if (tspi->is_clkon_always)
1515                 spi_pm_runtime_put_sync(&pdev->dev);
1516
1517 exit_pm_suspend:
1518         if (!spi_pm_runtime_status_suspended(&pdev->dev))
1519                 tegra_spi_runtime_idle(&pdev->dev);
1520
1521 exit_pm_disable:
1522         spi_pm_runtime_disable(&pdev->dev);
1523
1524         spi_tegra_deinit_dma_param(tspi, false);
1525
1526 exit_rx_dma_free:
1527         spi_tegra_deinit_dma_param(tspi, true);
1528
1529 exit_free_master:
1530         spi_master_put(master);
1531         return ret;
1532 }
1533
1534 static int __devexit spi_tegra_remove(struct platform_device *pdev)
1535 {
1536         struct spi_master       *master;
1537         struct spi_tegra_data   *tspi;
1538
1539         master = dev_get_drvdata(&pdev->dev);
1540         tspi = spi_master_get_devdata(master);
1541
1542         spi_unregister_master(master);
1543
1544         if (tspi->tx_dma)
1545                 spi_tegra_deinit_dma_param(tspi, false);
1546
1547         if (tspi->rx_dma)
1548                 spi_tegra_deinit_dma_param(tspi, true);
1549
1550         /* Disable clock if it is always enabled */
1551         if (tspi->is_clkon_always)
1552                 spi_pm_runtime_put_sync(&pdev->dev);
1553
1554         spi_pm_runtime_disable(&pdev->dev);
1555         if (!spi_pm_runtime_status_suspended(&pdev->dev))
1556                 tegra_spi_runtime_idle(&pdev->dev);
1557
1558         destroy_workqueue(tspi->spi_workqueue);
1559
1560         return 0;
1561 }
1562
1563 #ifdef CONFIG_PM
1564 static int spi_tegra_suspend(struct device *dev)
1565 {
1566         struct spi_master *master = dev_get_drvdata(dev);
1567         struct spi_tegra_data *tspi = spi_master_get_devdata(master);
1568         unsigned limit = 50;
1569         unsigned long flags;
1570
1571         spin_lock_irqsave(&tspi->lock, flags);
1572
1573         /* Wait for all transfer completes */
1574         if (!list_empty(&tspi->queue))
1575                 dev_warn(dev, "The transfer list is not empty "
1576                         "Waiting for time %d ms to complete transfer\n",
1577                         limit * 20);
1578
1579         while (!list_empty(&tspi->queue) && limit--) {
1580                 spin_unlock_irqrestore(&tspi->lock, flags);
1581                 msleep(20);
1582                 spin_lock_irqsave(&tspi->lock, flags);
1583         }
1584
1585         /* Wait for current transfer completes only */
1586         tspi->is_suspended = true;
1587         if (!list_empty(&tspi->queue)) {
1588                 limit = 50;
1589                 dev_err(dev, "All transfer has not completed, "
1590                         "Waiting for %d ms current transfer to complete\n",
1591                         limit * 20);
1592                 while (tspi->is_transfer_in_progress && limit--) {
1593                         spin_unlock_irqrestore(&tspi->lock, flags);
1594                         msleep(20);
1595                         spin_lock_irqsave(&tspi->lock, flags);
1596                 }
1597         }
1598
1599         if (tspi->is_transfer_in_progress) {
1600                 dev_err(dev,
1601                         "Spi transfer is in progress Avoiding suspend\n");
1602                 tspi->is_suspended = false;
1603                 spin_unlock_irqrestore(&tspi->lock, flags);
1604                 return -EBUSY;
1605         }
1606
1607         spin_unlock_irqrestore(&tspi->lock, flags);
1608
1609         /* Disable clock if it is always enabled */
1610         if (tspi->is_clkon_always)
1611                 spi_pm_runtime_put_sync(dev);
1612
1613         return 0;
1614 }
1615
1616 static int spi_tegra_resume(struct device *dev)
1617 {
1618         struct spi_master *master = dev_get_drvdata(dev);
1619         struct spi_tegra_data *tspi = spi_master_get_devdata(master);
1620         struct spi_message *m;
1621         struct spi_device *spi;
1622         struct spi_transfer *t = NULL;
1623         int single_xfer = 0;
1624         unsigned long flags;
1625
1626         /* Enable clock if it is always enabled */
1627         if (tspi->is_clkon_always)
1628                 spi_pm_runtime_get_sync(dev);
1629
1630         spi_pm_runtime_get_sync(dev);
1631         spi_tegra_writel(tspi, tspi->command_reg, SLINK_COMMAND);
1632         spi_pm_runtime_put_sync(dev);
1633
1634         spin_lock_irqsave(&tspi->lock, flags);
1635
1636         tspi->cur_speed = 0;
1637         tspi->is_suspended = false;
1638         if (!list_empty(&tspi->queue)) {
1639                 m = list_first_entry(&tspi->queue, struct spi_message, queue);
1640                 spi = m->state;
1641                 single_xfer = list_is_singular(&m->transfers);
1642                 m->actual_length = 0;
1643                 m->status = 0;
1644                 t = list_first_entry(&m->transfers, struct spi_transfer,
1645                                                 transfer_list);
1646                 tspi->is_transfer_in_progress = true;
1647         }
1648         spin_unlock_irqrestore(&tspi->lock, flags);
1649         if (t)
1650                 spi_tegra_start_transfer(spi, t, true, single_xfer);
1651         return 0;
1652 }
1653 #endif
1654
1655 static int tegra_spi_runtime_idle(struct device *dev)
1656 {
1657         struct spi_master *master = dev_get_drvdata(dev);
1658         struct spi_tegra_data *tspi = spi_master_get_devdata(master);
1659
1660         return tegra_spi_clk_disable(tspi);
1661 }
1662
1663 static int tegra_spi_runtime_resume(struct device *dev)
1664 {
1665         struct spi_master *master = dev_get_drvdata(dev);
1666         struct spi_tegra_data *tspi = spi_master_get_devdata(master);
1667
1668         return tegra_spi_clk_enable(tspi);
1669 }
1670
1671 static const struct dev_pm_ops tegra_spi_dev_pm_ops = {
1672 #if defined(CONFIG_PM_RUNTIME)
1673         .runtime_idle = tegra_spi_runtime_idle,
1674         .runtime_resume = tegra_spi_runtime_resume,
1675 #endif
1676 #ifdef CONFIG_PM
1677         .suspend = spi_tegra_suspend,
1678         .resume = spi_tegra_resume,
1679 #endif
1680 };
1681
1682 MODULE_ALIAS("platform:spi_tegra");
1683
1684 #ifdef CONFIG_OF
1685 static struct of_device_id spi_tegra_of_match_table[] __devinitdata = {
1686         { .compatible = "nvidia,tegra20-spi", },
1687         {}
1688 };
1689 MODULE_DEVICE_TABLE(of, spi_tegra_of_match_table);
1690 #else /* CONFIG_OF */
1691 #define spi_tegra_of_match_table NULL
1692 #endif /* CONFIG_OF */
1693
1694 static struct platform_driver spi_tegra_driver = {
1695         .driver = {
1696                 .name =         "spi_tegra",
1697                 .owner =        THIS_MODULE,
1698                 .pm =           &tegra_spi_dev_pm_ops,
1699                 .of_match_table = spi_tegra_of_match_table,
1700         },
1701         .probe =        spi_tegra_probe,
1702         .remove =       __devexit_p(spi_tegra_remove),
1703 };
1704
1705 static int __init spi_tegra_init(void)
1706 {
1707         return platform_driver_probe(&spi_tegra_driver, spi_tegra_probe);
1708 }
1709 subsys_initcall(spi_tegra_init);
1710
1711 static void __exit spi_tegra_exit(void)
1712 {
1713         platform_driver_unregister(&spi_tegra_driver);
1714 }
1715 module_exit(spi_tegra_exit);
1716
1717 MODULE_LICENSE("GPL");