361c84988004deef64013da0777970a1d9a9378c
[linux-3.10.git] / drivers / ata / ahci-tegra.c
1 /*
2  * ahci-tegra.c - AHCI SATA support for TEGRA AHCI device
3  *
4  * Copyright (c) 2011-2013, NVIDIA Corporation.  All rights reserved.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms and conditions of the GNU General Public License,
8  * version 2, as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope it will be useful, but WITHOUT
11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13  * more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
17  *
18  *
19  * libata documentation is available via 'make {ps|pdf}docs',
20  * as Documentation/DocBook/libata.*
21  *
22  * AHCI hardware documentation:
23  * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
24  * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
25  *
26  */
27
28 #include <linux/platform_device.h>
29 #include <linux/irq.h>
30 #include <linux/kernel.h>
31 #include <linux/module.h>
32 #include <linux/init.h>
33 #include <linux/blkdev.h>
34 #include <linux/delay.h>
35 #include <linux/interrupt.h>
36 #include <linux/dma-mapping.h>
37 #include <linux/device.h>
38 #include <linux/dmi.h>
39 #include <linux/gpio.h>
40 #include <scsi/scsi_host.h>
41 #include <scsi/scsi_cmnd.h>
42 #include <linux/libata.h>
43 #include <linux/regulator/machine.h>
44 #include <linux/pm_runtime.h>
45 #include "ahci.h"
46
47 #include <linux/clk.h>
48 #include <linux/clk/tegra.h>
49 #include <linux/tegra-powergate.h>
50 #include <linux/platform_data/tegra_ahci.h>
51
52 #include "../../arch/arm/mach-tegra/iomap.h"
53
54 #define DRV_NAME        "tegra-sata"
55 #define DRV_VERSION     "1.0"
56
57 #define ENABLE_AHCI_DBG_PRINT                   0
58 #if ENABLE_AHCI_DBG_PRINT
59 #define AHCI_DBG_PRINT(fmt, arg...)  printk(KERN_ERR fmt, ## arg)
60 #else
61 #define AHCI_DBG_PRINT(fmt, arg...) do {} while (0)
62 #endif
63
64 /* number of AHCI ports */
65 #define TEGRA_AHCI_NUM_PORTS                    1
66
67 /* idle timeout for PM in msec */
68 #define TEGRA_AHCI_MIN_IDLE_TIME                1000
69 #define TEGRA_AHCI_DEFAULT_IDLE_TIME            2000
70
71 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
72 static u32 tegra_ahci_idle_time = TEGRA_AHCI_DEFAULT_IDLE_TIME;
73 #endif
74
75 /* Bit 0 (EN_FPCI) to allow FPCI accesses to SATA */
76 #define SATA_CONFIGURATION_0_OFFSET             0x180
77 #define EN_FPCI                                 (1 << 0)
78 #define CLK_OVERRIDE                            (1 << 31)
79
80 #define SATA_INTR_MASK_0_OFFSET                 0x188
81 #define IP_INT_MASK                             (1 << 16)
82
83 /* Need to write 0x00400200 to 0x70020094 */
84 #define SATA_FPCI_BAR5_0_OFFSET                 0x094
85 #define PRI_ICTLR_CPU_IER_SET_0_OFFSET          0x024
86 #define CPU_IER_SATA_CTL                        (1 << 23)
87
88 #define AHCI_BAR5_CONFIG_LOCATION               0x24
89 #define TEGRA_SATA_BAR5_INIT_PROGRAM            0xFFFFFFFF
90 #define TEGRA_SATA_BAR5_FINAL_PROGRAM           0x40020000
91
92 #define FUSE_SATA_CALIB_OFFSET                  0x224
93 #define FUSE_SATA_CALIB_MASK                    0x3
94
95 #define T_SATA0_CFG_PHY_REG                     0x120
96 #define PHY_USE_7BIT_ALIGN_DET_FOR_SPD_MASK     (1 << 11)
97
98 #define T_SATA0_CFG_POWER_GATE                  0x4ac
99 #define POWER_GATE_SSTS_RESTORED_MASK           (1 << 23)
100 #define POWER_GATE_SSTS_RESTORED_YES            (1 << 23)
101 #define POWER_GATE_SSTS_RESTORED_NO             (0 << 23)
102
103 #define T_SATA0_DBG0_OFFSET                     0x550
104
105 #define T_SATA0_INDEX_OFFSET                    0x680
106 #define SATA0_NONE_SELECTED                     0
107 #define SATA0_CH1_SELECTED                      (1 << 0)
108
109 #define T_SATA0_CHX_PHY_CTRL1_GEN1_OFFSET       0x690
110 #define SATA0_CHX_PHY_CTRL1_GEN1_TX_AMP_SHIFT   0
111 #define SATA0_CHX_PHY_CTRL1_GEN1_TX_AMP_MASK    (0xff << 0)
112 #define SATA0_CHX_PHY_CTRL1_GEN1_TX_PEAK_SHIFT  8
113 #define SATA0_CHX_PHY_CTRL1_GEN1_TX_PEAK_MASK   (0xff << 8)
114
115 #define T_SATA0_CHX_PHY_CTRL1_GEN2_OFFSET       0x694
116 #define SATA0_CHX_PHY_CTRL1_GEN2_TX_AMP_SHIFT   0
117 #define SATA0_CHX_PHY_CTRL1_GEN2_TX_AMP_MASK    (0xff << 0)
118 #define SATA0_CHX_PHY_CTRL1_GEN2_TX_PEAK_SHIFT  12
119 #define SATA0_CHX_PHY_CTRL1_GEN2_TX_PEAK_MASK   (0xff << 12)
120 #define SATA0_CHX_PHY_CTRL1_GEN2_RX_EQ_SHIFT    24
121 #define SATA0_CHX_PHY_CTRL1_GEN2_RX_EQ_MASK     (0xf << 24)
122
123 /* AHCI config space defines */
124 #define TEGRA_PRIVATE_AHCI_CC_BKDR              0x4a4
125 #define TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE     0x54c
126 #define TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE_EN  (1 << 12)
127 #define TEGRA_PRIVATE_AHCI_CC_BKDR_PGM          0x01060100
128
129 /* AHCI HBA_CAP */
130 #define TEGRA_PRIVATE_AHCI_CAP_BKDR             0xa0
131 #define T_SATA0_AHCI_HBA_CAP_BKDR               0x300
132 #define AHCI_HBA_PLL_CTRL_0                     0xa8
133
134 #define CLAMP_TXCLK_ON_SLUMBER                  (1 << 13)
135 #define CLAMP_TXCLK_ON_DEVSLP                   (1 << 24)
136 #define NO_CLAMP_SHUT_DOWN                      (1 << 3)
137
138 #define TEGRA_SATA_IO_SPACE_OFFSET              4
139 #define TEGRA_SATA_ENABLE_IO_SPACE              (1 << 0)
140 #define TEGRA_SATA_ENABLE_MEM_SPACE             (1 << 1)
141 #define TEGRA_SATA_ENABLE_BUS_MASTER            (1 << 2)
142 #define TEGRA_SATA_ENABLE_SERR                  (1 << 8)
143
144 #define TEGRA_SATA_CORE_CLOCK_FREQ_HZ           (272*1000*1000)
145 #define TEGRA_SATA_OOB_CLOCK_FREQ_HZ            (272*1000*1000)
146
147 #define APB_PMC_SATA_PWRGT_0_REG                0x1ac
148 #define APBDEV_PMC_PWRGATE_TOGGLE_0             0x30
149 #define APBDEV_PMC_PWRGATE_STATUS_0             0x38
150 #define APBDEV_PMC_REMOVE_CLAMPING_CMD_0        0x34
151
152 #define CLK_RST_SATA_PLL_CFG0_REG               0x490
153 #define CLK_RST_SATA_PLL_CFG1_REG               0x494
154 #define CLK_RST_CONTROLLER_RST_DEVICES_V_0      0x358
155 #define CLK_RST_CONTROLLER_RST_DEVICES_W_0      0x35c
156 #define CLK_RST_CONTROLLER_RST_DEV_W_CLR_0      0x43c
157 #define CLK_RST_CONTROLLER_RST_DEV_V_CLR_0      0x434
158 #define CLK_RST_CONTROLLER_CLK_ENB_V_CLR_0      0x444
159 #define CLK_RST_CONTROLLER_CLK_ENB_V_SET_0      0x440
160
161 #define CLR_CLK_ENB_SATA_OOB                    (1 << 27)
162 #define CLR_CLK_ENB_SATA                        (1 << 28)
163
164 #define CLK_RST_CONTROLLER_PLLE_MISC_0          0x0ec
165 #define CLK_RST_CONTROLLER_PLLE_MISC_0_VALUE    0x00070300
166 #define CLK_RST_CONTROLLER_PLLE_BASE_0          0xe8
167 #define PLLE_ENABLE                             (1 << 30)
168 #define CLK_RST_CONTROLLER_PLLE_AUX_0           0x48c
169 #define CLK_RST_CONTROLLER_PLLE_AUX_0_MASK      (1 << 1)
170
171 #define CLR_SATACOLD_RST                        (1 << 1)
172 #define SWR_SATACOLD_RST                        (1 << 1)
173 #define SWR_SATA_RST                            (1 << 28)
174 #define SWR_SATA_OOB_RST                        (1 << 27)
175 #define DEVSLP_OVERRIDE                         (1 << 17)
176 #define SDS_SUPPORT                             (1 << 13)
177 #define DESO_SUPPORT                            (1 << 15)
178 #define SATA_AUX_PAD_PLL_CNTL_1_REG             0x1100
179 #define SATA_AUX_MISC_CNTL_1_REG                0x1108
180 #define SATA_AUX_SPARE_CFG0_0                   0x1118
181
182 /* for APB_PMC_SATA_PWRGT_0_REG */
183 #define PG_INFO_MASK                            (1 << 6)
184 #define PG_INFO_ON                              (1 << 6)
185 #define PG_INFO_OFF                             (0 << 6)
186 #define PLLE_IDDQ_SWCTL_MASK                    (1 << 4)
187 #define PLLE_IDDQ_SWCTL_ON                      (1 << 4)
188 #define PLLE_IDDQ_SWCTL_OFF                     (0 << 4)
189 #define PADPHY_IDDQ_OVERRIDE_VALUE_MASK         (1 << 3)
190 #define PADPHY_IDDQ_OVERRIDE_VALUE_ON           (1 << 3)
191 #define PADPHY_IDDQ_OVERRIDE_VALUE_OFF          (0 << 3)
192 #define PADPHY_IDDQ_SWCTL_MASK                  (1 << 2)
193 #define PADPHY_IDDQ_SWCTL_ON                    (1 << 2)
194 #define PADPHY_IDDQ_SWCTL_OFF                   (0 << 2)
195 #define PADPLL_IDDQ_OVERRIDE_VALUE_MASK         (1 << 1)
196 #define PADPLL_IDDQ_OVERRIDE_VALUE_ON           (1 << 1)
197 #define PADPLL_IDDQ_OVERRIDE_VALUE_OFF          (0 << 1)
198 #define PADPLL_IDDQ_SWCTL_MASK                  (1 << 0)
199 #define PADPLL_IDDQ_SWCTL_ON                    (1 << 0)
200 #define PADPLL_IDDQ_SWCTL_OFF                   (0 << 0)
201
202 #define START                                   (1 < 8)
203 #define PARTID_VALUE                            0x8
204
205 #define SAX_MASK                                (1 << 8)
206
207 /* for CLK_RST_SATA_PLL_CFG0_REG */
208 #define PADPLL_RESET_OVERRIDE_VALUE_MASK        (1 << 1)
209 #define PADPLL_RESET_OVERRIDE_VALUE_ON          (1 << 1)
210 #define PADPLL_RESET_OVERRIDE_VALUE_OFF         (0 << 1)
211 #define PADPLL_RESET_SWCTL_MASK                 (1 << 0)
212 #define PADPLL_RESET_SWCTL_ON                   (1 << 0)
213 #define PADPLL_RESET_SWCTL_OFF                  (0 << 0)
214
215 /* for CLK_RST_SATA_PLL_CFG1_REG */
216 #define IDDQ2LANE_SLUMBER_DLY_MASK              (0xffL << 16)
217 #define IDDQ2LANE_SLUMBER_DLY_SHIFT             16
218 #define IDDQ2LANE_SLUMBER_DLY_3MS               (3 << 16)
219 #define IDDQ2LANE_IDDQ_DLY_SHIFT                0
220 #define IDDQ2LANE_IDDQ_DLY_MASK                 (0xffL << 0)
221
222 /* for SATA_AUX_PAD_PLL_CNTL_1_REG */
223 #define REFCLK_SEL_MASK                         (3 << 11)
224 #define REFCLK_SEL_INT_CML                      (0 << 11)
225 #define LOCKDET_FIELD                           (1 << 6)
226
227 /* for SATA_AUX_MISC_CNTL_1_REG */
228 #define NVA2SATA_OOB_ON_POR_MASK                (1 << 7)
229 #define NVA2SATA_OOB_ON_POR_YES                 (1 << 7)
230 #define NVA2SATA_OOB_ON_POR_NO                  (0 << 7)
231 #define L0_RX_IDLE_T_SAX_SHIFT                  5
232 #define L0_RX_IDLE_T_SAX_MASK                   (3 << 5)
233 #define L0_RX_IDLE_T_NPG_SHIFT                  3
234 #define L0_RX_IDLE_T_NPG_MASK                   (3 << 3)
235 #define L0_RX_IDLE_T_MUX_MASK                   (1 << 2)
236 #define L0_RX_IDLE_T_MUX_FROM_APB_MISC          (1 << 2)
237 #define L0_RX_IDLE_T_MUX_FROM_SATA              (0 << 2)
238
239 #define SSTAT_IPM_STATE_MASK                    0xF00
240 #define SSTAT_IPM_SLUMBER_STATE                 0x600
241 #define XUSB_PADCTL_USB3_PAD_MUX_0              0x134
242 #define FORCE_SATA_PAD_IDDQ_DISABLE_MASK0       (1 << 6)
243 #define XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0         0x138
244 #define XUSB_PADCTL_PLL1_MODE                   (1 << 24)
245 #define XUSB_PADCTL_ELPG_PROGRAM_0              0x01c
246 #define AUX_ELPG_CLAMP_EN                       (1 << 24)
247 #define AUX_ELPG_CLAMP_EN_EARLY                 (1 << 25)
248 #define AUX_ELPG_VCORE_DOWN                     (1 << 26)
249 #define XUSB_PADCTL_IOPHY_MISC_PAD_S0_CTL_1_0   0x148
250 #define XUSB_PADCTL_IOPHY_MISC_IDDQ_OVRD        (1 << 1)
251 #define XUSB_PADCTL_IOPHY_MISC_IDDQ             (1 << 0)
252
253 #define SATA_AXI_BAR5_START_0                   0x54
254 #define SATA_AXI_BAR5_SZ_0                      0x14
255 #define SATA_AXI_BAR5_START_VALUE               0x70020
256 #define AXI_BAR5_SIZE_VALUE                     0x00008
257 #define FPCI_BAR5_0_START_VALUE                 0x0010000
258 #define FPCI_BAR5_0_FINAL_VALUE                 0x40020100
259 #define FPCI_BAR5_0_ACCESS_TYPE                 (1 << 0)
260
261 #define XUSB_PADCTL_IOPHY_MISC_PAD_S0_CTL_1_0   0x148
262 #define IDDQ_OVRD_MASK                          (1 << 1)
263 #define IDDQ_MASK                               (1 << 0)
264
265 /* for XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0 */
266 #define PLL_PWR_OVRD_MASK                       (1 << 3)
267 #define PLL_RST_MASK                            (1 << 1)
268 #define PLL_IDDQ_MASK                           (1 << 0)
269
270 /* for CLK_RST_CONTROLLER_PLLE_MISC_0 */
271 #define T124_PLLE_IDDQ_SWCTL_MASK               (1 << 14)
272 #define PLLE_IDDQ_OVERRIDE_VALUE_MASK           (1 << 13)
273
274 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
275
276 /* create a work for handling the async transfers */
277 static void tegra_ahci_work_handler(struct work_struct *work);
278 static DECLARE_WORK(tegra_ahci_work, tegra_ahci_work_handler);
279 static struct workqueue_struct *tegra_ahci_work_q;
280 #endif
281
282 enum {
283         AHCI_PCI_BAR = 5,
284 };
285
286 enum port_idle_status {
287         PORT_IS_NOT_IDLE,
288         PORT_IS_IDLE,
289         PORT_IS_IDLE_NOT_SLUMBER,
290         PORT_IS_SLUMBER,
291 };
292
293 enum sata_state {
294         SATA_ON,
295         SATA_OFF,
296         SATA_GOING_ON,
297         SATA_GOING_OFF,
298         SATA_ABORT_OFF,
299 };
300
301 char *sata_power_rails[] = {
302         "avdd_sata",
303         "vdd_sata",
304         "hvdd_sata",
305         "avdd_sata_pll"
306 };
307
308 #define NUM_SATA_POWER_RAILS    ARRAY_SIZE(sata_power_rails)
309
310 struct tegra_qc_list {
311         struct list_head list;
312         struct ata_queued_cmd *qc;
313 };
314
315 /*
316  *  tegra_ahci_host_priv is the extension of ahci_host_priv
317  *  with extra fields: idle_timer, pg_save, pg_state, etc.
318  */
319 struct tegra_ahci_host_priv {
320         struct ahci_host_priv   ahci_host_priv;
321         struct regulator        *power_rails[NUM_SATA_POWER_RAILS];
322         void __iomem            *bars_table[6];
323         struct ata_host         *host;
324         struct timer_list       idle_timer;
325         struct device           *dev;
326         void                    *pg_save;
327         enum sata_state         pg_state;
328         struct list_head        qc_list;
329 };
330
331 static int tegra_ahci_init_one(struct platform_device *pdev);
332 static int tegra_ahci_remove_one(struct platform_device *pdev);
333 static void tegra_ahci_set_clk_rst_cnt_rst_dev(void);
334 static void tegra_ahci_clr_clk_rst_cnt_rst_dev(void);
335 static void tegra_ahci_pad_config(void);
336 static void tegra_ahci_put_sata_in_iddq(void);
337 static void tegra_ahci_iddqlane_config(void);
338
339 #ifdef CONFIG_PM
340 static bool tegra_ahci_power_un_gate(struct ata_host *host);
341 static bool tegra_ahci_power_gate(struct ata_host *host);
342 static void tegra_ahci_abort_power_gate(struct ata_host *host);
343 static int tegra_ahci_controller_suspend(struct platform_device *pdev);
344 static int tegra_ahci_controller_resume(struct platform_device *pdev);
345 static int tegra_ahci_suspend(struct platform_device *pdev, pm_message_t mesg);
346 static int tegra_ahci_resume(struct platform_device *pdev);
347 static enum port_idle_status tegra_ahci_is_port_idle(struct ata_port *ap);
348 static bool tegra_ahci_are_all_ports_idle(struct ata_host *host);
349 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
350 static bool tegra_ahci_pad_resume(struct ata_host *host);
351 static bool tegra_ahci_pad_suspend(struct ata_host *host);
352 static void tegra_ahci_abort_pad_suspend(struct ata_host *host);
353 static unsigned int tegra_ahci_qc_issue(struct ata_queued_cmd *qc);
354 static int tegra_ahci_hardreset(struct ata_link *link, unsigned int *class,
355                                 unsigned long deadline);
356 static int tegra_ahci_runtime_suspend(struct device *dev);
357 static int tegra_ahci_runtime_resume(struct device *dev);
358 static void tegra_ahci_idle_timer(unsigned long arg);
359 static int tegra_ahci_queue_one_qc(struct tegra_ahci_host_priv *tegra_hpriv,
360                                    struct ata_queued_cmd *qc);
361 static void tegra_ahci_dequeue_qcs(struct tegra_ahci_host_priv *tegra_hpriv);
362 #endif
363 #else
364 #define tegra_ahci_controller_suspend   NULL
365 #define tegra_ahci_controller_resume    NULL
366 #define tegra_ahci_suspend              NULL
367 #define tegra_ahci_resume               NULL
368 #endif
369
370 static struct scsi_host_template ahci_sht = {
371         AHCI_SHT("tegra-sata"),
372 };
373
374 static struct ata_port_operations tegra_ahci_ops = {
375         .inherits       = &ahci_ops,
376 #ifdef CONFIG_PM
377 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
378         .qc_issue       = tegra_ahci_qc_issue,
379         .hardreset      = tegra_ahci_hardreset,
380 #endif
381 #endif
382 };
383
384 static const struct ata_port_info ahci_port_info = {
385         .flags          = AHCI_FLAG_COMMON,
386         .pio_mask       = 0x1f, /* pio0-4 */
387         .udma_mask      = ATA_UDMA6,
388         .port_ops       = &tegra_ahci_ops,
389 };
390
391 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
392 static const struct dev_pm_ops tegra_ahci_dev_rt_ops = {
393         .runtime_suspend = tegra_ahci_runtime_suspend,
394         .runtime_resume = tegra_ahci_runtime_resume,
395 };
396 #endif
397
398 static struct platform_driver tegra_platform_ahci_driver = {
399         .probe          = tegra_ahci_init_one,
400         .remove         = tegra_ahci_remove_one,
401 #ifdef CONFIG_PM
402         .suspend        = tegra_ahci_suspend,
403         .resume         = tegra_ahci_resume,
404 #endif
405         .driver = {
406                 .name = DRV_NAME,
407         }
408 };
409
410 struct tegra_ahci_host_priv *g_tegra_hpriv;
411
412 static inline u32 bar5_readl(u32 offset)
413 {
414         u32 val;
415         val = readl(IO_ADDRESS(TEGRA_SATA_BAR5_BASE + offset));
416         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_SATA_BAR5_BASE+offset,
417                         val);
418         return val;
419 }
420
421 static inline void bar5_writel(u32 val, u32 offset)
422 {
423         AHCI_DBG_PRINT("[0x%x] <= 0x%08x\n", TEGRA_SATA_BAR5_BASE+offset,
424                         val);
425         writel(val, IO_ADDRESS(TEGRA_SATA_BAR5_BASE + offset));
426 }
427
428
429 static inline u32 xusb_readl(u32 offset)
430 {
431         u32 val;
432         val = readl(IO_ADDRESS(TEGRA_XUSB_PADCTL_BASE + offset));
433         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_XUSB_PADCTL_BASE+offset,
434                         val);
435         return val;
436 }
437
438 static inline void xusb_writel(u32 val, u32 offset)
439 {
440         AHCI_DBG_PRINT("[0x%x] <= 0x%08x\n", TEGRA_XUSB_PADCTL_BASE+offset,
441                         val);
442         writel(val, IO_ADDRESS(TEGRA_XUSB_PADCTL_BASE + offset));
443 }
444
445
446 static inline u32 pmc_readl(u32 offset)
447 {
448         u32 val;
449         val = readl(IO_ADDRESS(TEGRA_PMC_BASE + offset));
450         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_PMC_BASE+offset, val);
451         return val;
452 }
453
454 static inline void pmc_writel(u32 val, u32 offset)
455 {
456         AHCI_DBG_PRINT("[0x%x] <= 0x%08x\n", TEGRA_PMC_BASE+offset, val);
457         writel(val, IO_ADDRESS(TEGRA_PMC_BASE + offset));
458 }
459
460 static inline u32 clk_readl(u32 offset)
461 {
462         u32 val;
463
464         val = readl(IO_ADDRESS(TEGRA_CLK_RESET_BASE + offset));
465         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_CLK_RESET_BASE+offset, val);
466         return val;
467 }
468
469 static inline void clk_writel(u32 val, u32 offset)
470 {
471         AHCI_DBG_PRINT("[0x%x] <= 0x%08x\n", TEGRA_CLK_RESET_BASE+offset, val);
472         writel(val, IO_ADDRESS(TEGRA_CLK_RESET_BASE + offset));
473 }
474
475 static inline u32 misc_readl(u32 offset)
476 {
477         u32 val;
478
479         val = readl(IO_ADDRESS(TEGRA_APB_MISC_BASE + offset));
480         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_APB_MISC_BASE+offset, val);
481         return val;
482 }
483
484 static inline void misc_writel(u32 val, u32 offset)
485 {
486         AHCI_DBG_PRINT("[0x%x] <= 0x%08x\n", TEGRA_APB_MISC_BASE+offset, val);
487         writel(val, IO_ADDRESS(TEGRA_APB_MISC_BASE + offset));
488 }
489
490 static inline u32 sata_readl(u32 offset)
491 {
492         u32 val;
493
494         val = readl(IO_ADDRESS(TEGRA_SATA_BASE + offset));
495         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_SATA_BASE+offset, val);
496         return val;
497 }
498
499 static inline void sata_writel(u32 val, u32 offset)
500 {
501         AHCI_DBG_PRINT("[0x%x] <= 0x%08x\n", TEGRA_SATA_BASE+offset, val);
502         writel(val, IO_ADDRESS(TEGRA_SATA_BASE + offset));
503 }
504
505 static inline u32 scfg_readl(u32 offset)
506 {
507         u32 val;
508
509         val = readl(IO_ADDRESS(TEGRA_SATA_CONFIG_BASE + offset));
510         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_SATA_CONFIG_BASE+offset,
511                                              val);
512         return val;
513 }
514
515 static inline void scfg_writel(u32 val, u32 offset)
516 {
517         AHCI_DBG_PRINT("[0x%x] <= 0x%08x\n", TEGRA_SATA_CONFIG_BASE+offset,
518                                              val);
519         writel(val, IO_ADDRESS(TEGRA_SATA_CONFIG_BASE + offset));
520 }
521
522 static inline u32 pictlr_readl(u32 offset)
523 {
524         u32 val;
525
526         val = readl(IO_ADDRESS(TEGRA_PRIMARY_ICTLR_BASE + offset));
527         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_PRIMARY_ICTLR_BASE+offset,
528                                              val);
529         return val;
530 }
531
532 static inline void pictlr_writel(u32 val, u32 offset)
533 {
534         AHCI_DBG_PRINT("[0x%x] <= 0x%08x\n", TEGRA_PRIMARY_ICTLR_BASE+offset,
535                                              val);
536         writel(val, IO_ADDRESS(TEGRA_PRIMARY_ICTLR_BASE + offset));
537 }
538
539 static inline u32 fuse_readl(u32 offset)
540 {
541         u32 val;
542
543         val = readl(IO_ADDRESS(TEGRA_FUSE_BASE + offset));
544         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_FUSE_BASE+offset, val);
545
546         return val;
547 }
548
549 /* Sata Pad Cntrl Values */
550 struct sata_pad_cntrl {
551         u8 gen1_tx_amp;
552         u8 gen1_tx_peak;
553         u8 gen2_tx_amp;
554         u8 gen2_tx_peak;
555 };
556
557 static const struct sata_pad_cntrl sata_calib_pad_val[] = {
558         {       /* SATA_CALIB[1:0]  = 00 */
559                 0x0c,
560                 0x04,
561                 0x0e,
562                 0x0a
563         },
564         {       /* SATA_CALIB[1:0]  = 01 */
565                 0x0e,
566                 0x04,
567                 0x14,
568                 0x0a
569         },
570         {       /* SATA_CALIB[1:0]  = 10 */
571                 0x0e,
572                 0x07,
573                 0x1a,
574                 0x0e
575         },
576         {       /* SATA_CALIB[1:0]  = 11 */
577                 0x14,
578                 0x0e,
579                 0x1a,
580                 0x0e
581         }
582 };
583
584 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
585 static void tegra_ahci_work_handler(struct work_struct *work)
586 {
587         tegra_ahci_runtime_resume(g_tegra_hpriv->dev);
588 }
589 #endif
590
591 static void tegra_ahci_set_pad_cntrl_regs(
592                         struct tegra_ahci_platform_data *ahci_pdata)
593 {
594         int     calib_val;
595         int     val;
596         int     i;
597
598         calib_val = fuse_readl(FUSE_SATA_CALIB_OFFSET) & FUSE_SATA_CALIB_MASK;
599
600         for (i = 0; i < TEGRA_AHCI_NUM_PORTS; ++i) {
601                 scfg_writel((1 << i), T_SATA0_INDEX_OFFSET);
602
603                 val = scfg_readl(T_SATA0_CHX_PHY_CTRL1_GEN1_OFFSET);
604                 val &= ~SATA0_CHX_PHY_CTRL1_GEN1_TX_AMP_MASK;
605                 val |= (sata_calib_pad_val[calib_val].gen1_tx_amp <<
606                         SATA0_CHX_PHY_CTRL1_GEN1_TX_AMP_SHIFT);
607                 scfg_writel(val, T_SATA0_CHX_PHY_CTRL1_GEN1_OFFSET);
608
609                 val = scfg_readl(T_SATA0_CHX_PHY_CTRL1_GEN1_OFFSET);
610                 val &= ~SATA0_CHX_PHY_CTRL1_GEN1_TX_PEAK_MASK;
611                 val |= (sata_calib_pad_val[calib_val].gen1_tx_peak <<
612                         SATA0_CHX_PHY_CTRL1_GEN1_TX_PEAK_SHIFT);
613                 scfg_writel(val, T_SATA0_CHX_PHY_CTRL1_GEN1_OFFSET);
614
615                 val = scfg_readl(T_SATA0_CHX_PHY_CTRL1_GEN2_OFFSET);
616                 val &= ~SATA0_CHX_PHY_CTRL1_GEN2_TX_AMP_MASK;
617                 val |= (sata_calib_pad_val[calib_val].gen2_tx_amp <<
618                         SATA0_CHX_PHY_CTRL1_GEN2_TX_AMP_SHIFT);
619                 scfg_writel(val, T_SATA0_CHX_PHY_CTRL1_GEN2_OFFSET);
620
621                 val = scfg_readl(T_SATA0_CHX_PHY_CTRL1_GEN2_OFFSET);
622                 val &= ~SATA0_CHX_PHY_CTRL1_GEN2_TX_PEAK_MASK;
623                 val |= (sata_calib_pad_val[calib_val].gen2_tx_peak <<
624                         SATA0_CHX_PHY_CTRL1_GEN2_TX_PEAK_SHIFT);
625                 scfg_writel(val, T_SATA0_CHX_PHY_CTRL1_GEN2_OFFSET);
626
627                 /* set 2 to SATA0_CHX_PHY_CTRL1_GEN2_RX_EQ field */
628                 val = scfg_readl(T_SATA0_CHX_PHY_CTRL1_GEN2_OFFSET);
629                 val &= ~SATA0_CHX_PHY_CTRL1_GEN2_RX_EQ_MASK;
630
631                 if ((ahci_pdata) && (ahci_pdata->gen2_rx_eq > 0)) {
632                         val |= (ahci_pdata->gen2_rx_eq <<
633                                         SATA0_CHX_PHY_CTRL1_GEN2_RX_EQ_SHIFT);
634                 } else {
635                         /* Default set 2 to SATA0_CHX_PHY_CTRL1_GEN2_RX_EQ
636                                                                 field */
637                         val |= (2 << SATA0_CHX_PHY_CTRL1_GEN2_RX_EQ_SHIFT);
638                 }
639
640                 scfg_writel(val, T_SATA0_CHX_PHY_CTRL1_GEN2_OFFSET);
641         }
642         scfg_writel(SATA0_NONE_SELECTED, T_SATA0_INDEX_OFFSET);
643 }
644
645 int tegra_ahci_get_rails(struct regulator *regulators[])
646 {
647         struct regulator *reg;
648         int i;
649         int ret = 0;
650
651         for (i = 0; i < NUM_SATA_POWER_RAILS; ++i) {
652                 reg = regulator_get(g_tegra_hpriv->dev, sata_power_rails[i]);
653                 if (IS_ERR_OR_NULL(reg)) {
654                         pr_err("%s: can't get regulator %s\n",
655                                 __func__, sata_power_rails[i]);
656                         WARN_ON(1);
657                         ret = PTR_ERR(reg);
658                         goto exit;
659                 }
660                 regulators[i] = reg;
661         }
662 exit:
663         return ret;
664 }
665
666 void tegra_ahci_put_rails(struct regulator *regulators[])
667 {
668         int i;
669
670         for (i = 0; i < NUM_SATA_POWER_RAILS; ++i)
671                 regulator_put(regulators[i]);
672 }
673
674 int tegra_ahci_power_on_rails(struct regulator *regulators[])
675 {
676         struct regulator *reg;
677         int i;
678         int ret = 0;
679
680         for (i = 0; i < NUM_SATA_POWER_RAILS; ++i) {
681                 reg = regulators[i];
682                 ret = regulator_enable(reg);
683                 if (ret) {
684                         pr_err("%s: can't enable regulator[%d]\n",
685                                 __func__, i);
686                         WARN_ON(1);
687                         goto exit;
688                 }
689         }
690
691 exit:
692         return ret;
693 }
694
695 int tegra_ahci_power_off_rails(struct regulator *regulators[])
696 {
697         struct regulator *reg;
698         int i;
699         int ret = 0;
700
701         for (i = 0; i < NUM_SATA_POWER_RAILS; ++i) {
702                 reg = regulators[i];
703                 if (!IS_ERR_OR_NULL(reg)) {
704                         ret = regulator_disable(reg);
705                         if (ret) {
706                                 pr_err("%s: can't disable regulator[%d]\n",
707                                         __func__, i);
708                                 WARN_ON(1);
709                                 goto exit;
710                         }
711                 }
712         }
713
714 exit:
715         return ret;
716 }
717 static int tegra_ahci_controller_init(struct tegra_ahci_host_priv *tegra_hpriv,
718                 int lp0)
719 {
720         int err;
721         struct clk *clk_sata = NULL;
722         struct clk *clk_sata_oob = NULL;
723         struct clk *clk_sata_cold = NULL;
724         struct clk *clk_pllp = NULL;
725         struct clk *clk_cml1 = NULL;
726         u32 val;
727         u32 timeout;
728
729         struct tegra_ahci_platform_data *ahci_pdata =
730                                         tegra_hpriv->dev->platform_data;
731
732         if (!lp0) {
733                 err = tegra_ahci_get_rails(tegra_hpriv->power_rails);
734                 if (err) {
735                         pr_err("%s: fails to get rails (%d)\n", __func__, err);
736                         goto exit;
737                 }
738
739         }
740         err = tegra_ahci_power_on_rails(tegra_hpriv->power_rails);
741         if (err) {
742                 pr_err("%s: fails to power on rails (%d)\n", __func__, err);
743                 goto exit;
744         }
745
746         /* pll_p is the parent of tegra_sata and tegra_sata_oob */
747         clk_pllp = clk_get_sys(NULL, "pll_p");
748         if (IS_ERR_OR_NULL(clk_pllp)) {
749                 pr_err("%s: unable to get PLL_P clock\n", __func__);
750                 err = PTR_ERR(clk_pllp);
751                 goto exit;
752         }
753
754         clk_sata = clk_get_sys("tegra_sata", NULL);
755         if (IS_ERR_OR_NULL(clk_sata)) {
756                 pr_err("%s: unable to get SATA clock\n", __func__);
757                 err = PTR_ERR(clk_sata);
758                 goto exit;
759         }
760
761         clk_sata_oob = clk_get_sys("tegra_sata_oob", NULL);
762         if (IS_ERR_OR_NULL(clk_sata_oob)) {
763                 pr_err("%s: unable to get SATA OOB clock\n", __func__);
764                 err = PTR_ERR(clk_sata_oob);
765                 goto exit;
766         }
767
768         clk_sata_cold = clk_get_sys("tegra_sata_cold", NULL);
769         if (IS_ERR_OR_NULL(clk_sata_cold)) {
770                 pr_err("%s: unable to get SATA COLD clock\n", __func__);
771                 err = PTR_ERR(clk_sata_cold);
772                 goto exit;
773         }
774
775         tegra_periph_reset_assert(clk_sata);
776         tegra_periph_reset_assert(clk_sata_oob);
777         tegra_periph_reset_assert(clk_sata_cold);
778         udelay(10);
779
780         /* need to establish both clocks divisors before setting clk sources */
781         clk_set_rate(clk_sata, clk_get_rate(clk_sata)/10);
782         clk_set_rate(clk_sata_oob, clk_get_rate(clk_sata_oob)/10);
783
784         /* set SATA clk and SATA_OOB clk source */
785         clk_set_parent(clk_sata, clk_pllp);
786         clk_set_parent(clk_sata_oob, clk_pllp);
787
788         /* Configure SATA clocks */
789         /* Core clock runs at 108MHz */
790         if (clk_set_rate(clk_sata, TEGRA_SATA_CORE_CLOCK_FREQ_HZ)) {
791                 err = -ENODEV;
792                 goto exit;
793         }
794         /* OOB clock runs at 216MHz */
795         if (clk_set_rate(clk_sata_oob, TEGRA_SATA_OOB_CLOCK_FREQ_HZ)) {
796                 err = -ENODEV;
797                 goto exit;
798         }
799
800         if (clk_prepare_enable(clk_sata)) {
801                 pr_err("%s: unable to enable SATA clock\n", __func__);
802                 err = -ENODEV;
803                 goto exit;
804         }
805
806         if (clk_prepare_enable(clk_sata_oob)) {
807                 pr_err("%s: unable to enable SATA OOB clock\n", __func__);
808                 err = -ENODEV;
809                 goto exit;
810         }
811
812         tegra_periph_reset_deassert(clk_sata);
813         tegra_periph_reset_deassert(clk_sata_oob);
814         tegra_periph_reset_deassert(clk_sata_cold);
815
816         tegra_ahci_clr_clk_rst_cnt_rst_dev();
817
818         if (ahci_pdata->pexp_gpio) {
819                 if (gpio_is_valid(ahci_pdata->pexp_gpio)) {
820                         val = gpio_request(ahci_pdata->pexp_gpio, "ahci-tegra");
821                         if (val) {
822                                 pr_err("failed to allocate Port expander gpio\n");
823                                 err = -ENODEV;
824                                 goto exit;
825                         }
826                         gpio_direction_output(ahci_pdata->pexp_gpio, 1);
827                 }
828         }
829
830         val = 0x100;
831         pmc_writel(val, APBDEV_PMC_REMOVE_CLAMPING_CMD_0);
832
833         /**** Init the SATA PAD PLL ****/
834         /* SATA_PADPLL_IDDQ_SWCTL=1 and SATA_PADPLL_IDDQ_OVERRIDE_VALUE=1 */
835         val = pmc_readl(APB_PMC_SATA_PWRGT_0_REG);
836         val &= ~(PADPLL_IDDQ_SWCTL_MASK | PADPLL_IDDQ_OVERRIDE_VALUE_MASK);
837         val |= (PADPLL_IDDQ_SWCTL_ON | PADPLL_IDDQ_OVERRIDE_VALUE_ON);
838         pmc_writel(val, APB_PMC_SATA_PWRGT_0_REG);
839
840         /* SATA_PADPLL_RESET_OVERRIDE_VALUE=1 and SATA_PADPLL_RESET_SWCTL=1 */
841         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
842         val &= ~(PADPLL_RESET_OVERRIDE_VALUE_MASK | PADPLL_RESET_SWCTL_MASK);
843         val |= (PADPLL_RESET_OVERRIDE_VALUE_ON | PADPLL_RESET_SWCTL_ON);
844         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
845
846         /* SATA_PADPHY_IDDQ_OVERRIDE_VALUE and SATA_PADPHY_IDDQ_SWCTL = 1 */
847         val = pmc_readl(APB_PMC_SATA_PWRGT_0_REG);
848         val &= ~(PADPHY_IDDQ_OVERRIDE_VALUE_MASK | PADPHY_IDDQ_SWCTL_MASK);
849         val |= (PADPHY_IDDQ_OVERRIDE_VALUE_ON | PADPHY_IDDQ_SWCTL_ON);
850         pmc_writel(val, APB_PMC_SATA_PWRGT_0_REG);
851
852         /* Get SATA pad PLL out of IDDQ mode */
853         val = pmc_readl(APB_PMC_SATA_PWRGT_0_REG);
854         val &= ~PADPLL_IDDQ_OVERRIDE_VALUE_MASK;
855         pmc_writel(val, APB_PMC_SATA_PWRGT_0_REG);
856         udelay(3);
857
858         /* select internal CML ref clk
859          * select PLLE as input to IO phy */
860         val = misc_readl(SATA_AUX_PAD_PLL_CNTL_1_REG);
861         val &= ~REFCLK_SEL_MASK;
862         val |= REFCLK_SEL_INT_CML;
863         misc_writel(val, SATA_AUX_PAD_PLL_CNTL_1_REG);
864
865         /* wait for SATA_PADPLL_IDDQ2LANE_SLUMBER_DLY = 3 microseconds. */
866         val = clk_readl(CLK_RST_SATA_PLL_CFG1_REG);
867         val &= ~IDDQ2LANE_SLUMBER_DLY_MASK;
868         val |= IDDQ2LANE_SLUMBER_DLY_3MS;
869         clk_writel(val, CLK_RST_SATA_PLL_CFG1_REG);
870         udelay(3);
871
872         /* de-assert IDDQ mode signal going to PHY */
873         val = pmc_readl(APB_PMC_SATA_PWRGT_0_REG);
874         val &= ~PADPHY_IDDQ_OVERRIDE_VALUE_MASK;
875         pmc_writel(val, APB_PMC_SATA_PWRGT_0_REG);
876 #if defined(CONFIG_TEGRA_SILICON_PLATFORM)
877         err = tegra_unpowergate_partition_with_clk_on(TEGRA_POWERGATE_SATA);
878         if (err) {
879                 pr_err("%s: ** failed to turn-on SATA (0x%x) **\n",
880                                 __func__, err);
881                 goto exit;
882         }
883 #endif
884
885         /*
886          * place SATA Pad PLL out of reset by writing
887          * SATA_PADPLL_RST_OVERRIDE_VALUE = 0
888          */
889         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
890         val &= ~PADPLL_RESET_OVERRIDE_VALUE_MASK;
891         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
892
893         /*
894          * Wait for SATA_AUX_PAD_PLL_CNTL_1_0_LOCKDET to turn 1 with a timeout
895          * of 15 us.
896          */
897         timeout = 15;
898         while (timeout--) {
899                 udelay(1);
900                 val = xusb_readl(XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0);
901                 if (val & LOCKDET_FIELD)
902                         break;
903         }
904         if (timeout == 0)
905                 pr_err("%s: AUX_PAD_PLL_CNTL_1 (0x%x) is not locked in 15us.\n",
906                         __func__, val);
907
908         tegra_ahci_pad_config();
909
910         clk_cml1 = clk_get_sys(NULL, "cml1");
911         if (IS_ERR_OR_NULL(clk_cml1)) {
912                 pr_err("%s: unable to get cml1 clock Errone is %d\n",
913                                         __func__, (int) PTR_ERR(clk_cml1));
914                 err = PTR_ERR(clk_cml1);
915                 goto exit;
916         }
917         if (clk_prepare_enable(clk_cml1)) {
918                 pr_err("%s: unable to enable cml1 clock\n", __func__);
919                 err = -ENODEV;
920                 goto exit;
921         }
922
923         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
924         val &= ~PADPLL_RESET_SWCTL_MASK;
925         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
926
927         /* PLLE Programing for SATA */
928
929         val = clk_readl(CLK_RST_CONTROLLER_PLLE_AUX_0);
930         val |= CLK_RST_CONTROLLER_PLLE_AUX_0_MASK;
931         clk_writel(val, CLK_RST_CONTROLLER_PLLE_AUX_0);
932
933         /* bring SATA IOPHY out of IDDQ */
934         val = xusb_readl(XUSB_PADCTL_USB3_PAD_MUX_0);
935         val |= FORCE_SATA_PAD_IDDQ_DISABLE_MASK0;
936         xusb_writel(val, XUSB_PADCTL_USB3_PAD_MUX_0);
937
938         val = xusb_readl(XUSB_PADCTL_ELPG_PROGRAM_0);
939         val &= ~(AUX_ELPG_CLAMP_EN | AUX_ELPG_CLAMP_EN_EARLY |
940                 AUX_ELPG_VCORE_DOWN);
941         xusb_writel(val, XUSB_PADCTL_ELPG_PROGRAM_0);
942
943         val = xusb_readl(XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0);
944         val = val | XUSB_PADCTL_PLL1_MODE;
945         xusb_writel(val, XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0);
946
947         /* clear NVA2SATA_OOB_ON_POR in SATA_AUX_MISC_CNTL_1_REG */
948         val = misc_readl(SATA_AUX_MISC_CNTL_1_REG);
949         val &= ~NVA2SATA_OOB_ON_POR_MASK;
950         misc_writel(val, SATA_AUX_MISC_CNTL_1_REG);
951
952         val = sata_readl(SATA_CONFIGURATION_0_OFFSET);
953         val |= EN_FPCI;
954         sata_writel(val, SATA_CONFIGURATION_0_OFFSET);
955
956         val = sata_readl(SATA_CONFIGURATION_0_OFFSET);
957         val |= CLK_OVERRIDE;
958         sata_writel(val, SATA_CONFIGURATION_0_OFFSET);
959
960         /* program sata pad control based on the fuse */
961         tegra_ahci_set_pad_cntrl_regs(ahci_pdata);
962
963         /*
964          * clear bit T_SATA0_CFG_PHY_0_USE_7BIT_ALIGN_DET_FOR_SPD of
965          * T_SATA0_CFG_PHY_0
966          */
967         val = scfg_readl(T_SATA0_CFG_PHY_REG);
968         val &= ~PHY_USE_7BIT_ALIGN_DET_FOR_SPD_MASK;
969         scfg_writel(val, T_SATA0_CFG_PHY_REG);
970
971         /*
972          * WAR: Before enabling SATA PLL shutdown, lockdet needs to be ignored.
973          *      To ignore lockdet, T_SATA0_DBG0_OFFSET register bit 10 needs to
974          *      be 1, and bit 8 needs to be 0.
975          */
976         val = scfg_readl(T_SATA0_DBG0_OFFSET);
977         val |= (1 << 10);
978         val &= ~(1 << 8);
979         scfg_writel(val, T_SATA0_DBG0_OFFSET);
980
981         /* program class code and programming interface for AHCI */
982         val = scfg_readl(TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE);
983         val |= TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE_EN;
984         scfg_writel(val, TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE);
985         scfg_writel(TEGRA_PRIVATE_AHCI_CC_BKDR_PGM, TEGRA_PRIVATE_AHCI_CC_BKDR);
986         val &= ~TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE_EN;
987         scfg_writel(val, TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE);
988
989         /* Program config space registers: */
990
991         /* Enable BUS_MASTER+MEM+IO space, and SERR */
992         val = scfg_readl(TEGRA_SATA_IO_SPACE_OFFSET);
993         val |= TEGRA_SATA_ENABLE_IO_SPACE | TEGRA_SATA_ENABLE_MEM_SPACE |
994                TEGRA_SATA_ENABLE_BUS_MASTER | TEGRA_SATA_ENABLE_SERR;
995         scfg_writel(val, TEGRA_SATA_IO_SPACE_OFFSET);
996
997         /* program bar5 space, by first writing 1's to bar5 register */
998         scfg_writel(TEGRA_SATA_BAR5_INIT_PROGRAM, AHCI_BAR5_CONFIG_LOCATION);
999         /* flush */
1000         val = scfg_readl(AHCI_BAR5_CONFIG_LOCATION);
1001
1002         /* then, write the BAR5_FINAL_PROGRAM address */
1003         scfg_writel(TEGRA_SATA_BAR5_FINAL_PROGRAM, AHCI_BAR5_CONFIG_LOCATION);
1004         /* flush */
1005         scfg_readl(AHCI_BAR5_CONFIG_LOCATION);
1006
1007         sata_writel((FPCI_BAR5_0_FINAL_VALUE >> 8),
1008                         SATA_FPCI_BAR5_0_OFFSET);
1009
1010         val = scfg_readl(T_SATA0_AHCI_HBA_CAP_BKDR);
1011         val |= (HOST_CAP_ALPM | HOST_CAP_SSC | HOST_CAP_PART);
1012         scfg_writel(val, T_SATA0_AHCI_HBA_CAP_BKDR);
1013
1014         /* Second Level Clock Gating*/
1015         val = bar5_readl(AHCI_HBA_PLL_CTRL_0);
1016         val |= (CLAMP_TXCLK_ON_SLUMBER | CLAMP_TXCLK_ON_DEVSLP);
1017         val &= ~NO_CLAMP_SHUT_DOWN;
1018         bar5_writel(val, AHCI_HBA_PLL_CTRL_0);
1019
1020         /* enable Interrupt channel */
1021         val = pictlr_readl(PRI_ICTLR_CPU_IER_SET_0_OFFSET);
1022         val |= CPU_IER_SATA_CTL;
1023         pictlr_writel(val, PRI_ICTLR_CPU_IER_SET_0_OFFSET);
1024
1025         /* set IP_INT_MASK */
1026         val = sata_readl(SATA_INTR_MASK_0_OFFSET);
1027         val |= IP_INT_MASK;
1028         sata_writel(val, SATA_INTR_MASK_0_OFFSET);
1029
1030 exit:
1031         if (!IS_ERR_OR_NULL(clk_pllp))
1032                 clk_put(clk_pllp);
1033         if (!IS_ERR_OR_NULL(clk_sata))
1034                 clk_put(clk_sata);
1035         if (!IS_ERR_OR_NULL(clk_sata_oob))
1036                 clk_put(clk_sata_oob);
1037         if (!IS_ERR_OR_NULL(clk_sata_cold))
1038                 clk_put(clk_sata_cold);
1039         if (!IS_ERR_OR_NULL(clk_cml1))
1040                 clk_put(clk_cml1);
1041
1042         if (err) {
1043                 /* turn off all SATA power rails; ignore returned status */
1044                 tegra_ahci_power_off_rails(tegra_hpriv->power_rails);
1045                 /* return regulators to system */
1046                 tegra_ahci_put_rails(tegra_hpriv->power_rails);
1047         }
1048         return err;
1049 }
1050
1051 static void tegra_ahci_save_initial_config(struct platform_device *pdev,
1052                                            struct ahci_host_priv *hpriv)
1053 {
1054         ahci_save_initial_config(&pdev->dev, hpriv, 0, 0);
1055 }
1056
1057 static void tegra_ahci_controller_remove(struct platform_device *pdev)
1058 {
1059         struct ata_host *host = dev_get_drvdata(&pdev->dev);
1060         struct tegra_ahci_host_priv *tegra_hpriv;
1061         int status;
1062
1063         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1064
1065 #ifdef CONFIG_PM
1066         /* call tegra_ahci_controller_suspend() to power-down the SATA */
1067         status = tegra_ahci_controller_suspend(pdev);
1068         if (status)
1069                 dev_err(host->dev, "remove: error suspend SATA (0x%x)\n",
1070                                    status);
1071 #else
1072         /* power off the sata */
1073         status = tegra_powergate_partition_with_clk_off(TEGRA_POWERGATE_SATA);
1074         if (status)
1075                 dev_err(host->dev, "remove: error turn-off SATA (0x%x)\n",
1076                                    status);
1077         tegra_ahci_power_off_rails(tegra_hpriv->power_rails);
1078 #endif
1079
1080         /* return system resources */
1081         tegra_ahci_put_rails(tegra_hpriv->power_rails);
1082 }
1083
1084 #ifdef CONFIG_PM
1085 static int tegra_ahci_controller_suspend(struct platform_device *pdev)
1086 {
1087         struct ata_host *host = dev_get_drvdata(&pdev->dev);
1088         struct tegra_ahci_host_priv *tegra_hpriv;
1089         struct tegra_ahci_platform_data *ahci_pdata;
1090         unsigned long flags;
1091
1092         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1093         ahci_pdata = tegra_hpriv->dev->platform_data;
1094
1095         /* stop the idle timer */
1096         if (timer_pending(&tegra_hpriv->idle_timer))
1097                 del_timer_sync(&tegra_hpriv->idle_timer);
1098
1099         spin_lock_irqsave(&host->lock, flags);
1100         if (tegra_hpriv->pg_state == SATA_OFF)
1101                 dev_dbg(host->dev, "suspend: SATA already power gated\n");
1102         else {
1103                 bool pg_ok;
1104
1105                 dev_dbg(host->dev, "suspend: power gating SATA...\n");
1106                 pg_ok = tegra_ahci_power_gate(host);
1107                 if (pg_ok) {
1108                         tegra_hpriv->pg_state = SATA_OFF;
1109                         dev_dbg(host->dev, "suspend: SATA is power gated\n");
1110                 } else {
1111                         tegra_ahci_abort_power_gate(host);
1112                         spin_unlock_irqrestore(&host->lock, flags);
1113                         return -EBUSY;
1114                 }
1115         }
1116
1117         if (ahci_pdata->pexp_gpio)
1118                 gpio_free(ahci_pdata->pexp_gpio);
1119         spin_unlock_irqrestore(&host->lock, flags);
1120
1121         return tegra_ahci_power_off_rails(tegra_hpriv->power_rails);
1122 }
1123
1124 static int tegra_ahci_controller_resume(struct platform_device *pdev)
1125 {
1126         struct ata_host *host = dev_get_drvdata(&pdev->dev);
1127         struct tegra_ahci_host_priv *tegra_hpriv;
1128         unsigned long flags;
1129         int err;
1130
1131         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1132
1133         err = tegra_ahci_power_on_rails(tegra_hpriv->power_rails);
1134         if (err) {
1135                 pr_err("%s: fails to power on rails (%d)\n", __func__, err);
1136                 return err;
1137         }
1138
1139         spin_lock_irqsave(&host->lock, flags);
1140         if (tegra_hpriv->pg_state == SATA_ON) {
1141                 dev_dbg(host->dev, "resume: SATA already powered on\n");
1142         } else {
1143                 dev_dbg(host->dev, "resume: powering on SATA...\n");
1144                 tegra_ahci_power_un_gate(host);
1145                 tegra_hpriv->pg_state = SATA_ON;
1146         }
1147         spin_unlock_irqrestore(&host->lock, flags);
1148
1149         return 0;
1150 }
1151
1152 static int tegra_ahci_suspend(struct platform_device *pdev, pm_message_t mesg)
1153 {
1154         struct ata_host *host = dev_get_drvdata(&pdev->dev);
1155         void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1156         u32 ctl;
1157         int rc;
1158
1159         dev_dbg(host->dev, "** entering %s: **\n", __func__);
1160         if (mesg.event & PM_EVENT_SLEEP) {
1161                 /*
1162                  * AHCI spec rev1.1 section 8.3.3:
1163                  * Software must disable interrupts prior to requesting a
1164                  * transition of the HBA to D3 state.
1165                  */
1166                 ctl = readl(mmio + HOST_CTL);
1167                 ctl &= ~HOST_IRQ_EN;
1168                 writel(ctl, mmio + HOST_CTL);
1169                 readl(mmio + HOST_CTL); /* flush */
1170         }
1171
1172         rc = ata_host_suspend(host, mesg);
1173         if (rc)
1174                 return rc;
1175
1176         return tegra_ahci_controller_suspend(pdev);
1177 }
1178
1179 static int tegra_ahci_resume(struct platform_device *pdev)
1180 {
1181         struct ata_host *host = dev_get_drvdata(&pdev->dev);
1182         int rc;
1183
1184         dev_dbg(host->dev, "** entering %s: **\n", __func__);
1185         rc = tegra_ahci_controller_resume(pdev);
1186         if (rc)
1187                 return rc;
1188
1189         rc = tegra_ahci_controller_init(g_tegra_hpriv, 1);
1190         if (rc != 0) {
1191                 dev_err(host->dev, "TEGRA SATA init failed in resume\n");
1192                 return rc;
1193         }
1194
1195         if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
1196                 rc = ahci_reset_controller(host);
1197                 if (rc)
1198                         return rc;
1199
1200                 ahci_init_controller(host);
1201         }
1202
1203         ata_host_resume(host);
1204
1205         return 0;
1206 }
1207
1208 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
1209 static int tegra_ahci_runtime_suspend(struct device *dev)
1210 {
1211         struct ata_host *host;
1212         struct tegra_ahci_host_priv *tegra_hpriv;
1213         bool pg_ok;
1214         unsigned long flags;
1215         int err = 0;
1216
1217         host = dev_get_drvdata(dev);
1218         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1219
1220         spin_lock_irqsave(&host->lock, flags);
1221
1222         switch (tegra_hpriv->pg_state) {
1223         case SATA_OFF:
1224                 dev_dbg(dev, "** rt-suspend: already power gated **\n");
1225                 break;
1226
1227         case SATA_ABORT_OFF:
1228                 dev_dbg(dev, "** rt-suspend: abort suspend **\n");
1229                 tegra_hpriv->pg_state = SATA_ON;
1230                 tegra_ahci_dequeue_qcs(tegra_hpriv);
1231                 err = -EBUSY;
1232                 break;
1233
1234         case SATA_ON:
1235         case SATA_GOING_OFF:
1236                 if (tegra_ahci_are_all_ports_idle(host)) {
1237                         /* if all ports are in idle, do power-gate */
1238                         dev_dbg(dev, "** rt-suspend: power-down sata (%u) **\n",
1239                                         tegra_hpriv->pg_state);
1240 #ifdef TEGRA_AHCI_CONTEXT_RESTORE
1241                         pg_ok = tegra_ahci_power_gate(host);
1242 #else
1243                         pg_ok = tegra_ahci_pad_suspend(host);
1244 #endif
1245                         dev_dbg(dev, "** rt-suspend: done **\n");
1246                         if (pg_ok) {
1247                                 tegra_hpriv->pg_state = SATA_OFF;
1248                         } else {
1249                                 dev_err(dev, "** rt-suspend: abort pg **\n");
1250 #ifdef TEGRA_AHCI_CONTEXT_RESTORE
1251                                 tegra_ahci_abort_power_gate(host);
1252 #else
1253                                 tegra_ahci_abort_pad_suspend(host);
1254 #endif
1255                                 tegra_hpriv->pg_state = SATA_ON;
1256                                 err = -EBUSY;
1257                         }
1258                 } else {
1259                         dev_dbg(dev, "** rt-suspend: port not idle (%u) **\n",
1260                                         tegra_hpriv->pg_state);
1261                         err = -EBUSY;
1262                 }
1263                 break;
1264
1265         case SATA_GOING_ON:
1266         default:
1267                 dev_err(dev, "** rt-suspend: bad state (%u) **\n",
1268                         tegra_hpriv->pg_state);
1269                 WARN_ON(1);
1270                 err = -EBUSY;
1271                 break;
1272
1273         }
1274
1275         spin_unlock_irqrestore(&host->lock, flags);
1276
1277         return err;
1278 }
1279
1280 static int tegra_ahci_runtime_resume(struct device *dev)
1281 {
1282         struct ata_host *host;
1283         struct tegra_ahci_host_priv *tegra_hpriv;
1284         int err = 0;
1285
1286         host = dev_get_drvdata(dev);
1287         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1288
1289         if (tegra_hpriv->pg_state == SATA_ON) {
1290                 dev_dbg(dev, "** rt-resume: already power ungated **\n");
1291                 goto exit;
1292         }
1293
1294         if ((tegra_hpriv->pg_state == SATA_OFF) ||
1295             (tegra_hpriv->pg_state == SATA_GOING_ON)) {
1296                 dev_dbg(dev, "** rt-resume: power-up sata (%u) **\n",
1297                                 tegra_hpriv->pg_state);
1298 #ifdef TEGRA_AHCI_CONTEXT_RESTORE
1299                 tegra_ahci_power_un_gate(host);
1300 #else
1301                 tegra_ahci_pad_resume(host);
1302 #endif
1303                 dev_dbg(dev, "** rt-resume: done **\n");
1304                 tegra_hpriv->pg_state = SATA_ON;
1305
1306                 /* now qc_issue all qcs in the qc_list */
1307                 tegra_ahci_dequeue_qcs(tegra_hpriv);
1308
1309         } else {
1310                 dev_err(dev, "** rt-resume: bad state (%u) **\n",
1311                                 tegra_hpriv->pg_state);
1312                 WARN_ON(1);
1313                 err = -EBUSY;
1314         }
1315
1316
1317 exit:
1318
1319         return err;
1320 }
1321
1322 #endif
1323
1324 #ifdef TEGRA_AHCI_CONTEXT_RESTORE
1325 static u16 pg_save_bar5_registers[] = {
1326         0x018,  /* T_AHCI_HBA_CCC_PORTS */
1327         0x004,  /* T_AHCI_HBA_GHC */
1328         0x014,  /* T_AHCI_HBA_CCC_CTL - OP (optional) */
1329         0x01C,  /* T_AHCI_HBA_EM_LOC */
1330         0x020   /* T_AHCI_HBA_EM_CTL - OP */
1331 };
1332
1333 static u16 pg_save_bar5_port_registers[] = {
1334         0x100,  /* T_AHCI_PORT_PXCLB */
1335         0x104,  /* T_AHCI_PORT_PXCLBU */
1336         0x108,  /* T_AHCI_PORT_PXFB */
1337         0x10C,  /* T_AHCI_PORT_PXFBU */
1338         0x114,  /* T_AHCI_PORT_PXIE */
1339         0x118,  /* T_AHCI_PORT_PXCMD */
1340         0x12C,  /* T_AHCI_PORT_PXSCTL */
1341         0x144   /* T_AHCI_PORT_PXDEVSLP */
1342 };
1343
1344 /*
1345  * pg_save_bar5_bkdr_registers:
1346  *    These registers in BAR5 are read only.
1347  * To restore back those register values, write the saved value
1348  *    to the registers specified in pg_restore_bar5_bkdr_registers[].
1349  *    These pg_restore_bar5_bkdr_registers[] are in SATA_CONFIG space.
1350  */
1351 static u16 pg_save_bar5_bkdr_registers[] = {
1352         /* Save and restore via bkdr writes */
1353         0x000,  /* T_AHCI_HBA_CAP */
1354         0x00C,  /* T_AHCI_HBA_PI */
1355         0x024   /* T_AHCI_HBA_CAP2 */
1356 };
1357
1358 static u16 pg_restore_bar5_bkdr_registers[] = {
1359         /* Save and restore via bkdr writes */
1360         0x300,  /* BKDR of T_AHCI_HBA_CAP */
1361         0x33c,  /* BKDR of T_AHCI_HBA_PI */
1362         0x330   /* BKDR of T_AHCI_HBA_CAP2 */
1363 };
1364
1365 /* These registers are saved for each port */
1366 static u16 pg_save_bar5_bkdr_port_registers[] = {
1367         0x120,  /* NV_PROJ__SATA0_CHX_AHCI_PORT_PXTFD  */
1368         0x124,  /* NV_PROJ__SATA0_CHX_AHCI_PORT_PXSIG */
1369         0x128   /* NV_PROJ__SATA0_CHX_AHCI_PORT_PXSSTS */
1370 };
1371
1372 static u16 pg_restore_bar5_bkdr_port_registers[] = {
1373         /* Save and restore via bkdr writes */
1374         0x790,  /* BKDR of NV_PROJ__SATA0_CHX_AHCI_PORT_PXTFD  */
1375         0x794,  /* BKDR of NV_PROJ__SATA0_CHX_AHCI_PORT_PXSIG */
1376         0x798   /* BKDR of NV_PROJ__SATA0_CHX_AHCI_PORT_PXSSTS */
1377 };
1378
1379 static u16 pg_save_config_registers[] = {
1380         0x004,  /* T_SATA0_CFG_1 */
1381         0x00C,  /* T_SATA0_CFG_3 */
1382         0x024,  /* T_SATA0_CFG_9 */
1383         0x028,  /* T_SATA0_CFG_10 */
1384         0x030,  /* T_SATA0_CFG_12 */
1385         0x034,  /* T_SATA0_CFG_13 */
1386         0x038,  /* T_SATA0_CFG_14 */
1387         0x03C,  /* T_SATA0_CFG_15 */
1388         0x040,  /* T_SATA0_CFG_16 */
1389         0x044,  /* T_SATA0_CFG_17 */
1390         0x048,  /* T_SATA0_CFG_18 */
1391         0x0B0,  /* T_SATA0_MSI_CTRL */
1392         0x0B4,  /* T_SATA0_MSI_ADDR1 */
1393         0x0B8,  /* T_SATA0_MSI_ADDR2 */
1394         0x0BC,  /* T_SATA0_MSI_DATA */
1395         0x0C0,  /* T_SATA0_MSI_QUEUE */
1396         0x0EC,  /* T_SATA0_MSI_MAP */
1397         0x124,  /* T_SATA0_CFG_PHY_POWER */
1398         0x128,  /* T_SATA0_CFG_PHY_POWER_1 */
1399         0x12C,  /* T_SATA0_CFG_PHY_1 */
1400         0x174,  /* T_SATA0_CFG_LINK_0 */
1401         0x178,  /* T_SATA0_CFG_LINK_1 */
1402         0x1D0,  /* MCP_SATA0_CFG_TRANS_0 */
1403         0x238,  /* T_SATA0_ALPM_CTRL */
1404         0x30C,  /* T_SATA0_AHCI_HBA_CYA_0 */
1405         0x320,  /* T_SATA0_AHCI_HBA_SPARE_1 */
1406         0x324,  /* T_SATA0_AHCI_HBA_SPARE_2 */
1407         0x328,  /* T_SATA0_AHCI_HBA_DYN_CLK_CLAMP */
1408         0x32C,  /* T_SATA0_AHCI_CFG_ERR_CTRL */
1409         0x338,  /* T_SATA0_AHCI_HBA_CYA_1 */
1410         0x340,  /* T_SATA0_AHCI_HBA_PRE_STAGING_CONTROL */
1411         0x430,  /* T_SATA0_CFG_FPCI_0 */
1412         0x494,  /* T_SATA0_CFG_ESATA_CTRL */
1413         0x4A0,  /* T_SATA0_CYA1 */
1414         0x4B0,  /* T_SATA0_CFG_GLUE */
1415         0x534,  /* T_SATA0_PHY_CTRL */
1416         0x540,  /* T_SATA0_CTRL */
1417         0x550,  /* T_SATA0_DBG0 */
1418         0x554   /* T_SATA0_LOW_POWER_COUNT */
1419 };
1420
1421 static u16 pg_save_config_port_registers[] = {
1422         /* Save and restore per port */
1423         /* need to have port selected */
1424         0x530,  /* T_SATA0_CHXCFG1 */
1425         0x684,  /* T_SATA0_CHX_MISC */
1426         0x700,  /* T_SATA0_CHXCFG3 */
1427         0x704,  /* T_SATA0_CHXCFG4_CHX */
1428         0x690,  /* T_SATA0_CHX_PHY_CTRL1_GEN1 */
1429         0x694,  /* T_SATA0_CHX_PHY_CTRL1_GEN2 */
1430         0x698,  /* T_SATA0_CHX_PHY_CTRL1_GEN3 */
1431         0x69C,  /* T_SATA0_CHX_PHY_CTRL_2 */
1432         0x6B0,  /* T_SATA0_CHX_PHY_CTRL_3 */
1433         0x6B4,  /* T_SATA0_CHX_PHY_CTRL_4 */
1434         0x6B8,  /* T_SATA0_CHX_PHY_CTRL_5 */
1435         0x6BC,  /* T_SATA0_CHX_PHY_CTRL_6 */
1436         0x714,  /* T_SATA0_PRBS_CHX - OP */
1437         0x750,  /* T_SATA0_CHX_LINK0 */
1438         0x7F0   /* T_SATA0_CHX_GLUE */
1439 };
1440
1441 static u16 pg_save_ipfs_registers[] = {
1442         0x094,  /* SATA_FPCI_BAR5_0 */
1443         0x0C0,  /* SATA_MSI_BAR_SZ_0 */
1444         0x0C4,  /* SATA_MSI_AXI_BAR_ST_0 */
1445         0x0C8,  /* SATA_MSI_FPCI_BAR_ST_0 */
1446         0x140,  /* SATA_MSI_EN_VEC0_0 */
1447         0x144,  /* SATA_MSI_EN_VEC1_0 */
1448         0x148,  /* SATA_MSI_EN_VEC2_0 */
1449         0x14C,  /* SATA_MSI_EN_VEC3_0 */
1450         0x150,  /* SATA_MSI_EN_VEC4_0 */
1451         0x154,  /* SATA_MSI_EN_VEC5_0 */
1452         0x158,  /* SATA_MSI_EN_VEC6_0 */
1453         0x15C,  /* SATA_MSI_EN_VEC7_0 */
1454         0x180,  /* SATA_CONFIGURATION_0 */
1455         0x184,  /* SATA_FPCI_ERROR_MASKS_0 */
1456         0x188,  /* SATA_INTR_MASK_0 */
1457         0x1A0,  /* SATA_CFG_REVID_0 */
1458         0x198,  /* SATA_IPFS_INTR_ENABLE_0 */
1459         0x1BC,  /* SATA_CLKGATE_HYSTERSIS_0 */
1460         0x1DC   /* SATA_SATA_MCCIF_FIFOCTRL_0 */
1461 };
1462
1463 static void tegra_ahci_save_regs(u32 **save_addr,
1464                                  u32 reg_base,
1465                                  u16 reg_array[],
1466                                  u32 regs)
1467 {
1468         u32 i;
1469         u32 *dest = *save_addr;
1470         void __iomem *base = IO_ADDRESS(reg_base);
1471
1472         for (i = 0; i < regs; ++i, ++dest) {
1473                 *dest = readl(base + (u32)reg_array[i]);
1474                 AHCI_DBG_PRINT("save: [0x%x]=0x%08x\n",
1475                                (reg_base+(u32)reg_array[i]), *dest);
1476         }
1477         *save_addr = dest;
1478 }
1479
1480 static void tegra_ahci_restore_regs(void **save_addr,
1481                                     u32 reg_base,
1482                                     u16 reg_array[],
1483                                     u32 regs)
1484 {
1485         u32 i;
1486         u32 *src = *save_addr;
1487         void __iomem *base = IO_ADDRESS(reg_base);
1488
1489         for (i = 0; i < regs; ++i, ++src) {
1490                 writel(*src, base + (u32)reg_array[i]);
1491                 AHCI_DBG_PRINT("restore: [0x%x]=0x%08x\n",
1492                                 (reg_base+(u32)reg_array[i]), *src);
1493         }
1494         *save_addr = src;
1495 }
1496
1497 static void tegra_ahci_pg_save_registers(struct ata_host *host)
1498 {
1499         struct tegra_ahci_host_priv *tegra_hpriv;
1500         u32 *pg_save;
1501         u32 regs;
1502         int i;
1503
1504         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1505         pg_save = tegra_hpriv->pg_save;
1506
1507         /*
1508          * Driver should save/restore the registers in the order of
1509          * IPFS, CFG, Ext CFG, BAR5.
1510          */
1511
1512         /* save IPFS registers */
1513         regs = ARRAY_SIZE(pg_save_ipfs_registers);
1514         tegra_ahci_save_regs(&pg_save, TEGRA_SATA_BASE,
1515                              pg_save_ipfs_registers, regs);
1516         /* after the call, pg_save should point to the next address to save */
1517
1518         /* save CONFIG registers */
1519         regs = ARRAY_SIZE(pg_save_config_registers);
1520         tegra_ahci_save_regs(&pg_save, TEGRA_SATA_CONFIG_BASE,
1521                              pg_save_config_registers, regs);
1522
1523         /* save CONFIG per port registers */
1524         for (i = 0; i < TEGRA_AHCI_NUM_PORTS; ++i) {
1525                 scfg_writel((1 << i), T_SATA0_INDEX_OFFSET);
1526                 regs = ARRAY_SIZE(pg_save_config_port_registers);
1527                 tegra_ahci_save_regs(&pg_save, TEGRA_SATA_CONFIG_BASE,
1528                                      pg_save_config_port_registers, regs);
1529         }
1530         scfg_writel(SATA0_NONE_SELECTED, T_SATA0_INDEX_OFFSET);
1531
1532         /* save BAR5 registers */
1533         regs = ARRAY_SIZE(pg_save_bar5_registers);
1534         tegra_ahci_save_regs(&pg_save, TEGRA_SATA_BAR5_BASE,
1535                              pg_save_bar5_registers, regs);
1536
1537         /* save BAR5 port_registers */
1538         regs = ARRAY_SIZE(pg_save_bar5_port_registers);
1539         for (i = 0; i < TEGRA_AHCI_NUM_PORTS; ++i)
1540                 tegra_ahci_save_regs(&pg_save, TEGRA_SATA_BAR5_BASE + (0x80*i),
1541                                      pg_save_bar5_port_registers, regs);
1542
1543         /* save bkdr registers */
1544         regs = ARRAY_SIZE(pg_save_bar5_bkdr_registers);
1545         tegra_ahci_save_regs(&pg_save, TEGRA_SATA_BAR5_BASE,
1546                              pg_save_bar5_bkdr_registers, regs);
1547
1548         /* and save bkdr per_port registers */
1549         for (i = 0; i < TEGRA_AHCI_NUM_PORTS; ++i) {
1550                 scfg_writel((1 << i), T_SATA0_INDEX_OFFSET);
1551                 regs = ARRAY_SIZE(pg_save_bar5_bkdr_port_registers);
1552                 tegra_ahci_save_regs(&pg_save, TEGRA_SATA_BAR5_BASE + (0x80*i),
1553                                      pg_save_bar5_bkdr_port_registers,
1554                                      regs);
1555         }
1556         scfg_writel(SATA0_NONE_SELECTED, T_SATA0_INDEX_OFFSET);
1557 }
1558
1559 static void tegra_ahci_pg_restore_registers(struct ata_host *host)
1560 {
1561         struct tegra_ahci_host_priv *tegra_hpriv;
1562         void *pg_save;
1563         u32 regs, val;
1564         int i;
1565
1566         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1567         pg_save = tegra_hpriv->pg_save;
1568
1569         /*
1570          * Driver should restore the registers in the order of
1571          * IPFS, CFG, Ext CFG, BAR5.
1572          */
1573
1574         /* restore IPFS registers */
1575         regs = ARRAY_SIZE(pg_save_ipfs_registers);
1576         tegra_ahci_restore_regs(&pg_save, TEGRA_SATA_BASE,
1577                                 pg_save_ipfs_registers, regs);
1578         /* after the call, pg_save should point to the next addr to restore */
1579
1580         /* restore CONFIG registers */
1581         regs = ARRAY_SIZE(pg_save_config_registers);
1582         tegra_ahci_restore_regs(&pg_save, TEGRA_SATA_CONFIG_BASE,
1583                                 pg_save_config_registers, regs);
1584
1585         /* restore CONFIG per port registers */
1586         for (i = 0; i < TEGRA_AHCI_NUM_PORTS; ++i) {
1587                 scfg_writel((1 << i), T_SATA0_INDEX_OFFSET);
1588                 regs = ARRAY_SIZE(pg_save_config_port_registers);
1589                 tegra_ahci_restore_regs(&pg_save, TEGRA_SATA_CONFIG_BASE,
1590                                         pg_save_config_port_registers,
1591                                         regs);
1592         }
1593         scfg_writel(SATA0_NONE_SELECTED, T_SATA0_INDEX_OFFSET);
1594
1595         /* restore BAR5 registers */
1596         regs = ARRAY_SIZE(pg_save_bar5_registers);
1597         tegra_ahci_restore_regs(&pg_save, TEGRA_SATA_BAR5_BASE,
1598                                 pg_save_bar5_registers, regs);
1599
1600         /* restore BAR5 port_registers */
1601         regs = ARRAY_SIZE(pg_save_bar5_port_registers);
1602         for (i = 0; i < TEGRA_AHCI_NUM_PORTS; ++i)
1603                 tegra_ahci_restore_regs(&pg_save, TEGRA_SATA_BAR5_BASE+(0x80*i),
1604                                         pg_save_bar5_port_registers, regs);
1605
1606         /* restore bkdr registers */
1607         regs = ARRAY_SIZE(pg_restore_bar5_bkdr_registers);
1608         tegra_ahci_restore_regs(&pg_save, TEGRA_SATA_CONFIG_BASE,
1609                              pg_restore_bar5_bkdr_registers, regs);
1610
1611         /* and restore BAR5 bkdr per_port registers */
1612         for (i = 0; i < TEGRA_AHCI_NUM_PORTS; ++i) {
1613                 scfg_writel((1 << i), T_SATA0_INDEX_OFFSET);
1614                 regs = ARRAY_SIZE(pg_restore_bar5_bkdr_port_registers);
1615                 tegra_ahci_restore_regs(&pg_save, TEGRA_SATA_CONFIG_BASE,
1616                                         pg_restore_bar5_bkdr_port_registers,
1617                                         regs);
1618         }
1619         scfg_writel(SATA0_NONE_SELECTED, T_SATA0_INDEX_OFFSET);
1620
1621         /* program class code and programming interface for AHCI */
1622         val = scfg_readl(TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE);
1623         val |= TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE_EN;
1624         scfg_writel(val, TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE);
1625         scfg_writel(TEGRA_PRIVATE_AHCI_CC_BKDR_PGM, TEGRA_PRIVATE_AHCI_CC_BKDR);
1626         val &= ~TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE_EN;
1627         scfg_writel(val, TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE);
1628 }
1629 #endif
1630 static u32 tegra_ahci_port_error(struct ata_port *ap)
1631 {
1632         void __iomem *port_mmio = ahci_port_base(ap);
1633         u32 err_status;
1634
1635         err_status = readl(port_mmio + PORT_IRQ_STAT);
1636         /* excludes PhyRdy and Connect Change status */
1637         err_status &= (PORT_IRQ_ERROR & (~(PORT_IRQ_PHYRDY|PORT_IRQ_CONNECT)));
1638         return err_status;
1639 }
1640
1641 static bool tegra_ahci_check_errors(struct ata_host *host)
1642 {       int i;
1643         struct ata_port *ap;
1644         u32 err;
1645
1646         for (i = 0; i < host->n_ports; i++) {
1647                 ap = host->ports[i];
1648                 err = tegra_ahci_port_error(ap);
1649                 if (err) {
1650                         dev_err(host->dev,
1651                                 "pg-chk-err = 0x%08x on port %d\n", err, i);
1652                         return true;
1653                 }
1654         }
1655         return false;
1656 }
1657
1658 void tegra_ahci_iddqlane_config(void)
1659 {
1660         u32 val;
1661         u32 dat;
1662
1663         /* wait for SATA_PADPLL_IDDQ2LANE_SLUMBER_DLY = 3 microseconds. */
1664         val = clk_readl(CLK_RST_SATA_PLL_CFG1_REG);
1665         val &= ~IDDQ2LANE_SLUMBER_DLY_MASK;
1666         val |= IDDQ2LANE_SLUMBER_DLY_3MS;
1667         clk_writel(val, CLK_RST_SATA_PLL_CFG1_REG);
1668
1669         /* get sata phy and pll out of iddq: */
1670         val = pmc_readl(APB_PMC_SATA_PWRGT_0_REG);
1671         val &= ~(PADPLL_IDDQ_OVERRIDE_VALUE_MASK | PADPLL_IDDQ_SWCTL_MASK);
1672         pmc_writel(val, APB_PMC_SATA_PWRGT_0_REG);
1673
1674         /* wait for delay of IDDQ2LAND_SLUMBER_DLY */
1675         val = clk_readl(CLK_RST_SATA_PLL_CFG1_REG);
1676         dat = (val & IDDQ2LANE_SLUMBER_DLY_MASK) >> IDDQ2LANE_SLUMBER_DLY_SHIFT;
1677         udelay(dat);
1678         val = pmc_readl(APB_PMC_SATA_PWRGT_0_REG);
1679         val &= ~(PADPHY_IDDQ_OVERRIDE_VALUE_MASK | PADPHY_IDDQ_SWCTL_MASK);
1680         pmc_writel(val, APB_PMC_SATA_PWRGT_0_REG);
1681
1682 }
1683 void tegra_ahci_put_sata_in_iddq()
1684 {
1685         u32 val;
1686         u32 dat;
1687
1688         /*
1689          * Hw wake up is not needed:
1690          * Driver/RM shall place the SATA PHY and SATA PADPLL in IDDQ.
1691          * SATA_PADPLL_RESET_SWCTL =1
1692          * SATA_PADPLL_RESET_OVERRIDE_VALUE=1
1693          * SATA_PADPHY_IDDQ_SWCTL=1
1694          * SATA_PADPHY_IDDQ_OVERRIDE_VALUE=1
1695          */
1696
1697         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
1698         val &= ~(PADPLL_RESET_SWCTL_MASK | PADPLL_RESET_OVERRIDE_VALUE_MASK);
1699         val |= (PADPLL_RESET_SWCTL_ON | PADPLL_RESET_OVERRIDE_VALUE_ON);
1700         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
1701
1702         val = pmc_readl(APB_PMC_SATA_PWRGT_0_REG);
1703         val &= ~(PADPHY_IDDQ_OVERRIDE_VALUE_MASK | PADPHY_IDDQ_SWCTL_MASK);
1704         val |= (PADPHY_IDDQ_SWCTL_ON | PADPHY_IDDQ_OVERRIDE_VALUE_ON);
1705         pmc_writel(val, APB_PMC_SATA_PWRGT_0_REG);
1706
1707         /* Wait for time specified in SATA_LANE_IDDQ2_PADPLL_IDDQ */
1708         val = clk_readl(CLK_RST_SATA_PLL_CFG1_REG);
1709         dat = (val & IDDQ2LANE_IDDQ_DLY_MASK) >> IDDQ2LANE_IDDQ_DLY_SHIFT;
1710         udelay(dat);
1711
1712         /* SATA_PADPLL_IDDQ_SWCTL=1 & SATA_PADPLL_IDDQ_OVERRIDE_VALUE=1 */
1713         val = pmc_readl(APB_PMC_SATA_PWRGT_0_REG);
1714         val &= ~(PADPLL_IDDQ_OVERRIDE_VALUE_MASK | PADPLL_IDDQ_SWCTL_MASK);
1715         val |= (PADPLL_IDDQ_SWCTL_ON | PADPLL_IDDQ_OVERRIDE_VALUE_ON);
1716         pmc_writel(val, APB_PMC_SATA_PWRGT_0_REG);
1717
1718 }
1719 void tegra_ahci_clr_clk_rst_cnt_rst_dev(void)
1720 {
1721         u32 val;
1722
1723         val = clk_readl(CLK_RST_CONTROLLER_RST_DEV_V_CLR_0);
1724         val |= (SWR_SATA_OOB_RST | SWR_SATA_RST);
1725         clk_writel(val, CLK_RST_CONTROLLER_RST_DEV_V_CLR_0);
1726
1727         val = clk_readl(CLK_RST_CONTROLLER_RST_DEV_W_CLR_0);
1728         val |= SWR_SATACOLD_RST;
1729         clk_writel(val, CLK_RST_CONTROLLER_RST_DEV_W_CLR_0);
1730
1731 }
1732 void tegra_ahci_set_clk_rst_cnt_rst_dev(void)
1733 {
1734
1735         u32 val;
1736
1737         val = clk_readl(CLK_RST_CONTROLLER_RST_DEV_V_CLR_0);
1738         val &= ~(SWR_SATA_OOB_RST | SWR_SATA_RST);
1739         clk_writel(val, CLK_RST_CONTROLLER_RST_DEV_V_CLR_0);
1740
1741         val = clk_readl(CLK_RST_CONTROLLER_RST_DEV_W_CLR_0);
1742         val &= ~SWR_SATACOLD_RST;
1743         clk_writel(val, CLK_RST_CONTROLLER_RST_DEV_W_CLR_0);
1744
1745 }
1746
1747 static void tegra_ahci_pad_config(void)
1748 {
1749         u32 val;
1750
1751         /* clear SW control of SATA PADPLL, SATA PHY and PLLE */
1752
1753         /* for SATA PHY IDDQ */
1754         val = xusb_readl(XUSB_PADCTL_IOPHY_MISC_PAD_S0_CTL_1_0);
1755         val &= ~(IDDQ_OVRD_MASK | IDDQ_MASK);
1756         xusb_writel(val, XUSB_PADCTL_IOPHY_MISC_PAD_S0_CTL_1_0);
1757
1758         /* for SATA PADPLL IDDQ */
1759         val = xusb_readl(XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0);
1760         val &= ~(PLL_PWR_OVRD_MASK | PLL_IDDQ_MASK | PLL_RST_MASK);
1761         xusb_writel(val, XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0);
1762
1763         /* PLLE related stuff*/
1764
1765         val = clk_readl(CLK_RST_CONTROLLER_PLLE_MISC_0);
1766         val &= ~(T124_PLLE_IDDQ_SWCTL_MASK | PLLE_IDDQ_OVERRIDE_VALUE_MASK);
1767         clk_writel(val, CLK_RST_CONTROLLER_PLLE_MISC_0);
1768
1769         clk_writel(CLK_RST_CONTROLLER_PLLE_MISC_0_VALUE,
1770                         CLK_RST_CONTROLLER_PLLE_MISC_0);
1771
1772         val = clk_readl(CLK_RST_CONTROLLER_PLLE_BASE_0);
1773         val |= PLLE_ENABLE;
1774         clk_writel(val, CLK_RST_CONTROLLER_PLLE_BASE_0);
1775
1776 }
1777
1778 static void tegra_ahci_abort_power_gate(struct ata_host *host)
1779 {
1780         u32 val;
1781         val = pmc_readl(APB_PMC_SATA_PWRGT_0_REG);
1782         val &= ~PG_INFO_MASK;
1783         pmc_writel(val, APB_PMC_SATA_PWRGT_0_REG);
1784 }
1785
1786 static bool tegra_ahci_power_gate(struct ata_host *host)
1787 {
1788         u32 val;
1789         u32 dat;
1790         struct tegra_ahci_host_priv *tegra_hpriv;
1791         int status;
1792
1793         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1794
1795         val = pmc_readl(APB_PMC_SATA_PWRGT_0_REG);
1796         val &= ~PG_INFO_MASK;
1797         val |= PG_INFO_ON;
1798         pmc_writel(val, APB_PMC_SATA_PWRGT_0_REG);
1799
1800 #ifdef TEGRA_AHCI_CONTEXT_RESTORE
1801         tegra_ahci_pg_save_registers(host);
1802 #endif
1803         val = xusb_readl(XUSB_PADCTL_IOPHY_MISC_PAD_S0_CTL_1_0);
1804         val |= XUSB_PADCTL_IOPHY_MISC_IDDQ |
1805                 XUSB_PADCTL_IOPHY_MISC_IDDQ_OVRD;
1806         xusb_writel(val, XUSB_PADCTL_IOPHY_MISC_PAD_S0_CTL_1_0);
1807
1808         /*
1809          * Read SATA_AUX_MISC_CNTL_1_0 register L0_RX_IDLE_T_SAX field and
1810          * write that value into same register L0_RX_IDLE_T_NPG field.
1811          * And write 1 to L0_RX_IDLE_T_MUX field.
1812          */
1813         val = misc_readl(SATA_AUX_MISC_CNTL_1_REG);
1814         dat = val;
1815         dat &= L0_RX_IDLE_T_SAX_MASK;
1816         dat >>= L0_RX_IDLE_T_SAX_SHIFT;
1817         dat <<= L0_RX_IDLE_T_NPG_SHIFT;
1818         val &= ~L0_RX_IDLE_T_NPG_MASK;
1819         val |= dat;
1820         val |= L0_RX_IDLE_T_MUX_FROM_APB_MISC;
1821         val |= DEVSLP_OVERRIDE;
1822         misc_writel(val, SATA_AUX_MISC_CNTL_1_REG);
1823
1824         tegra_ahci_set_clk_rst_cnt_rst_dev();
1825
1826         /* abort PG if there are errors occurred */
1827         if (tegra_ahci_check_errors(host)) {
1828                 dev_err(host->dev, "** pg: errors; abort power gating **\n");
1829                 return false;
1830         }
1831         /* make sure all ports have no outstanding commands and are idle. */
1832         if (!tegra_ahci_are_all_ports_idle(host)) {
1833                 dev_err(host->dev, "** pg: cmds; abort power gating **\n");
1834                 return false;
1835         }
1836         tegra_ahci_put_sata_in_iddq();
1837
1838         val = pmc_readl(APBDEV_PMC_PWRGATE_TOGGLE_0);
1839         val |= PARTID_VALUE;
1840         val |= START;
1841         pmc_writel(val, APBDEV_PMC_PWRGATE_TOGGLE_0);
1842
1843         val = pmc_readl(APBDEV_PMC_PWRGATE_STATUS_0);
1844         val &= ~SAX_MASK;
1845         pmc_writel(val, APBDEV_PMC_PWRGATE_STATUS_0);
1846
1847         /* power off the sata */
1848         status = tegra_powergate_partition_with_clk_off(TEGRA_POWERGATE_SATA);
1849         if (status) {
1850                 dev_err(host->dev, "** failed to turn-off SATA (0x%x) **\n",
1851                                    status);
1852                 return false;
1853         }
1854
1855         return true;
1856 }
1857
1858 static bool tegra_ahci_power_un_gate(struct ata_host *host)
1859 {
1860         u32 val;
1861         u32 timeout;
1862         struct tegra_ahci_host_priv *tegra_hpriv;
1863         int status;
1864
1865         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1866
1867         tegra_ahci_iddqlane_config();
1868
1869         val = pmc_readl(APBDEV_PMC_PWRGATE_TOGGLE_0);
1870         val |= PARTID_VALUE;
1871         val |= START;
1872         pmc_writel(val, APBDEV_PMC_PWRGATE_TOGGLE_0);
1873
1874         val = pmc_readl(APBDEV_PMC_PWRGATE_STATUS_0);
1875         val |= SAX_MASK;
1876         pmc_writel(val, APBDEV_PMC_PWRGATE_STATUS_0);
1877
1878         status = tegra_unpowergate_partition_with_clk_on(TEGRA_POWERGATE_SATA);
1879         if (status) {
1880                 dev_err(host->dev, "** failed to turn-on SATA (0x%x) **\n",
1881                                    status);
1882                 return false;
1883         }
1884
1885         /* deasset PADPLL and wait until it locks. */
1886         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
1887         val &= ~PADPLL_RESET_OVERRIDE_VALUE_MASK;
1888         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
1889
1890         tegra_ahci_clr_clk_rst_cnt_rst_dev();
1891
1892 #ifdef TEGRA_AHCI_CONTEXT_RESTORE
1893         /* restore registers */
1894         tegra_ahci_pg_restore_registers(host);
1895 #endif
1896         tegra_ahci_pad_config();
1897
1898         /*
1899          * Wait for SATA_AUX_PAD_PLL_CNTL_1_0_LOCKDET to turn 1 with a timeout
1900          * of 15 us.
1901          */
1902         timeout = 15;
1903         while (timeout--) {
1904                 udelay(1);
1905                 val = xusb_readl(XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0);
1906                 if (val & LOCKDET_FIELD)
1907                         break;
1908         }
1909         if (timeout == 0)
1910                 pr_err("%s: SATA_PAD_PLL is not locked in 15us.\n", __func__);
1911
1912         /*
1913          * During the restoration of the registers, the driver would now need to
1914          * restore the register T_SATA0_CFG_POWER_GATE_SSTS_RESTORED after the
1915          * ssts_det, ssts_spd are restored. This register is used to tell the
1916          * controller whether a drive existed earlier or not and move the PHY
1917          * state machines into either HR_slumber or not.
1918          */
1919         val = scfg_readl(T_SATA0_CFG_POWER_GATE);
1920         val &= ~POWER_GATE_SSTS_RESTORED_MASK;
1921         val |= POWER_GATE_SSTS_RESTORED_YES;
1922         scfg_writel(val, T_SATA0_CFG_POWER_GATE);
1923
1924
1925         /*
1926          * Driver needs to switch the rx_idle_t driven source back to from
1927          * Sata controller after SAX is power-ungated.
1928          */
1929         val = misc_readl(SATA_AUX_MISC_CNTL_1_REG);
1930         val &= ~DEVSLP_OVERRIDE;
1931         val &= ~L0_RX_IDLE_T_MUX_MASK;
1932         val |= L0_RX_IDLE_T_MUX_FROM_SATA;
1933         misc_writel(val, SATA_AUX_MISC_CNTL_1_REG);
1934
1935         /*
1936          * Driver can start to use main SATA interrupt instead of the
1937          * rx_stat_t interrupt.
1938          */
1939         val = pictlr_readl(PRI_ICTLR_CPU_IER_SET_0_OFFSET);
1940         val |= CPU_IER_SATA_CTL;
1941         pictlr_writel(val, PRI_ICTLR_CPU_IER_SET_0_OFFSET);
1942
1943         /* Set the bits in the CAR to allow HW based low power sequencing. */
1944         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
1945         val &= ~PADPLL_RESET_SWCTL_MASK;
1946         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
1947
1948         /*
1949          * power un-gating process is complete by clearing
1950          * APBDEV_PMC_SATA_PWRGT_0.Pmc2sata_pg_info = 0
1951          */
1952         val = pmc_readl(APB_PMC_SATA_PWRGT_0_REG);
1953         val &= ~PG_INFO_MASK;
1954         pmc_writel(val, APB_PMC_SATA_PWRGT_0_REG);
1955
1956         return true;
1957 }
1958
1959 static enum port_idle_status tegra_ahci_is_port_idle(struct ata_port *ap)
1960 {
1961         void __iomem *port_mmio = ahci_port_base(ap);
1962
1963         if (readl(port_mmio + PORT_CMD_ISSUE) ||
1964             readl(port_mmio + PORT_SCR_ACT))
1965                 return PORT_IS_NOT_IDLE;
1966         return PORT_IS_IDLE;
1967 }
1968
1969 /* check if all supported ports are idle (no outstanding commands) */
1970 static bool tegra_ahci_are_all_ports_idle(struct ata_host *host)
1971 {       int i;
1972         struct ata_port *ap;
1973
1974         for (i = 0; i < host->n_ports; i++) {
1975                 ap = host->ports[i];
1976                 if (ap && (tegra_ahci_is_port_idle(ap) == PORT_IS_NOT_IDLE))
1977                         return false;
1978         }
1979         return true;
1980 }
1981
1982
1983 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
1984 static void tegra_ahci_abort_pad_suspend(struct ata_host *host)
1985 {
1986         /*No implementation*/
1987 }
1988
1989 static bool tegra_ahci_pad_suspend(struct ata_host *host)
1990 {
1991         u32 val;
1992         struct tegra_ahci_host_priv *tegra_hpriv;
1993
1994         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1995
1996         val = xusb_readl(XUSB_PADCTL_IOPHY_MISC_PAD_S0_CTL_1_0);
1997         val |= XUSB_PADCTL_IOPHY_MISC_IDDQ |
1998                 XUSB_PADCTL_IOPHY_MISC_IDDQ_OVRD;
1999         xusb_writel(val, XUSB_PADCTL_IOPHY_MISC_PAD_S0_CTL_1_0);
2000
2001         /* abort PG if there are errors occurred */
2002         if (tegra_ahci_check_errors(host)) {
2003                 dev_err(host->dev, "** pg: errors; abort power gating **\n");
2004                 return false;
2005         }
2006         /* make sure all ports have no outstanding commands and are idle. */
2007         if (!tegra_ahci_are_all_ports_idle(host)) {
2008                 dev_err(host->dev, "** pg: cmds; abort power gating **\n");
2009                 return false;
2010         }
2011         tegra_ahci_put_sata_in_iddq();
2012
2013         return true;
2014 }
2015
2016 static bool tegra_ahci_pad_resume(struct ata_host *host)
2017 {
2018         u32 val;
2019         u32 timeout;
2020         struct tegra_ahci_host_priv *tegra_hpriv;
2021
2022         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
2023
2024         tegra_ahci_iddqlane_config();
2025
2026         /* deasset PADPLL and wait until it locks. */
2027         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
2028         val &= ~PADPLL_RESET_OVERRIDE_VALUE_MASK;
2029         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
2030
2031         tegra_ahci_pad_config();
2032
2033         /*
2034          * Wait for SATA_AUX_PAD_PLL_CNTL_1_0_LOCKDET to turn 1 with a timeout
2035          * of 15 us.
2036          */
2037         timeout = 15;
2038         while (timeout--) {
2039                 udelay(1);
2040                 val = xusb_readl(XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0);
2041                 if (val & LOCKDET_FIELD)
2042                         break;
2043         }
2044         if (timeout == 0)
2045                 pr_err("%s: SATA_PAD_PLL is not locked in 15us.\n", __func__);
2046
2047         /*
2048          * Driver can start to use main SATA interrupt instead of the
2049          * rx_stat_t interrupt.
2050          */
2051         val = pictlr_readl(PRI_ICTLR_CPU_IER_SET_0_OFFSET);
2052         val |= CPU_IER_SATA_CTL;
2053         pictlr_writel(val, PRI_ICTLR_CPU_IER_SET_0_OFFSET);
2054
2055         /* Set the bits in the CAR to allow HW based low power sequencing. */
2056         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
2057         val &= ~PADPLL_RESET_SWCTL_MASK;
2058         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
2059
2060         return true;
2061 }
2062
2063 static void tegra_ahci_to_add_idle_timer(struct ata_host *host)
2064 {
2065         struct tegra_ahci_host_priv *tegra_hpriv;
2066
2067         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
2068
2069         /* note: the routine is called from interrupt context */
2070         spin_lock(&host->lock);
2071         /* start idle-timer if all ports have no outstanding commands */
2072         if (tegra_ahci_are_all_ports_idle(host)) {
2073                 /* adjust tegra_ahci_idle_time to minimum if it is too small */
2074                 tegra_ahci_idle_time = max((u32)TEGRA_AHCI_MIN_IDLE_TIME,
2075                                            tegra_ahci_idle_time);
2076                 tegra_hpriv->idle_timer.expires =
2077                         ata_deadline(jiffies, tegra_ahci_idle_time);
2078                 mod_timer(&tegra_hpriv->idle_timer,
2079                           tegra_hpriv->idle_timer.expires);
2080         }
2081         spin_unlock(&host->lock);
2082 }
2083
2084 static void tegra_ahci_idle_timer(unsigned long arg)
2085 {
2086         struct ata_host *host = (void *)arg;
2087         struct tegra_ahci_host_priv *tegra_hpriv;
2088         unsigned long flags;
2089
2090         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
2091
2092         spin_lock_irqsave(&host->lock, flags);
2093         if (tegra_hpriv->pg_state == SATA_ON)
2094                 tegra_hpriv->pg_state = SATA_GOING_OFF;
2095         else {
2096                 dev_err(host->dev, "idle_timer: bad state (%u)\n",
2097                                 tegra_hpriv->pg_state);
2098                 WARN_ON(1);
2099                 spin_unlock_irqrestore(&host->lock, flags);
2100                 return;
2101         }
2102         spin_unlock_irqrestore(&host->lock, flags);
2103         tegra_ahci_runtime_suspend(tegra_hpriv->dev);
2104 }
2105
2106 static int tegra_ahci_queue_one_qc(struct tegra_ahci_host_priv *tegra_hpriv,
2107                                    struct ata_queued_cmd *qc)
2108 {
2109         struct tegra_qc_list *qc_list;
2110
2111         qc_list = kmalloc(sizeof(struct tegra_qc_list), GFP_ATOMIC);
2112         if (!qc_list) {
2113                 dev_err(tegra_hpriv->dev, "failed to alloc qc_list\n");
2114                 return AC_ERR_SYSTEM;
2115         }
2116         qc_list->qc = qc;
2117         list_add_tail(&(qc_list->list), &(tegra_hpriv->qc_list));
2118         dev_dbg(tegra_hpriv->dev, "queuing qc=%x\n", (unsigned int)qc);
2119         return 0;
2120 }
2121
2122 static void tegra_ahci_dequeue_qcs(struct tegra_ahci_host_priv *tegra_hpriv)
2123 {
2124         struct list_head *list, *next;
2125         struct tegra_qc_list *qc_list;
2126         struct ata_queued_cmd *qc;
2127
2128         /* now qc_issue all qcs in the qc_list */
2129         list_for_each_safe(list, next, &tegra_hpriv->qc_list) {
2130                 qc_list = list_entry(list, struct tegra_qc_list, list);
2131                 qc = qc_list->qc;
2132                 dev_dbg(tegra_hpriv->dev, "dequeue qc=%x\n", (unsigned int)qc);
2133                 ahci_ops.qc_issue(qc);
2134                 list_del(list);
2135                 kfree(qc_list);
2136         }
2137 }
2138
2139 static unsigned int tegra_ahci_qc_issue(struct ata_queued_cmd *qc)
2140 {
2141         struct ata_port *ap = qc->ap;
2142         struct ata_host *host = ap->host;
2143         struct tegra_ahci_host_priv *tegra_hpriv = host->private_data;
2144
2145         /* stop the idle timer */
2146         if (timer_pending(&tegra_hpriv->idle_timer))
2147                 del_timer_sync(&tegra_hpriv->idle_timer);
2148
2149         /* note: host->lock is locked */
2150         switch (tegra_hpriv->pg_state) {
2151         case SATA_ON:
2152                 /* normal case, issue the qc */
2153                 return ahci_ops.qc_issue(qc);
2154         case SATA_GOING_OFF:
2155         case SATA_ABORT_OFF:
2156                 /* SATA is going OFF, let's abort the suspend */
2157                 dev_dbg(host->dev, "** qc_issue: going OFF **\n");
2158                 tegra_hpriv->pg_state = SATA_ABORT_OFF;
2159                 return tegra_ahci_queue_one_qc(tegra_hpriv, qc);
2160         case SATA_OFF:
2161                 dev_dbg(host->dev, "** qc_issue: request power-up sata **\n");
2162                 queue_work(tegra_ahci_work_q, &tegra_ahci_work);
2163                 tegra_hpriv->pg_state = SATA_GOING_ON;
2164                 /* continue with the following code to queue the qc */
2165         case SATA_GOING_ON:
2166                 return tegra_ahci_queue_one_qc(tegra_hpriv, qc);
2167         default:
2168                 dev_err(host->dev, "** qc_issue: bad state (%u) **\n",
2169                                         tegra_hpriv->pg_state);
2170                 WARN_ON(1);
2171                 return AC_ERR_SYSTEM;
2172         }
2173 }
2174
2175 static int tegra_ahci_hardreset(struct ata_link *link, unsigned int *class,
2176                                 unsigned long deadline)
2177 {
2178         struct ata_port *ap = link->ap;
2179         struct ata_host *host = ap->host;
2180         struct tegra_ahci_host_priv *tegra_hpriv = host->private_data;
2181         unsigned long flags;
2182         int rc;
2183
2184         if (tegra_hpriv->pg_state == SATA_OFF) {
2185                 dev_dbg(host->dev, "** hreset: request power-up sata **\n");
2186                 spin_lock_irqsave(&host->lock, flags);
2187                 rc = tegra_ahci_runtime_resume(tegra_hpriv->dev);
2188                 spin_unlock_irqrestore(&host->lock, flags);
2189                 /* rc == 0 means the request has been run successfully */
2190                 if (rc) {
2191                         dev_err(host->dev, "** hreset: rt_get()=%d **\n", rc);
2192                         WARN_ON(1);
2193                         return AC_ERR_SYSTEM;
2194                 }
2195                 tegra_hpriv->pg_state = SATA_ON;
2196         }
2197
2198         return ahci_ops.hardreset(link, class, deadline);
2199 }
2200
2201 static irqreturn_t tegra_ahci_interrupt(int irq, void *dev_instance)
2202 {
2203         irqreturn_t irq_retval;
2204
2205         irq_retval = ahci_interrupt(irq, dev_instance);
2206         if (irq_retval == IRQ_NONE)
2207                 return IRQ_NONE;
2208
2209 #ifdef CONFIG_PM
2210         tegra_ahci_to_add_idle_timer((struct ata_host *)dev_instance);
2211 #endif
2212
2213         return irq_retval;
2214 }
2215 #endif
2216 #endif
2217
2218 static int tegra_ahci_remove_one(struct platform_device *pdev)
2219 {
2220         struct ata_host *host = dev_get_drvdata(&pdev->dev);
2221         struct ahci_host_priv *hpriv;
2222
2223         BUG_ON(host == NULL);
2224         BUG_ON(host->iomap[AHCI_PCI_BAR] == NULL);
2225         hpriv = host->private_data;
2226
2227         tegra_ahci_controller_remove(pdev);
2228
2229         devm_iounmap(&pdev->dev, host->iomap[AHCI_PCI_BAR]);
2230         devres_free(host);
2231
2232 #ifdef TEGRA_AHCI_CONTEXT_RESTORE
2233         /* Free PG save/restore area */
2234         devm_kfree(&pdev->dev, ((struct tegra_ahci_host_priv *)hpriv)->pg_save);
2235
2236 #endif
2237 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
2238         cancel_work_sync(&tegra_ahci_work);
2239         if (tegra_ahci_work_q)
2240                 destroy_workqueue(tegra_ahci_work_q);
2241 #endif
2242
2243         devm_kfree(&pdev->dev, hpriv);
2244
2245         return 0;
2246 }
2247
2248 static int tegra_ahci_init_one(struct platform_device *pdev)
2249 {
2250         struct ata_port_info pi = ahci_port_info;
2251         const struct ata_port_info *ppi[] = { &pi, NULL };
2252         struct device *dev = &pdev->dev;
2253         struct ahci_host_priv *hpriv = NULL;
2254         struct tegra_ahci_host_priv *tegra_hpriv;
2255         struct ata_host *host = NULL;
2256         int n_ports, i, rc = 0;
2257         struct resource *res, *irq_res;
2258         void __iomem *mmio;
2259 #if defined(TEGRA_AHCI_CONTEXT_RESTORE)
2260         u32 save_size;
2261 #endif
2262         irq_handler_t irq_handler = ahci_interrupt;
2263
2264         VPRINTK("ENTER\n");
2265
2266         WARN_ON((int)ATA_MAX_QUEUE > AHCI_MAX_CMDS);
2267
2268         ata_print_version_once(&pdev->dev, DRV_VERSION);
2269
2270         /* Simple resource validation */
2271         if (pdev->num_resources != 3) {
2272                 dev_err(dev, "invalid number of resources\n");
2273                 dev_err(dev, "not enough SATA resources\n");
2274                 return -EINVAL;
2275         }
2276
2277         /* acquire bar resources */
2278         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2279         if (res == NULL)
2280                 return -EINVAL;
2281
2282         /* acquire IRQ resource */
2283         irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
2284         if (irq_res == NULL)
2285                 return -EINVAL;
2286         if (irq_res->start <= 0)
2287                 return -EINVAL;
2288
2289         /* allocate sizeof tegra_ahci_host_priv, which contains extra fields */
2290         hpriv = devm_kzalloc(dev, sizeof(struct tegra_ahci_host_priv),
2291                              GFP_KERNEL);
2292         if (!hpriv) {
2293                 rc = -ENOMEM;
2294                 goto fail;
2295         }
2296
2297 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
2298         tegra_ahci_work_q = alloc_workqueue("tegra_ahci_work_q",
2299                                         WQ_HIGHPRI | WQ_UNBOUND, 16);
2300         if (!tegra_ahci_work_q) {
2301                 dev_err(dev, "TEGRA WORKQUEUE SATA init failed\n");
2302                 goto fail;
2303         }
2304 #endif
2305
2306         hpriv->flags |= (unsigned long)pi.private_data;
2307         tegra_hpriv = (struct tegra_ahci_host_priv *)hpriv;
2308         tegra_hpriv->dev = dev;
2309         g_tegra_hpriv = tegra_hpriv;
2310
2311         /* Call tegra init routine */
2312         rc = tegra_ahci_controller_init(tegra_hpriv, 0);
2313         if (rc != 0) {
2314                 dev_err(dev, "TEGRA SATA init failed\n");
2315                 goto fail;
2316         }
2317
2318         /*
2319          * We reserve a table of 6 BARs in tegra_hpriv to store BARs.
2320          * Save the mapped AHCI_PCI_BAR address to the table.
2321          */
2322         mmio = devm_ioremap(dev, res->start, (res->end-res->start+1));
2323         tegra_hpriv->bars_table[AHCI_PCI_BAR] = mmio;
2324         hpriv->mmio = mmio;
2325
2326         /* save initial config */
2327         tegra_ahci_save_initial_config(pdev, hpriv);
2328         dev_dbg(dev, "past save init config\n");
2329
2330         /* prepare host */
2331         if (hpriv->cap & HOST_CAP_NCQ) {
2332                 pi.flags |= ATA_FLAG_NCQ;
2333                 pi.flags |= ATA_FLAG_FPDMA_AA;
2334         }
2335
2336         /*
2337          * CAP.NP sometimes indicate the index of the last enabled
2338          * port, at other times, that of the last possible port, so
2339          * determining the maximum port number requires looking at
2340          * both CAP.NP and port_map.
2341          */
2342         n_ports = max(ahci_nr_ports(hpriv->cap), fls(hpriv->port_map));
2343         host = ata_host_alloc_pinfo(dev, ppi, n_ports);
2344         if (!host) {
2345                 rc = -ENOMEM;
2346                 goto fail;
2347         }
2348         host->private_data = hpriv;
2349         tegra_hpriv->host = host;
2350         host->iomap = tegra_hpriv->bars_table;
2351
2352         if (!(hpriv->cap & HOST_CAP_SSS))
2353                 host->flags |= ATA_HOST_PARALLEL_SCAN;
2354         else
2355                 pr_info("ahci: SSS flag set, parallel bus scan disabled\n");
2356
2357         for (i = 0; i < host->n_ports; i++) {
2358                 struct ata_port *ap = host->ports[i];
2359
2360                 /* set initial link pm policy */
2361                 ap->target_lpm_policy = ATA_LPM_UNKNOWN;
2362
2363                 /* disabled/not-implemented port */
2364                 if (!(hpriv->port_map & (1 << i)))
2365                         ap->ops = &ata_dummy_port_ops;
2366                 else
2367                         ap->target_lpm_policy = ATA_LPM_MIN_POWER;
2368         }
2369
2370         rc = ahci_reset_controller(host);
2371         if (rc) {
2372                 dev_err(dev, "Reset controller failed! (rc=%d)\n", rc);
2373                 goto fail;
2374         }
2375
2376         ahci_init_controller(host);
2377         ahci_print_info(host, "TEGRA-SATA");
2378         dev_dbg(dev, "controller init okay\n");
2379
2380 #if defined(TEGRA_AHCI_CONTEXT_RESTORE)
2381         /* Setup PG save/restore area: */
2382
2383         /* calculate the size */
2384         save_size = ARRAY_SIZE(pg_save_ipfs_registers) +
2385                             ARRAY_SIZE(pg_save_config_registers) +
2386                             ARRAY_SIZE(pg_save_bar5_registers) +
2387                             ARRAY_SIZE(pg_save_bar5_bkdr_registers);
2388
2389         /* and add save port_registers for all the ports */
2390         save_size += TEGRA_AHCI_NUM_PORTS *
2391                      (ARRAY_SIZE(pg_save_config_port_registers) +
2392                       ARRAY_SIZE(pg_save_bar5_port_registers) +
2393                       ARRAY_SIZE(pg_save_bar5_bkdr_port_registers));
2394
2395         /*
2396          * save_size is number of registers times number of bytes per
2397          * register to get total save size.
2398          */
2399         save_size *= sizeof(u32);
2400         tegra_hpriv->pg_save = devm_kzalloc(dev, save_size, GFP_KERNEL);
2401         if (!tegra_hpriv->pg_save) {
2402                 rc = -ENOMEM;
2403                 goto fail;
2404         }
2405 #endif
2406 #ifdef CONFIG_PM
2407 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
2408         tegra_hpriv->pg_state = SATA_ON;
2409
2410         /* setup sata idle timer */
2411         init_timer_deferrable(&tegra_hpriv->idle_timer);
2412         tegra_hpriv->idle_timer.function = tegra_ahci_idle_timer;
2413         tegra_hpriv->idle_timer.data = (unsigned long)host;
2414
2415         INIT_LIST_HEAD(&tegra_hpriv->qc_list);
2416
2417         /* use our own irq handler */
2418         irq_handler = tegra_ahci_interrupt;
2419 #endif
2420
2421 #endif
2422
2423         rc = ata_host_activate(host, irq_res->start, irq_handler, 0, &ahci_sht);
2424         if (rc == 0)
2425                 return 0;
2426 fail:
2427         if (host) {
2428                 if (host->iomap[AHCI_PCI_BAR])
2429                         devm_iounmap(dev, host->iomap[AHCI_PCI_BAR]);
2430                 devres_free(host);
2431         }
2432         if (hpriv)
2433                 devm_kfree(dev, hpriv);
2434
2435 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
2436         if (tegra_ahci_work_q)
2437                 destroy_workqueue(tegra_ahci_work_q);
2438 #endif
2439
2440         return rc;
2441 }
2442
2443 static int __init ahci_init(void)
2444 {
2445         return platform_driver_register(&tegra_platform_ahci_driver);
2446 }
2447
2448 static void __exit ahci_exit(void)
2449 {
2450         platform_driver_unregister(&tegra_platform_ahci_driver);
2451 }
2452
2453 #ifdef  CONFIG_DEBUG_FS
2454
2455 #include <linux/debugfs.h>
2456 #include <linux/seq_file.h>
2457
2458 static void dbg_ahci_dump_regs(struct seq_file *s, u32 *ptr, u32 base, u32 regs)
2459 {
2460 #define REGS_PER_LINE   4
2461
2462         u32 i, j;
2463         u32 lines = regs / REGS_PER_LINE;
2464
2465         for (i = 0; i < lines; i++) {
2466                 seq_printf(s, "0x%08x: ", base+(i*16));
2467                 for (j = 0; j < REGS_PER_LINE; ++j) {
2468                         seq_printf(s, "0x%08x ", readl(ptr));
2469                         ++ptr;
2470                 }
2471                 seq_puts(s, "\n");
2472         }
2473 #undef REGS_PER_LINE
2474 }
2475
2476 static int dbg_ahci_dump_show(struct seq_file *s, void *unused)
2477 {
2478         u32 base;
2479         u32 *ptr;
2480         u32 i;
2481
2482         base = TEGRA_SATA_CONFIG_BASE;
2483         ptr = (u32 *)IO_TO_VIRT(base);
2484         seq_puts(s, "SATA CONFIG Registers:\n");
2485         seq_puts(s, "----------------------\n");
2486         dbg_ahci_dump_regs(s, ptr, base, 0x200);
2487
2488         base = TEGRA_SATA_BAR5_BASE;
2489         ptr = (u32 *)IO_TO_VIRT(base);
2490         seq_puts(s, "\nAHCI HBA Registers:\n");
2491         seq_puts(s, "-------------------\n");
2492         dbg_ahci_dump_regs(s, ptr, base, 64);
2493
2494         for (i = 0; i < TEGRA_AHCI_NUM_PORTS; ++i) {
2495                 base = TEGRA_SATA_BAR5_BASE + 0x100 + (0x80*i);
2496                 ptr = (u32 *)IO_TO_VIRT(base);
2497                 seq_printf(s, "\nPort %u Registers:\n", i);
2498                 seq_puts(s, "---------------\n");
2499                 dbg_ahci_dump_regs(s, ptr, base, 20);
2500         }
2501
2502 #ifdef  CONFIG_TEGRA_SATA_IDLE_POWERGATE
2503         /* adjust tegra_ahci_idle_time to minimum if it is too small */
2504         tegra_ahci_idle_time = max((u32)TEGRA_AHCI_MIN_IDLE_TIME,
2505                                    tegra_ahci_idle_time);
2506         seq_printf(s, "\nIdle Timeout = %u milli-seconds.\n",
2507                       tegra_ahci_idle_time);
2508 #endif
2509
2510         if (tegra_powergate_is_powered(TEGRA_POWERGATE_SATA))
2511                 seq_puts(s, "\n=== SATA controller is powered on ===\n\n");
2512         else
2513                 seq_puts(s, "\n=== SATA controller is powered off ===\n\n");
2514
2515         return 0;
2516 }
2517
2518 static int dbg_ahci_dump_open(struct inode *inode, struct file *file)
2519 {
2520         return single_open(file, dbg_ahci_dump_show, &inode->i_private);
2521 }
2522
2523 static const struct file_operations debug_fops = {
2524         .open           = dbg_ahci_dump_open,
2525         .read           = seq_read,
2526         .llseek         = seq_lseek,
2527         .release        = single_release,
2528 };
2529
2530 static int __init tegra_ahci_dump_debuginit(void)
2531 {
2532         (void) debugfs_create_file("tegra_ahci", S_IRUGO,
2533                                    NULL, NULL, &debug_fops);
2534 #ifdef  CONFIG_TEGRA_SATA_IDLE_POWERGATE
2535         (void) debugfs_create_u32("tegra_ahci_idle_ms", S_IRWXUGO,
2536                                    NULL, &tegra_ahci_idle_time);
2537 #endif
2538         return 0;
2539 }
2540 late_initcall(tegra_ahci_dump_debuginit);
2541 #endif
2542
2543 MODULE_AUTHOR("NVIDIA");
2544 MODULE_DESCRIPTION("Tegra AHCI SATA low-level driver");
2545 MODULE_LICENSE("GPL v2");
2546 MODULE_VERSION(DRV_VERSION);
2547
2548 module_init(ahci_init);
2549 module_exit(ahci_exit);