16482f8d311dbd2d8c667c96a3f2ad4b6e03406e
[linux-4.9.git] / drivers / platform / tegra / powergate / powergate-t21x.c
1 /*
2  * Copyright (c) 2014-2017, NVIDIA CORPORATION.  All rights reserved.
3  *
4  * This software is licensed under the terms of the GNU General Public
5  * License version 2, as published by the Free Software Foundation, and
6  * may be copied, distributed, and modified under those terms.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11  * GNU General Public License for more details.
12  *
13  */
14
15 #include <dt-bindings/clock/tegra210-car.h>
16 #include <linux/err.h>
17 #include <linux/spinlock.h>
18 #include <linux/delay.h>
19 #include <linux/io.h>
20 #include <linux/regulator/consumer.h>
21 #include <soc/tegra/tegra_powergate.h>
22 #include <soc/tegra/tegra-powergate-driver.h>
23 #include <soc/tegra/chip-id.h>
24 #include <linux/tegra_soctherm.h>
25 #include <soc/tegra/tegra-dvfs.h>
26 #include <trace/events/power.h>
27 #include <linux/clk.h>
28 #include <linux/clk-provider.h>
29 #include <linux/clk/tegra.h>
30 #include <soc/tegra/reset.h>
31 #include <soc/tegra/common.h>
32
33 #define MAX_CLK_EN_NUM                  15
34 #define MAX_HOTRESET_CLIENT_NUM         4
35
36 enum clk_type {
37         CLK_AND_RST,
38         RST_ONLY,
39         CLK_ONLY,
40 };
41
42 struct partition_clk_info {
43         const char *clk_name;
44         enum clk_type clk_type;
45         struct clk *clk_ptr;
46 };
47
48 struct powergate_partition_info {
49         const char *name;
50         struct partition_clk_info clk_info[MAX_CLK_EN_NUM];
51         struct partition_clk_info slcg_info[MAX_CLK_EN_NUM];
52         unsigned long reset_id[MAX_CLK_EN_NUM];
53         int reset_id_num;
54         struct raw_notifier_head slcg_notifier;
55         int refcount;
56         bool disable_after_boot;
57         struct mutex pg_mutex;
58         bool skip_reset;
59 };
60
61 #define EMULATION_MC_FLUSH_TIMEOUT 100
62 #define TEGRA210_POWER_DOMAIN_NVENC TEGRA210_POWER_DOMAIN_MPE
63
64 enum mc_client {
65         MC_CLIENT_AFI           = 0,
66         MC_CLIENT_AVPC          = 1,
67         MC_CLIENT_DC            = 2,
68         MC_CLIENT_DCB           = 3,
69         MC_CLIENT_HC            = 6,
70         MC_CLIENT_HDA           = 7,
71         MC_CLIENT_ISP2          = 8,
72         MC_CLIENT_NVENC         = 11,
73         MC_CLIENT_SATA          = 15,
74         MC_CLIENT_VI            = 17,
75         MC_CLIENT_VIC           = 18,
76         MC_CLIENT_XUSB_HOST     = 19,
77         MC_CLIENT_XUSB_DEV      = 20,
78         MC_CLIENT_BPMP          = 21,
79         MC_CLIENT_TSEC          = 22,
80         MC_CLIENT_SDMMC1        = 29,
81         MC_CLIENT_SDMMC2        = 30,
82         MC_CLIENT_SDMMC3        = 31,
83         MC_CLIENT_SDMMC4        = 32,
84         MC_CLIENT_ISP2B         = 33,
85         MC_CLIENT_GPU           = 34,
86         MC_CLIENT_NVDEC         = 37,
87         MC_CLIENT_APE           = 38,
88         MC_CLIENT_SE            = 39,
89         MC_CLIENT_NVJPG         = 40,
90         MC_CLIENT_TSECB         = 45,
91         MC_CLIENT_LAST          = -1,
92 };
93
94 struct tegra210_mc_client_info {
95         enum mc_client  hot_reset_clients[MAX_HOTRESET_CLIENT_NUM];
96 };
97
98 static struct tegra210_mc_client_info tegra210_pg_mc_info[] = {
99         [TEGRA210_POWER_DOMAIN_CRAIL] = {
100                 .hot_reset_clients = {
101                         [0] = MC_CLIENT_LAST,
102                 },
103         },
104         [TEGRA210_POWER_DOMAIN_VENC] = {
105                 .hot_reset_clients = {
106                         [0] = MC_CLIENT_ISP2,
107                         [1] = MC_CLIENT_VI,
108                         [2] = MC_CLIENT_LAST,
109                 },
110         },
111 #ifdef CONFIG_ARCH_TEGRA_HAS_PCIE
112         [TEGRA210_POWER_DOMAIN_PCIE] = {
113                 .hot_reset_clients = {
114                         [0] = MC_CLIENT_AFI,
115                         [1] = MC_CLIENT_LAST,
116                 },
117         },
118 #endif
119 #ifdef CONFIG_ARCH_TEGRA_HAS_SATA
120         [TEGRA210_POWER_DOMAIN_SATA] = {
121                 .hot_reset_clients = {
122                         [0] = MC_CLIENT_SATA,
123                         [1] = MC_CLIENT_LAST,
124                 },
125         },
126 #endif
127         [TEGRA210_POWER_DOMAIN_NVENC] = {
128                 .hot_reset_clients = {
129                         [0] = MC_CLIENT_NVENC,
130                         [1] = MC_CLIENT_LAST,
131                 },
132         },
133         [TEGRA210_POWER_DOMAIN_SOR] = {
134                 .hot_reset_clients = {
135                         [0] = MC_CLIENT_LAST,
136                 },
137         },
138         [TEGRA210_POWER_DOMAIN_DISA] = {
139                 .hot_reset_clients = {
140                         [0] = MC_CLIENT_DC,
141                         [1] = MC_CLIENT_LAST,
142                 },
143         },
144         [TEGRA210_POWER_DOMAIN_DISB] = {
145                 .hot_reset_clients = {
146                         [0] = MC_CLIENT_DCB,
147                         [1] = MC_CLIENT_LAST,
148                 },
149         },
150         [TEGRA210_POWER_DOMAIN_XUSBA] = {
151                 .hot_reset_clients = {
152                         [0] = MC_CLIENT_LAST,
153                 },
154         },
155         [TEGRA210_POWER_DOMAIN_XUSBB] = {
156                 .hot_reset_clients = {
157                         [0] = MC_CLIENT_XUSB_DEV,
158                         [1] = MC_CLIENT_LAST,
159                 },
160         },
161         [TEGRA210_POWER_DOMAIN_XUSBC] = {
162                 .hot_reset_clients = {
163                         [0] = MC_CLIENT_XUSB_HOST,
164                         [1] = MC_CLIENT_LAST,
165                 },
166         },
167 #ifdef CONFIG_ARCH_TEGRA_VIC
168         [TEGRA210_POWER_DOMAIN_VIC] = {
169                 .hot_reset_clients = {
170                         [0] = MC_CLIENT_VIC,
171                         [1] = MC_CLIENT_LAST,
172                 },
173         },
174 #endif
175         [TEGRA210_POWER_DOMAIN_NVDEC] = {
176                 .hot_reset_clients = {
177                         [0] = MC_CLIENT_NVDEC,
178                         [1] = MC_CLIENT_LAST,
179                 },
180         },
181         [TEGRA210_POWER_DOMAIN_NVJPG] = {
182                 .hot_reset_clients = {
183                         [0] = MC_CLIENT_NVJPG,
184                         [1] = MC_CLIENT_LAST,
185                 },
186         },
187         [TEGRA210_POWER_DOMAIN_APE] = {
188                 .hot_reset_clients = {
189                         [0] = MC_CLIENT_APE,
190                         [1] = MC_CLIENT_LAST,
191                 },
192         },
193         [TEGRA210_POWER_DOMAIN_VE2] = {
194                 .hot_reset_clients = {
195                         [0] = MC_CLIENT_ISP2B,
196                         [1] = MC_CLIENT_LAST,
197                 },
198         },
199         [TEGRA210_POWER_DOMAIN_GPU] = {
200                 .hot_reset_clients = {
201                         [0] = MC_CLIENT_GPU,
202                         [1] = MC_CLIENT_LAST,
203                 },
204         },
205 };
206
207 static struct powergate_partition_info tegra210_pg_partition_info[] = {
208         [TEGRA210_POWER_DOMAIN_VENC] = {
209                 .name = "ve",
210                 .clk_info = {
211                         [0] = { .clk_name = "ispa", .clk_type = CLK_ONLY },
212                         [1] = { .clk_name = "vi", .clk_type = CLK_ONLY },
213                         [2] = { .clk_name = "csi", .clk_type = CLK_ONLY },
214                         [3] = { .clk_name = "vii2c", .clk_type = CLK_ONLY },
215                         [4] = { .clk_name = "cilab", .clk_type = CLK_ONLY },
216                         [5] = { .clk_name = "cilcd", .clk_type = CLK_ONLY },
217                         [6] = { .clk_name = "cile", .clk_type = CLK_ONLY },
218                 },
219                 .slcg_info = {
220                         [0] = { .clk_name = "mc_capa" },
221                         [1] = { .clk_name = "mc_cbpa" },
222                         [2] = { .clk_name = "mc_ccpa" },
223                         [3] = { .clk_name = "mc_cdpa" },
224                         [4] = { .clk_name = "host1x" },
225                         [5] = { .clk_name = "vi_slcg_ovr" },
226                         [6] = { .clk_name = "ispa_slcg_ovr" },
227                 },
228                 .reset_id = { TEGRA210_CLK_ISPA, TEGRA210_CLK_VI,
229                               TEGRA210_CLK_CSI, TEGRA210_CLK_VI_I2C },
230                 .reset_id_num = 4,
231         },
232 #ifdef CONFIG_ARCH_TEGRA_HAS_PCIE
233         [TEGRA210_POWER_DOMAIN_PCIE] = {
234                 .name = "pcie",
235                 .clk_info = {
236                         [0] = { .clk_name = "afi", .clk_type = CLK_ONLY },
237                         [1] = { .clk_name = "pcie", .clk_type = CLK_ONLY },
238                 },
239                 .reset_id = { TEGRA210_CLK_AFI, TEGRA210_CLK_PCIE,
240                               TEGRA210_CLK_PCIEX },
241                 .reset_id_num = 3,
242                 .skip_reset = true,
243         },
244 #endif
245 #ifdef CONFIG_ARCH_TEGRA_HAS_SATA
246         [TEGRA210_POWER_DOMAIN_SATA] = {
247                 .name = "sata",
248                 .disable_after_boot = false,
249                 .clk_info = {
250                         [0] = { .clk_name = "sata_oob", .clk_type = CLK_ONLY },
251                         [1] = { .clk_name = "cml1", .clk_type = CLK_ONLY },
252                         [3] = { .clk_name = "sata_aux", .clk_type = CLK_ONLY },
253                         [4] = { .clk_name = "sata", .clk_type = CLK_ONLY },
254                 },
255                 .slcg_info = {
256                         [0] = { .clk_name = "mc_capa" },
257                         [1] = { .clk_name = "mc_cbpa" },
258                         [2] = { .clk_name = "mc_ccpa" },
259                         [3] = { .clk_name = "mc_cdpa" },
260                         [4] = { .clk_name = "sata_slcg_fpci" },
261                         [5] = { .clk_name = "sata_slcg_ipfs" },
262                         [6] = { .clk_name = "sata_slcg_ovr" },
263                 },
264                 .reset_id = { TEGRA210_CLK_SATA_OOB, TEGRA210_CLK_SATA_COLD,
265                               TEGRA210_CLK_SATA },
266                 .reset_id_num = 3,
267         },
268 #endif
269         [TEGRA210_POWER_DOMAIN_NVENC] = {
270                 .name = "nvenc",
271                 .clk_info = {
272                         [0] = { .clk_name = "nvenc.cbus", .clk_type = CLK_ONLY },
273                 },
274                 .slcg_info = {
275                         [0] = { .clk_name = "mc_capa" },
276                         [1] = { .clk_name = "mc_cbpa" },
277                         [2] = { .clk_name = "mc_ccpa" },
278                         [3] = { .clk_name = "mc_cdpa" },
279                         [4] = { .clk_name = "nvenc_slcg_ovr" },
280                 },
281                 .reset_id = { TEGRA210_CLK_NVENC },
282                 .reset_id_num = 1,
283         },
284         [TEGRA210_POWER_DOMAIN_SOR] = {
285                 .name = "sor",
286                 .clk_info = {
287                         [0] = { .clk_name = "sor0", .clk_type = CLK_ONLY },
288                         [1] = { .clk_name = "dsia", .clk_type = CLK_ONLY },
289                         [2] = { .clk_name = "dsib", .clk_type = CLK_ONLY },
290                         [3] = { .clk_name = "sor1", .clk_type = CLK_ONLY },
291                         [4] = { .clk_name = "mipi-cal", .clk_type = CLK_ONLY },
292                         [5] = { .clk_name = "dpaux", .clk_type = CLK_ONLY },
293                         [6] = { .clk_name = "dpaux1", .clk_type = CLK_ONLY },
294                 },
295                 .slcg_info = {
296                         [0] = { .clk_name = "mc_capa" },
297                         [1] = { .clk_name = "mc_cbpa" },
298                         [2] = { .clk_name = "mc_ccpa" },
299                         [3] = { .clk_name = "mc_cdpa" },
300                         [4] = { .clk_name = "hda2hdmi" },
301                         [5] = { .clk_name = "hda2codec_2x" },
302                         [6] = { .clk_name = "disp1" },
303                         [7] = { .clk_name = "disp2" },
304                         [8] = { .clk_name = "disp1_slcg_ovr" },
305                         [9] = { .clk_name = "disp2_slcg_ovr" },
306                 },
307                 .reset_id = { TEGRA210_CLK_SOR0, TEGRA210_CLK_DISP1,
308                               TEGRA210_CLK_DSIB, TEGRA210_CLK_SOR1,
309                               TEGRA210_CLK_MIPI_CAL },
310                 .reset_id_num = 5,
311         },
312         [TEGRA210_POWER_DOMAIN_DISA] = {
313                 .name = "disa",
314                 .clk_info = {
315                         [0] = { .clk_name = "disp1", .clk_type = CLK_ONLY },
316                 },
317                 .slcg_info = {
318                         [0] = { .clk_name = "mc_capa" },
319                         [1] = { .clk_name = "mc_cbpa" },
320                         [2] = { .clk_name = "mc_ccpa" },
321                         [3] = { .clk_name = "mc_cdpa" },
322                         [4] = { .clk_name = "la" },
323                         [5] = { .clk_name = "host1x" },
324                         [6] = { .clk_name = "disp1_slcg_ovr" },
325                 },
326                 .reset_id = { TEGRA210_CLK_DISP1 },
327                 .reset_id_num = 1,
328         },
329         [TEGRA210_POWER_DOMAIN_DISB] = {
330                 .name = "disb",
331                 .disable_after_boot = true,
332                 .clk_info = {
333                         [0] = { .clk_name = "disp2", .clk_type = CLK_ONLY },
334                 },
335                 .slcg_info = {
336                         [0] = { .clk_name = "mc_capa" },
337                         [1] = { .clk_name = "mc_cbpa" },
338                         [2] = { .clk_name = "mc_ccpa" },
339                         [3] = { .clk_name = "mc_cdpa" },
340                         [4] = { .clk_name = "la" },
341                         [5] = { .clk_name = "host1x" },
342                         [6] = { .clk_name = "disp2_slcg_ovr" },
343                 },
344                 .reset_id = { TEGRA210_CLK_DISP2 },
345                 .reset_id_num = 1,
346         },
347         [TEGRA210_POWER_DOMAIN_XUSBA] = {
348                 .name = "xusba",
349                 .clk_info = {
350                         [0] = { .clk_name = "xusb_ss", .clk_type = CLK_ONLY },
351                         [1] = { .clk_name = "xusb_ssp_src", .clk_type = CLK_ONLY },
352                         [2] = { .clk_name = "xusb_hs_src", .clk_type = CLK_ONLY },
353                         [3] = { .clk_name = "xusb_fs_src", .clk_type = CLK_ONLY },
354                         [4] = { .clk_name = "xusb_dev_src", .clk_type = CLK_ONLY },
355                 },
356                 .slcg_info = {
357                         [0] = { .clk_name = "mc_capa" },
358                         [1] = { .clk_name = "mc_cbpa" },
359                         [2] = { .clk_name = "mc_ccpa" },
360                         [3] = { .clk_name = "mc_cdpa" },
361                         [4] = { .clk_name = "xusb_host" },
362                         [5] = { .clk_name = "xusb_dev" },
363                         [6] = { .clk_name = "xusb_host_slcg" },
364                         [7] = { .clk_name = "xusb_dev_slcg" },
365                 },
366                 .reset_id = { TEGRA210_CLK_XUSB_SS },
367                 .reset_id_num = 1,
368         },
369         [TEGRA210_POWER_DOMAIN_XUSBB] = {
370                 .name = "xusbb",
371                 .clk_info = {
372                         [0] = { .clk_name = "xusb_dev", .clk_type = CLK_ONLY },
373                 },
374                 .slcg_info = {
375                         [0] = { .clk_name = "mc_capa" },
376                         [1] = { .clk_name = "mc_cbpa" },
377                         [2] = { .clk_name = "mc_ccpa" },
378                         [3] = { .clk_name = "mc_cdpa" },
379                         [4] = { .clk_name = "xusb_ss" },
380                         [5] = { .clk_name = "xusb_host" },
381                         [6] = { .clk_name = "xusb_host_slcg" },
382                         [7] = { .clk_name = "xusb_dev_slcg" },
383                 },
384                 .reset_id = { 95 },
385                 .reset_id_num = 1,
386         },
387         [TEGRA210_POWER_DOMAIN_XUSBC] = {
388                 .name = "xusbc",
389                 .clk_info = {
390                         [0] = { .clk_name = "xusb_host", .clk_type = CLK_ONLY },
391                 },
392                 .slcg_info = {
393                         [0] = { .clk_name = "mc_capa" },
394                         [1] = { .clk_name = "mc_cbpa" },
395                         [2] = { .clk_name = "mc_ccpa" },
396                         [3] = { .clk_name = "mc_cdpa" },
397                         [4] = { .clk_name = "xusb_ss" },
398                         [5] = { .clk_name = "xusb_dev" },
399                         [6] = { .clk_name = "xusb_dev_slcg" },
400                         [7] = { .clk_name = "xusb_host_slcg" },
401                 },
402                 .reset_id = { TEGRA210_CLK_XUSB_HOST },
403                 .reset_id_num = 1,
404         },
405 #ifdef CONFIG_ARCH_TEGRA_VIC
406         [TEGRA210_POWER_DOMAIN_VIC] = {
407                 .name = "vic",
408                 .clk_info = {
409                         [0] = { .clk_name = "vic03.cbus", .clk_type = CLK_ONLY },
410                 },
411                 .slcg_info = {
412                         [0] = { .clk_name = "mc_capa" },
413                         [1] = { .clk_name = "mc_cbpa" },
414                         [2] = { .clk_name = "mc_ccpa" },
415                         [3] = { .clk_name = "mc_cdpa" },
416                         [4] = { .clk_name = "vic03_slcg_ovr" },
417                         [5] = { .clk_name = "host1x" },
418                 },
419                 .reset_id = { TEGRA210_CLK_VIC03 },
420                 .reset_id_num = 1,
421         },
422 #endif
423         [TEGRA210_POWER_DOMAIN_NVDEC] = {
424                 .name = "nvdec",
425                 .clk_info = {
426                         [0] = { .clk_name = "nvdec", .clk_type = CLK_ONLY },
427                 },
428                 .slcg_info = {
429                         [0] = { .clk_name = "mc_capa" },
430                         [1] = { .clk_name = "mc_cbpa" },
431                         [2] = { .clk_name = "mc_ccpa" },
432                         [3] = { .clk_name = "mc_cdpa" },
433                         [4] = { .clk_name = "nvdec_slcg_ovr" },
434                         [5] = { .clk_name = "nvjpg" },
435                         [6] = { .clk_name = "nvjpg_slcg_ovr" },
436                 },
437                 .reset_id = { TEGRA210_CLK_NVDEC },
438                 .reset_id_num = 1,
439         },
440         [TEGRA210_POWER_DOMAIN_NVJPG] = {
441                 .name = "nvjpg",
442                 .clk_info = {
443                         [0] = { .clk_name = "nvjpg", .clk_type = CLK_ONLY },
444                 },
445                 .slcg_info = {
446                         [0] = { .clk_name = "mc_capa" },
447                         [1] = { .clk_name = "mc_cbpa" },
448                         [2] = { .clk_name = "mc_ccpa" },
449                         [3] = { .clk_name = "mc_cdpa" },
450                         [4] = { .clk_name = "nvjpg_slcg_ovr" },
451                         [5] = { .clk_name = "nvdec" },
452                         [6] = { .clk_name = "nvdec_slcg_ovr" },
453                 },
454                 .reset_id = { TEGRA210_CLK_NVJPG },
455                 .reset_id_num = 1,
456         },
457         [TEGRA210_POWER_DOMAIN_APE] = {
458                 .name = "ape",
459                 .clk_info = {
460                         [0] = { .clk_name = "ape", .clk_type = CLK_ONLY },
461                 },
462                 .slcg_info = {
463                         [0] = { .clk_name = "mc_capa" },
464                         [1] = { .clk_name = "mc_cbpa" },
465                         [2] = { .clk_name = "mc_ccpa" },
466                         [3] = { .clk_name = "mc_cdpa" },
467                         [4] = { .clk_name = "aclk" },
468                         [5] = { .clk_name = "i2s0" },
469                         [6] = { .clk_name = "i2s1" },
470                         [7] = { .clk_name = "i2s2" },
471                         [8] = { .clk_name = "i2s3" },
472                         [9] = { .clk_name = "i2s4" },
473                         [10] = { .clk_name = "spdif_out" },
474                         [11] = { .clk_name = "d_audio" },
475                         [12] = { .clk_name = "ape_slcg_ovr" },
476                         [13] = { .clk_name = "aclk_slcg_ovr" },
477                         [14] = { .clk_name = "daudio_slcg_ovr" },
478
479                 },
480                 .reset_id = { TEGRA210_CLK_APE },
481                 .reset_id_num = 1,
482         },
483         [TEGRA210_POWER_DOMAIN_VE2] = {
484                 .name = "ve2",
485                 .clk_info = {
486                         [0] = { .clk_name = "ispb", .clk_type = CLK_ONLY },
487                 },
488                 .slcg_info = {
489                         [0] = { .clk_name = "mc_capa" },
490                         [1] = { .clk_name = "mc_cbpa" },
491                         [2] = { .clk_name = "mc_ccpa" },
492                         [3] = { .clk_name = "mc_cdpa" },
493                         [4] = { .clk_name = "ispb_slcg_ovr" },
494                 },
495                 .reset_id = { TEGRA210_CLK_ISPB },
496                 .reset_id_num = 1,
497         },
498         [TEGRA210_POWER_DOMAIN_GPU] = {
499                 .name = "gpu",
500                 .clk_info = {
501                         [0] = { .clk_name = "gpu_gate", .clk_type = CLK_AND_RST },
502                         [1] = { .clk_name = "gpu_ref", .clk_type = CLK_ONLY },
503                         [2] = { .clk_name = "pll_p_out5", .clk_type = CLK_ONLY },
504                 },
505         },
506 };
507
508 struct mc_client_hotreset_reg {
509         u32 control_reg;
510         u32 status_reg;
511 };
512
513 static struct mc_client_hotreset_reg tegra210_mc_reg[] = {
514         [0] = { .control_reg = 0x200, .status_reg = 0x204 },
515         [1] = { .control_reg = 0x970, .status_reg = 0x974 },
516 };
517
518 struct tegra210_powergate_info {
519         bool valid;
520         unsigned int mask;
521         int part_id;
522         struct tegra210_mc_client_info *mc_info;
523         struct powergate_partition_info *part_info;
524 };
525
526 #define T210_POWERGATE_INFO(_pg_id, _pg_bit, _part_id, _mc_id)          \
527 [TEGRA210_POWER_DOMAIN_##_pg_id] = {                                            \
528         .valid = true,                                                  \
529         .mask = BIT(_pg_bit),                                           \
530         .part_id = TEGRA210_POWER_DOMAIN_##_part_id,                            \
531         .mc_info = &tegra210_pg_mc_info[TEGRA210_POWER_DOMAIN_##_mc_id],        \
532         .part_info = &tegra210_pg_partition_info[TEGRA210_POWER_DOMAIN_##_part_id], \
533 }
534
535 static struct tegra210_powergate_info t210_pg_info[TEGRA210_POWER_DOMAIN_MAX] = {
536         T210_POWERGATE_INFO(CRAIL, 0, CRAIL, CRAIL),
537         T210_POWERGATE_INFO(GPU, 1, GPU, GPU),
538         T210_POWERGATE_INFO(VENC, 2, VENC, VENC),
539         T210_POWERGATE_INFO(PCIE, 3, PCIE, PCIE),
540         T210_POWERGATE_INFO(VDEC, 4, VDEC, VDEC),
541         T210_POWERGATE_INFO(L2, 5, L2, L2),
542         T210_POWERGATE_INFO(MPE, 6, MPE, MPE),
543         T210_POWERGATE_INFO(HEG, 7, HEG, HEG),
544         T210_POWERGATE_INFO(SATA, 8, SATA, SATA),
545         T210_POWERGATE_INFO(CPU1, 9, CPU1, CPU1),
546         T210_POWERGATE_INFO(CPU2, 10, CPU2, CPU2),
547         T210_POWERGATE_INFO(CPU3, 11, CPU3, CPU3),
548         T210_POWERGATE_INFO(CELP, 12, CELP, CELP),
549         T210_POWERGATE_INFO(3D1, 13, 3D1, 3D1),
550         T210_POWERGATE_INFO(CPU0, 14, CPU0, CPU0),
551         T210_POWERGATE_INFO(C0NC, 15, C0NC, C0NC),
552         T210_POWERGATE_INFO(C1NC, 16, C1NC, C1NC),
553         T210_POWERGATE_INFO(SOR, 17, SOR, SOR),
554         T210_POWERGATE_INFO(DISA, 18, DISA, DISA),
555         T210_POWERGATE_INFO(DISB, 19, DISB, DISB),
556         T210_POWERGATE_INFO(XUSBA, 20, XUSBA, XUSBA),
557         T210_POWERGATE_INFO(XUSBB, 21, XUSBB, XUSBB),
558         T210_POWERGATE_INFO(XUSBC, 22, XUSBC, XUSBC),
559         T210_POWERGATE_INFO(VIC, 23, VIC, VIC),
560         T210_POWERGATE_INFO(NVDEC, 25, NVDEC, NVDEC),
561         T210_POWERGATE_INFO(NVJPG, 26, NVJPG, NVJPG),
562         T210_POWERGATE_INFO(APE, 27, APE, APE),
563         T210_POWERGATE_INFO(VE2, 29, VE2, VE2),
564 };
565
566 #define PMC_GPU_RG_CONTROL              0x2d4
567
568 #define PWRGATE_CLAMP_STATUS            0x2c
569 #define PWRGATE_TOGGLE                  0x30
570 #define PWRGATE_TOGGLE_START            (1 << 8)
571 #define REMOVE_CLAMPING                 0x34
572 #define PWRGATE_STATUS                  0x38
573
574 static DEFINE_SPINLOCK(tegra210_pg_lock);
575
576 static struct dvfs_rail *gpu_rail;
577 static void __iomem *tegra_mc;
578 static void __iomem *tegra_pmc;
579
580 static int tegra210_pg_powergate_partition(int id);
581 static int tegra210_pg_unpowergate_partition(int id);
582 static int tegra210_pg_mc_flush(int id);
583 static int tegra210_pg_mc_flush_done(int id);
584
585 static u32 mc_read(unsigned long reg)
586 {
587         return readl(tegra_mc + reg);
588 }
589
590 static void mc_write(u32 val, unsigned long reg)
591 {
592         writel_relaxed(val, tegra_mc + reg);
593 }
594
595 /* PMC register read/write */
596 static u32 pmc_read(unsigned long reg)
597 {
598         return readl(tegra_pmc + reg);
599 }
600
601 static void pmc_write(u32 val, unsigned long reg)
602 {
603         writel_relaxed(val, tegra_pmc + reg);
604 }
605
606 #define HOTRESET_READ_COUNTS            5
607
608 static const char *tegra210_pg_get_name(int id)
609 {
610         return t210_pg_info[id].part_info->name;
611 }
612
613 static spinlock_t *tegra210_pg_get_lock(void)
614 {
615         return &tegra210_pg_lock;
616 }
617
618 static bool tegra210_pg_skip(int id)
619 {
620         switch (t210_pg_info[id].part_id) {
621         case TEGRA210_POWER_DOMAIN_GPU:
622                 return true;
623         default:
624                 return false;
625         }
626 }
627
628 static bool tegra210_pg_is_powered(int id)
629 {
630         u32 status = 0;
631
632         if (id == TEGRA210_POWER_DOMAIN_GPU) {
633                 if (gpu_rail)
634                         status = tegra_dvfs_is_rail_up(gpu_rail);
635         } else {
636                 status = pmc_read(PWRGATE_STATUS) & t210_pg_info[id].mask;
637         }
638
639         return !!status;
640 }
641
642 static int tegra_powergate_set(int id, bool new_state)
643 {
644         bool status;
645         unsigned long flags;
646         spinlock_t *lock;
647         u32 reg;
648
649         /* 10us timeout for toggle operation if it takes affect*/
650         int toggle_timeout = 10;
651
652         /* 100 * 10 = 1000us timeout for toggle command to take affect in case
653            of contention with h/w initiated CPU power gating */
654         int contention_timeout = 100;
655
656         if (tegra_cpu_is_asim())
657                 return 0;
658
659         lock = tegra210_pg_get_lock();
660
661         spin_lock_irqsave(lock, flags);
662
663         status = !!(pmc_read(PWRGATE_STATUS) & t210_pg_info[id].mask);
664
665         if (status == new_state) {
666                 spin_unlock_irqrestore(lock, flags);
667                 return 0;
668         }
669
670         switch (id) {
671         case TEGRA210_POWER_DOMAIN_CPU0:
672         case TEGRA210_POWER_DOMAIN_CPU1:
673         case TEGRA210_POWER_DOMAIN_CPU2:
674         case TEGRA210_POWER_DOMAIN_CPU3:
675                 /* CPU ungated in s/w only during boot/resume with outer
676                    waiting loop and no contention from other CPUs */
677                 pmc_write(PWRGATE_TOGGLE_START | id, PWRGATE_TOGGLE);
678                 pmc_read(PWRGATE_TOGGLE);
679                 spin_unlock_irqrestore(lock, flags);
680                 return 0;
681
682         default:
683                 break;
684         }
685
686         /* Wait if PMC is already processing some other power gating request */
687         do {
688                 udelay(1);
689                 reg = pmc_read(PWRGATE_TOGGLE);
690                 contention_timeout--;
691         } while ((contention_timeout > 0) && (reg & PWRGATE_TOGGLE_START));
692
693         if (contention_timeout <= 0)
694                 pr_err(" Timed out waiting for PMC to submit \
695                                 new power gate request \n");
696         contention_timeout = 100;
697
698         /* Submit power gate request */
699         pmc_write(PWRGATE_TOGGLE_START | id, PWRGATE_TOGGLE);
700
701         /* Wait while PMC accepts the request */
702         do {
703                 udelay(1);
704                 reg = pmc_read(PWRGATE_TOGGLE);
705                 contention_timeout--;
706         } while ((contention_timeout > 0) && (reg & PWRGATE_TOGGLE_START));
707
708         if (contention_timeout <= 0)
709                 pr_err(" Timed out waiting for PMC to accept \
710                                 new power gate request \n");
711         contention_timeout = 100;
712
713         /* Check power gate status */
714         do {
715                 do {
716                         udelay(1);
717                         status = !!(pmc_read(PWRGATE_STATUS) &
718                                     t210_pg_info[id].mask);
719
720                         toggle_timeout--;
721                 } while ((status != new_state) && (toggle_timeout > 0));
722
723                 toggle_timeout = 10;
724                 contention_timeout--;
725         } while ((status != new_state) && (contention_timeout > 0));
726
727         spin_unlock_irqrestore(lock, flags);
728
729         if (status != new_state) {
730                 WARN(1, "Could not set powergate %d to %d", id, new_state);
731                 return -EBUSY;
732         }
733
734         trace_power_domain_target(tegra210_pg_get_name(id), new_state,
735                         raw_smp_processor_id());
736
737         return 0;
738 }
739
740 static const char *clk_get_name(struct clk *clk)
741 {
742         return __clk_get_name(clk);
743 }
744
745 static int powergate_module(int id)
746 {
747         tegra210_pg_mc_flush(id);
748
749         return tegra_powergate_set(id, false);
750 }
751
752 static int partition_clk_enable(struct powergate_partition_info *pg_info)
753 {
754         int ret;
755         u32 idx;
756         struct clk *clk;
757         struct partition_clk_info *clk_info;
758
759         for (idx = 0; idx < MAX_CLK_EN_NUM; idx++) {
760                 clk_info = &pg_info->clk_info[idx];
761                 clk = clk_info->clk_ptr;
762                 if (IS_ERR(clk) || !clk)
763                         break;
764
765                 if (clk_info->clk_type != RST_ONLY) {
766                         ret = clk_prepare_enable(clk);
767                         if (ret)
768                                 goto err_clk_en;
769                 }
770         }
771
772         return 0;
773
774 err_clk_en:
775         WARN(1, "Could not enable clk %s, error %d", clk_get_name(clk), ret);
776         while (idx--) {
777                 clk_info = &pg_info->clk_info[idx];
778                 if (clk_info->clk_type != RST_ONLY)
779                         clk_disable_unprepare(clk_info->clk_ptr);
780         }
781
782         return ret;
783 }
784
785 static void partition_clk_disable(struct powergate_partition_info *pg_info)
786 {
787         u32 idx;
788         struct clk *clk;
789         struct partition_clk_info *clk_info;
790
791         for (idx = 0; idx < MAX_CLK_EN_NUM; idx++) {
792                 clk_info = &pg_info->clk_info[idx];
793                 clk = clk_info->clk_ptr;
794
795                 if (IS_ERR(clk) || !clk)
796                         break;
797
798                 if (clk_info->clk_type != RST_ONLY)
799                         clk_disable_unprepare(clk);
800         }
801 }
802
803 static void get_clk_info(struct powergate_partition_info *pg_info)
804 {
805         int idx;
806
807         for (idx = 0; idx < MAX_CLK_EN_NUM; idx++) {
808                 if (!pg_info->clk_info[idx].clk_name)
809                         break;
810
811                 pg_info->clk_info[idx].clk_ptr =
812                         clk_get_sys(NULL, pg_info->clk_info[idx].clk_name);
813
814                 if (IS_ERR_OR_NULL(pg_info->clk_info[idx].clk_ptr))
815                         WARN(1, "Could not find clock %s for %s partition\n",
816                                 pg_info->clk_info[idx].clk_name,
817                                 pg_info->name);
818         }
819 }
820
821 static void powergate_partition_assert_reset(struct powergate_partition_info *pg_info)
822 {
823         tegra_rst_assertv(&pg_info->reset_id[0], pg_info->reset_id_num);
824 }
825
826 static void powergate_partition_deassert_reset(struct powergate_partition_info *pg_info)
827 {
828         tegra_rst_deassertv(&pg_info->reset_id[0], pg_info->reset_id_num);
829 }
830
831 static int slcg_clk_enable(struct powergate_partition_info *pg_info)
832 {
833         int ret;
834         u32 idx;
835         struct clk *clk;
836         struct partition_clk_info *slcg_info;
837
838         for (idx = 0; idx < MAX_CLK_EN_NUM; idx++) {
839                 slcg_info = &pg_info->slcg_info[idx];
840                 clk = slcg_info->clk_ptr;
841                 if (IS_ERR(clk) || !clk)
842                         break;
843
844                 ret = clk_prepare_enable(clk);
845                 if (ret)
846                         goto err_clk_en;
847         }
848
849         return 0;
850
851 err_clk_en:
852         WARN(1, "Could not enable clk %s, error %d", __clk_get_name(clk), ret);
853         while (idx--) {
854                 slcg_info = &pg_info->slcg_info[idx];
855                 clk_disable_unprepare(slcg_info->clk_ptr);
856         }
857
858         return ret;
859 }
860
861 static void slcg_clk_disable(struct powergate_partition_info *pg_info)
862 {
863         u32 idx;
864         struct clk *clk;
865         struct partition_clk_info *slcg_info;
866
867         for (idx = 0; idx < MAX_CLK_EN_NUM; idx++) {
868                 slcg_info = &pg_info->slcg_info[idx];
869                 clk = slcg_info->clk_ptr;
870
871                 if (IS_ERR(clk) || !clk)
872                         break;
873
874                 clk_disable_unprepare(clk);
875         }
876 }
877
878 static void get_slcg_info(struct powergate_partition_info *pg_info)
879 {
880         int idx;
881
882         for (idx = 0; idx < MAX_CLK_EN_NUM; idx++) {
883                 if (!pg_info->slcg_info[idx].clk_name)
884                         break;
885
886                 pg_info->slcg_info[idx].clk_ptr =
887                         clk_get_sys(NULL, pg_info->slcg_info[idx].clk_name);
888
889                 if (IS_ERR_OR_NULL(pg_info->slcg_info[idx].clk_ptr))
890                         pr_err("### Could not find clock %s for %s partition\n",
891                                 pg_info->slcg_info[idx].clk_name,
892                                 pg_info->name);
893         }
894 }
895
896 static int tegra210_slcg_register_notifier(int id, struct notifier_block *nb)
897 {
898         struct powergate_partition_info *pg_info = t210_pg_info[id].part_info;
899
900         if (!pg_info || !nb)
901                 return -EINVAL;
902
903         return raw_notifier_chain_register(&pg_info->slcg_notifier, nb);
904 }
905
906 static int tegra210_slcg_unregister_notifier(int id, struct notifier_block *nb)
907 {
908         struct powergate_partition_info *pg_info = t210_pg_info[id].part_info;
909
910         if (!pg_info || !nb)
911                 return -EINVAL;
912
913         return raw_notifier_chain_unregister(&pg_info->slcg_notifier, nb);
914 }
915
916 static int tegra210_powergate_remove_clamping(int id)
917 {
918         u32 mask;
919         int contention_timeout = 100;
920
921         /*
922          * PCIE and VDE clamping masks are swapped with respect to their
923          * partition ids
924          */
925         if (id ==  TEGRA210_POWER_DOMAIN_VDEC)
926                 mask = t210_pg_info[TEGRA210_POWER_DOMAIN_PCIE].mask;
927         else if (id == TEGRA210_POWER_DOMAIN_PCIE)
928                 mask = t210_pg_info[TEGRA210_POWER_DOMAIN_VDEC].mask;
929         else
930                 mask = t210_pg_info[id].mask;
931
932         pmc_write(mask, REMOVE_CLAMPING);
933         /* Wait until clamp is removed */
934         do {
935                 udelay(1);
936                 contention_timeout--;
937         } while ((contention_timeout > 0)
938                         && (pmc_read(PWRGATE_CLAMP_STATUS) & BIT(id)));
939
940         WARN(contention_timeout <= 0, "Couldn't remove clamping");
941
942         return 0;
943 }
944
945 static int __tegra1xx_powergate(int id, struct powergate_partition_info *pg_info,
946                                 bool clk_enable)
947 {
948         int ret;
949
950         /* If first clk_ptr is null, fill clk info for the partition */
951         if (!pg_info->clk_info[0].clk_ptr)
952                 get_clk_info(pg_info);
953
954         if (clk_enable) {
955                 /*
956                  * Enable clocks only if clocks are not expected to
957                  * be off when power gating is done
958                  */
959                 ret = partition_clk_enable(pg_info);
960                 if (ret) {
961                         WARN(1, "Couldn't enable clock");
962                         return ret;
963                 }
964
965                 udelay(10);
966         }
967
968         tegra210_pg_mc_flush(id);
969
970         udelay(10);
971
972         powergate_partition_assert_reset(pg_info);
973
974         udelay(10);
975
976         /* Powergating is done only if refcnt of all clks is 0 */
977         partition_clk_disable(pg_info);
978
979         udelay(10);
980
981         ret = tegra_powergate_set(id, false);
982         if (ret)
983                 goto err_power_off;
984
985         return 0;
986
987 err_power_off:
988         WARN(1, "Could not Powergate Partition %d", id);
989         return ret;
990 }
991
992 static int tegra1xx_powergate(int id, struct powergate_partition_info *pg_info)
993 {
994         return __tegra1xx_powergate(id, pg_info, true);
995 }
996
997 static int __tegra1xx_unpowergate(int id, struct powergate_partition_info *pg_info,
998                                 bool clk_disable)
999 {
1000         int ret;
1001
1002         /* If first clk_ptr is null, fill clk info for the partition */
1003         if (!pg_info->clk_info[0].clk_ptr)
1004                 get_clk_info(pg_info);
1005
1006         if (!pg_info->slcg_info[0].clk_ptr)
1007                 get_slcg_info(pg_info);
1008
1009         if (tegra210_pg_is_powered(id)) {
1010                 if (!clk_disable) {
1011                         ret = partition_clk_enable(pg_info);
1012                         if (ret)
1013                                 return ret;
1014                         if (!pg_info->skip_reset) {
1015                                 powergate_partition_assert_reset(pg_info);
1016                                 udelay(10);
1017                                 powergate_partition_deassert_reset(pg_info);
1018                         }
1019                 }
1020                 return 0;
1021         }
1022
1023         ret = tegra_powergate_set(id, true);
1024         if (ret)
1025                 goto err_power;
1026
1027         udelay(10);
1028
1029         /* Un-Powergating fails if all clks are not enabled */
1030         ret = partition_clk_enable(pg_info);
1031         if (ret)
1032                 goto err_clk_on;
1033
1034         powergate_partition_assert_reset(pg_info);
1035
1036         udelay(10);
1037
1038         ret = tegra210_powergate_remove_clamping(id);
1039         if (ret)
1040                 goto err_clamp;
1041
1042         udelay(10);
1043
1044         if (!pg_info->skip_reset) {
1045                 powergate_partition_deassert_reset(pg_info);
1046
1047                 udelay(10);
1048         }
1049
1050         tegra210_pg_mc_flush_done(id);
1051
1052         udelay(10);
1053
1054         slcg_clk_enable(pg_info);
1055
1056         raw_notifier_call_chain(&pg_info->slcg_notifier, 0, NULL);
1057
1058         slcg_clk_disable(pg_info);
1059
1060         /* Disable all clks enabled earlier. Drivers should enable clks */
1061         if (clk_disable)
1062                 partition_clk_disable(pg_info);
1063
1064         return 0;
1065
1066 err_clamp:
1067         partition_clk_disable(pg_info);
1068 err_clk_on:
1069         powergate_module(id);
1070 err_power:
1071         WARN(1, "Could not Un-Powergate %d", id);
1072         return ret;
1073 }
1074
1075 static int tegra1xx_unpowergate(int id,
1076                                 struct powergate_partition_info *pg_info)
1077 {
1078         return __tegra1xx_unpowergate(id, pg_info, true);
1079 }
1080
1081 static int tegra1xx_powergate_partition_with_clk_off(int id,
1082                 struct powergate_partition_info *pg_info)
1083 {
1084         int ret = 0;
1085
1086         ret = __tegra1xx_powergate(id, pg_info, false);
1087         if (ret)
1088                 WARN(1, "Could not Powergate Partition %d", id);
1089
1090         return ret;
1091 }
1092
1093 static int tegra1xx_unpowergate_partition_with_clk_on(int id,
1094         struct powergate_partition_info *pg_info)
1095 {
1096         int ret = 0;
1097
1098         ret = __tegra1xx_unpowergate(id, pg_info, false);
1099         if (ret)
1100                 WARN(1, "Could not Un-Powergate %d", id);
1101
1102         return ret;
1103 }
1104
1105
1106 static bool tegra210_pg_hotreset_check(u32 status_reg, u32 *status)
1107 {
1108         int i;
1109         u32 curr_status;
1110         u32 prev_status;
1111         unsigned long flags;
1112
1113         spin_lock_irqsave(&tegra210_pg_lock, flags);
1114         prev_status = mc_read(status_reg);
1115         for (i = 0; i < HOTRESET_READ_COUNTS; i++) {
1116                 curr_status = mc_read(status_reg);
1117                 if (curr_status != prev_status) {
1118                         spin_unlock_irqrestore(&tegra210_pg_lock, flags);
1119                         return false;
1120                 }
1121         }
1122         *status = curr_status;
1123         spin_unlock_irqrestore(&tegra210_pg_lock, flags);
1124
1125         return true;
1126 }
1127
1128 static int tegra210_pg_mc_flush(int id)
1129 {
1130         u32 idx, rst_control, rst_status;
1131         u32 rst_control_reg, rst_status_reg;
1132         enum mc_client mc_client_bit;
1133         unsigned long flags;
1134         unsigned int timeout;
1135         bool ret;
1136         int reg_idx;
1137
1138         for (idx = 0; idx < MAX_HOTRESET_CLIENT_NUM; idx++) {
1139                 mc_client_bit =
1140                         t210_pg_info[id].mc_info->hot_reset_clients[idx];
1141                 if (mc_client_bit == MC_CLIENT_LAST)
1142                         break;
1143
1144                 reg_idx = mc_client_bit / 32;
1145                 mc_client_bit %= 32;
1146                 rst_control_reg = tegra210_mc_reg[reg_idx].control_reg;
1147                 rst_status_reg = tegra210_mc_reg[reg_idx].status_reg;
1148
1149                 spin_lock_irqsave(&tegra210_pg_lock, flags);
1150                 rst_control = mc_read(rst_control_reg);
1151                 rst_control |= (1 << mc_client_bit);
1152                 mc_write(rst_control, rst_control_reg);
1153                 spin_unlock_irqrestore(&tegra210_pg_lock, flags);
1154
1155                 timeout = 0;
1156                 do {
1157                         udelay(10);
1158                         rst_status = 0;
1159                         ret = tegra210_pg_hotreset_check(rst_status_reg,
1160                                                                 &rst_status);
1161                         if ((timeout++ > EMULATION_MC_FLUSH_TIMEOUT) &&
1162                                 (tegra_platform_is_qt() ||
1163                                 tegra_platform_is_fpga())) {
1164                                 pr_warn("%s flush %d timeout\n", __func__, id);
1165                                 break;
1166                         }
1167                         if (!ret)
1168                                 continue;
1169                 } while (!(rst_status & (1 << mc_client_bit)));
1170         }
1171
1172         return 0;
1173 }
1174
1175 static int tegra210_pg_mc_flush_done(int id)
1176 {
1177         u32 idx, rst_control, rst_control_reg;
1178         enum mc_client mc_client_bit;
1179         unsigned long flags;
1180         int reg_idx;
1181
1182         for (idx = 0; idx < MAX_HOTRESET_CLIENT_NUM; idx++) {
1183                 mc_client_bit =
1184                         t210_pg_info[id].mc_info->hot_reset_clients[idx];
1185                 if (mc_client_bit == MC_CLIENT_LAST)
1186                         break;
1187
1188                 reg_idx = mc_client_bit / 32;
1189                 mc_client_bit %= 32;
1190                 rst_control_reg = tegra210_mc_reg[reg_idx].control_reg;
1191
1192                 spin_lock_irqsave(&tegra210_pg_lock, flags);
1193                 rst_control = mc_read(rst_control_reg);
1194                 rst_control &= ~(1 << mc_client_bit);
1195                 mc_write(rst_control, rst_control_reg);
1196                 spin_unlock_irqrestore(&tegra210_pg_lock, flags);
1197
1198         }
1199         wmb();
1200
1201         return 0;
1202 }
1203
1204 static int tegra210_pg_powergate(int id)
1205 {
1206         struct powergate_partition_info *partition = t210_pg_info[id].part_info;
1207         int ret = 0;
1208
1209         trace_powergate(__func__, tegra210_pg_get_name(id), id, 1, 0);
1210         mutex_lock(&partition->pg_mutex);
1211
1212         if (--partition->refcount > 0)
1213                 goto exit_unlock;
1214
1215         if ((partition->refcount < 0) || !tegra210_pg_is_powered(id)) {
1216                 WARN(1, "Partition %s already powergated, refcount and status mismatch\n",
1217                      partition->name);
1218                 goto exit_unlock;
1219         }
1220
1221         ret = tegra1xx_powergate(id, partition);
1222
1223 exit_unlock:
1224         mutex_unlock(&partition->pg_mutex);
1225         trace_powergate(__func__, tegra210_pg_get_name(id), id, 0, ret);
1226         return ret;
1227 }
1228
1229 static int tegra210_pg_unpowergate(int id)
1230 {
1231         struct powergate_partition_info *partition = t210_pg_info[id].part_info;
1232         int ret = 0;
1233
1234         trace_powergate(__func__, tegra210_pg_get_name(id), id, 1, 0);
1235         mutex_lock(&partition->pg_mutex);
1236
1237         if (partition->refcount++ > 0)
1238                 goto exit_unlock;
1239
1240         if (tegra210_pg_is_powered(id)) {
1241                 WARN(1, "Partition %s is already unpowergated, refcount and status mismatch\n",
1242                      partition->name);
1243                 goto exit_unlock;
1244         }
1245
1246         ret = tegra1xx_unpowergate(id, partition);
1247
1248 exit_unlock:
1249         mutex_unlock(&partition->pg_mutex);
1250         trace_powergate(__func__, tegra210_pg_get_name(id), id, 0, ret);
1251         return ret;
1252 }
1253
1254 static int tegra210_pg_gpu_powergate(int id)
1255 {
1256         int ret = 0;
1257         struct powergate_partition_info *partition = t210_pg_info[id].part_info;
1258
1259         trace_powergate(__func__, tegra210_pg_get_name(id), id, 1, 0);
1260         mutex_lock(&partition->pg_mutex);
1261
1262         if (--partition->refcount > 0)
1263                 goto exit_unlock;
1264
1265         if (!tegra210_pg_is_powered(id)) {
1266                 WARN(1, "GPU rail is already off, refcount and status mismatch\n");
1267                 goto exit_unlock;
1268         }
1269
1270         if (!partition->clk_info[0].clk_ptr)
1271                 get_clk_info(partition);
1272
1273         tegra210_pg_mc_flush(id);
1274
1275         udelay(10);
1276
1277         powergate_partition_assert_reset(partition);
1278
1279         udelay(10);
1280
1281         pmc_write(0x1, PMC_GPU_RG_CONTROL);
1282         pmc_read(PMC_GPU_RG_CONTROL);
1283
1284         udelay(10);
1285
1286         partition_clk_disable(partition);
1287
1288         udelay(10);
1289
1290         tegra_soctherm_gpu_tsens_invalidate(1);
1291
1292         if (gpu_rail) {
1293                 ret = tegra_dvfs_rail_power_down(gpu_rail);
1294                 if (ret) {
1295                         WARN(1, "Could not power down GPU rail\n");
1296                         goto exit_unlock;
1297                 }
1298         } else {
1299                 pr_info("No GPU regulator?\n");
1300         }
1301
1302 exit_unlock:
1303         mutex_unlock(&partition->pg_mutex);
1304         trace_powergate(__func__, tegra210_pg_get_name(id), id, 0, ret);
1305         return ret;
1306 }
1307
1308 static int tegra210_pg_gpu_unpowergate(int id)
1309 {
1310         int ret = 0;
1311         bool first = false;
1312         struct powergate_partition_info *partition = t210_pg_info[id].part_info;
1313
1314         trace_powergate(__func__, tegra210_pg_get_name(id), id, 1, 0);
1315         mutex_lock(&partition->pg_mutex);
1316
1317         if (partition->refcount++ > 0)
1318                 goto exit_unlock;
1319
1320         if (!gpu_rail) {
1321                 gpu_rail = tegra_dvfs_get_rail_by_name("vdd_gpu");
1322                 if (IS_ERR_OR_NULL(gpu_rail)) {
1323                         WARN(1, "No GPU regulator?\n");
1324                         goto err_power;
1325                 }
1326                 first = true;
1327         }
1328
1329         if (tegra210_pg_is_powered(id)) {
1330                 WARN(1, "GPU rail is already on, refcount and status mismatch\n");
1331                 goto exit_unlock;
1332         }
1333
1334         ret = tegra_dvfs_rail_power_up(gpu_rail);
1335         if (ret) {
1336                 WARN(1, "Could not turn on GPU rail\n");
1337                 goto err_power;
1338         }
1339
1340         tegra_soctherm_gpu_tsens_invalidate(0);
1341
1342         if (!partition->clk_info[0].clk_ptr)
1343                 get_clk_info(partition);
1344
1345         if (!first) {
1346                 ret = partition_clk_enable(partition);
1347                 if (ret) {
1348                         WARN(1, "Could not turn on partition clocks\n");
1349                         goto err_clk_on;
1350                 }
1351         }
1352
1353         udelay(10);
1354
1355         powergate_partition_assert_reset(partition);
1356
1357         udelay(10);
1358
1359         pmc_write(0, PMC_GPU_RG_CONTROL);
1360         pmc_read(PMC_GPU_RG_CONTROL);
1361
1362         udelay(10);
1363
1364         /*
1365          * Make sure all clok branches into GPU, except reference clock are
1366          * gated across resert de-assertion.
1367          */
1368         clk_disable_unprepare(partition->clk_info[0].clk_ptr);
1369         powergate_partition_deassert_reset(partition);
1370         clk_prepare_enable(partition->clk_info[0].clk_ptr);
1371
1372         /* Flush MC after boot/railgate/SC7 */
1373         tegra210_pg_mc_flush(id);
1374
1375         udelay(10);
1376
1377         tegra210_pg_mc_flush_done(id);
1378
1379         udelay(10);
1380
1381 exit_unlock:
1382         mutex_unlock(&partition->pg_mutex);
1383         trace_powergate(__func__, tegra210_pg_get_name(id), id, 0, ret);
1384         return ret;
1385
1386 err_clk_on:
1387         powergate_module(id);
1388 err_power:
1389         mutex_unlock(&partition->pg_mutex);
1390
1391         trace_powergate(__func__, tegra210_pg_get_name(id), id, 0, ret);
1392         return ret;
1393 }
1394
1395 static int tegra210_pg_powergate_sor(int id)
1396 {
1397         int ret;
1398
1399         ret = tegra210_pg_powergate(id);
1400         if (ret)
1401                 return ret;
1402
1403         ret = tegra210_pg_powergate_partition(TEGRA210_POWER_DOMAIN_SOR);
1404         if (ret)
1405                 return ret;
1406
1407         return 0;
1408 }
1409
1410 static int tegra210_pg_unpowergate_sor(int id)
1411 {
1412         int ret;
1413
1414         ret = tegra210_pg_unpowergate_partition(TEGRA210_POWER_DOMAIN_SOR);
1415         if (ret)
1416                 return ret;
1417
1418         ret = tegra210_pg_unpowergate(id);
1419         if (ret) {
1420                 tegra210_pg_powergate_partition(TEGRA210_POWER_DOMAIN_SOR);
1421                 return ret;
1422         }
1423
1424         return 0;
1425 }
1426
1427 static int tegra210_pg_nvdec_powergate(int id)
1428 {
1429         tegra210_pg_powergate(TEGRA210_POWER_DOMAIN_NVDEC);
1430         tegra210_pg_powergate_partition(TEGRA210_POWER_DOMAIN_NVJPG);
1431
1432         return 0;
1433 }
1434
1435 static int tegra210_pg_nvdec_unpowergate(int id)
1436 {
1437         tegra210_pg_unpowergate_partition(TEGRA210_POWER_DOMAIN_NVJPG);
1438         tegra210_pg_unpowergate(TEGRA210_POWER_DOMAIN_NVDEC);
1439
1440         return 0;
1441 }
1442
1443 static int tegra210_pg_sata_powergate(int id)
1444 {
1445         tegra210_set_sata_pll_seq_sw(true);
1446         tegra210_pg_powergate(TEGRA210_POWER_DOMAIN_SATA);
1447
1448         return 0;
1449 }
1450
1451 static int tegra210_pg_sata_unpowergate(int id)
1452 {
1453         tegra210_pg_unpowergate(TEGRA210_POWER_DOMAIN_SATA);
1454         tegra210_set_sata_pll_seq_sw(false);
1455
1456         return 0;
1457 }
1458
1459 static int tegra210_pg_powergate_partition(int id)
1460 {
1461         int ret;
1462
1463         switch (t210_pg_info[id].part_id) {
1464                 case TEGRA210_POWER_DOMAIN_GPU:
1465                         ret = tegra210_pg_gpu_powergate(id);
1466                         break;
1467                 case TEGRA210_POWER_DOMAIN_DISA:
1468                 case TEGRA210_POWER_DOMAIN_DISB:
1469                 case TEGRA210_POWER_DOMAIN_VENC:
1470                         ret = tegra210_pg_powergate_sor(id);
1471                         break;
1472                 case TEGRA210_POWER_DOMAIN_NVDEC:
1473                         ret = tegra210_pg_nvdec_powergate(id);
1474                         break;
1475                 case TEGRA210_POWER_DOMAIN_SATA:
1476                         ret = tegra210_pg_sata_powergate(id);
1477                         break;
1478                 default:
1479                         ret = tegra210_pg_powergate(id);
1480         }
1481
1482         return ret;
1483 }
1484
1485 static int tegra210_pg_unpowergate_partition(int id)
1486 {
1487         int ret;
1488
1489         switch (t210_pg_info[id].part_id) {
1490                 case TEGRA210_POWER_DOMAIN_GPU:
1491                         ret = tegra210_pg_gpu_unpowergate(id);
1492                         break;
1493                 case TEGRA210_POWER_DOMAIN_DISA:
1494                 case TEGRA210_POWER_DOMAIN_DISB:
1495                 case TEGRA210_POWER_DOMAIN_VENC:
1496                         ret = tegra210_pg_unpowergate_sor(id);
1497                         break;
1498                 case TEGRA210_POWER_DOMAIN_NVDEC:
1499                         ret = tegra210_pg_nvdec_unpowergate(id);
1500                         break;
1501                 case TEGRA210_POWER_DOMAIN_SATA:
1502                         ret = tegra210_pg_sata_unpowergate(id);
1503                         break;
1504                 default:
1505                         ret = tegra210_pg_unpowergate(id);
1506         }
1507
1508         return ret;
1509 }
1510
1511 static int tegra210_pg_powergate_clk_off(int id)
1512 {
1513         int ret = 0;
1514         struct powergate_partition_info *partition = t210_pg_info[id].part_info;
1515
1516         BUG_ON(id == TEGRA210_POWER_DOMAIN_GPU);
1517
1518         trace_powergate(__func__, tegra210_pg_get_name(id), id, 1, 0);
1519         mutex_lock(&partition->pg_mutex);
1520
1521         if (--partition->refcount > 0)
1522                 goto exit_unlock;
1523
1524         if ((partition->refcount < 0) || !tegra210_pg_is_powered(id)) {
1525                 WARN(1, "Partition %s already powergated, refcount and status mismatch\n",
1526                      partition->name);
1527                 goto exit_unlock;
1528         }
1529
1530         if (t210_pg_info[id].part_id == TEGRA210_POWER_DOMAIN_SATA)
1531                 tegra210_set_sata_pll_seq_sw(true);
1532
1533         ret = tegra1xx_powergate_partition_with_clk_off(id,
1534                         t210_pg_info[id].part_info);
1535
1536 exit_unlock:
1537         mutex_unlock(&partition->pg_mutex);
1538         trace_powergate(__func__, tegra210_pg_get_name(id), id, 0, ret);
1539
1540         return ret;
1541 }
1542
1543 static int tegra210_pg_unpowergate_clk_on(int id)
1544 {
1545         int ret = 0;
1546         struct powergate_partition_info *partition = t210_pg_info[id].part_info;
1547
1548         BUG_ON(id == TEGRA210_POWER_DOMAIN_GPU);
1549         trace_powergate(__func__, tegra210_pg_get_name(id), id, 1, 0);
1550         mutex_lock(&partition->pg_mutex);
1551
1552         if (partition->refcount++ > 0)
1553                 goto exit_unlock;
1554
1555         ret = tegra1xx_unpowergate_partition_with_clk_on(id,
1556                         t210_pg_info[id].part_info);
1557
1558         if (t210_pg_info[id].part_id == TEGRA210_POWER_DOMAIN_SATA)
1559                 tegra210_set_sata_pll_seq_sw(false);
1560
1561 exit_unlock:
1562         mutex_unlock(&partition->pg_mutex);
1563         trace_powergate(__func__, tegra210_pg_get_name(id), id, 0, ret);
1564
1565         return ret;
1566 }
1567
1568 static int tegra210_pg_init_refcount(void)
1569 {
1570         int i;
1571
1572         for (i = 0; i < TEGRA210_POWER_DOMAIN_MAX; i++) {
1573                 if (!t210_pg_info[i].valid)
1574                         continue;
1575
1576                 /* Consider main partion ID only */
1577                 if (i != t210_pg_info[i].part_id)
1578                         continue;
1579
1580                 if (tegra210_pg_is_powered(i))
1581                         t210_pg_info[i].part_info->refcount = 1;
1582                 else
1583                         t210_pg_info[i].part_info->disable_after_boot = 0;
1584
1585                 mutex_init(&t210_pg_info[i].part_info->pg_mutex);
1586         }
1587
1588         /* SOR refcount depends on other units */
1589         t210_pg_info[TEGRA210_POWER_DOMAIN_SOR].part_info->refcount =
1590                 (tegra210_pg_is_powered(TEGRA210_POWER_DOMAIN_DISA) ? 1 : 0) +
1591                 (tegra210_pg_is_powered(TEGRA210_POWER_DOMAIN_DISB) ? 1 : 0) +
1592                 (tegra210_pg_is_powered(TEGRA210_POWER_DOMAIN_VENC) ? 1 : 0);
1593
1594         tegra210_pg_powergate_partition(TEGRA210_POWER_DOMAIN_XUSBA);
1595         tegra210_pg_powergate_partition(TEGRA210_POWER_DOMAIN_XUSBB);
1596         tegra210_pg_powergate_partition(TEGRA210_POWER_DOMAIN_XUSBC);
1597
1598         return 0;
1599 }
1600
1601 static bool tegra210_powergate_id_is_valid(int id)
1602 {
1603         if ((id < 0) || (id >= TEGRA210_POWER_DOMAIN_MAX))
1604                 return false;
1605
1606         return t210_pg_info[id].valid;
1607 }
1608
1609 static int tegra210_powergate_cpuid_to_powergate_id(int cpu)
1610 {
1611         switch (cpu) {
1612         case 0:
1613                 return TEGRA210_POWER_DOMAIN_CPU0;
1614         case 1:
1615                 return TEGRA210_POWER_DOMAIN_CPU1;
1616         case 2:
1617                 return TEGRA210_POWER_DOMAIN_CPU2;
1618         case 3:
1619                 return TEGRA210_POWER_DOMAIN_CPU3;
1620         default:
1621                 return -1;
1622         }
1623
1624         return -1;
1625 }
1626
1627 static struct tegra_powergate_driver_ops tegra210_pg_ops = {
1628         .soc_name = "tegra210",
1629
1630         .num_powerdomains = TEGRA210_POWER_DOMAIN_MAX,
1631         .powergate_id_is_soc_valid = tegra210_powergate_id_is_valid,
1632         .powergate_cpuid_to_powergate_id =
1633                                 tegra210_powergate_cpuid_to_powergate_id,
1634
1635         .get_powergate_lock = tegra210_pg_get_lock,
1636         .get_powergate_domain_name = tegra210_pg_get_name,
1637
1638         .powergate_partition = tegra210_pg_powergate_partition,
1639         .unpowergate_partition = tegra210_pg_unpowergate_partition,
1640
1641         .powergate_partition_with_clk_off = tegra210_pg_powergate_clk_off,
1642         .unpowergate_partition_with_clk_on = tegra210_pg_unpowergate_clk_on,
1643
1644         .powergate_mc_flush = tegra210_pg_mc_flush,
1645         .powergate_mc_flush_done = tegra210_pg_mc_flush_done,
1646
1647         .powergate_skip = tegra210_pg_skip,
1648
1649         .powergate_is_powered = tegra210_pg_is_powered,
1650
1651         .powergate_init_refcount = tegra210_pg_init_refcount,
1652         .powergate_remove_clamping = tegra210_powergate_remove_clamping,
1653         .slcg_register_notifier = tegra210_slcg_register_notifier,
1654         .slcg_unregister_notifier = tegra210_slcg_unregister_notifier,
1655 };
1656
1657 #define TEGRA_PMC_BASE  0x7000E400
1658 #define TEGRA_MC_BASE   0x70019000
1659 struct tegra_powergate_driver_ops *tegra210_powergate_init_chip_support(void)
1660 {
1661         tegra_pmc = ioremap(TEGRA_PMC_BASE, 4096);
1662         tegra_mc = ioremap(TEGRA_MC_BASE, 4096);
1663
1664         if (tegra_platform_is_linsim())
1665                 return NULL;
1666
1667         return &tegra210_pg_ops;
1668 }
1669
1670 static int __init tegra210_disable_boot_partitions(void)
1671 {
1672         int i;
1673
1674         if (!soc_is_tegra210_n_before())
1675                 return 0;
1676
1677         pr_info("Disable partitions left on by BL\n");
1678         for (i = 0; i < TEGRA210_POWER_DOMAIN_MAX; i++) {
1679                 if (!t210_pg_info[i].valid)
1680                         continue;
1681
1682                 /* consider main partion ID only */
1683                 if (i != t210_pg_info[i].part_id)
1684                         continue;
1685
1686                 if (t210_pg_info[i].part_info->disable_after_boot &&
1687                         (t210_pg_info[i].part_id != TEGRA210_POWER_DOMAIN_GPU)) {
1688                         pr_info("  %s\n", t210_pg_info[i].part_info->name);
1689                         tegra210_pg_powergate_partition(i);
1690                 }
1691         }
1692
1693         return 0;
1694 }
1695 late_initcall(tegra210_disable_boot_partitions);