usb: storage: add usb storage shutdown ops
[linux-4.9.git] / drivers / soc / tegra / tegra210-dvfs.c
1 /*
2  * Copyright (c) 2014-2018, NVIDIA CORPORATION.  All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify it
5  * under the terms and conditions of the GNU General Public License,
6  * version 2, as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope it will be useful, but WITHOUT
9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
11  * more details.
12  *
13  * You should have received a copy of the GNU General Public License
14  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
15  */
16
17 #include <linux/kernel.h>
18 #include <linux/module.h>
19 #include <linux/clk.h>
20 #include <linux/clk-provider.h>
21 #include <linux/of_address.h>
22 #include <linux/thermal.h>
23 #include <linux/regulator/consumer.h>
24
25 #include <soc/tegra/cvb.h>
26 #include <soc/tegra/tegra-dfll.h>
27 #include <soc/tegra/tegra-dvfs.h>
28 #include <soc/tegra/fuse.h>
29 #include <soc/tegra/tegra_emc.h>
30
31 #include <dt-bindings/thermal/tegra210b01-trips.h>
32
33 #define KHZ             1000
34 #define MHZ             1000000
35 #define VDD_SAFE_STEP   100
36
37 /* Margin % applied to PLL CVB tables */
38 #define CVB_PLL_MARGIN  30
39
40 #define VDD_CPU_INDEX   0
41 #define VDD_CORE_INDEX  1
42 #define VDD_GPU_INDEX   2
43
44 struct tegra_dvfs_data {
45         struct dvfs_rail **rails;
46         int rails_num;
47         struct cpu_dvfs *cpu_fv_table;
48         int cpu_fv_table_size;
49         struct cvb_dvfs *gpu_cvb_table;
50         int gpu_cvb_table_size;
51         struct dvb_dvfs *emc_dvb_table;
52         int emc_dvb_table_size;
53
54         const int *core_mv;
55         struct dvfs *core_vf_table;
56         int core_vf_table_size;
57         struct dvfs *spi_vf_table;
58         struct dvfs *spi_slave_vf_table;
59         struct dvfs *qspi_sdr_vf_table;
60         struct dvfs *qspi_ddr_vf_table;
61         struct dvfs *sor1_dp_vf_table;
62         int sor1_dp_vf_table_size;
63         int (*get_core_min_mv)(void);
64         int (*get_core_max_mv)(void);
65
66         struct dvfs_therm_limits *core_floors;
67         struct dvfs_therm_limits *core_caps;
68         struct dvfs_therm_limits *core_caps_ucm2;
69         const char *core_dvfs_ver;
70 };
71
72 static struct tegra_dvfs_data *dvfs_data;
73
74 static bool tegra_dvfs_cpu_disabled;
75 static bool tegra_dvfs_core_disabled;
76 static bool tegra_dvfs_gpu_disabled;
77 static int cpu_millivolts[MAX_DVFS_FREQS];
78 static int cpu_dfll_millivolts[MAX_DVFS_FREQS];
79 static int cpu_lp_millivolts[MAX_DVFS_FREQS];
80
81 static struct dvfs_therm_limits
82 tegra210_core_therm_floors[MAX_THERMAL_LIMITS] = {
83         {15, 950},
84         {0, 0},
85 };
86
87 static struct dvfs_therm_limits
88 tegra210_core_therm_caps[MAX_THERMAL_LIMITS] = {
89         {86, 1132},
90         {0, 0},
91 };
92
93 static struct dvfs_therm_limits
94 tegra210_core_therm_caps_ucm2[MAX_THERMAL_LIMITS] = {
95         {86, 1090},
96         {0, 0},
97 };
98
99 static struct dvfs_therm_limits
100 tegra210b01_core_therm_floors[MAX_THERMAL_LIMITS] = {
101         {TEGRA210B01_SOC_THERMAL_FLOOR_0 / 1000, 800},
102         {0, 0},
103 };
104
105 static struct dvfs_therm_limits
106 tegra210b01_core_therm_caps[MAX_THERMAL_LIMITS] = {
107         {TEGRA210B01_SOC_THERMAL_CAP_0 / 1000, 1010},
108         {0, 0},
109 };
110
111 static struct dvfs_therm_limits
112 tegra210b01_core_therm_caps_ucm2[MAX_THERMAL_LIMITS] = {
113         {TEGRA210B01_SOC_THERMAL_CAP_0 / 1000, 1010},
114         {0, 0},
115 };
116
117
118 static struct dvfs_rail tegra210_dvfs_rail_vdd_cpu = {
119         .reg_id = "vdd-cpu",
120         .max_millivolts = 1300,
121         .min_millivolts = 800,
122         .step = VDD_SAFE_STEP,
123         .step_up = 1300,
124         .jmp_to_zero = true,
125         .dfll_mode = true,
126         .alignment = {
127                 .step_uv = 6250, /* 6.25mV */
128         },
129         .stats = {
130                 .bin_uv = 6250, /* 6.25mV */
131         },
132         .is_ready = false,
133 };
134
135 static struct dvfs_rail tegra210_dvfs_rail_vdd_core = {
136         .reg_id = "vdd-core",
137         .max_millivolts = 1300,
138         .step = VDD_SAFE_STEP,
139         .step_up = 1300,
140         .alignment = {
141                 .step_uv = 12500, /* 12.5mV */
142         },
143         .stats = {
144                 .bin_uv = 12500, /* 12.5mV */
145         },
146         .is_ready = false,
147 };
148
149 static struct dvfs_rail tegra210_dvfs_rail_vdd_gpu = {
150         .reg_id = "vdd-gpu",
151         .max_millivolts = 1300,
152         .step = VDD_SAFE_STEP,
153         .step_up = 1300,
154         .alignment = {
155                 .step_uv = 10000, /* 10mV */
156         },
157         .stats = {
158                 .bin_uv = 10000, /* 10mV */
159         },
160         .in_band_pm = true,
161 };
162
163 static struct dvfs_rail *tegra210_dvfs_rails[] = {
164         [VDD_CPU_INDEX] = &tegra210_dvfs_rail_vdd_cpu,
165         [VDD_CORE_INDEX] = &tegra210_dvfs_rail_vdd_core,
166         [VDD_GPU_INDEX] = &tegra210_dvfs_rail_vdd_gpu,
167 };
168
169 static struct dvfs_rail tegra210b01_dvfs_rail_vdd_core = {
170         .reg_id = "vdd-core",
171         .max_millivolts = 1300,
172         .step = VDD_SAFE_STEP,
173         .step_up = 1300,
174         .alignment = {
175                 .step_uv = 12500, /* 12.5mV */
176         },
177         .stats = {
178                 .bin_uv = 12500, /* 12.5mV */
179         },
180 };
181
182 static struct dvfs_rail tegra210b01_dvfs_rail_vdd_cpu = {
183         .reg_id = "vdd-cpu",
184         .max_millivolts = 1125,
185         .step = VDD_SAFE_STEP,
186         .step_up = 1125,
187         .jmp_to_zero = true,
188         .dfll_mode = true,
189         .alignment = {
190                 .step_uv = 5000, /* 5.0mV */
191         },
192         .stats = {
193                 .bin_uv = 5000, /* 5.0mV */
194         },
195 };
196
197 static struct dvfs_rail tegra210b01_dvfs_rail_vdd_gpu = {
198         .reg_id = "vdd-gpu",
199         .max_millivolts = 1125,
200         .step = VDD_SAFE_STEP,
201         .step_up = 1125,
202         .alignment = {
203                 .step_uv = 5000, /* 5mV */
204         },
205         .stats = {
206                 .bin_uv = 5000, /* 10mV */
207         },
208         .vts_floors_table =             /* applied if no vts cdev */
209                 { {TEGRA210B01_GPU_DVFS_THERMAL_MIN / 1000, 800 }, },
210         .in_band_pm = true,
211 };
212
213 static struct dvfs_rail *tegra210b01_dvfs_rails[] = {
214         [VDD_CPU_INDEX] = &tegra210b01_dvfs_rail_vdd_cpu,
215         [VDD_CORE_INDEX] = &tegra210b01_dvfs_rail_vdd_core,
216         [VDD_GPU_INDEX] = &tegra210b01_dvfs_rail_vdd_gpu,
217 };
218
219 static struct dvfs_rail vdd_cpu_rail;
220 static struct dvfs_rail vdd_gpu_rail;
221 static struct dvfs_rail vdd_core_rail;
222
223 static struct dvfs_rail *vdd_dvfs_rails[] = {
224         [VDD_CPU_INDEX] = &vdd_cpu_rail,
225         [VDD_CORE_INDEX] = &vdd_core_rail,
226         [VDD_GPU_INDEX] = &vdd_gpu_rail,
227 };
228
229 static struct dvfs cpu_dvfs = {
230         .clk_name       = "cclk_g",
231         .millivolts     = cpu_millivolts,
232         .dfll_millivolts = cpu_dfll_millivolts,
233         .auto_dvfs      = true,
234         .dvfs_rail      = &vdd_cpu_rail,
235 };
236
237 /* CPU DVFS tables */
238 #define CPU_PLL_CVB_TABLE \
239         .pll_min_millivolts = 950, \
240         .speedo_scale = 100,    \
241         .voltage_scale = 1000,  \
242         .cvb_pll_table = {              \
243                 {204000000UL,   {        0,        0,        0 } }, \
244                 {306000000UL,   {        0,        0,        0 } }, \
245                 {408000000UL,   {        0,        0,        0 } }, \
246                 {510000000UL,   {        0,        0,        0 } }, \
247                 {612000000UL,   {        0,        0,        0 } }, \
248                 {714000000UL,   {        0,        0,        0 } }, \
249                 {816000000UL,   {        0,        0,        0 } }, \
250                 {918000000UL,   {        0,        0,        0 } }, \
251                 {1020000000UL,  { -2875621,   358099,    -8585 } }, \
252                 {1122000000UL,  {   -52225,   104159,    -2816 } }, \
253                 {1224000000UL,  {  1076868,     8356,     -727 } }, \
254                 {1326000000UL,  {  2208191,   -84659,     1240 } }, \
255                 {1428000000UL,  {  2519460,  -105063,     1611 } }, \
256                 {1530000000UL,  {  2639809,  -108729,     1626 } }, \
257                 {1632000000UL,  {  2889664,  -122173,     1834 } }, \
258                 {1734000000UL,  {  3386160,  -154021,     2393 } }, \
259                 {1836000000UL,  {  5100873,  -279186,     4747 } }, \
260                 {1912500000UL,  {  5100873,  -279186,     4747 } }, \
261                 {2014500000UL,  {  5100873,  -279186,     4747 } }, \
262                 {2218500000UL,  {  5100873,  -279186,     4747 } }, \
263                 {0,             { } }, \
264         }
265
266 #define CPU_PLL_CVB_TABLE_XA \
267         .pll_min_millivolts = 950, \
268         .speedo_scale = 100,    \
269         .voltage_scale = 1000,  \
270         .cvb_pll_table = {              \
271                 {204000000UL,   {        0,        0,        0 } }, \
272                 {306000000UL,   {        0,        0,        0 } }, \
273                 {408000000UL,   {        0,        0,        0 } }, \
274                 {510000000UL,   {        0,        0,        0 } }, \
275                 {612000000UL,   {        0,        0,        0 } }, \
276                 {714000000UL,   {        0,        0,        0 } }, \
277                 {816000000UL,   {        0,        0,        0 } }, \
278                 {918000000UL,   {        0,        0,        0 } }, \
279                 {1020000000UL,  { -2875621,   358099,    -8585 } }, \
280                 {1122000000UL,  {   -52225,   104159,    -2816 } }, \
281                 {1224000000UL,  {  1076868,     8356,     -727 } }, \
282                 {1326000000UL,  {  2208191,   -84659,     1240 } }, \
283                 {1428000000UL,  {  2519460,  -105063,     1611 } }, \
284                 {1530000000UL,  {  2639809,  -108729,     1626 } }, \
285                 {1606500000UL,  {  2889664,  -122173,     1834 } }, \
286                 {1632000000UL,  {  3386160,  -154021,     2393 } }, \
287                 {0,             {      0,      0,   0} }, \
288         }
289
290 #define CPU_PLL_CVB_TABLE_EUCM1 \
291         .pll_min_millivolts = 950, \
292         .speedo_scale = 100,    \
293         .voltage_scale = 1000,  \
294         .cvb_pll_table = {              \
295                 {204000000UL,   {        0,        0,        0 } }, \
296                 {306000000UL,   {        0,        0,        0 } }, \
297                 {408000000UL,   {        0,        0,        0 } }, \
298                 {510000000UL,   {        0,        0,        0 } }, \
299                 {612000000UL,   {        0,        0,        0 } }, \
300                 {714000000UL,   {        0,        0,        0 } }, \
301                 {816000000UL,   {        0,        0,        0 } }, \
302                 {918000000UL,   {        0,        0,        0 } }, \
303                 {1020000000UL,  { -2875621,   358099,    -8585 } }, \
304                 {1122000000UL,  {   -52225,   104159,    -2816 } }, \
305                 {1224000000UL,  {  1076868,     8356,     -727 } }, \
306                 {1326000000UL,  {  2208191,   -84659,     1240 } }, \
307                 {1428000000UL,  {  2519460,  -105063,     1611 } }, \
308                 {1555500000UL,  {  2639809,  -108729,     1626 } }, \
309                 {1632000000UL,  {  2889664,  -122173,     1834 } }, \
310                 {1734000000UL,  {  3386160,  -154021,     2393 } }, \
311                 {0,             { } }, \
312         }
313
314 #define CPU_PLL_CVB_TABLE_EUCM2 \
315         .pll_min_millivolts = 950, \
316         .speedo_scale = 100,    \
317         .voltage_scale = 1000,  \
318         .cvb_pll_table = {              \
319                 {204000000UL,   {        0,        0,        0 } }, \
320                 {306000000UL,   {        0,        0,        0 } }, \
321                 {408000000UL,   {        0,        0,        0 } }, \
322                 {510000000UL,   {        0,        0,        0 } }, \
323                 {612000000UL,   {        0,        0,        0 } }, \
324                 {714000000UL,   {        0,        0,        0 } }, \
325                 {816000000UL,   {        0,        0,        0 } }, \
326                 {918000000UL,   {        0,        0,        0 } }, \
327                 {1020000000UL,  { -2875621,   358099,    -8585 } }, \
328                 {1122000000UL,  {   -52225,   104159,    -2816 } }, \
329                 {1224000000UL,  {  1076868,     8356,     -727 } }, \
330                 {1326000000UL,  {  2208191,   -84659,     1240 } }, \
331                 {1479000000UL,  {  2519460,  -105063,     1611 } }, \
332                 {1555500000UL,  {  2639809,  -108729,     1626 } }, \
333                 {1683000000UL,  {  2889664,  -122173,     1834 } }, \
334                 {0,             { } }, \
335         }
336
337 #define CPU_PLL_CVB_TABLE_EUCM2_JOINT_RAIL \
338         .pll_min_millivolts = 950, \
339         .speedo_scale = 100,    \
340         .voltage_scale = 1000,  \
341         .cvb_pll_table = {              \
342                 {204000000UL,   {        0,        0,        0 } }, \
343                 {306000000UL,   {        0,        0,        0 } }, \
344                 {408000000UL,   {        0,        0,        0 } }, \
345                 {510000000UL,   {        0,        0,        0 } }, \
346                 {612000000UL,   {        0,        0,        0 } }, \
347                 {714000000UL,   {        0,        0,        0 } }, \
348                 {816000000UL,   {        0,        0,        0 } }, \
349                 {918000000UL,   {        0,        0,        0 } }, \
350                 {1020000000UL,  { -2875621,   358099,    -8585 } }, \
351                 {1122000000UL,  {   -52225,   104159,    -2816 } }, \
352                 {1224000000UL,  {  1076868,     8356,     -727 } }, \
353                 {1326000000UL,  {  2208191,   -84659,     1240 } }, \
354                 {1479000000UL,  {  2519460,  -105063,     1611 } }, \
355                 {1504500000UL,  {  2639809,  -108729,     1626 } }, \
356                 {0,             { } }, \
357         }
358
359 #define CPU_PLL_CVB_TABLE_ODN \
360         .pll_min_millivolts = 950, \
361         .speedo_scale = 100,    \
362         .voltage_scale = 1000,  \
363         .cvb_pll_table = {              \
364                 {204000000UL,   {        0,        0,        0 } }, \
365                 {306000000UL,   {        0,        0,        0 } }, \
366                 {408000000UL,   {        0,        0,        0 } }, \
367                 {510000000UL,   {        0,        0,        0 } }, \
368                 {612000000UL,   {        0,        0,        0 } }, \
369                 {714000000UL,   {        0,        0,        0 } }, \
370                 {816000000UL,   {        0,        0,        0 } }, \
371                 {918000000UL,   {        0,        0,        0 } }, \
372                 {1020000000UL,  { -2875621,   358099,    -8585 } }, \
373                 {1122000000UL,  {   -52225,   104159,    -2816 } }, \
374                 {1224000000UL,  {  1076868,     8356,     -727 } }, \
375                 {1326000000UL,  {  2208191,   -84659,     1240 } }, \
376                 {1428000000UL,  {  2519460,  -105063,     1611 } }, \
377                 {1581000000UL,  {  2889664,  -122173,     1834 } }, \
378                 {1683000000UL,  {  5100873,  -279186,     4747 } }, \
379                 {1785000000UL,  {  5100873,  -279186,     4747 } }, \
380                 {0,             { } }, \
381         }
382
383 static struct cpu_dvfs cpu_fv_dvfs_table[] = {
384         {
385                 .speedo_id = 10,
386                 .process_id = 0,
387                 .min_mv = 840,
388                 .max_mv = 1120,
389                 CPU_PLL_CVB_TABLE_EUCM2_JOINT_RAIL,
390         },
391         {
392                 .speedo_id = 10,
393                 .process_id = 1,
394                 .min_mv = 840,
395                 .max_mv = 1120,
396                 CPU_PLL_CVB_TABLE_EUCM2_JOINT_RAIL,
397         },
398         {
399                 .speedo_id = 9,
400                 .process_id = 0,
401                 .min_mv = 900,
402                 .max_mv = 1162,
403                 CPU_PLL_CVB_TABLE_EUCM2,
404         },
405         {
406                 .speedo_id = 9,
407                 .process_id = 1,
408                 .min_mv = 900,
409                 .max_mv = 1162,
410                 CPU_PLL_CVB_TABLE_EUCM2,
411         },
412         {
413                 .speedo_id = 8,
414                 .process_id = 0,
415                 .min_mv = 900,
416                 .max_mv = 1195,
417                 CPU_PLL_CVB_TABLE_EUCM2,
418         },
419         {
420                 .speedo_id = 8,
421                 .process_id = 1,
422                 .min_mv = 900,
423                 .max_mv = 1195,
424                 CPU_PLL_CVB_TABLE_EUCM2,
425         },
426         {
427                 .speedo_id = 7,
428                 .process_id = 0,
429                 .min_mv = 841,
430                 .max_mv = 1227,
431                 CPU_PLL_CVB_TABLE_EUCM1,
432         },
433         {
434                 .speedo_id = 7,
435                 .process_id = 1,
436                 .min_mv = 841,
437                 .max_mv = 1227,
438                 CPU_PLL_CVB_TABLE_EUCM1,
439         },
440         {
441                 .speedo_id = 6,
442                 .process_id = 0,
443                 .min_mv = 870,
444                 .max_mv = 1150,
445                 CPU_PLL_CVB_TABLE,
446         },
447         {
448                 .speedo_id = 6,
449                 .process_id = 1,
450                 .min_mv = 870,
451                 .max_mv = 1150,
452                 CPU_PLL_CVB_TABLE,
453         },
454         {
455                 .speedo_id = 5,
456                 .process_id = 0,
457                 .min_mv = 818,
458                 .max_mv = 1227,
459                 CPU_PLL_CVB_TABLE,
460         },
461         {
462                 .speedo_id = 5,
463                 .process_id = 1,
464                 .min_mv = 818,
465                 .max_mv = 1227,
466                 CPU_PLL_CVB_TABLE,
467         },
468         {
469                 .speedo_id = 4,
470                 .process_id = -1,
471                 .min_mv = 918,
472                 .max_mv = 1113,
473                 CPU_PLL_CVB_TABLE_XA,
474         },
475         {
476                 .speedo_id = 3,
477                 .process_id = 0,
478                 .min_mv = 825,
479                 .max_mv = 1227,
480                 CPU_PLL_CVB_TABLE_ODN,
481         },
482         {
483                 .speedo_id = 3,
484                 .process_id = 1,
485                 .min_mv = 825,
486                 .max_mv = 1227,
487                 CPU_PLL_CVB_TABLE_ODN,
488         },
489         {
490                 .speedo_id = 2,
491                 .process_id = 0,
492                 .min_mv = 870,
493                 .max_mv = 1227,
494                 CPU_PLL_CVB_TABLE,
495         },
496         {
497                 .speedo_id = 2,
498                 .process_id = 1,
499                 .min_mv = 870,
500                 .max_mv = 1227,
501                 CPU_PLL_CVB_TABLE,
502         },
503         {
504                 .speedo_id = 1,
505                 .process_id = 0,
506                 .min_mv = 837,
507                 .max_mv = 1227,
508                 CPU_PLL_CVB_TABLE,
509         },
510         {
511                 .speedo_id = 1,
512                 .process_id = 1,
513                 .min_mv = 837,
514                 .max_mv = 1227,
515                 CPU_PLL_CVB_TABLE,
516         },
517         {
518                 .speedo_id = 0,
519                 .process_id = 0,
520                 .min_mv = 850,
521                 .max_mv = 1170,
522                 CPU_PLL_CVB_TABLE,
523         },
524         {
525                 .speedo_id = 0,
526                 .process_id = 1,
527                 .min_mv = 850,
528                 .max_mv = 1170,
529                 CPU_PLL_CVB_TABLE,
530         },
531 };
532
533 #define CPUB01_PLL_CVB_TABLE_SLT        \
534         .speedo_scale = 100,    \
535         .voltage_scale = 1000,  \
536         .cvb_pll_table = {      \
537                 /* f                    c0,       c1,       c2 */   \
538                 {  204000000UL, {        0,        0,        0 } }, \
539                 {  306000000UL, {        0,        0,        0 } }, \
540                 {  408000000UL, {        0,        0,        0 } }, \
541                 {  510000000UL, {        0,        0,        0 } }, \
542                 {  612000000UL, {        0,        0,        0 } }, \
543                 {  714000000UL, {        0,        0,        0 } }, \
544                 {  816000000UL, {        0,        0,        0 } }, \
545                 {  918000000UL, {        0,        0,        0 } }, \
546                 { 1020000000UL, {  1120000,        0,        0 } }, \
547                 { 1122000000UL, {  1120000,        0,        0 } }, \
548                 { 1224000000UL, {  1120000,        0,        0 } }, \
549                 { 1326000000UL, {  1120000,        0,        0 } }, \
550                 { 1428000000UL, {  1120000,        0,        0 } }, \
551                 { 1581000000UL, {  1120000,        0,        0 } }, \
552                 { 1683000000UL, {  1120000,        0,        0 } }, \
553                 { 1785000000UL, {  1120000,        0,        0 } }, \
554                 { 1887000000UL, {  1120000,        0,        0 } }, \
555                 { 1963500000UL, {  1120000,        0,        0 } }, \
556                 { 2091000000UL, {  1120000,        0,        0 } }, \
557                 { 0,            { } }, \
558         }, \
559         .pll_min_millivolts = 800
560
561 #define CPUB01_PLL_CVB_TABLE    \
562         .speedo_scale = 100,    \
563         .voltage_scale = 1000,  \
564         .cvb_pll_table = {      \
565                 /* f                    c0,       c1,       c2 */   \
566                 {  204000000UL, {        0,        0,        0 } }, \
567                 {  306000000UL, {        0,        0,        0 } }, \
568                 {  408000000UL, {        0,        0,        0 } }, \
569                 {  510000000UL, {        0,        0,        0 } }, \
570                 {  612000000UL, {        0,        0,        0 } }, \
571                 {  714000000UL, {        0,        0,        0 } }, \
572                 {  816000000UL, {        0,        0,        0 } }, \
573                 {  918000000UL, {        0,        0,        0 } }, \
574                 { 1020000000UL, {  1120000,        0,        0 } }, \
575                 { 1122000000UL, {  1120000,        0,        0 } }, \
576                 { 1224000000UL, {  1120000,        0,        0 } }, \
577                 { 1326000000UL, {  1120000,        0,        0 } }, \
578                 { 1428000000UL, {  1120000,        0,        0 } }, \
579                 { 1581000000UL, {  1120000,        0,        0 } }, \
580                 { 1683000000UL, {  1120000,        0,        0 } }, \
581                 { 1785000000UL, {  1120000,        0,        0 } }, \
582                 { 1887000000UL, {  1120000,        0,        0 } }, \
583                 { 1963500000UL, {  1120000,        0,        0 } }, \
584                 { 2014500000UL, {  1120000,        0,        0 } }, \
585                 { 0,            { } }, \
586         }, \
587         .pll_min_millivolts = 800
588
589 static struct cpu_dvfs cpub01_fv_dvfs_table[] = {
590         {
591                 .speedo_id = 2,
592                 .process_id = -1,
593                 .max_mv = 1120,
594                 CPUB01_PLL_CVB_TABLE_SLT,
595         },
596         {
597                 .speedo_id = -1,
598                 .process_id = -1,
599                 .max_mv = 1120,
600                 CPUB01_PLL_CVB_TABLE,
601         },
602 };
603
604
605 /* CPU LP DVFS tables */
606 static unsigned long cpu_lp_max_freq[] = {
607 /* speedo_id    0        1        2        3        4       5 */
608                 1132800, 1132800, 1132800, 1132800, 940800, 1132800
609 };
610
611 #define CPU_LP_FV_TABLE          \
612         .fv_table = {            \
613                 {51000,   850},  \
614                 {102000,  850},  \
615                 {204000,  850},  \
616                 {307200,  850},  \
617                 {403200,  850},  \
618                 {518400,  850},  \
619                 {614400,  868},  \
620                 {710400,  912},  \
621                 {825600,  962},  \
622                 {921600,  1006}, \
623                 {1036800, 1062}, \
624                 {1132800, 1118}, \
625                 {1228800, 1168}, \
626         }
627
628 static struct cpu_dvfs cpu_lp_fv_dvfs_table[] = {
629         {
630                 .speedo_id = 5,
631                 .process_id = -1,
632                 .min_mv = 818,
633                 .max_mv = 1227,
634                 CPU_LP_FV_TABLE,
635         },
636         {
637                 .speedo_id = 2,
638                 .process_id = -1,
639                 .min_mv = 804,
640                 .max_mv = 1170,
641                 CPU_LP_FV_TABLE,
642         },
643         {
644                 .speedo_id = 1,
645                 .process_id = -1,
646                 .min_mv = 837,
647                 .max_mv = 1227,
648                 CPU_LP_FV_TABLE,
649         },
650         {
651                 .speedo_id = -1,
652                 .process_id = -1,
653                 .min_mv = 850,
654                 .max_mv = 1170,
655                 CPU_LP_FV_TABLE,
656         },
657 };
658
659 static struct dvfs cpu_lp_dvfs = {
660         .clk_name       = "cclk_lp",
661         .millivolts     = cpu_lp_millivolts,
662         .auto_dvfs      = true,
663         .dvfs_rail      = &vdd_cpu_rail,
664         .freqs_mult     = KHZ,
665 };
666
667 /* GPU DVFS tables */
668 #define NA_FREQ_CVB_TABLE       \
669         .freqs_mult = KHZ,      \
670         .speedo_scale = 100,    \
671         .thermal_scale = 10,    \
672         .voltage_scale = 1000,  \
673         .cvb_table = {          \
674                 /* f       dfll pll:    c0,       c1,       c2,       c3,       c4,       c5 */    \
675                 {   76800, { }, {   814294,     8144,     -940,      808,   -21583,      226 }, }, \
676                 {  153600, { }, {   856185,     8144,     -940,      808,   -21583,      226 }, }, \
677                 {  230400, { }, {   898077,     8144,     -940,      808,   -21583,      226 }, }, \
678                 {  307200, { }, {   939968,     8144,     -940,      808,   -21583,      226 }, }, \
679                 {  384000, { }, {   981860,     8144,     -940,      808,   -21583,      226 }, }, \
680                 {  460800, { }, {  1023751,     8144,     -940,      808,   -21583,      226 }, }, \
681                 {  537600, { }, {  1065642,     8144,     -940,      808,   -21583,      226 }, }, \
682                 {  614400, { }, {  1107534,     8144,     -940,      808,   -21583,      226 }, }, \
683                 {  691200, { }, {  1149425,     8144,     -940,      808,   -21583,      226 }, }, \
684                 {  768000, { }, {  1191317,     8144,     -940,      808,   -21583,      226 }, }, \
685                 {  844800, { }, {  1233208,     8144,     -940,      808,   -21583,      226 }, }, \
686                 {  921600, { }, {  1275100,     8144,     -940,      808,   -21583,      226 }, }, \
687                 {  998400, { }, {  1316991,     8144,     -940,      808,   -21583,      226 }, }, \
688                 { 0,       { }, { }, }, \
689         }
690
691 #define NA_FREQ_CVB_TABLE_XA    \
692         .freqs_mult = KHZ,      \
693         .speedo_scale = 100,    \
694         .thermal_scale = 10,    \
695         .voltage_scale = 1000,  \
696         .cvb_table = {          \
697                 /* f       dfll pll:    c0,       c1,       c2,       c3,       c4,       c5 */    \
698                 {   76800, { }, {  1526811,   -59106,      963,      238,   -11292,      185 }, }, \
699                 {  153600, { }, {  1543573,   -57798,      910,      179,    -9918,      191 }, }, \
700                 {  230400, { }, {  1567838,   -56991,      869,       60,    -8545,      203 }, }, \
701                 {  307200, { }, {  1600241,   -56742,      841,        0,    -7019,      209 }, }, \
702                 {  384000, { }, {  1635184,   -56501,      813,        0,    -5493,      221 }, }, \
703                 {  460800, { }, {  1672308,   -56300,      787,     -119,    -3662,      226 }, }, \
704                 {  537600, { }, {  1712114,   -56093,      759,     -179,    -1526,      238 }, }, \
705                 {  614400, { }, {  1756009,   -56048,      737,     -298,      610,      244 }, }, \
706                 {  691200, { }, {  1790251,   -54860,      687,     -358,     3204,      238 }, }, \
707                 {  768000, { }, {  1783830,   -49449,      532,     -477,     6714,      197 }, }, \
708                 {  844800, { }, {  1819706,   -45928,      379,     -358,     7019,       89 }, }, \
709                 { 0,       { }, { }, }, \
710         }
711
712 #define FIXED_FREQ_CVB_TABLE    \
713         .freqs_mult = KHZ,      \
714         .speedo_scale = 100,    \
715         .thermal_scale = 10,    \
716         .voltage_scale = 1000,  \
717         .cvb_table = {          \
718                 /* f       dfll pll:    c0,       c1,       c2 */    \
719                 {   76800, { }, {  1786666,   -85625,     1632 }, }, \
720                 {  153600, { }, {  1846729,   -87525,     1632 }, }, \
721                 {  230400, { }, {  1910480,   -89425,     1632 }, }, \
722                 {  307200, { }, {  1977920,   -91325,     1632 }, }, \
723                 {  384000, { }, {  2049049,   -93215,     1632 }, }, \
724                 {  460800, { }, {  2122872,   -95095,     1632 }, }, \
725                 {  537600, { }, {  2201331,   -96985,     1632 }, }, \
726                 {  614400, { }, {  2283479,   -98885,     1632 }, }, \
727                 {  691200, { }, {  2369315,  -100785,     1632 }, }, \
728                 {  768000, { }, {  2458841,  -102685,     1632 }, }, \
729                 {  844800, { }, {  2550821,  -104555,     1632 }, }, \
730                 {  921600, { }, {  2647676,  -106455,     1632 }, }, \
731                 { 0,       { }, { }, }, \
732         }
733
734 static struct dvfs gpu_dvfs = {
735         .clk_name       = "gbus",
736         .auto_dvfs      = true,
737         .dvfs_rail      = &vdd_gpu_rail,
738 };
739
740 static struct cvb_dvfs gpu_cvb_dvfs_table[] = {
741         {
742                 .speedo_id = 4,
743                 .process_id = -1,
744                 .pll_min_millivolts = 918,
745                 .max_mv = 1113,
746                 .max_freq = 844800,
747 #ifdef CONFIG_TEGRA_USE_NA_GPCPLL
748                 NA_FREQ_CVB_TABLE_XA,
749 #else
750                 FIXED_FREQ_CVB_TABLE,
751 #endif
752         },
753
754         {
755                 .speedo_id = 3,
756                 .process_id = -1,
757                 .pll_min_millivolts = 810,
758                 .max_mv = 1150,
759                 .max_freq = 921600,
760 #ifdef CONFIG_TEGRA_USE_NA_GPCPLL
761                 NA_FREQ_CVB_TABLE,
762 #else
763                 FIXED_FREQ_CVB_TABLE,
764 #endif
765         },
766
767         {
768                 .speedo_id = 2,
769                 .process_id = -1,
770                 .pll_min_millivolts = 818,
771                 .max_mv = 1150,
772                 .max_freq = 998400,
773 #ifdef CONFIG_TEGRA_USE_NA_GPCPLL
774                 NA_FREQ_CVB_TABLE,
775 #else
776                 FIXED_FREQ_CVB_TABLE,
777 #endif
778         },
779
780         {
781                 .speedo_id = 1,
782                 .process_id = -1,
783                 .pll_min_millivolts = 840,
784                 .max_mv = 1150,
785                 .max_freq = 998400,
786 #ifdef CONFIG_TEGRA_USE_NA_GPCPLL
787                 NA_FREQ_CVB_TABLE,
788 #else
789                 FIXED_FREQ_CVB_TABLE,
790 #endif
791         },
792
793         {
794                 .speedo_id = 0,
795                 .process_id = -1,
796                 .pll_min_millivolts = 950,
797 #ifdef CONFIG_TEGRA_GPU_DVFS
798                 .max_mv = 1150,
799 #else
800                 .max_mv = 1000,
801 #endif
802                 .max_freq = 921600,
803                 FIXED_FREQ_CVB_TABLE,
804         },
805 };
806
807 #define GPUB01_NA_CVB_TABLE_SLT \
808         .freqs_mult = KHZ,      \
809         .speedo_scale = 100,    \
810         .thermal_scale = 10,    \
811         .voltage_scale = 1000,  \
812         .cvb_table = {          \
813                 /* f       dfll pll:    c0,       c1,       c2,       c3,       c4,       c5 */    \
814                 {   76800, { }, {   590000,        0,        0,        0,        0,        0 }, }, \
815                 {  153600, { }, {   590000,        0,        0,        0,        0,        0 }, }, \
816                 {  230400, { }, {   590000,        0,        0,        0,        0,        0 }, }, \
817                 {  307200, { }, {   590000,        0,        0,        0,        0,        0 }, }, \
818                 {  384000, { }, {   590000,        0,        0,        0,        0,        0 }, }, \
819                 {  460800, { }, {   795089,   -11096,     -163,      298,   -10421,      162 }, }, \
820                 {  537600, { }, {   795089,   -11096,     -163,      298,   -10421,      162 }, }, \
821                 {  614400, { }, {   820606,    -6285,     -452,      238,    -6182,       81 }, }, \
822                 {  691200, { }, {   846289,    -4565,     -552,      119,    -3958,       -2 }, }, \
823                 {  768000, { }, {   888720,    -5110,     -584,        0,    -2849,       39 }, }, \
824                 {  844800, { }, {   936634,    -6089,     -602,      -60,      -99,      -93 }, }, \
825                 {  921600, { }, {   982562,    -7373,     -614,     -179,     1797,      -13 }, }, \
826                 {  998400, { }, {  1090179,   -14125,     -497,     -179,     3518,        9 }, }, \
827                 { 1075200, { }, {  1155798,   -13465,     -648,        0,     1077,       40 }, }, \
828                 { 1152000, { }, {  1198568,   -10904,     -830,        0,     1469,      110 }, }, \
829                 { 1228800, { }, {  1269988,   -12707,     -859,        0,     3722,      313 }, }, \
830                 { 1267200, { }, {  1308155,   -13694,     -867,        0,     3681,      559 }, }, \
831                 { 0,       { }, { }, }, \
832         }, \
833         .cvb_vmin = {   0, { }, {   590000,        0,        0 }, }, \
834         .cvb_version = "NAPLL En - p4v2-AggressiveSLT"
835
836 #define GPUB01_NA_CVB_TABLE     \
837         .freqs_mult = KHZ,      \
838         .speedo_scale = 100,    \
839         .thermal_scale = 10,    \
840         .voltage_scale = 1000,  \
841         .cvb_table = {          \
842                 /* f       dfll pll:    c0,       c1,       c2,       c3,       c4,       c5 */    \
843                 {   76800, { }, {   610000,        0,        0,        0,        0,        0 }, }, \
844                 {  153600, { }, {   610000,        0,        0,        0,        0,        0 }, }, \
845                 {  230400, { }, {   610000,        0,        0,        0,        0,        0 }, }, \
846                 {  307200, { }, {   610000,        0,        0,        0,        0,        0 }, }, \
847                 {  384000, { }, {   610000,        0,        0,        0,        0,        0 }, }, \
848                 {  460800, { }, {   610000,        0,        0,        0,        0,        0 }, }, \
849                 {  537600, { }, {   801688,   -10900,     -163,      298,   -10599,      162 }, }, \
850                 {  614400, { }, {   824214,    -5743,     -452,      238,    -6325,       81 }, }, \
851                 {  691200, { }, {   848830,    -3903,     -552,      119,    -4030,       -2 }, }, \
852                 {  768000, { }, {   891575,    -4409,     -584,        0,    -2849,       39 }, }, \
853                 {  844800, { }, {   940071,    -5367,     -602,      -60,      -63,      -93 }, }, \
854                 {  921600, { }, {   986765,    -6637,     -614,     -179,     1905,      -13 }, }, \
855                 {  998400, { }, {  1098475,   -13529,     -497,     -179,     3626,        9 }, }, \
856                 { 1075200, { }, {  1163644,   -12688,     -648,        0,     1077,       40 }, }, \
857                 { 1152000, { }, {  1204812,    -9908,     -830,        0,     1469,      110 }, }, \
858                 { 1228800, { }, {  1277303,   -11675,     -859,        0,     3722,      313 }, }, \
859                 { 1267200, { }, {  1335531,   -12567,     -867,        0,     3681,      559 }, }, \
860                 { 0,       { }, { }, }, \
861         }, \
862         .cvb_vmin = {   0, { }, {   610000,        0,        0 }, }, \
863         .cvb_version = "NAPLL En - p4v3"
864
865 static struct cvb_dvfs gpub01_cvb_dvfs_table[] = {
866         {
867                 .speedo_id = 2,
868                 .process_id = -1,
869                 .max_mv = 1050,
870                 .max_freq = 1267200,
871                 GPUB01_NA_CVB_TABLE_SLT,
872         },
873         {
874                 .speedo_id = -1,
875                 .process_id = -1,
876                 .max_mv = 1050,
877                 .max_freq = 1267200,
878                 GPUB01_NA_CVB_TABLE,
879         },
880 };
881
882 static int gpu_vmin[MAX_THERMAL_RANGES];
883 static int gpu_peak_millivolts[MAX_DVFS_FREQS];
884 static int gpu_millivolts[MAX_THERMAL_RANGES][MAX_DVFS_FREQS];
885 static struct dvfs_therm_limits vdd_gpu_therm_caps_table[MAX_THERMAL_LIMITS];
886 static struct dvfs_therm_limits vdd_gpu_therm_caps_ucm2_table[MAX_THERMAL_LIMITS];
887 static unsigned long gpu_cap_rates[MAX_THERMAL_LIMITS];
888 static struct clk *vgpu_cap_clk;
889
890 /* Core DVFS tables */
891 static int core_millivolts[MAX_DVFS_FREQS];
892
893 #define CORE_DVFS(_clk_name, _speedo_id, _process_id, _auto, _mult, _freqs...) \
894         {                                                       \
895                 .clk_name       = _clk_name,                    \
896                 .speedo_id      = _speedo_id,                   \
897                 .process_id     = _process_id,                  \
898                 .freqs          = {_freqs},                     \
899                 .freqs_mult     = _mult,                        \
900                 .millivolts     = core_millivolts,              \
901                 .auto_dvfs      = _auto,                        \
902                 .dvfs_rail      = &vdd_core_rail,       \
903         }
904
905
906 /* Include T210 core DVFS tables generated from characterization data */
907 #include "tegra210-core-dvfs.c"
908
909 /* Include T210b01 core DVFS tables generated from characterization data */
910 #include "tegra210b01-core-dvfs.c"
911 #include "tegra210b01-slt-core-dvfs.c"
912
913 int tegra_dvfs_disable_core_set(const char *arg, const struct kernel_param *kp)
914 {
915         int ret;
916
917         ret = param_set_bool(arg, kp);
918         if (ret)
919                 return ret;
920
921         if (tegra_dvfs_core_disabled)
922                 tegra_dvfs_rail_disable(&vdd_core_rail);
923         else
924                 tegra_dvfs_rail_enable(&vdd_core_rail);
925
926         return 0;
927 }
928
929 int tegra_dvfs_disable_cpu_set(const char *arg, const struct kernel_param *kp)
930 {
931         int ret;
932
933         ret = param_set_bool(arg, kp);
934         if (ret)
935                 return ret;
936
937         if (tegra_dvfs_cpu_disabled)
938                 tegra_dvfs_rail_disable(&vdd_cpu_rail);
939         else
940                 tegra_dvfs_rail_enable(&vdd_cpu_rail);
941
942         return 0;
943 }
944
945 int tegra_dvfs_disable_gpu_set(const char *arg, const struct kernel_param *kp)
946 {
947         int ret;
948
949         ret = param_set_bool(arg, kp);
950         if (ret)
951                 return ret;
952
953         if (tegra_dvfs_gpu_disabled)
954                 tegra_dvfs_rail_disable(&vdd_gpu_rail);
955         else
956                 tegra_dvfs_rail_enable(&vdd_gpu_rail);
957
958         return 0;
959 }
960
961 int tegra_dvfs_disable_get(char *buffer, const struct kernel_param *kp)
962 {
963         return param_get_bool(buffer, kp);
964 }
965
966 static struct kernel_param_ops tegra_dvfs_disable_core_ops = {
967         .set = tegra_dvfs_disable_core_set,
968         .get = tegra_dvfs_disable_get,
969 };
970
971 static struct kernel_param_ops tegra_dvfs_disable_cpu_ops = {
972         .set = tegra_dvfs_disable_cpu_set,
973         .get = tegra_dvfs_disable_get,
974 };
975
976 static struct kernel_param_ops tegra_dvfs_disable_gpu_ops = {
977         .set = tegra_dvfs_disable_gpu_set,
978         .get = tegra_dvfs_disable_get,
979 };
980
981 module_param_cb(disable_core, &tegra_dvfs_disable_core_ops,
982         &tegra_dvfs_core_disabled, 0644);
983 module_param_cb(disable_cpu, &tegra_dvfs_disable_cpu_ops,
984         &tegra_dvfs_cpu_disabled, 0644);
985 module_param_cb(disable_gpu, &tegra_dvfs_disable_gpu_ops,
986         &tegra_dvfs_gpu_disabled, 0644);
987
988 static void init_dvfs_one(struct dvfs *d, int max_freq_index)
989 {
990         int ret;
991         struct clk *c = clk_get_sys(d->clk_name, d->clk_name);
992
993         if (IS_ERR(c)) {
994                 pr_info("tegra210_dvfs: no clock found for %s\n",
995                         d->clk_name);
996                 return;
997         }
998
999         d->max_millivolts = d->dvfs_rail->nominal_millivolts;
1000         d->num_freqs = max_freq_index + 1;
1001
1002         ret = tegra_setup_dvfs(c, d);
1003         if (ret)
1004                 pr_err("tegra210_dvfs: failed to enable dvfs on %s\n",
1005                                 __clk_get_name(c));
1006 }
1007
1008 static bool match_dvfs_one(const char *name, int dvfs_speedo_id,
1009                            int dvfs_process_id, int speedo_id, int process_id)
1010 {
1011         if ((dvfs_process_id != -1 && dvfs_process_id != process_id) ||
1012                 (dvfs_speedo_id != -1 && dvfs_speedo_id != speedo_id)) {
1013                 pr_debug("tegra210_dvfs: rejected %s speedo %d, process %d\n",
1014                          name, dvfs_speedo_id, dvfs_process_id);
1015                 return false;
1016         }
1017         return true;
1018 }
1019
1020 static int set_cpu_dvfs_data(struct cpu_dvfs *d,
1021                              struct dvfs *cpu_dvfs, int *max_freq_index)
1022 {
1023         int i, mv, dfll_mv, min_mv, min_dfll_mv, num_freqs;
1024         unsigned long fmax_at_vmin = 0;
1025         unsigned long fmin_use_dfll = 0;
1026         unsigned long *freqs;
1027         int *dfll_millivolts;
1028         struct rail_alignment *align = tegra_dfll_get_alignment();
1029         const char *version = tegra_dfll_get_cvb_version();
1030         int speedo = tegra_sku_info.cpu_speedo_value;
1031
1032         if (align == ERR_PTR(-EPROBE_DEFER))
1033                 return -EPROBE_DEFER;
1034
1035         vdd_cpu_rail.nvver = version;
1036
1037         min_dfll_mv = d->min_mv;
1038         if (min_dfll_mv < vdd_cpu_rail.min_millivolts) {
1039                 pr_debug("tegra210_dvfs: dfll min %dmV below rail min %dmV\n",
1040                          min_dfll_mv, vdd_cpu_rail.min_millivolts);
1041                 min_dfll_mv = vdd_cpu_rail.min_millivolts;
1042         }
1043         min_dfll_mv = tegra_round_voltage(min_dfll_mv, align, true);
1044         d->max_mv = tegra_round_voltage(d->max_mv, align, false);
1045
1046         min_mv = d->pll_min_millivolts;
1047         if (min_mv < vdd_cpu_rail.min_millivolts) {
1048                 pr_debug("tegra210_dvfs: pll min %dmV below rail min %dmV\n",
1049                          min_mv, vdd_cpu_rail.min_millivolts);
1050                 min_mv = vdd_cpu_rail.min_millivolts;
1051         }
1052         min_mv = tegra_round_voltage(min_mv, align, true);
1053
1054         if (tegra_get_cpu_fv_table(&num_freqs, &freqs, &dfll_millivolts))
1055                 return -EPROBE_DEFER;
1056
1057         for (i = 0; i < num_freqs; i++) {
1058                 if (freqs[i] != d->cvb_pll_table[i].freq) {
1059                         pr_err("Err: DFLL freq ladder does not match PLL's\n");
1060                         return -EINVAL;
1061                 }
1062
1063                 /*
1064                  * Check maximum frequency at minimum voltage for dfll source;
1065                  * round down unless all table entries are above Vmin, then use
1066                  * the 1st entry as is.
1067                  */
1068                 dfll_mv = max(dfll_millivolts[i] / 1000, min_dfll_mv);
1069                 if (dfll_mv > min_dfll_mv) {
1070                         if (!i)
1071                                 fmax_at_vmin = freqs[i];
1072                         if (!fmax_at_vmin)
1073                                 fmax_at_vmin = freqs[i - 1];
1074                 }
1075
1076                 /* Clip maximum frequency at maximum voltage for pll source */
1077                 mv = tegra_get_cvb_voltage(speedo, d->speedo_scale,
1078                                            &d->cvb_pll_table[i].coefficients);
1079                 mv = (100 + CVB_PLL_MARGIN) * mv / 100;
1080                 mv = tegra_round_cvb_voltage(mv, d->voltage_scale, align);
1081                 mv = max(mv, min_mv);
1082                 if ((mv > d->max_mv) && !i) {
1083                         pr_err("Err: volt of 1st entry is higher than Vmax\n");
1084                         return -EINVAL;
1085                 }
1086
1087                 /* Minimum rate with pll source voltage above dfll Vmin */
1088                 if ((mv >= min_dfll_mv) && !fmin_use_dfll)
1089                         fmin_use_dfll = freqs[i];
1090
1091                 /* fill in dvfs tables */
1092                 cpu_dvfs->freqs[i] = freqs[i];
1093                 cpu_millivolts[i] = mv;
1094                 cpu_dfll_millivolts[i] = min(dfll_mv, d->max_mv);
1095         }
1096
1097         /*
1098          * In the dfll operating range dfll voltage at any rate should be
1099          * better (below) than pll voltage
1100          */
1101         if (!fmin_use_dfll || (fmin_use_dfll > fmax_at_vmin))
1102                 fmin_use_dfll = fmax_at_vmin;
1103
1104         /* dvfs tables are successfully populated - fill in the rest */
1105         cpu_dvfs->speedo_id = d->speedo_id;
1106         cpu_dvfs->process_id = d->process_id;
1107         cpu_dvfs->dvfs_rail->nominal_millivolts = min(d->max_mv,
1108                 max(cpu_millivolts[i - 1], cpu_dfll_millivolts[i - 1]));
1109         *max_freq_index = i - 1;
1110
1111         cpu_dvfs->use_dfll_rate_min = fmin_use_dfll;
1112
1113         return 0;
1114 }
1115
1116 /*
1117  * Setup slow CPU (a.k.a LP CPU) DVFS table from FV data. Only PLL is used as
1118  * a clock source for slow CPU. Its maximum frequency must be reached within
1119  * nominal voltage -- FV frequency list is cut off at rate that exceeds either
1120  * sku-based maximum limit or requires voltage above nominal. Error when DVFS
1121  * table can not be constructed must never happen.
1122  *
1123  * Final CPU rail nominal voltage is set as maximum of fast and slow CPUs
1124  * nominal voltages.
1125  */
1126 static int set_cpu_lp_dvfs_data(unsigned long max_freq, struct cpu_dvfs *d,
1127                                 struct dvfs *cpu_lp_dvfs, int *max_freq_index)
1128 {
1129         int i, mv, min_mv;
1130         struct rail_alignment *align = &vdd_cpu_rail.alignment;
1131
1132         min_mv = d->min_mv;
1133         if (min_mv < vdd_cpu_rail.min_millivolts) {
1134                 pr_debug("tegra210_dvfs: scpu min %dmV below rail min %dmV\n",
1135                          min_mv, vdd_cpu_rail.min_millivolts);
1136                 min_mv = vdd_cpu_rail.min_millivolts;
1137         }
1138         min_mv = tegra_round_voltage(min_mv, align, true);
1139
1140         d->max_mv = tegra_round_voltage(d->max_mv, align, false);
1141         BUG_ON(d->max_mv > vdd_cpu_rail.max_millivolts);
1142         cpu_lp_dvfs->dvfs_rail->nominal_millivolts =
1143                 max(cpu_lp_dvfs->dvfs_rail->nominal_millivolts, d->max_mv);
1144
1145         for (i = 0; i < MAX_DVFS_FREQS; i++) {
1146                 struct cpu_pll_fv_table *t = &d->fv_table[i];
1147                 if (!t->freq || t->freq > max_freq)
1148                         break;
1149
1150                 mv = t->volt;
1151                 mv = max(mv, min_mv);
1152                 if (mv > d->max_mv) {
1153                         pr_warn("tegra210_dvfs: %dmV for %s rate %d above limit %dmV\n",
1154                              mv, cpu_lp_dvfs->clk_name, t->freq, d->max_mv);
1155                         break;
1156                 }
1157
1158                 /* fill in dvfs tables */
1159                 cpu_lp_dvfs->freqs[i] = t->freq;
1160                 cpu_lp_millivolts[i] = mv;
1161         }
1162
1163         /* Table must not be empty */
1164         if (!i) {
1165                 pr_err("tegra210_dvfs: invalid cpu lp dvfs table\n");
1166                 return -ENOENT;
1167         }
1168
1169         /* dvfs tables are successfully populated - fill in the rest */
1170         cpu_lp_dvfs->speedo_id = d->speedo_id;
1171         cpu_lp_dvfs->process_id = d->process_id;
1172         *max_freq_index = i - 1;
1173
1174         return 0;
1175 }
1176
1177 int of_tegra_dvfs_init(const struct of_device_id *matches)
1178 {
1179         int ret;
1180         struct device_node *np;
1181
1182         for_each_matching_node(np, matches) {
1183                 const struct of_device_id *match = of_match_node(matches, np);
1184                 int (*dvfs_init_cb)(struct device_node *) = match->data;
1185
1186                 ret = dvfs_init_cb(np);
1187                 if (ret) {
1188                         pr_err("dt: Failed to read %s data from DT\n",
1189                                match->compatible);
1190                         return ret;
1191                 }
1192         }
1193         return 0;
1194 }
1195
1196 /*
1197  * QSPI DVFS tables are different in SDR and DDR modes. Use SDR tables by
1198  * default. Check if DDR mode is specified for enabled QSPI device in DT,
1199  * and overwrite DVFS table, respectively.
1200  */
1201
1202 static struct dvfs *qspi_dvfs;
1203
1204 static int of_update_qspi_dvfs(struct device_node *dn)
1205 {
1206         if (of_device_is_available(dn)) {
1207                 if (of_get_property(dn, "nvidia,x4-is-ddr", NULL))
1208                         qspi_dvfs = &dvfs_data->qspi_ddr_vf_table[0];
1209         }
1210         return 0;
1211 }
1212
1213 static struct of_device_id tegra210_dvfs_qspi_of_match[] = {
1214         { .compatible = "nvidia,tegra210-qspi", .data = of_update_qspi_dvfs, },
1215         { },
1216 };
1217
1218 static void init_qspi_dvfs(int soc_speedo_id, int core_process_id,
1219                                   int core_nominal_mv_index)
1220 {
1221         qspi_dvfs = &dvfs_data->qspi_sdr_vf_table[0];
1222
1223         of_tegra_dvfs_init(tegra210_dvfs_qspi_of_match);
1224
1225         if (match_dvfs_one(qspi_dvfs->clk_name, qspi_dvfs->speedo_id,
1226                 qspi_dvfs->process_id, soc_speedo_id, core_process_id))
1227                 init_dvfs_one(qspi_dvfs, core_nominal_mv_index);
1228 }
1229
1230 /*
1231  * SPI DVFS tables are different in master and in slave mode. Use master tables
1232  * by default. Check if slave mode is specified for enabled SPI devices in DT,
1233  * and overwrite master table for the respective SPI controller.
1234  */
1235
1236 static struct {
1237         u64 address;
1238         struct dvfs *d;
1239 } spi_map[] = {
1240         { 0x7000d400, },
1241         { 0x7000d600, },
1242         { 0x7000d800, },
1243         { 0x7000da00, },
1244 };
1245
1246 static int of_update_spi_slave_dvfs(struct device_node *dn)
1247 {
1248         int i;
1249         u64 addr = 0;
1250         const __be32 *reg;
1251
1252         if (!of_device_is_available(dn))
1253                 return 0;
1254
1255         reg = of_get_property(dn, "reg", NULL);
1256         if (reg)
1257                 addr = of_translate_address(dn, reg);
1258
1259         for (i = 0; i < ARRAY_SIZE(spi_map); i++) {
1260                 if (spi_map[i].address == addr) {
1261                         spi_map[i].d = &dvfs_data->spi_slave_vf_table[i];
1262                         break;
1263                 }
1264         }
1265         return 0;
1266 }
1267
1268 static struct of_device_id tegra21_dvfs_spi_slave_of_match[] = {
1269         { .compatible = "nvidia,tegra210-spi-slave",
1270           .data = of_update_spi_slave_dvfs, },
1271         { },
1272 };
1273
1274 static void init_spi_dvfs(int soc_speedo_id, int core_process_id,
1275                           int core_nominal_mv_index)
1276 {
1277         int i;
1278
1279         for (i = 0; i <  ARRAY_SIZE(spi_map); i++)
1280                 spi_map[i].d = &dvfs_data->spi_vf_table[i];
1281
1282         of_tegra_dvfs_init(tegra21_dvfs_spi_slave_of_match);
1283
1284         for (i = 0; i <  ARRAY_SIZE(spi_map); i++) {
1285                 struct dvfs *d = spi_map[i].d;
1286                 if (!match_dvfs_one(d->clk_name, d->speedo_id,
1287                         d->process_id, soc_speedo_id, core_process_id))
1288                         continue;
1289                 init_dvfs_one(d, core_nominal_mv_index);
1290         }
1291 }
1292
1293 static void init_sor1_dvfs(int soc_speedo_id, int core_process_id,
1294                            int core_nominal_mv_index)
1295 {
1296         struct dvfs *sor1_dp_dvfs = &dvfs_data->sor1_dp_vf_table[0];
1297         struct clk *c;
1298         int i, n = dvfs_data->sor1_dp_vf_table_size;
1299
1300         c = clk_get_sys(sor1_dp_dvfs->clk_name, sor1_dp_dvfs->clk_name);
1301         if (IS_ERR(c)) {
1302                 pr_debug("init_sor1_dvfs: no clock found for %s\n",
1303                         sor1_dp_dvfs->clk_name);
1304                 return;
1305         }
1306
1307         for (i = 0; i < n; i++, sor1_dp_dvfs++) {
1308                 if (match_dvfs_one(sor1_dp_dvfs->clk_name,
1309                         sor1_dp_dvfs->speedo_id, sor1_dp_dvfs->process_id,
1310                         soc_speedo_id, core_process_id)) {
1311                         tegra_dvfs_add_alt_freqs(c, sor1_dp_dvfs);
1312                         break;
1313                 }
1314         }
1315
1316         return;
1317
1318 }
1319
1320 static int get_core_sku_max_mv(void)
1321 {
1322         int speedo_rev = tegra_sku_info.speedo_rev;
1323
1324         switch (tegra_sku_info.soc_process_id) {
1325         case 0:
1326                 if (tegra_sku_info.soc_speedo_id == 1)
1327                         return 1100;
1328                 if (speedo_rev <= 1)
1329                         return 1000;
1330
1331                 return 1125;
1332         case 1:
1333                 return 1075;
1334         case 2:
1335                 return 1000;
1336         default:
1337                 pr_err("Un-supported Tegra210 speedo %d\n",
1338                                 tegra_sku_info.soc_speedo_id);
1339                 return -EINVAL;
1340         }
1341 }
1342
1343 static int get_core_sku_min_mv(void)
1344 {
1345         int rev = tegra_sku_info.revision;
1346         bool a02 = (rev == TEGRA_REVISION_A02) || (rev == TEGRA_REVISION_A02p);
1347
1348         if (tegra_sku_info.soc_speedo_id == 1)
1349                 return 1100;
1350
1351         switch (tegra_sku_info.sku_id) {
1352         case 0x0:
1353         case 0x1:
1354         case 0x7:
1355         case 0x17:
1356         case 0x13:
1357                 if (!a02)
1358                         return 825;
1359                 return 800;
1360         case 0x87:
1361                 return 825;
1362         case 0x83:
1363         case 0x8f:
1364                 return 800;
1365         default:
1366                 return 950;
1367         }
1368 }
1369
1370 static int get_coreb01_sku_max_mv(void)
1371 {
1372         switch (tegra_sku_info.soc_process_id) {
1373         case 0:
1374                 return 1050;
1375         case 1:
1376                 return 1025;
1377         case 2:
1378                 return 1000;
1379         default:
1380                 pr_err("Un-supported Tegra210b01 process id %d\n",
1381                                 tegra_sku_info.soc_process_id);
1382                 return -EINVAL;
1383         }
1384 }
1385
1386 static int get_coreb01slt_sku_max_mv(void)
1387 {
1388         return get_coreb01_sku_max_mv();
1389 }
1390
1391 static int get_coreb01_sku_min_mv(void)
1392 {
1393         return 637;
1394 }
1395
1396 static int get_coreb01slt_sku_min_mv(void)
1397 {
1398         return 600;
1399 }
1400
1401 static int get_core_nominal_mv_index(int speedo_id)
1402 {
1403         int i;
1404         int mv = dvfs_data->get_core_max_mv();
1405
1406         if (mv < 0)
1407                 return mv;
1408
1409         /* Round nominal level down to the nearest core scaling step */
1410         for (i = 0; i < MAX_DVFS_FREQS; i++) {
1411                 if ((core_millivolts[i] == 0) || (mv < core_millivolts[i]))
1412                         break;
1413         }
1414
1415         if (i == 0) {
1416                 pr_err("tegra210-dvfs: failed to get nominal idx at volt %d\n",
1417                        mv);
1418                 return -ENOSYS;
1419         }
1420
1421         return i - 1;
1422 }
1423
1424 static int get_core_min_mv_index(void)
1425 {
1426         int i;
1427         int mv = dvfs_data->get_core_min_mv();
1428
1429         if (mv < 0)
1430                 return mv;
1431
1432         /* Round minimum level up to the nearest core scaling step */
1433         for (i = 0; i < MAX_DVFS_FREQS - 1; i++) {
1434                 if ((core_millivolts[i+1] == 0) || (mv <= core_millivolts[i]))
1435                         break;
1436         }
1437
1438         return i;
1439 }
1440
1441 static int init_cpu_dvfs_table(struct cpu_dvfs *fv_dvfs_table,
1442                                int table_size, int *cpu_max_freq_index)
1443 {
1444         int i, ret;
1445         int cpu_speedo_id = tegra_sku_info.cpu_speedo_id;
1446         int cpu_process_id = tegra_sku_info.cpu_process_id;
1447
1448         for (ret = 0, i = 0; i < table_size; i++) {
1449                 struct cpu_dvfs *d = &fv_dvfs_table[i];
1450
1451                 if (match_dvfs_one("cpu dvfs", d->speedo_id, d->process_id,
1452                                    cpu_speedo_id, cpu_process_id)) {
1453                         ret = set_cpu_dvfs_data(
1454                                 d, &cpu_dvfs, cpu_max_freq_index);
1455                         if (ret)
1456                                 return ret;
1457                         break;
1458                 }
1459         }
1460         BUG_ON(i == table_size);
1461
1462         return ret;
1463 }
1464
1465 static int init_cpu_lp_dvfs_table(int *cpu_lp_max_freq_index)
1466 {
1467         int i, ret;
1468         unsigned int cpu_lp_speedo_id = tegra_sku_info.cpu_speedo_id;
1469         int cpu_lp_process_id = tegra_sku_info.cpu_process_id;
1470         unsigned long max_freq;
1471
1472         if (cpu_lp_speedo_id >= ARRAY_SIZE(cpu_lp_max_freq))
1473                 max_freq = cpu_lp_max_freq[0];
1474         else
1475                 max_freq = cpu_lp_max_freq[cpu_lp_speedo_id];
1476
1477         for (ret = 0, i = 0; i <  ARRAY_SIZE(cpu_lp_fv_dvfs_table); i++) {
1478                 struct cpu_dvfs *d = &cpu_lp_fv_dvfs_table[i];
1479                 if (match_dvfs_one("cpu lp dvfs", d->speedo_id, d->process_id,
1480                                    cpu_lp_speedo_id, cpu_lp_process_id)) {
1481                         ret = set_cpu_lp_dvfs_data(max_freq,
1482                                 d, &cpu_lp_dvfs, cpu_lp_max_freq_index);
1483                         break;
1484                 }
1485         }
1486
1487         return ret;
1488 }
1489
1490 static void adjust_emc_dvfs_from_timing_table(struct dvfs *d)
1491 {
1492         unsigned long rate;
1493         int i;
1494
1495         for (i = 0; i < ARRAY_SIZE(core_millivolts); i++) {
1496                 if (core_millivolts[i] == 0)
1497                         return;
1498
1499                 rate = tegra210_predict_emc_rate(core_millivolts[i]);
1500                 if (IS_ERR_VALUE(rate))
1501                         return;
1502
1503                 if (rate)
1504                         d->freqs[i] = rate;
1505         }
1506 }
1507
1508 static unsigned long dvb_predict_rate(
1509         int process_id, struct dvb_dvfs *dvbd, int mv)
1510 {
1511         int i;
1512         unsigned long rate = 0;
1513
1514         for (i = 0; i < MAX_DVFS_FREQS; i++) {
1515                 if (!dvbd->dvb_table[i].freq)
1516                         break;
1517                 if (dvbd->dvb_table[i].mvolts[process_id] > mv)
1518                         break;
1519                 rate = dvbd->dvb_table[i].freq * dvbd->freqs_mult;
1520         }
1521
1522         return rate;
1523 }
1524
1525 static void adjust_emc_dvfs_from_dvb_table(
1526         int process_id, struct dvb_dvfs *dvbd, struct dvfs *d)
1527 {
1528         unsigned long rate;
1529         int i;
1530
1531         if (process_id > MAX_PROCESS_ID) {
1532                 WARN(1, "Process id %d above emc dvb table max\n", process_id);
1533                 return;
1534         }
1535
1536         for (i = 0; i < ARRAY_SIZE(core_millivolts); i++) {
1537                 if (core_millivolts[i] == 0)
1538                         return;
1539
1540                 rate = dvb_predict_rate(process_id, dvbd, core_millivolts[i]);
1541                 if (rate)
1542                         d->freqs[i] = rate;
1543         }
1544 }
1545
1546 static struct dvb_dvfs *get_emc_dvb_dvfs(int speedo_id)
1547 {
1548         int i;
1549
1550         if (dvfs_data->emc_dvb_table) {
1551                 for (i = 0; i < dvfs_data->emc_dvb_table_size; i++) {
1552                         struct dvb_dvfs *dvbd = &dvfs_data->emc_dvb_table[i];
1553
1554                         if (dvbd->speedo_id == -1 ||
1555                             dvbd->speedo_id == speedo_id)
1556                                 return dvbd;
1557                 }
1558         }
1559         return NULL;
1560 }
1561
1562 /*
1563  * Find maximum GPU frequency that can be reached at minimum voltage across all
1564  * temperature ranges.
1565  */
1566 static unsigned long find_gpu_fmax_at_vmin(
1567         struct dvfs *gpu_dvfs, int thermal_ranges, int freqs_num)
1568 {
1569         int i, j;
1570         unsigned long fmax = ULONG_MAX;
1571
1572         /*
1573          * For voltage scaling row in each temperature range, as well as peak
1574          * voltage row find maximum frequency at lowest voltage, and return
1575          * minimax. On Tegra21 all GPU DVFS thermal dependencies are integrated
1576          * into thermal DVFS table (i.e., there is no separate thermal floors
1577          * applied in the rail level). Hence, returned frequency specifies max
1578          * frequency safe at minimum voltage across all temperature ranges.
1579          */
1580         for (j = 0; j < thermal_ranges; j++) {
1581                 for (i = 1; i < freqs_num; i++) {
1582                         if (gpu_millivolts[j][i] > gpu_millivolts[j][0])
1583                                 break;
1584                 }
1585                 fmax = min(fmax, gpu_dvfs->freqs[i - 1]);
1586         }
1587
1588         for (i = 1; i < freqs_num; i++) {
1589                 if (gpu_peak_millivolts[i] > gpu_peak_millivolts[0])
1590                         break;
1591         }
1592         fmax = min(fmax, gpu_dvfs->freqs[i - 1]);
1593
1594         return fmax;
1595 }
1596
1597 /*
1598  * Determine minimum voltage safe at maximum frequency across all temperature
1599  * ranges.
1600  */
1601 static int find_gpu_vmin_at_fmax(
1602         struct dvfs *gpu_dvfs, int thermal_ranges, int freqs_num)
1603 {
1604         int j, vmin;
1605
1606         /*
1607          * For voltage scaling row in each temperature range find minimum
1608          * voltage at maximum frequency and return max Vmin across ranges.
1609          */
1610         for (vmin = 0, j = 0; j < thermal_ranges; j++)
1611                 vmin = max(vmin, gpu_millivolts[j][freqs_num-1]);
1612
1613         return vmin;
1614 }
1615
1616 static int of_parse_dvfs_rail_cdev_trips(struct device_node *node,
1617                 int *therm_trips_table,
1618                 struct dvfs_therm_limits *therm_limits_table,
1619                 struct dvfs_therm_limits *therm_limits_ucm2_table,
1620                 struct rail_alignment *align, bool up)
1621 {
1622         struct of_phandle_iter iter;
1623         int cells_num, i = 0, t;
1624
1625         /* 1 cell per trip-point, if constraint is specified */
1626         cells_num = of_property_read_bool(node, "nvidia,constraint-ucm2") ? 2 :
1627                 of_property_read_bool(node, "nvidia,constraint") ? 1 : 0;
1628
1629         if (of_phandle_iterator_init(&iter, node, "nvidia,trips",
1630                                      NULL, cells_num))
1631                 return -ENOENT;
1632
1633         while (!of_phandle_iterator_next(&iter)) {
1634                 struct device_node *trip_dn = iter.node;
1635
1636                 if (i >= MAX_THERMAL_LIMITS) {
1637                         pr_err("tegra_dvfs: list of scaling cdev trips exceeds max limit\n");
1638                         return -EINVAL;
1639                 }
1640
1641                 if (of_property_read_s32(trip_dn, "temperature", &t)) {
1642                         pr_err("tegra_dvfs: failed to read scalings cdev trip %d\n", i);
1643                         return -ENODATA;
1644                 }
1645
1646                 if (therm_trips_table)
1647                         therm_trips_table[i] = t / 1000; /* convert mC to C */
1648
1649                 if (cells_num && therm_limits_table) {
1650                         int mv = 0;
1651                         struct of_phandle_args dvfs_args;
1652                         dvfs_args.args_count = of_phandle_iterator_args(&iter,
1653                                                 dvfs_args.args, MAX_PHANDLE_ARGS);
1654
1655                         mv = dvfs_args.args[0];
1656                         mv = tegra_round_voltage(mv, align, up);
1657                         therm_limits_table[i].temperature = t / 1000;
1658                         therm_limits_table[i].mv = mv;
1659                         if (cells_num == 2 && therm_limits_ucm2_table) {
1660                                 mv = dvfs_args.args[1];
1661                                 mv = tegra_round_voltage(mv, align, up);
1662                                 therm_limits_ucm2_table[i].temperature = t/1000;
1663                                 therm_limits_ucm2_table[i].mv = mv;
1664                         }
1665                 }
1666                 i++;
1667         }
1668
1669         return i;
1670 }
1671
1672 static int init_gpu_rail_thermal_scaling(struct device_node *node,
1673                                          struct dvfs_rail *rail,
1674                                          struct cvb_dvfs *d)
1675 {
1676         int thermal_ranges;
1677         struct device_node *cdev_node;
1678
1679         cdev_node = of_find_compatible_node(NULL, NULL,
1680                                 "nvidia,tegra210-rail-scaling-cdev");
1681         if (!cdev_node || !of_device_is_available(cdev_node))
1682                 return 1;
1683
1684         rail->vts_of_node = cdev_node;
1685
1686         thermal_ranges = of_parse_dvfs_rail_cdev_trips(cdev_node,
1687                 &rail->vts_trips_table[0], &rail->vts_floors_table[0],
1688                 NULL, &rail->alignment, true);
1689
1690         if (thermal_ranges <= 0)
1691                 return 1;
1692
1693         rail->vts_number_of_trips = thermal_ranges - 1;
1694
1695         return thermal_ranges;
1696 }
1697
1698 /* cooling device to limit GPU frequenct based on the vmax thermal profile */
1699 #define GPU_MAX_RATE 1300000000UL
1700 static int gpu_dvfs_rail_get_vmax_cdev_max_state(
1701         struct thermal_cooling_device *cdev, unsigned long *max_state)
1702 {
1703         struct dvfs_rail *rail = cdev->devdata;
1704
1705         *max_state = rail->therm_caps_size;
1706
1707         return 0;
1708 }
1709
1710 static int gpu_dvfs_rail_get_vmax_cdev_cur_state(
1711         struct thermal_cooling_device *cdev, unsigned long *cur_state)
1712 {
1713         struct dvfs_rail *rail = cdev->devdata;
1714
1715         *cur_state = rail->therm_cap_idx;
1716
1717         return 0;
1718 }
1719
1720 static int gpu_dvfs_rail_set_vmax_cdev_cur_state(
1721         struct thermal_cooling_device *cdev, unsigned long cur_state)
1722 {
1723         struct dvfs_rail *rail = cdev->devdata;
1724         int level = 0, err = -EINVAL;
1725         unsigned long cap_rate = GPU_MAX_RATE;
1726
1727         if (cur_state)
1728                 level = rail->therm_caps[cur_state - 1].mv;
1729
1730         if (level) {
1731                 if (rail->vts_cdev && gpu_dvfs.therm_dvfs)
1732                         cap_rate = gpu_cap_rates[cur_state - 1];
1733                 else
1734                         cap_rate = tegra_dvfs_predict_hz_at_mv_max_tfloor(
1735                                 gpu_dvfs.clk, level);
1736         }
1737
1738         if (!IS_ERR_VALUE(cap_rate)) {
1739                 err = clk_set_rate(vgpu_cap_clk, cap_rate);
1740                 if (err)
1741                         pr_err("tegra_dvfs: Failed to set GPU cap rate %lu\n",
1742                                cap_rate);
1743         } else {
1744                 pr_err("tegra_dvfs: Failed to find GPU cap rate for %dmV\n",
1745                        level);
1746         }
1747
1748         rail->therm_cap_idx = cur_state;
1749
1750         return err;
1751 }
1752
1753 static struct thermal_cooling_device_ops gpu_dvfs_rail_vmax_cooling_ops = {
1754         .get_max_state = gpu_dvfs_rail_get_vmax_cdev_max_state,
1755         .get_cur_state = gpu_dvfs_rail_get_vmax_cdev_cur_state,
1756         .set_cur_state = gpu_dvfs_rail_set_vmax_cdev_cur_state,
1757 };
1758
1759 #define CAP_TRIP_ON_SCALING_MARGIN 5
1760
1761 static int init_gpu_rail_thermal_caps(struct device_node *node,
1762                 struct dvfs *dvfs, struct dvfs_rail *rail, int thermal_ranges,
1763                 int freqs_num)
1764 {
1765         struct device_node *cdev_node;
1766         int num_trips, i, j, k;
1767         bool ucm2 = tegra_sku_info.ucm == TEGRA_UCM2;
1768
1769         if (thermal_ranges <= 1 )
1770                 return 0;
1771
1772         cdev_node = of_find_compatible_node(NULL, NULL,
1773                                 "nvidia,tegra210-rail-vmax-cdev");
1774         if (!cdev_node || !of_device_is_available(cdev_node))
1775                 return 0;
1776
1777         rail->vmax_of_node = cdev_node;
1778
1779         vgpu_cap_clk = of_clk_get_by_name(cdev_node, "cap-clk");
1780         if (IS_ERR(vgpu_cap_clk))
1781                 return 0;
1782
1783         num_trips = of_parse_dvfs_rail_cdev_trips(cdev_node,
1784                 NULL, vdd_gpu_therm_caps_table, vdd_gpu_therm_caps_ucm2_table,
1785                 &rail->alignment, false);
1786         if (num_trips <= 0)
1787                 return 0;
1788
1789         rail->therm_caps =
1790                 ucm2 ? vdd_gpu_therm_caps_ucm2_table : vdd_gpu_therm_caps_table;
1791         if (!rail->therm_caps[0].mv) {
1792                 pr_err("tegra_dvfs: invalid gpu cap table\n");
1793                 rail->therm_caps = NULL;
1794                 return 0;
1795         }
1796         rail->therm_caps_size = num_trips;
1797         rail->therm_cap_idx = num_trips;
1798
1799         for (k = 0; k < num_trips; k++) {
1800                 int cap_tempr = rail->therm_caps[k].temperature;
1801                 int cap_level = rail->therm_caps[k].mv;
1802                 unsigned long cap_freq = GPU_MAX_RATE;
1803
1804                 for (j = 0; j < thermal_ranges; j++) {
1805                         if ((j < thermal_ranges - 1) && /* vts trips=ranges-1 */
1806                             (rail->vts_trips_table[j] +
1807                             CAP_TRIP_ON_SCALING_MARGIN < cap_tempr))
1808                                 continue;
1809
1810                         for (i = 1; i < freqs_num; i++) {
1811                                 if (gpu_millivolts[j][i] > cap_level)
1812                                         break;
1813                         }
1814                         cap_freq = min(cap_freq, dvfs->freqs[i - 1]);
1815                 }
1816                 gpu_cap_rates[k] = cap_freq * dvfs->freqs_mult;
1817         }
1818
1819
1820         clk_set_rate(vgpu_cap_clk, gpu_cap_rates[num_trips - 1]);
1821         rail->vmax_cdev = thermal_of_cooling_device_register(rail->vmax_of_node,
1822                 "GPU-cap", rail, &gpu_dvfs_rail_vmax_cooling_ops);
1823         pr_info("tegra_dvfs: GPU-cap: %sregistered\n",
1824                 IS_ERR_OR_NULL(rail->vmax_cdev) ? "not " : "");
1825
1826         return num_trips;
1827 }
1828
1829 /*
1830  * Setup gpu dvfs tables from cvb data, determine nominal voltage for gpu rail,
1831  * and gpu maximum frequency. Error when gpu dvfs table can not be constructed
1832  * must never happen.
1833  */
1834 static int set_gpu_dvfs_data(struct device_node *node, unsigned long max_freq,
1835         struct cvb_dvfs *d, struct dvfs *gpu_dvfs, int *max_freq_index)
1836 {
1837         int i, j, thermal_ranges, mv, min_mv, err;
1838         struct cvb_dvfs_table *table = NULL;
1839         int speedo = tegra_sku_info.gpu_speedo_value;
1840         struct dvfs_rail *rail = &vdd_gpu_rail;
1841         struct rail_alignment *align = &rail->alignment;
1842
1843         rail->nvver = d->cvb_version;
1844
1845         d->max_mv = tegra_round_voltage(d->max_mv, align, false);
1846         min_mv = d->pll_min_millivolts;
1847         mv = tegra_get_cvb_voltage(
1848                 speedo, d->speedo_scale, &d->cvb_vmin.cvb_pll_param);
1849         mv = tegra_round_cvb_voltage(mv, d->voltage_scale, align);
1850         min_mv = max(min_mv, mv);
1851         if (min_mv < rail->min_millivolts) {
1852                 pr_debug("tegra21_dvfs: gpu min %dmV below rail min %dmV\n",
1853                          min_mv, rail->min_millivolts);
1854                 min_mv = rail->min_millivolts;
1855         }
1856
1857         /*
1858          * Get scaling thermal ranges; 1 range implies no thermal dependency.
1859          * Invalidate scaling cooling device in the latter case.
1860          */
1861         thermal_ranges = init_gpu_rail_thermal_scaling(node, rail, d);
1862         if (thermal_ranges == 1)
1863                 rail->vts_cdev = NULL;
1864
1865         /*
1866          * Apply fixed thermal floor for each temperature range
1867          */
1868         for (j = 0; j < thermal_ranges; j++) {
1869                 mv = max(min_mv, (int)rail->vts_floors_table[j].mv);
1870                 gpu_vmin[j] = tegra_round_voltage(mv, align, true);
1871         }
1872
1873         /*
1874          * Use CVB table to fill in gpu dvfs frequencies and voltages. Each
1875          * CVB entry specifies gpu frequency and CVB coefficients to calculate
1876          * the respective voltage.
1877          */
1878         for (i = 0; i < MAX_DVFS_FREQS; i++) {
1879                 table = &d->cvb_table[i];
1880                 if (!table->freq || (table->freq > max_freq))
1881                         break;
1882
1883                 mv = tegra_get_cvb_voltage(
1884                         speedo, d->speedo_scale, &table->cvb_pll_param);
1885
1886                 for (j = 0; j < thermal_ranges; j++) {
1887                         int mvj = mv;
1888                         int t = 0;
1889
1890                         if (thermal_ranges > 1)
1891                                 t = rail->vts_trips_table[j];
1892
1893                         /* get thermal offset for this trip-point */
1894                         mvj += tegra_get_cvb_t_voltage(speedo, d->speedo_scale,
1895                                 t, d->thermal_scale, &table->cvb_pll_param);
1896                         mvj = tegra_round_cvb_voltage(mvj, d->voltage_scale, align);
1897
1898                         /* clip to minimum, abort if above maximum */
1899                         mvj = max(mvj, gpu_vmin[j]);
1900                         if (mvj > d->max_mv)
1901                                 break;
1902
1903                         /*
1904                          * Update voltage for adjacent ranges bounded by this
1905                          * trip-point (cvb & dvfs are transpose matrices, and
1906                          * cvb freq row index is column index for dvfs matrix)
1907                          */
1908                         gpu_millivolts[j][i] = mvj;
1909                         if (j && (gpu_millivolts[j-1][i] < mvj))
1910                                 gpu_millivolts[j-1][i] = mvj;
1911                 }
1912                 /* Make sure all voltages for this frequency are below max */
1913                 if (j < thermal_ranges)
1914                         break;
1915
1916                 /* fill in gpu dvfs tables */
1917                 gpu_dvfs->freqs[i] = table->freq;
1918         }
1919
1920         gpu_dvfs->millivolts = &gpu_millivolts[0][0];
1921
1922         /*
1923          * Table must not be empty, must have at least one entry in range, and
1924          * must specify monotonically increasing voltage on frequency dependency
1925          * in each temperature range.
1926          */
1927         err = tegra_dvfs_init_thermal_dvfs_voltages(&gpu_millivolts[0][0],
1928                         gpu_peak_millivolts, i, thermal_ranges, gpu_dvfs);
1929
1930         if (err || !i) {
1931                 pr_err("tegra21_dvfs: invalid gpu dvfs table\n");
1932                 return -ENOENT;
1933         }
1934
1935         /* Shift out the 1st trip-point */
1936         for (j = 1; j < thermal_ranges; j++)
1937                 rail->vts_trips_table[j - 1] = rail->vts_trips_table[j];
1938
1939         /* dvfs tables are successfully populated - fill in the gpu dvfs */
1940         gpu_dvfs->speedo_id = d->speedo_id;
1941         gpu_dvfs->process_id = d->process_id;
1942         gpu_dvfs->freqs_mult = d->freqs_mult;
1943
1944         *max_freq_index = i - 1;
1945
1946         gpu_dvfs->dvfs_rail->nominal_millivolts = min(d->max_mv,
1947                 find_gpu_vmin_at_fmax(gpu_dvfs, thermal_ranges, i));
1948
1949         gpu_dvfs->fmax_at_vmin_safe_t = d->freqs_mult *
1950                 find_gpu_fmax_at_vmin(gpu_dvfs, thermal_ranges, i);
1951
1952         /* Initialize thermal capping */
1953         init_gpu_rail_thermal_caps(node, gpu_dvfs, rail, thermal_ranges, i);
1954
1955 #ifdef CONFIG_TEGRA_USE_NA_GPCPLL
1956         /*
1957          * Set NA DVFS flag, if GPCPLL NA mode is enabled. This is necessary to
1958          * make sure that GPCPLL configuration is updated by tegra core DVFS
1959          * when thermal DVFS cooling device state is changed. Since tegra core
1960          * DVFS does not support NA operations for Vmin cooling device, GPU Vmin
1961          * thermal floors have been integrated with thermal DVFS, and no Vmin
1962          * cooling device is installed.
1963          */
1964         if (tegra_sku_info.gpu_speedo_id)
1965                 gpu_dvfs->na_dvfs = 1;
1966 #else
1967         if (of_device_is_compatible(node, "nvidia,tegra210b01-dvfs"))
1968                 WARN(1, "TEGRA_USE_NA_GPCPLL must be set on T210b01\n");
1969 #endif
1970         return 0;
1971 }
1972
1973 static void init_gpu_dvfs_table(struct device_node *node,
1974         struct cvb_dvfs *cvb_dvfs_table, int table_size,
1975         int *gpu_max_freq_index)
1976 {
1977         int i, ret;
1978         int gpu_speedo_id = tegra_sku_info.gpu_speedo_id;
1979         int gpu_process_id = tegra_sku_info.gpu_process_id;
1980
1981         for (ret = 0, i = 0; i < table_size; i++) {
1982                 struct cvb_dvfs *d = &cvb_dvfs_table[i];
1983                 unsigned long max_freq = d->max_freq;
1984                 u32 f;
1985
1986                 if (!of_property_read_u32(node, "nvidia,gpu-max-freq-khz", &f))
1987                         max_freq = min(max_freq, (unsigned long)f);
1988
1989                 if (match_dvfs_one("gpu cvb", d->speedo_id, d->process_id,
1990                                    gpu_speedo_id, gpu_process_id)) {
1991                         ret = set_gpu_dvfs_data(node, max_freq,
1992                                 d, &gpu_dvfs, gpu_max_freq_index);
1993                         break;
1994                 }
1995         }
1996         BUG_ON((i == table_size) || ret);
1997 }
1998
1999 static void init_core_dvfs_table(int soc_speedo_id, int core_process_id)
2000 {
2001         int core_nominal_mv_index;
2002         int core_min_mv_index;
2003         int i;
2004         static bool initialized;
2005
2006         if (initialized)
2007                 return;
2008
2009         initialized = true;
2010
2011         /*
2012          * Find nominal voltages for core (1st) and cpu rails before rail
2013          * init. Nominal voltage index in core scaling ladder can also be
2014          * used to determine max dvfs frequencies for all core clocks. In
2015          * case of error disable core scaling and set index to 0, so that
2016          * core clocks would not exceed rates allowed at minimum voltage.
2017          */
2018         core_nominal_mv_index = get_core_nominal_mv_index(soc_speedo_id);
2019         if (core_nominal_mv_index < 0) {
2020                 vdd_core_rail.disabled = true;
2021                 tegra_dvfs_core_disabled = true;
2022                 core_nominal_mv_index = 0;
2023         }
2024
2025         core_min_mv_index = get_core_min_mv_index();
2026         if (core_min_mv_index < 0) {
2027                 vdd_core_rail.disabled = true;
2028                 tegra_dvfs_core_disabled = true;
2029                 core_min_mv_index = 0;
2030         }
2031
2032         vdd_core_rail.nominal_millivolts =
2033                 core_millivolts[core_nominal_mv_index];
2034         vdd_core_rail.min_millivolts =
2035                 core_millivolts[core_min_mv_index];
2036         vdd_core_rail.nvver = dvfs_data->core_dvfs_ver;
2037
2038         /*
2039          * Search core dvfs table for speedo/process matching entries and
2040          * initialize dvfs-ed clocks
2041          */
2042         for (i = 0; i < dvfs_data->core_vf_table_size; i++) {
2043                 struct dvfs *d = &dvfs_data->core_vf_table[i];
2044                 if (!match_dvfs_one(d->clk_name, d->speedo_id,
2045                         d->process_id, soc_speedo_id, core_process_id))
2046                         continue;
2047                 init_dvfs_one(d, core_nominal_mv_index);
2048
2049                 /*
2050                  * If EMC DVB table is installed, use it to set EMC VF opps.
2051                  * Otherwise, EMC dvfs is board dependent, EMC frequencies are
2052                  * determined by the Tegra BCT and the board specific EMC DFS
2053                  * table owned by EMC driver.
2054                  */
2055                 if (!strcmp(d->clk_name, "emc") && tegra210_emc_is_ready()) {
2056                         struct dvb_dvfs *dvbd = get_emc_dvb_dvfs(soc_speedo_id);
2057
2058                         if (dvbd)
2059                                 adjust_emc_dvfs_from_dvb_table(
2060                                         core_process_id, dvbd, d);
2061                         else
2062                                 adjust_emc_dvfs_from_timing_table(d);
2063                 }
2064         }
2065
2066         init_qspi_dvfs(soc_speedo_id, core_process_id, core_nominal_mv_index);
2067         init_sor1_dvfs(soc_speedo_id, core_process_id, core_nominal_mv_index);
2068         init_spi_dvfs(soc_speedo_id, core_process_id, core_nominal_mv_index);
2069 }
2070
2071 static void init_dvfs_data(struct tegra_dvfs_data *data)
2072 {
2073         int i;
2074         static bool initialized;
2075
2076         if (initialized)
2077                 return;
2078
2079         initialized = true;
2080
2081         dvfs_data = data;
2082
2083         BUG_ON(dvfs_data->rails_num != ARRAY_SIZE(vdd_dvfs_rails));
2084         vdd_cpu_rail = *dvfs_data->rails[VDD_CPU_INDEX];
2085         vdd_core_rail = *dvfs_data->rails[VDD_CORE_INDEX];
2086         vdd_gpu_rail = *dvfs_data->rails[VDD_GPU_INDEX];
2087
2088         for (i = 0; i < MAX_DVFS_FREQS; i++)
2089                 core_millivolts[i] = dvfs_data->core_mv[i];
2090 }
2091
2092 static struct tegra_dvfs_data tegra210_dvfs_data = {
2093         .rails = tegra210_dvfs_rails,
2094         .rails_num = ARRAY_SIZE(tegra210_dvfs_rails),
2095         .cpu_fv_table = cpu_fv_dvfs_table,
2096         .cpu_fv_table_size = ARRAY_SIZE(cpu_fv_dvfs_table),
2097         .gpu_cvb_table = gpu_cvb_dvfs_table,
2098         .gpu_cvb_table_size = ARRAY_SIZE(gpu_cvb_dvfs_table),
2099
2100         .core_mv = core_voltages_mv,
2101         .core_vf_table = core_dvfs_table,
2102         .core_vf_table_size = ARRAY_SIZE(core_dvfs_table),
2103         .spi_vf_table = spi_dvfs_table,
2104         .spi_slave_vf_table = spi_slave_dvfs_table,
2105         .qspi_sdr_vf_table = qspi_sdr_dvfs_table,
2106         .qspi_ddr_vf_table = qspi_ddr_dvfs_table,
2107         .sor1_dp_vf_table = sor1_dp_dvfs_table,
2108         .sor1_dp_vf_table_size = ARRAY_SIZE(sor1_dp_dvfs_table),
2109         .get_core_min_mv = get_core_sku_min_mv,
2110         .get_core_max_mv = get_core_sku_max_mv,
2111
2112         .core_floors = tegra210_core_therm_floors,
2113         .core_caps = tegra210_core_therm_caps,
2114         .core_caps_ucm2 = tegra210_core_therm_caps_ucm2,
2115 };
2116
2117 static struct tegra_dvfs_data tegra210b01_dvfs_data = {
2118         .rails = tegra210b01_dvfs_rails,
2119         .rails_num = ARRAY_SIZE(tegra210b01_dvfs_rails),
2120         .cpu_fv_table = cpub01_fv_dvfs_table,
2121         .cpu_fv_table_size = ARRAY_SIZE(cpub01_fv_dvfs_table),
2122         .gpu_cvb_table = gpub01_cvb_dvfs_table,
2123         .gpu_cvb_table_size = ARRAY_SIZE(gpub01_cvb_dvfs_table),
2124
2125         .emc_dvb_table = emcb01_dvb_dvfs_table,
2126         .emc_dvb_table_size = ARRAY_SIZE(emcb01_dvb_dvfs_table),
2127
2128         .core_mv = coreb01_voltages_mv,
2129         .core_vf_table = coreb01_dvfs_table,
2130         .core_vf_table_size = ARRAY_SIZE(coreb01_dvfs_table),
2131         .spi_vf_table = spib01_dvfs_table,
2132         .spi_slave_vf_table = spi_slaveb01_dvfs_table,
2133         .qspi_sdr_vf_table = qspi_sdrb01_dvfs_table,
2134         .qspi_ddr_vf_table = qspi_ddrb01_dvfs_table,
2135         .sor1_dp_vf_table = sor1_dpb01_dvfs_table,
2136         .sor1_dp_vf_table_size = ARRAY_SIZE(sor1_dpb01_dvfs_table),
2137         .get_core_min_mv = get_coreb01_sku_min_mv,
2138         .get_core_max_mv = get_coreb01_sku_max_mv,
2139
2140         .core_floors = tegra210b01_core_therm_floors,
2141         .core_caps = tegra210b01_core_therm_caps,
2142         .core_caps_ucm2 = tegra210b01_core_therm_caps_ucm2,
2143
2144         .core_dvfs_ver = coreb01_dvfs_table_ver,
2145 };
2146
2147 static struct tegra_dvfs_data tegra210b01slt_dvfs_data = {
2148         .rails = tegra210b01_dvfs_rails,
2149         .rails_num = ARRAY_SIZE(tegra210b01_dvfs_rails),
2150         .cpu_fv_table = cpub01_fv_dvfs_table,
2151         .cpu_fv_table_size = ARRAY_SIZE(cpub01_fv_dvfs_table),
2152         .gpu_cvb_table = gpub01_cvb_dvfs_table,
2153         .gpu_cvb_table_size = ARRAY_SIZE(gpub01_cvb_dvfs_table),
2154
2155         .emc_dvb_table = emcb01slt_dvb_dvfs_table,
2156         .emc_dvb_table_size = ARRAY_SIZE(emcb01slt_dvb_dvfs_table),
2157
2158         .core_mv = coreb01slt_voltages_mv,
2159         .core_vf_table = coreb01slt_dvfs_table,
2160         .core_vf_table_size = ARRAY_SIZE(coreb01slt_dvfs_table),
2161         .spi_vf_table = spib01slt_dvfs_table,
2162         .spi_slave_vf_table = spi_slaveb01slt_dvfs_table,
2163         .qspi_sdr_vf_table = qspi_sdrb01slt_dvfs_table,
2164         .qspi_ddr_vf_table = qspi_ddrb01slt_dvfs_table,
2165         .sor1_dp_vf_table = sor1_dpb01slt_dvfs_table,
2166         .sor1_dp_vf_table_size = ARRAY_SIZE(sor1_dpb01slt_dvfs_table),
2167         .get_core_min_mv = get_coreb01slt_sku_min_mv,
2168         .get_core_max_mv = get_coreb01slt_sku_max_mv,
2169
2170         .core_floors = tegra210b01_core_therm_floors,
2171         .core_caps = tegra210b01_core_therm_caps,
2172         .core_caps_ucm2 = tegra210b01_core_therm_caps_ucm2,
2173
2174         .core_dvfs_ver = coreb01slt_dvfs_table_ver,
2175 };
2176
2177 static void disable_rail_scaling(struct device_node *np)
2178 {
2179         /* With DFLL as clock source CPU rail scaling cannot be disabled */
2180
2181         if (tegra_dvfs_core_disabled ||
2182             of_property_read_bool(np, "nvidia,core-rail-scaling-disabled")) {
2183                 vdd_dvfs_rails[VDD_CORE_INDEX]->disabled = true;
2184         }
2185         if (tegra_dvfs_gpu_disabled ||
2186             of_property_read_bool(np, "nvidia,gpu-rail-scaling-disabled")) {
2187                 vdd_dvfs_rails[VDD_GPU_INDEX]->disabled = true;
2188         }
2189 }
2190
2191 static int tegra210x_init_dvfs(struct device *dev, bool cpu_lp_init)
2192 {
2193         int soc_speedo_id = tegra_sku_info.soc_speedo_id;
2194         int core_process_id = tegra_sku_info.soc_process_id;
2195         bool ucm2 = tegra_sku_info.ucm == TEGRA_UCM2;
2196         int i, ret;
2197         int cpu_max_freq_index = 0;
2198         int cpu_lp_max_freq_index = 0;
2199         int gpu_max_freq_index = 0;
2200         struct device_node *node = dev->of_node;
2201
2202         tegra_dvfs_init_rails_lists(vdd_dvfs_rails, dvfs_data->rails_num);
2203         init_core_dvfs_table(soc_speedo_id, core_process_id);
2204
2205         /* Get rails alignment, defer probe if regulators are not ready */
2206         for (i = 0; i <  dvfs_data->rails_num; i++) {
2207                 struct regulator *reg;
2208                 unsigned int step_uv;
2209                 int min_uV, max_uV, ret;
2210
2211                 reg = regulator_get(dev, vdd_dvfs_rails[i]->reg_id);
2212                 if (IS_ERR(reg)) {
2213                         pr_info("tegra_dvfs: Unable to get %s rail for step info, defering probe\n",
2214                                         vdd_dvfs_rails[i]->reg_id);
2215                         return -EPROBE_DEFER;
2216                 }
2217
2218                 ret = regulator_get_constraint_voltages(reg, &min_uV, &max_uV);
2219                 if (!ret)
2220                         vdd_dvfs_rails[i]->alignment.offset_uv = min_uV;
2221
2222                 step_uv = regulator_get_linear_step(reg); /* 1st try get step */
2223                 if (!step_uv && !ret) {    /* if no step, try to calculate it */
2224                         int n_voltages = regulator_count_voltages(reg);
2225
2226                         if (n_voltages > 1)
2227                                 step_uv = (max_uV - min_uV) / (n_voltages - 1);
2228                 }
2229                 if (step_uv) {
2230                         vdd_dvfs_rails[i]->alignment.step_uv = step_uv;
2231                         vdd_dvfs_rails[i]->stats.bin_uv = step_uv;
2232                 }
2233                 regulator_put(reg);
2234         }
2235
2236         /*
2237          * Construct fast and slow CPU DVFS tables from FV data; find maximum
2238          * frequency, minimum  and nominal voltage for each CPU cluster, and
2239          * combined rail limits (fast CPU should be initialized 1st).
2240          */
2241         ret = init_cpu_dvfs_table(dvfs_data->cpu_fv_table,
2242                 dvfs_data->cpu_fv_table_size, &cpu_max_freq_index);
2243         if (ret)
2244                 goto out;
2245
2246         if (cpu_lp_init) {
2247                 ret = init_cpu_lp_dvfs_table(&cpu_lp_max_freq_index);
2248                 if (ret)
2249                         goto out;
2250         }
2251
2252         /*
2253          * Construct GPU DVFS table from CVB data; find GPU maximum frequency,
2254          * and nominal voltage.
2255          */
2256         init_gpu_dvfs_table(node, dvfs_data->gpu_cvb_table,
2257                 dvfs_data->gpu_cvb_table_size, &gpu_max_freq_index);
2258
2259         /* Init core thermal floors abd caps */
2260         vdd_core_rail.therm_floors = dvfs_data->core_floors;
2261         vdd_core_rail.therm_caps =
2262                 ucm2 ? dvfs_data->core_caps_ucm2 : dvfs_data->core_caps;
2263         tegra_dvfs_core_init_therm_limits(&vdd_core_rail);
2264
2265         /* Init rail structures and dependencies */
2266         tegra_dvfs_init_rails(vdd_dvfs_rails, dvfs_data->rails_num);
2267
2268         if (of_property_read_bool(of_chosen, "nvidia,tegra-joint_xpu_rail"))
2269                 vdd_dvfs_rails[VDD_GPU_INDEX]->joint_rail_with_dfll = true;
2270
2271         /*
2272          * Initialize matching cpu dvfs entry already found when nominal
2273          * voltage was determined
2274          */
2275         init_dvfs_one(&cpu_dvfs, cpu_max_freq_index);
2276         if (cpu_lp_init)
2277                 init_dvfs_one(&cpu_lp_dvfs, cpu_lp_max_freq_index);
2278         init_dvfs_one(&gpu_dvfs, gpu_max_freq_index);
2279
2280         disable_rail_scaling(node);
2281
2282         for (i = 0; i < dvfs_data->rails_num; i++) {
2283                 struct dvfs_rail *rail = vdd_dvfs_rails[i];
2284                 pr_info("tegra dvfs: %s: nominal %dmV, offset %duV, step %duV, scaling %s\n",
2285                         rail->reg_id, rail->nominal_millivolts,
2286                         rail->alignment.offset_uv, rail->alignment.step_uv,
2287                         rail->disabled ? "disabled" : "enabled");
2288         }
2289         return 0;
2290 out:
2291         return ret;
2292 }
2293
2294 int tegra210_init_dvfs(struct device *dev)
2295 {
2296         init_dvfs_data(&tegra210_dvfs_data);
2297         return tegra210x_init_dvfs(dev, true);
2298 }
2299
2300 int tegra210b01_init_dvfs(struct device *dev)
2301 {
2302         if (tegra_sku_info.soc_speedo_id == 2)
2303                 init_dvfs_data(&tegra210b01slt_dvfs_data);
2304         else
2305                 init_dvfs_data(&tegra210b01_dvfs_data);
2306
2307         return tegra210x_init_dvfs(dev, false);
2308 }