ARM: tegra: power: Add dynamic CPU regulator mode control
[linux-3.10.git] / arch / arm / mach-tegra / dvfs.c
1 /*
2  *
3  * Copyright (C) 2010 Google, Inc.
4  *
5  * Author:
6  *      Colin Cross <ccross@google.com>
7  *
8  * Copyright (C) 2010-2013 NVIDIA CORPORATION. All rights reserved.
9  *
10  * This software is licensed under the terms of the GNU General Public
11  * License version 2, as published by the Free Software Foundation, and
12  * may be copied, distributed, and modified under those terms.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/clk.h>
23 #include <linux/clkdev.h>
24 #include <linux/debugfs.h>
25 #include <linux/init.h>
26 #include <linux/list.h>
27 #include <linux/list_sort.h>
28 #include <linux/module.h>
29 #include <linux/regulator/consumer.h>
30 #include <linux/seq_file.h>
31 #include <linux/slab.h>
32 #include <linux/suspend.h>
33 #include <linux/delay.h>
34 #include <linux/clk/tegra.h>
35 #include <linux/reboot.h>
36 #include <linux/clk/tegra.h>
37 #include <linux/tegra-soc.h>
38
39 #include "board.h"
40 #include "clock.h"
41 #include "dvfs.h"
42
43 #define DVFS_RAIL_STATS_BIN     12500
44
45 struct dvfs_rail *tegra_cpu_rail;
46 struct dvfs_rail *tegra_core_rail;
47 struct dvfs_rail *tegra_gpu_rail;
48
49 static LIST_HEAD(dvfs_rail_list);
50 static DEFINE_MUTEX(dvfs_lock);
51 static DEFINE_MUTEX(rail_disable_lock);
52
53 static int dvfs_rail_update(struct dvfs_rail *rail);
54
55 static inline int tegra_dvfs_rail_get_disable_level(struct dvfs_rail *rail)
56 {
57         return rail->disable_millivolts ? : rail->nominal_millivolts;
58 }
59
60 static inline int tegra_dvfs_rail_get_suspend_level(struct dvfs_rail *rail)
61 {
62         return rail->suspend_millivolts ? : rail->nominal_millivolts;
63 }
64
65 void tegra_dvfs_add_relationships(struct dvfs_relationship *rels, int n)
66 {
67         int i;
68         struct dvfs_relationship *rel;
69
70         mutex_lock(&dvfs_lock);
71
72         for (i = 0; i < n; i++) {
73                 rel = &rels[i];
74                 list_add_tail(&rel->from_node, &rel->to->relationships_from);
75                 list_add_tail(&rel->to_node, &rel->from->relationships_to);
76         }
77
78         mutex_unlock(&dvfs_lock);
79 }
80
81 /* Make sure there is a matching cooling device for thermal limit profile. */
82 static void dvfs_validate_cdevs(struct dvfs_rail *rail)
83 {
84         if (!rail->therm_mv_caps != !rail->therm_mv_caps_num) {
85                 rail->therm_mv_caps_num = 0;
86                 rail->therm_mv_caps = NULL;
87                 WARN(1, "%s: not matching thermal caps/num\n", rail->reg_id);
88         }
89
90         if (rail->therm_mv_caps && !rail->vmax_cdev)
91                 WARN(1, "%s: missing vmax cooling device\n", rail->reg_id);
92
93         if (!rail->therm_mv_floors != !rail->therm_mv_floors_num) {
94                 rail->therm_mv_floors_num = 0;
95                 rail->therm_mv_floors = NULL;
96                 WARN(1, "%s: not matching thermal floors/num\n", rail->reg_id);
97         }
98
99         if (rail->therm_mv_floors && !rail->vmin_cdev)
100                 WARN(1, "%s: missing vmin cooling device\n", rail->reg_id);
101
102         /* Limit override range to maximum floor */
103         if (rail->therm_mv_floors)
104                 rail->min_override_millivolts = rail->therm_mv_floors[0];
105
106         /* Only GPU thermal dvfs is supported */
107         if (rail->vts_cdev && (rail != tegra_gpu_rail)) {
108                 rail->vts_cdev = NULL;
109                 WARN(1, "%s: thermal dvfs is not supported\n", rail->reg_id);
110         }
111 }
112
113 int tegra_dvfs_init_rails(struct dvfs_rail *rails[], int n)
114 {
115         int i, mv;
116
117         mutex_lock(&dvfs_lock);
118
119         for (i = 0; i < n; i++) {
120                 INIT_LIST_HEAD(&rails[i]->dvfs);
121                 INIT_LIST_HEAD(&rails[i]->relationships_from);
122                 INIT_LIST_HEAD(&rails[i]->relationships_to);
123
124                 mv = rails[i]->nominal_millivolts;
125                 if (rails[i]->boot_millivolts > mv)
126                         WARN(1, "%s: boot voltage %d above nominal %d\n",
127                              rails[i]->reg_id, rails[i]->boot_millivolts, mv);
128                 if (rails[i]->disable_millivolts > mv)
129                         rails[i]->disable_millivolts = mv;
130                 if (rails[i]->suspend_millivolts > mv)
131                         rails[i]->suspend_millivolts = mv;
132
133                 mv = tegra_dvfs_rail_get_boot_level(rails[i]);
134                 rails[i]->millivolts = mv;
135                 rails[i]->new_millivolts = mv;
136                 if (!rails[i]->step)
137                         rails[i]->step = rails[i]->max_millivolts;
138                 if (!rails[i]->step_up)
139                         rails[i]->step_up = rails[i]->step;
140
141                 list_add_tail(&rails[i]->node, &dvfs_rail_list);
142
143                 if (!strcmp("vdd_cpu", rails[i]->reg_id))
144                         tegra_cpu_rail = rails[i];
145                 else if (!strcmp("vdd_gpu", rails[i]->reg_id))
146                         tegra_gpu_rail = rails[i];
147                 else if (!strcmp("vdd_core", rails[i]->reg_id))
148                         tegra_core_rail = rails[i];
149
150                 dvfs_validate_cdevs(rails[i]);
151         }
152
153         mutex_unlock(&dvfs_lock);
154
155         return 0;
156 };
157
158 static int dvfs_solve_relationship(struct dvfs_relationship *rel)
159 {
160         return rel->solve(rel->from, rel->to);
161 }
162
163 /* rail statistic - called during rail init, or under dfs_lock, or with
164    CPU0 only on-line, and interrupts disabled */
165 static void dvfs_rail_stats_init(struct dvfs_rail *rail, int millivolts)
166 {
167         int dvfs_rail_stats_range;
168
169         if (!rail->stats.bin_uV)
170                 rail->stats.bin_uV = DVFS_RAIL_STATS_BIN;
171
172         dvfs_rail_stats_range =
173                 (DVFS_RAIL_STATS_TOP_BIN - 1) * rail->stats.bin_uV / 1000;
174
175         rail->stats.last_update = ktime_get();
176         if (millivolts >= rail->min_millivolts) {
177                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
178                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
179                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
180         }
181
182         if (rail->max_millivolts >
183             rail->min_millivolts + dvfs_rail_stats_range)
184                 pr_warn("tegra_dvfs: %s: stats above %d mV will be squashed\n",
185                         rail->reg_id,
186                         rail->min_millivolts + dvfs_rail_stats_range);
187 }
188
189 static void dvfs_rail_stats_update(
190         struct dvfs_rail *rail, int millivolts, ktime_t now)
191 {
192         rail->stats.time_at_mv[rail->stats.last_index] = ktime_add(
193                 rail->stats.time_at_mv[rail->stats.last_index], ktime_sub(
194                         now, rail->stats.last_update));
195         rail->stats.last_update = now;
196
197         if (rail->stats.off)
198                 return;
199
200         if (millivolts >= rail->min_millivolts) {
201                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
202                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
203                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
204         } else if (millivolts == 0)
205                         rail->stats.last_index = 0;
206 }
207
208 static void dvfs_rail_stats_pause(struct dvfs_rail *rail,
209                                   ktime_t delta, bool on)
210 {
211         int i = on ? rail->stats.last_index : 0;
212         rail->stats.time_at_mv[i] = ktime_add(rail->stats.time_at_mv[i], delta);
213 }
214
215 void tegra_dvfs_rail_off(struct dvfs_rail *rail, ktime_t now)
216 {
217         if (rail) {
218                 dvfs_rail_stats_update(rail, 0, now);
219                 rail->stats.off = true;
220         }
221 }
222
223 void tegra_dvfs_rail_on(struct dvfs_rail *rail, ktime_t now)
224 {
225         if (rail) {
226                 rail->stats.off = false;
227                 dvfs_rail_stats_update(rail, rail->millivolts, now);
228         }
229 }
230
231 void tegra_dvfs_rail_pause(struct dvfs_rail *rail, ktime_t delta, bool on)
232 {
233         if (rail)
234                 dvfs_rail_stats_pause(rail, delta, on);
235 }
236
237 static int dvfs_rail_set_voltage_reg(struct dvfs_rail *rail, int millivolts)
238 {
239         int ret;
240
241         /*
242          * safely return success for low voltage requests on fixed regulator
243          * (higher requests will go through and fail, as they should)
244          */
245         if (rail->fixed_millivolts && (millivolts <= rail->fixed_millivolts))
246                 return 0;
247
248         rail->updating = true;
249         rail->reg_max_millivolts = rail->reg_max_millivolts ==
250                 rail->max_millivolts ?
251                 rail->max_millivolts + 1 : rail->max_millivolts;
252         ret = regulator_set_voltage(rail->reg,
253                 millivolts * 1000,
254                 rail->reg_max_millivolts * 1000);
255         rail->updating = false;
256
257         return ret;
258 }
259
260 /* Sets the voltage on a dvfs rail to a specific value, and updates any
261  * rails that depend on this rail. */
262 static int dvfs_rail_set_voltage(struct dvfs_rail *rail, int millivolts)
263 {
264         int ret = 0;
265         struct dvfs_relationship *rel;
266         int step, offset;
267         int i;
268         int steps;
269         bool jmp_to_zero;
270
271         if (!rail->reg) {
272                 if (millivolts == rail->millivolts)
273                         return 0;
274                 else
275                         return -EINVAL;
276         }
277
278         if (millivolts > rail->millivolts) {
279                 step = rail->step_up;
280                 offset = step;
281         } else {
282                 step = rail->step;
283                 offset = -step;
284         }
285
286         /*
287          * DFLL adjusts rail voltage automatically, but not exactly to the
288          * expected level - update stats, anyway.
289          */
290         if (rail->dfll_mode) {
291                 rail->millivolts = rail->new_millivolts = millivolts;
292                 dvfs_rail_stats_update(rail, millivolts, ktime_get());
293                 return 0;
294         }
295
296         if (rail->disabled)
297                 return 0;
298
299         rail->resolving_to = true;
300         jmp_to_zero = rail->jmp_to_zero &&
301                         ((millivolts == 0) || (rail->millivolts == 0));
302         steps = jmp_to_zero ? 1 :
303                 DIV_ROUND_UP(abs(millivolts - rail->millivolts), step);
304
305         for (i = 0; i < steps; i++) {
306                 if (!jmp_to_zero &&
307                     (abs(millivolts - rail->millivolts) > step))
308                         rail->new_millivolts = rail->millivolts + offset;
309                 else
310                         rail->new_millivolts = millivolts;
311
312                 /* Before changing the voltage, tell each rail that depends
313                  * on this rail that the voltage will change.
314                  * This rail will be the "from" rail in the relationship,
315                  * the rail that depends on this rail will be the "to" rail.
316                  * from->millivolts will be the old voltage
317                  * from->new_millivolts will be the new voltage */
318                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
319                         ret = dvfs_rail_update(rel->to);
320                         if (ret)
321                                 goto out;
322                 }
323
324                 ret = dvfs_rail_set_voltage_reg(rail, rail->new_millivolts);
325                 if (ret) {
326                         pr_err("Failed to set dvfs regulator %s\n", rail->reg_id);
327                         goto out;
328                 }
329
330                 rail->millivolts = rail->new_millivolts;
331                 dvfs_rail_stats_update(rail, rail->millivolts, ktime_get());
332
333                 /* After changing the voltage, tell each rail that depends
334                  * on this rail that the voltage has changed.
335                  * from->millivolts and from->new_millivolts will be the
336                  * new voltage */
337                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
338                         ret = dvfs_rail_update(rel->to);
339                         if (ret)
340                                 goto out;
341                 }
342         }
343
344         if (unlikely(rail->millivolts != millivolts)) {
345                 pr_err("%s: rail didn't reach target %d in %d steps (%d)\n",
346                         __func__, millivolts, steps, rail->millivolts);
347                 ret = -EINVAL;
348         }
349
350 out:
351         rail->resolving_to = false;
352         return ret;
353 }
354
355 /* Determine the minimum valid voltage for a rail, taking into account
356  * the dvfs clocks and any rails that this rail depends on.  Calls
357  * dvfs_rail_set_voltage with the new voltage, which will call
358  * dvfs_rail_update on any rails that depend on this rail. */
359 static inline int dvfs_rail_apply_limits(struct dvfs_rail *rail, int millivolts)
360 {
361         int min_mv = rail->min_millivolts;
362
363         if (rail->therm_mv_floors) {
364                 int i = rail->therm_floor_idx;
365                 if (i < rail->therm_mv_floors_num)
366                         min_mv = rail->therm_mv_floors[i];
367         }
368
369         if (rail->override_millivolts) {
370                 millivolts = rail->override_millivolts;
371         } else {
372                 /* apply offset and clip up to pll mode fixed mv */
373                 millivolts += rail->dbg_mv_offs;
374                 if (!rail->dfll_mode && rail->fixed_millivolts &&
375                     (millivolts < rail->fixed_millivolts))
376                         millivolts = rail->fixed_millivolts;
377         }
378
379         if (millivolts < min_mv)
380                 millivolts = min_mv;
381
382         return millivolts;
383 }
384
385 static int dvfs_rail_update(struct dvfs_rail *rail)
386 {
387         int millivolts = 0;
388         struct dvfs *d;
389         struct dvfs_relationship *rel;
390         int ret = 0;
391         int steps;
392
393         /* if dvfs is suspended, return and handle it during resume */
394         if (rail->suspended)
395                 return 0;
396
397         /* if regulators are not connected yet, return and handle it later */
398         if (!rail->reg)
399                 return 0;
400
401         /* if no clock has requested voltage since boot, defer update */
402         if (!rail->rate_set)
403                 return 0;
404
405         /* if rail update is entered while resolving circular dependencies,
406            abort recursion */
407         if (rail->resolving_to)
408                 return 0;
409
410         /* Find the maximum voltage requested by any clock */
411         list_for_each_entry(d, &rail->dvfs, reg_node)
412                 millivolts = max(d->cur_millivolts, millivolts);
413
414         /* Apply offset and min/max limits if any clock is requesting voltage */
415         if (millivolts)
416                 millivolts = dvfs_rail_apply_limits(rail, millivolts);
417         /* Keep current voltage if regulator is to be disabled via explicitly */
418         else if (rail->in_band_pm)
419                 return 0;
420         /* Keep current voltage if regulator must not be disabled at run time */
421         else if (!rail->jmp_to_zero) {
422                 WARN(1, "%s cannot be turned off by dvfs\n", rail->reg_id);
423                 return 0;
424         }
425         /* else: fall thru if regulator is turned off by side band signaling */
426
427         /* retry update if limited by from-relationship to account for
428            circular dependencies */
429         steps = DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
430         for (; steps >= 0; steps--) {
431                 rail->new_millivolts = millivolts;
432
433                 /* Check any rails that this rail depends on */
434                 list_for_each_entry(rel, &rail->relationships_from, from_node)
435                         rail->new_millivolts = dvfs_solve_relationship(rel);
436
437                 if (rail->new_millivolts == rail->millivolts)
438                         break;
439
440                 ret = dvfs_rail_set_voltage(rail, rail->new_millivolts);
441         }
442
443         return ret;
444 }
445
446 static struct regulator *get_fixed_regulator(struct dvfs_rail *rail)
447 {
448         struct regulator *reg;
449         char reg_id[80];
450         struct dvfs *d;
451         int v, i;
452         unsigned long dfll_boost;
453
454         strcpy(reg_id, rail->reg_id);
455         strcat(reg_id, "_fixed");
456         reg = regulator_get(NULL, reg_id);
457         if (IS_ERR(reg))
458                 return reg;
459
460         v = regulator_get_voltage(reg) / 1000;
461         if ((v < rail->min_millivolts) || (v > rail->nominal_millivolts) ||
462             (rail->therm_mv_floors && v < rail->therm_mv_floors[0])) {
463                 pr_err("tegra_dvfs: ivalid fixed %s voltage %d\n",
464                        rail->reg_id, v);
465                 return ERR_PTR(-EINVAL);
466         }
467
468         /*
469          * Only fixed at nominal voltage vdd_core regulator is allowed, same
470          * is true for cpu rail if dfll mode is not supported at all. No thermal
471          * capping can be implemented in this case.
472          */
473         if (!IS_ENABLED(CONFIG_ARCH_TEGRA_HAS_CL_DVFS) ||
474             (rail != tegra_cpu_rail)) {
475                 if (v != rail->nominal_millivolts) {
476                         pr_err("tegra_dvfs: %s fixed below nominal at %d\n",
477                                rail->reg_id, v);
478                         return ERR_PTR(-EINVAL);
479                 }
480                 if (rail->therm_mv_caps) {
481                         pr_err("tegra_dvfs: cannot fix %s with thermal caps\n",
482                                rail->reg_id);
483                         return ERR_PTR(-ENOSYS);
484                 }
485                 return reg;
486         }
487
488         /*
489          * If dfll mode is supported, fixed vdd_cpu regulator may be below
490          * nominal in pll mode - maximum cpu rate in pll mode is limited
491          * respectively. Regulator is required to allow automatic scaling
492          * in dfll mode.
493          *
494          * FIXME: platform data to explicitly identify such "hybrid" regulator?
495          */
496         d = list_first_entry(&rail->dvfs, struct dvfs, reg_node);
497         for (i = 0; i < d->num_freqs; i++) {
498                 if (d->millivolts[i] > v)
499                         break;
500         }
501
502         if (!i) {
503                 pr_err("tegra_dvfs: %s fixed at %d: too low for min rate\n",
504                        rail->reg_id, v);
505                 return ERR_PTR(-EINVAL);
506         }
507
508         dfll_boost = (d->freqs[d->num_freqs - 1] - d->freqs[i - 1]);
509         if (d->dfll_data.max_rate_boost < dfll_boost)
510                 d->dfll_data.max_rate_boost = dfll_boost;
511
512         rail->fixed_millivolts = v;
513         return reg;
514 }
515
516 static int dvfs_rail_connect_to_regulator(struct dvfs_rail *rail)
517 {
518         struct regulator *reg;
519         int v;
520
521         if (!rail->reg) {
522                 reg = regulator_get(NULL, rail->reg_id);
523                 if (IS_ERR(reg)) {
524                         reg = get_fixed_regulator(rail);
525                         if (IS_ERR(reg)) {
526                                 pr_err("tegra_dvfs: failed to connect %s rail\n",
527                                        rail->reg_id);
528                                 return PTR_ERR(reg);
529                         }
530                 }
531                 rail->reg = reg;
532         }
533
534         v = regulator_enable(rail->reg);
535         if (v < 0) {
536                 pr_err("tegra_dvfs: failed on enabling regulator %s\n, err %d",
537                         rail->reg_id, v);
538                 return v;
539         }
540
541         v = regulator_get_voltage(rail->reg);
542         if (v < 0) {
543                 pr_err("tegra_dvfs: failed initial get %s voltage\n",
544                        rail->reg_id);
545                 return v;
546         }
547         rail->millivolts = v / 1000;
548         rail->new_millivolts = rail->millivolts;
549         dvfs_rail_stats_init(rail, rail->millivolts);
550
551         if (rail->boot_millivolts &&
552             (rail->boot_millivolts != rail->millivolts)) {
553                 WARN(1, "%s boot voltage %d does not match expected %d\n",
554                      rail->reg_id, rail->millivolts, rail->boot_millivolts);
555                 rail->boot_millivolts = rail->millivolts;
556         }
557         return 0;
558 }
559
560 static inline unsigned long *dvfs_get_freqs(struct dvfs *d)
561 {
562         return d->alt_freqs ? : &d->freqs[0];
563 }
564
565 static inline const int *dvfs_get_millivolts(struct dvfs *d, unsigned long rate)
566 {
567         if (tegra_dvfs_is_dfll_scale(d, rate))
568                 return d->dfll_millivolts;
569
570         return tegra_dvfs_get_millivolts_pll(d);
571 }
572
573 static int
574 __tegra_dvfs_set_rate(struct dvfs *d, unsigned long rate)
575 {
576         int i = 0;
577         int ret, mv, detach_mv;
578         unsigned long *freqs = dvfs_get_freqs(d);
579         const int *millivolts = dvfs_get_millivolts(d, rate);
580
581         if (freqs == NULL || millivolts == NULL)
582                 return -ENODEV;
583
584         /* On entry to dfll range limit 1st step to range bottom (full ramp of
585            voltage/rate is completed automatically in dfll mode) */
586         if (tegra_dvfs_is_dfll_range_entry(d, rate))
587                 rate = d->dfll_data.use_dfll_rate_min;
588
589         if (rate > freqs[d->num_freqs - 1]) {
590                 pr_warn("tegra_dvfs: rate %lu too high for dvfs on %s\n", rate,
591                         d->clk_name);
592                 return -EINVAL;
593         }
594
595         if (rate == 0) {
596                 d->cur_millivolts = 0;
597         } else {
598                 while (i < d->num_freqs && rate > freqs[i])
599                         i++;
600
601                 if ((d->max_millivolts) &&
602                     (millivolts[i] > d->max_millivolts)) {
603                         pr_warn("tegra_dvfs: voltage %d too high for dvfs on"
604                                 " %s\n", millivolts[i], d->clk_name);
605                         return -EINVAL;
606                 }
607
608                 mv = millivolts[i];
609                 detach_mv = tegra_dvfs_rail_get_boot_level(d->dvfs_rail);
610                 if (!d->dvfs_rail->reg && (mv > detach_mv)) {
611                         pr_warn("%s: %s: voltage %d above boot limit %d\n",
612                                 __func__, d->clk_name, mv, detach_mv);
613                         return -EINVAL;
614                 }
615
616                 detach_mv = tegra_dvfs_rail_get_disable_level(d->dvfs_rail);
617                 if (d->dvfs_rail->disabled && (mv > detach_mv)) {
618                         pr_warn("%s: %s: voltage %d above disable limit %d\n",
619                                 __func__, d->clk_name, mv, detach_mv);
620                         return -EINVAL;
621                 }
622
623                 detach_mv = tegra_dvfs_rail_get_suspend_level(d->dvfs_rail);
624                 if (d->dvfs_rail->suspended && (mv > detach_mv)) {
625                         pr_warn("%s: %s: voltage %d above disable limit %d\n",
626                                 __func__, d->clk_name, mv, detach_mv);
627                         return -EINVAL;
628                 }
629
630                 detach_mv = d->dvfs_rail->override_millivolts;
631                 if (detach_mv && (mv > detach_mv)) {
632                         pr_warn("%s: %s: voltage %d above override level %d\n",
633                                 __func__, d->clk_name, mv, detach_mv);
634                         return -EINVAL;
635                 }
636                 d->cur_millivolts = millivolts[i];
637         }
638
639         d->cur_rate = rate;
640
641         d->dvfs_rail->rate_set = true;
642         ret = dvfs_rail_update(d->dvfs_rail);
643         if (ret)
644                 pr_err("Failed to set regulator %s for clock %s to %d mV\n",
645                         d->dvfs_rail->reg_id, d->clk_name, d->cur_millivolts);
646
647         return ret;
648 }
649
650 int tegra_dvfs_alt_freqs_set(struct dvfs *d, unsigned long *alt_freqs)
651 {
652         int ret = 0;
653
654         mutex_lock(&dvfs_lock);
655
656         if (d->alt_freqs != alt_freqs) {
657                 d->alt_freqs = alt_freqs;
658                 ret = __tegra_dvfs_set_rate(d, d->cur_rate);
659         }
660
661         mutex_unlock(&dvfs_lock);
662         return ret;
663 }
664
665 static int predict_millivolts(struct clk *c, const int *millivolts,
666                               unsigned long rate)
667 {
668         int i;
669
670         if (!millivolts)
671                 return -ENODEV;
672         /*
673          * Predicted voltage can not be used across the switch to alternative
674          * frequency limits. For now, just fail the call for clock that has
675          * alternative limits initialized.
676          */
677         if (c->dvfs->alt_freqs)
678                 return -ENOSYS;
679
680         for (i = 0; i < c->dvfs->num_freqs; i++) {
681                 if (rate <= c->dvfs->freqs[i])
682                         break;
683         }
684
685         if (i == c->dvfs->num_freqs)
686                 return -EINVAL;
687
688         return millivolts[i];
689 }
690
691 int tegra_dvfs_predict_millivolts(struct clk *c, unsigned long rate)
692 {
693         const int *millivolts;
694
695         if (!rate || !c->dvfs)
696                 return 0;
697
698         millivolts = tegra_dvfs_is_dfll_range(c->dvfs, rate) ?
699                 c->dvfs->dfll_millivolts :
700                 tegra_dvfs_get_millivolts_pll(c->dvfs);
701         return predict_millivolts(c, millivolts, rate);
702 }
703
704 int tegra_dvfs_predict_peak_millivolts(struct clk *c, unsigned long rate)
705 {
706         const int *millivolts;
707
708         if (!rate || !c->dvfs)
709                 return 0;
710
711         millivolts = tegra_dvfs_is_dfll_range(c->dvfs, rate) ?
712                         c->dvfs->dfll_millivolts : c->dvfs->peak_millivolts ? :
713                         tegra_dvfs_get_millivolts_pll(c->dvfs);
714         return predict_millivolts(c, millivolts, rate);
715 }
716
717 int tegra_dvfs_predict_millivolts_pll(struct clk *c, unsigned long rate)
718 {
719         const int *millivolts;
720
721         if (!rate || !c->dvfs)
722                 return 0;
723
724         millivolts = tegra_dvfs_get_millivolts_pll(c->dvfs);
725         return predict_millivolts(c, millivolts, rate);
726 }
727
728 int tegra_dvfs_predict_millivolts_dfll(struct clk *c, unsigned long rate)
729 {
730         const int *millivolts;
731
732         if (!rate || !c->dvfs)
733                 return 0;
734
735         millivolts = c->dvfs->dfll_millivolts;
736         return predict_millivolts(c, millivolts, rate);
737 }
738
739 const int *tegra_dvfs_get_millivolts_pll(struct dvfs *d)
740 {
741         if (d->therm_dvfs) {
742                 int therm_idx = d->dvfs_rail->therm_scale_idx;
743                 return d->millivolts + therm_idx * MAX_DVFS_FREQS;
744         }
745         return d->millivolts;
746 }
747
748 int tegra_dvfs_set_rate(struct clk *c, unsigned long rate)
749 {
750         int ret;
751
752         if (!c->dvfs)
753                 return -EINVAL;
754
755         mutex_lock(&dvfs_lock);
756         ret = __tegra_dvfs_set_rate(c->dvfs, rate);
757         mutex_unlock(&dvfs_lock);
758
759         return ret;
760 }
761 EXPORT_SYMBOL(tegra_dvfs_set_rate);
762
763 int tegra_dvfs_get_freqs(struct clk *c, unsigned long **freqs, int *num_freqs)
764 {
765         if (!c->dvfs)
766                 return -ENOSYS;
767
768         if (c->dvfs->alt_freqs)
769                 return -ENOSYS;
770
771         *num_freqs = c->dvfs->num_freqs;
772         *freqs = c->dvfs->freqs;
773
774         return 0;
775 }
776 EXPORT_SYMBOL(tegra_dvfs_get_freqs);
777
778 #ifdef CONFIG_TEGRA_VDD_CORE_OVERRIDE
779 static DEFINE_MUTEX(rail_override_lock);
780
781 static int dvfs_override_core_voltage(int override_mv)
782 {
783         int ret, floor, ceiling;
784         struct dvfs_rail *rail = tegra_core_rail;
785
786         if (!rail)
787                 return -ENOENT;
788
789         if (rail->fixed_millivolts)
790                 return -ENOSYS;
791
792         floor = rail->min_override_millivolts;
793         ceiling = rail->nominal_millivolts;
794         if (override_mv && ((override_mv < floor) || (override_mv > ceiling))) {
795                 pr_err("%s: override level %d outside the range [%d...%d]\n",
796                        __func__, override_mv, floor, ceiling);
797                 return -EINVAL;
798         }
799
800         mutex_lock(&rail_override_lock);
801
802         if (override_mv == rail->override_millivolts) {
803                 ret = 0;
804                 goto out;
805         }
806
807         if (override_mv) {
808                 ret = tegra_dvfs_core_cap_level_apply(override_mv);
809                 if (ret) {
810                         pr_err("%s: failed to set cap for override level %d\n",
811                                __func__, override_mv);
812                         goto out;
813                 }
814         }
815
816         mutex_lock(&dvfs_lock);
817         if (rail->disabled || rail->suspended) {
818                 pr_err("%s: cannot scale %s rail\n", __func__,
819                        rail->disabled ? "disabled" : "suspended");
820                 ret = -EPERM;
821                 if (!override_mv) {
822                         mutex_unlock(&dvfs_lock);
823                         goto out;
824                 }
825         } else {
826                 rail->override_millivolts = override_mv;
827                 ret = dvfs_rail_update(rail);
828                 if (ret) {
829                         pr_err("%s: failed to set override level %d\n",
830                                __func__, override_mv);
831                         rail->override_millivolts = 0;
832                         dvfs_rail_update(rail);
833                 }
834         }
835         mutex_unlock(&dvfs_lock);
836
837         if (!override_mv || ret)
838                 tegra_dvfs_core_cap_level_apply(0);
839 out:
840         mutex_unlock(&rail_override_lock);
841         return ret;
842 }
843 #else
844 static int dvfs_override_core_voltage(int override_mv)
845 {
846         pr_err("%s: vdd core override is not supported\n", __func__);
847         return -ENOSYS;
848 }
849 #endif
850
851 int tegra_dvfs_override_core_voltage(struct clk *c, int override_mv)
852 {
853         if (!c->dvfs || !c->dvfs->can_override) {
854                 pr_err("%s: %s cannot override vdd core\n", __func__, c->name);
855                 return -EPERM;
856         }
857         return dvfs_override_core_voltage(override_mv);
858 }
859 EXPORT_SYMBOL(tegra_dvfs_override_core_voltage);
860
861 /* May only be called during clock init, does not take any locks on clock c. */
862 int __init tegra_enable_dvfs_on_clk(struct clk *c, struct dvfs *d)
863 {
864         int i;
865
866         if (c->dvfs) {
867                 pr_err("Error when enabling dvfs on %s for clock %s:\n",
868                         d->dvfs_rail->reg_id, c->name);
869                 pr_err("DVFS already enabled for %s\n",
870                         c->dvfs->dvfs_rail->reg_id);
871                 return -EINVAL;
872         }
873
874         for (i = 0; i < MAX_DVFS_FREQS; i++) {
875                 if (d->millivolts[i] == 0)
876                         break;
877
878                 d->freqs[i] *= d->freqs_mult;
879
880                 /* If final frequencies are 0, pad with previous frequency */
881                 if (d->freqs[i] == 0 && i > 1)
882                         d->freqs[i] = d->freqs[i - 1];
883         }
884         d->num_freqs = i;
885
886         if (d->auto_dvfs) {
887                 c->auto_dvfs = true;
888                 clk_set_cansleep(c);
889         }
890
891         c->dvfs = d;
892
893         /*
894          * Minimum core override level is determined as maximum voltage required
895          * for clocks outside shared buses (shared bus rates can be capped to
896          * safe levels when override limit is set)
897          */
898         if (i && c->ops && !c->ops->shared_bus_update &&
899             !(c->flags & PERIPH_ON_CBUS) && !d->can_override) {
900                 int mv = tegra_dvfs_predict_peak_millivolts(c, d->freqs[i-1]);
901                 if (d->dvfs_rail->min_override_millivolts < mv)
902                         d->dvfs_rail->min_override_millivolts = mv;
903         }
904
905         mutex_lock(&dvfs_lock);
906         list_add_tail(&d->reg_node, &d->dvfs_rail->dvfs);
907         mutex_unlock(&dvfs_lock);
908
909         return 0;
910 }
911
912 static bool tegra_dvfs_all_rails_suspended(void)
913 {
914         struct dvfs_rail *rail;
915         bool all_suspended = true;
916
917         list_for_each_entry(rail, &dvfs_rail_list, node)
918                 if (!rail->suspended && !rail->disabled)
919                         all_suspended = false;
920
921         return all_suspended;
922 }
923
924 static bool tegra_dvfs_from_rails_suspended_or_solved(struct dvfs_rail *to)
925 {
926         struct dvfs_relationship *rel;
927         bool all_suspended = true;
928
929         list_for_each_entry(rel, &to->relationships_from, from_node)
930                 if (!rel->from->suspended && !rel->from->disabled &&
931                         !rel->solved_at_nominal)
932                         all_suspended = false;
933
934         return all_suspended;
935 }
936
937 static int tegra_dvfs_suspend_one(void)
938 {
939         struct dvfs_rail *rail;
940         int ret, mv;
941
942         list_for_each_entry(rail, &dvfs_rail_list, node) {
943                 if (!rail->suspended && !rail->disabled &&
944                     tegra_dvfs_from_rails_suspended_or_solved(rail)) {
945                         /* Safe, as pll mode rate is capped to fixed level */
946                         if (!rail->dfll_mode && rail->fixed_millivolts) {
947                                 mv = rail->fixed_millivolts;
948                         } else {
949                                 mv = tegra_dvfs_rail_get_suspend_level(rail);
950                                 mv = dvfs_rail_apply_limits(rail, mv);
951                         }
952
953                         /* apply suspend limit only if it is above current mv */
954                         ret = -EPERM;
955                         if (mv >= rail->millivolts)
956                                 ret = dvfs_rail_set_voltage(rail, mv);
957                         if (ret) {
958                                 pr_err("tegra_dvfs: failed %s suspend at %d\n",
959                                        rail->reg_id, rail->millivolts);
960                                 return ret;
961                         }
962
963                         rail->suspended = true;
964                         return 0;
965                 }
966         }
967
968         return -EINVAL;
969 }
970
971 static void tegra_dvfs_resume(void)
972 {
973         struct dvfs_rail *rail;
974
975         mutex_lock(&dvfs_lock);
976
977         list_for_each_entry(rail, &dvfs_rail_list, node)
978                 rail->suspended = false;
979
980         list_for_each_entry(rail, &dvfs_rail_list, node)
981                 dvfs_rail_update(rail);
982
983         mutex_unlock(&dvfs_lock);
984 }
985
986 static int tegra_dvfs_suspend(void)
987 {
988         int ret = 0;
989
990         mutex_lock(&dvfs_lock);
991
992         while (!tegra_dvfs_all_rails_suspended()) {
993                 ret = tegra_dvfs_suspend_one();
994                 if (ret)
995                         break;
996         }
997
998         mutex_unlock(&dvfs_lock);
999
1000         if (ret)
1001                 tegra_dvfs_resume();
1002
1003         return ret;
1004 }
1005
1006 static int tegra_dvfs_pm_suspend(struct notifier_block *nb,
1007                                  unsigned long event, void *data)
1008 {
1009         if (event == PM_SUSPEND_PREPARE) {
1010                 if (tegra_dvfs_suspend())
1011                         return NOTIFY_STOP;
1012                 pr_info("tegra_dvfs: suspended\n");
1013         }
1014         return NOTIFY_OK;
1015 };
1016
1017 static int tegra_dvfs_pm_resume(struct notifier_block *nb,
1018                                 unsigned long event, void *data)
1019 {
1020         if (event == PM_POST_SUSPEND) {
1021                 tegra_dvfs_resume();
1022                 pr_info("tegra_dvfs: resumed\n");
1023         }
1024         return NOTIFY_OK;
1025 };
1026
1027 static struct notifier_block tegra_dvfs_suspend_nb = {
1028         .notifier_call = tegra_dvfs_pm_suspend,
1029         .priority = -1,
1030 };
1031
1032 static struct notifier_block tegra_dvfs_resume_nb = {
1033         .notifier_call = tegra_dvfs_pm_resume,
1034         .priority = 1,
1035 };
1036
1037 static int tegra_dvfs_reboot_notify(struct notifier_block *nb,
1038                                 unsigned long event, void *data)
1039 {
1040         switch (event) {
1041         case SYS_RESTART:
1042         case SYS_HALT:
1043         case SYS_POWER_OFF:
1044                 tegra_dvfs_suspend();
1045                 return NOTIFY_OK;
1046         }
1047         return NOTIFY_DONE;
1048 }
1049
1050 static struct notifier_block tegra_dvfs_reboot_nb = {
1051         .notifier_call = tegra_dvfs_reboot_notify,
1052 };
1053
1054 /* must be called with dvfs lock held */
1055 static void __tegra_dvfs_rail_disable(struct dvfs_rail *rail)
1056 {
1057         int ret = -EPERM;
1058         int mv;
1059
1060         /* don't set voltage in DFLL mode - won't work, but break stats */
1061         if (rail->dfll_mode) {
1062                 rail->disabled = true;
1063                 return;
1064         }
1065
1066         /* Safe, as pll mode rate is capped to fixed level */
1067         if (!rail->dfll_mode && rail->fixed_millivolts) {
1068                 mv = rail->fixed_millivolts;
1069         } else {
1070                 mv = tegra_dvfs_rail_get_disable_level(rail);
1071                 mv = dvfs_rail_apply_limits(rail, mv);
1072         }
1073
1074         /* apply detach mode limit provided it is above current volatge */
1075         if (mv >= rail->millivolts)
1076                 ret = dvfs_rail_set_voltage(rail, mv);
1077         if (ret) {
1078                 pr_err("tegra_dvfs: failed to disable %s at %d\n",
1079                        rail->reg_id, rail->millivolts);
1080                 return;
1081         }
1082         rail->disabled = true;
1083 }
1084
1085 /* must be called with dvfs lock held */
1086 static void __tegra_dvfs_rail_enable(struct dvfs_rail *rail)
1087 {
1088         rail->disabled = false;
1089         dvfs_rail_update(rail);
1090 }
1091
1092 void tegra_dvfs_rail_enable(struct dvfs_rail *rail)
1093 {
1094         if (!rail)
1095                 return;
1096
1097         mutex_lock(&rail_disable_lock);
1098
1099         if (rail->disabled) {
1100                 mutex_lock(&dvfs_lock);
1101                 __tegra_dvfs_rail_enable(rail);
1102                 mutex_unlock(&dvfs_lock);
1103
1104                 tegra_dvfs_rail_post_enable(rail);
1105         }
1106         mutex_unlock(&rail_disable_lock);
1107 }
1108
1109 void tegra_dvfs_rail_disable(struct dvfs_rail *rail)
1110 {
1111         if (!rail)
1112                 return;
1113
1114         mutex_lock(&rail_disable_lock);
1115         if (rail->disabled)
1116                 goto out;
1117
1118         /* rail disable will set it to nominal voltage underneath clock
1119            framework - need to re-configure clock rates that are not safe
1120            at nominal (yes, unsafe at nominal is ugly, but possible). Rate
1121            change must be done outside of dvfs lock. */
1122         if (tegra_dvfs_rail_disable_prepare(rail)) {
1123                 pr_info("dvfs: failed to prepare regulator %s to disable\n",
1124                         rail->reg_id);
1125                 goto out;
1126         }
1127
1128         mutex_lock(&dvfs_lock);
1129         __tegra_dvfs_rail_disable(rail);
1130         mutex_unlock(&dvfs_lock);
1131 out:
1132         mutex_unlock(&rail_disable_lock);
1133 }
1134
1135 int tegra_dvfs_rail_disable_by_name(const char *reg_id)
1136 {
1137         struct dvfs_rail *rail = tegra_dvfs_get_rail_by_name(reg_id);
1138         if (!rail)
1139                 return -EINVAL;
1140
1141         tegra_dvfs_rail_disable(rail);
1142         return 0;
1143 }
1144
1145 struct dvfs_rail *tegra_dvfs_get_rail_by_name(const char *reg_id)
1146 {
1147         struct dvfs_rail *rail;
1148
1149         mutex_lock(&dvfs_lock);
1150         list_for_each_entry(rail, &dvfs_rail_list, node) {
1151                 if (!strcmp(reg_id, rail->reg_id)) {
1152                         mutex_unlock(&dvfs_lock);
1153                         return rail;
1154                 }
1155         }
1156         mutex_unlock(&dvfs_lock);
1157         return NULL;
1158 }
1159
1160 int tegra_dvfs_rail_power_up(struct dvfs_rail *rail)
1161 {
1162         int ret = -ENOENT;
1163
1164         if (!rail || !rail->in_band_pm)
1165                 return -ENOSYS;
1166
1167         mutex_lock(&dvfs_lock);
1168         if (rail->reg) {
1169                 ret = regulator_enable(rail->reg);
1170                 if (!ret && !timekeeping_suspended)
1171                         tegra_dvfs_rail_on(rail, ktime_get());
1172         }
1173         mutex_unlock(&dvfs_lock);
1174         return ret;
1175 }
1176
1177 int tegra_dvfs_rail_power_down(struct dvfs_rail *rail)
1178 {
1179         int ret = -ENOENT;
1180
1181         if (!rail || !rail->in_band_pm)
1182                 return -ENOSYS;
1183
1184         mutex_lock(&dvfs_lock);
1185         if (rail->reg) {
1186                 ret = regulator_disable(rail->reg);
1187                 if (!ret && !timekeeping_suspended)
1188                         tegra_dvfs_rail_off(rail, ktime_get());
1189         }
1190         mutex_unlock(&dvfs_lock);
1191         return ret;
1192 }
1193
1194 bool tegra_dvfs_is_rail_up(struct dvfs_rail *rail)
1195 {
1196         bool ret = false;
1197
1198         if (!rail)
1199                 return false;
1200
1201         if (!rail->in_band_pm)
1202                 return true;
1203
1204         mutex_lock(&dvfs_lock);
1205         if (rail->reg)
1206                 ret = regulator_is_enabled(rail->reg) > 0;
1207         mutex_unlock(&dvfs_lock);
1208         return ret;
1209 }
1210
1211 int tegra_dvfs_rail_set_mode(struct dvfs_rail *rail, unsigned int mode)
1212 {
1213         int ret = -ENOENT;
1214
1215         pr_debug("%s: updating %s mode from %u to %u\n", __func__,
1216                  rail->reg_id, regulator_get_mode(rail->reg), mode);
1217
1218         if (rail && rail->reg)
1219                 ret = regulator_set_mode(rail->reg, mode);
1220
1221         if (ret)
1222                 pr_err("Failed to set dvfs regulator %s mode %u\n",
1223                        rail->reg_id, mode);
1224         return ret;
1225 }
1226
1227 bool tegra_dvfs_rail_updating(struct clk *clk)
1228 {
1229         return (!clk ? false :
1230                 (!clk->dvfs ? false :
1231                  (!clk->dvfs->dvfs_rail ? false :
1232                   (clk->dvfs->dvfs_rail->updating ||
1233                    clk->dvfs->dvfs_rail->dfll_mode_updating))));
1234 }
1235
1236 #ifdef CONFIG_OF
1237 int __init of_tegra_dvfs_init(const struct of_device_id *matches)
1238 {
1239         int ret;
1240         struct device_node *np;
1241
1242         for_each_matching_node(np, matches) {
1243                 const struct of_device_id *match = of_match_node(matches, np);
1244                 of_tegra_dvfs_init_cb_t dvfs_init_cb = match->data;
1245                 ret = dvfs_init_cb(np);
1246                 if (ret) {
1247                         pr_err("dt: Failed to read %s tables from DT\n",
1248                                                         match->compatible);
1249                         return ret;
1250                 }
1251         }
1252         return 0;
1253 }
1254 #endif
1255 int tegra_dvfs_dfll_mode_set(struct dvfs *d, unsigned long rate)
1256 {
1257         mutex_lock(&dvfs_lock);
1258         if (!d->dvfs_rail->dfll_mode) {
1259                 d->dvfs_rail->dfll_mode = true;
1260                 __tegra_dvfs_set_rate(d, rate);
1261         }
1262         mutex_unlock(&dvfs_lock);
1263         return 0;
1264 }
1265
1266 int tegra_dvfs_dfll_mode_clear(struct dvfs *d, unsigned long rate)
1267 {
1268         int ret = 0;
1269
1270         mutex_lock(&dvfs_lock);
1271         if (d->dvfs_rail->dfll_mode) {
1272                 d->dvfs_rail->dfll_mode = false;
1273                 /* avoid false detection of matching target (voltage in dfll
1274                    mode is fluctuating, and recorded level is just estimate) */
1275                 d->dvfs_rail->millivolts--;
1276                 if (d->dvfs_rail->disabled) {
1277                         d->dvfs_rail->disabled = false;
1278                         __tegra_dvfs_rail_disable(d->dvfs_rail);
1279                 }
1280                 ret = __tegra_dvfs_set_rate(d, rate);
1281         }
1282         mutex_unlock(&dvfs_lock);
1283         return ret;
1284 }
1285
1286 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmax_cdev(void)
1287 {
1288         if (tegra_cpu_rail)
1289                 return tegra_cpu_rail->vmax_cdev;
1290         return NULL;
1291 }
1292
1293 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmin_cdev(void)
1294 {
1295         if (tegra_cpu_rail)
1296                 return tegra_cpu_rail->vmin_cdev;
1297         return NULL;
1298 }
1299
1300 struct tegra_cooling_device *tegra_dvfs_get_core_vmin_cdev(void)
1301 {
1302         if (tegra_core_rail)
1303                 return tegra_core_rail->vmin_cdev;
1304         return NULL;
1305 }
1306
1307 struct tegra_cooling_device *tegra_dvfs_get_gpu_vmin_cdev(void)
1308 {
1309         if (tegra_gpu_rail)
1310                 return tegra_gpu_rail->vmin_cdev;
1311         return NULL;
1312 }
1313
1314 struct tegra_cooling_device *tegra_dvfs_get_gpu_vts_cdev(void)
1315 {
1316         if (tegra_gpu_rail)
1317                 return tegra_gpu_rail->vts_cdev;
1318         return NULL;
1319 }
1320
1321 static void make_safe_thermal_dvfs(struct dvfs_rail *rail)
1322 {
1323         struct dvfs *d;
1324
1325         mutex_lock(&dvfs_lock);
1326         list_for_each_entry(d, &rail->dvfs, reg_node) {
1327                 if (d->therm_dvfs) {
1328                         BUG_ON(!d->peak_millivolts);
1329                         d->millivolts = d->peak_millivolts;
1330                         d->therm_dvfs = false;
1331                 }
1332         }
1333         mutex_unlock(&dvfs_lock);
1334 }
1335
1336 #ifdef CONFIG_THERMAL
1337 /* Cooling device limits minimum rail voltage at cold temperature in pll mode */
1338 static int tegra_dvfs_rail_get_vmin_cdev_max_state(
1339         struct thermal_cooling_device *cdev, unsigned long *max_state)
1340 {
1341         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1342         *max_state = rail->vmin_cdev->trip_temperatures_num;
1343         return 0;
1344 }
1345
1346 static int tegra_dvfs_rail_get_vmin_cdev_cur_state(
1347         struct thermal_cooling_device *cdev, unsigned long *cur_state)
1348 {
1349         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1350         *cur_state = rail->therm_floor_idx;
1351         return 0;
1352 }
1353
1354 static int tegra_dvfs_rail_set_vmin_cdev_state(
1355         struct thermal_cooling_device *cdev, unsigned long cur_state)
1356 {
1357         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1358
1359         mutex_lock(&dvfs_lock);
1360         if (rail->therm_floor_idx != cur_state) {
1361                 rail->therm_floor_idx = cur_state;
1362                 dvfs_rail_update(rail);
1363         }
1364         mutex_unlock(&dvfs_lock);
1365         return 0;
1366 }
1367
1368 static struct thermal_cooling_device_ops tegra_dvfs_vmin_cooling_ops = {
1369         .get_max_state = tegra_dvfs_rail_get_vmin_cdev_max_state,
1370         .get_cur_state = tegra_dvfs_rail_get_vmin_cdev_cur_state,
1371         .set_cur_state = tegra_dvfs_rail_set_vmin_cdev_state,
1372 };
1373
1374 static void tegra_dvfs_rail_register_vmin_cdev(struct dvfs_rail *rail)
1375 {
1376         if (!rail->vmin_cdev)
1377                 return;
1378
1379         /* just report error - initialized for cold temperature, anyway */
1380         if (IS_ERR_OR_NULL(thermal_cooling_device_register(
1381                 rail->vmin_cdev->cdev_type, (void *)rail,
1382                 &tegra_dvfs_vmin_cooling_ops)))
1383                 pr_err("tegra cooling device %s failed to register\n",
1384                        rail->vmin_cdev->cdev_type);
1385 }
1386
1387 /* Cooling device to scale voltage with temperature in pll mode */
1388 static int tegra_dvfs_rail_get_vts_cdev_max_state(
1389         struct thermal_cooling_device *cdev, unsigned long *max_state)
1390 {
1391         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1392         *max_state = rail->vts_cdev->trip_temperatures_num;
1393         return 0;
1394 }
1395
1396 static int tegra_dvfs_rail_get_vts_cdev_cur_state(
1397         struct thermal_cooling_device *cdev, unsigned long *cur_state)
1398 {
1399         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1400         *cur_state = rail->therm_scale_idx;
1401         return 0;
1402 }
1403
1404 static int tegra_dvfs_rail_set_vts_cdev_state(
1405         struct thermal_cooling_device *cdev, unsigned long cur_state)
1406 {
1407         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1408         struct dvfs *d;
1409
1410         mutex_lock(&dvfs_lock);
1411         if (rail->therm_scale_idx != cur_state) {
1412                 rail->therm_scale_idx = cur_state;
1413                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1414                         if (d->therm_dvfs)
1415                                 __tegra_dvfs_set_rate(d, d->cur_rate);
1416                 }
1417         }
1418         mutex_unlock(&dvfs_lock);
1419         return 0;
1420 }
1421
1422 static struct thermal_cooling_device_ops tegra_dvfs_vts_cooling_ops = {
1423         .get_max_state = tegra_dvfs_rail_get_vts_cdev_max_state,
1424         .get_cur_state = tegra_dvfs_rail_get_vts_cdev_cur_state,
1425         .set_cur_state = tegra_dvfs_rail_set_vts_cdev_state,
1426 };
1427
1428 static void tegra_dvfs_rail_register_vts_cdev(struct dvfs_rail *rail)
1429 {
1430         struct thermal_cooling_device *dev;
1431
1432         if (!rail->vts_cdev)
1433                 return;
1434
1435         dev = thermal_cooling_device_register(rail->vts_cdev->cdev_type,
1436                 (void *)rail, &tegra_dvfs_vts_cooling_ops);
1437         /* report error & set max limits across thermal ranges as safe dvfs */
1438         if (IS_ERR_OR_NULL(dev) || list_empty(&dev->thermal_instances)) {
1439                 pr_err("tegra cooling device %s failed to register\n",
1440                        rail->vts_cdev->cdev_type);
1441                 make_safe_thermal_dvfs(rail);
1442         }
1443 }
1444
1445 #else
1446 #define tegra_dvfs_rail_register_vmin_cdev(rail)
1447 static inline void tegra_dvfs_rail_register_vts_cdev(struct dvfs_rail *rail)
1448 {
1449         make_safe_thermal_dvfs(rail);
1450 }
1451 #endif
1452
1453 /*
1454  * Validate rail thermal profile, and get its size. Valid profile:
1455  * - voltage limits are descending with temperature increasing
1456  * - the lowest limit is above rail minimum voltage in pll and
1457  *   in dfll mode (if applicable)
1458  * - the highest limit is below rail nominal voltage
1459  */
1460 static int __init get_thermal_profile_size(
1461         int *trips_table, int *limits_table,
1462         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1463 {
1464         int i, min_mv;
1465
1466         for (i = 0; i < MAX_THERMAL_LIMITS - 1; i++) {
1467                 if (!limits_table[i+1])
1468                         break;
1469
1470                 if ((trips_table[i] >= trips_table[i+1]) ||
1471                     (limits_table[i] < limits_table[i+1])) {
1472                         pr_warn("%s: not ordered profile\n", rail->reg_id);
1473                         return -EINVAL;
1474                 }
1475         }
1476
1477         min_mv = max(rail->min_millivolts, d ? d->min_millivolts : 0);
1478         if (limits_table[i] < min_mv) {
1479                 pr_warn("%s: thermal profile below Vmin\n", rail->reg_id);
1480                 return -EINVAL;
1481         }
1482
1483         if (limits_table[0] > rail->nominal_millivolts) {
1484                 pr_warn("%s: thermal profile above Vmax\n", rail->reg_id);
1485                 return -EINVAL;
1486         }
1487         return i + 1;
1488 }
1489
1490 void __init tegra_dvfs_rail_init_vmax_thermal_profile(
1491         int *therm_trips_table, int *therm_caps_table,
1492         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1493 {
1494         int i = get_thermal_profile_size(therm_trips_table,
1495                                          therm_caps_table, rail, d);
1496         if (i <= 0) {
1497                 rail->vmax_cdev = NULL;
1498                 WARN(1, "%s: invalid Vmax thermal profile\n", rail->reg_id);
1499                 return;
1500         }
1501
1502         /* Install validated thermal caps */
1503         rail->therm_mv_caps = therm_caps_table;
1504         rail->therm_mv_caps_num = i;
1505
1506         /* Setup trip-points if applicable */
1507         if (rail->vmax_cdev) {
1508                 rail->vmax_cdev->trip_temperatures_num = i;
1509                 rail->vmax_cdev->trip_temperatures = therm_trips_table;
1510         }
1511 }
1512
1513 void __init tegra_dvfs_rail_init_vmin_thermal_profile(
1514         int *therm_trips_table, int *therm_floors_table,
1515         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1516 {
1517         int i = get_thermal_profile_size(therm_trips_table,
1518                                          therm_floors_table, rail, d);
1519         if (i <= 0) {
1520                 rail->vmin_cdev = NULL;
1521                 WARN(1, "%s: invalid Vmin thermal profile\n", rail->reg_id);
1522                 return;
1523         }
1524
1525         /* Install validated thermal floors */
1526         rail->therm_mv_floors = therm_floors_table;
1527         rail->therm_mv_floors_num = i;
1528
1529         /* Setup trip-points if applicable */
1530         if (rail->vmin_cdev) {
1531                 rail->vmin_cdev->trip_temperatures_num = i;
1532                 rail->vmin_cdev->trip_temperatures = therm_trips_table;
1533         }
1534 }
1535
1536 /*
1537  * Validate thermal dvfs settings:
1538  * - trip-points are montonically increasing
1539  * - voltages in any temperature range are montonically increasing with
1540  *   frequency (can go up/down across ranges at iso frequency)
1541  * - voltage for any frequency/thermal range combination must be within
1542  *   rail minimum/maximum limits
1543  */
1544 int __init tegra_dvfs_rail_init_thermal_dvfs_trips(
1545         int *therm_trips_table, struct dvfs_rail *rail)
1546 {
1547         int i;
1548
1549         if (!rail->vts_cdev) {
1550                 WARN(1, "%s: missing thermal dvfs cooling device\n",
1551                      rail->reg_id);
1552                 return -ENOENT;
1553         }
1554
1555         for (i = 0; i < MAX_THERMAL_LIMITS - 1; i++) {
1556                 if (therm_trips_table[i] >= therm_trips_table[i+1])
1557                         break;
1558         }
1559
1560         rail->vts_cdev->trip_temperatures_num = i + 1;
1561         rail->vts_cdev->trip_temperatures = therm_trips_table;
1562         return 0;
1563 }
1564
1565 int __init tegra_dvfs_init_thermal_dvfs_voltages(int *therm_voltages,
1566         int *peak_voltages, int freqs_num, int ranges_num, struct dvfs *d)
1567 {
1568         int *millivolts;
1569         int freq_idx, therm_idx;
1570
1571         for (therm_idx = 0; therm_idx < ranges_num; therm_idx++) {
1572                 millivolts = therm_voltages + therm_idx * MAX_DVFS_FREQS;
1573                 for (freq_idx = 0; freq_idx < freqs_num; freq_idx++) {
1574                         int mv = millivolts[freq_idx];
1575                         if ((mv > d->dvfs_rail->max_millivolts) ||
1576                             (mv < d->dvfs_rail->min_millivolts) ||
1577                             (freq_idx && (mv < millivolts[freq_idx - 1]))) {
1578                                 WARN(1, "%s: invalid thermal dvfs entry %d(%d, %d)\n",
1579                                      d->clk_name, mv, freq_idx, therm_idx);
1580                                 return -EINVAL;
1581                         }
1582                         if (mv > peak_voltages[freq_idx])
1583                                 peak_voltages[freq_idx] = mv;
1584                 }
1585         }
1586
1587         d->millivolts = therm_voltages;
1588         d->peak_millivolts = peak_voltages;
1589         d->therm_dvfs = true;
1590         return 0;
1591 }
1592
1593 /* Directly set cold temperature limit in dfll mode */
1594 int tegra_dvfs_rail_dfll_mode_set_cold(struct dvfs_rail *rail)
1595 {
1596         int ret = 0;
1597
1598         /* No thermal floors - nothing to do */
1599         if (!rail || !rail->therm_mv_floors)
1600                 return ret;
1601
1602         /*
1603          * Since cooling thresholds are the same in pll and dfll modes, pll mode
1604          * thermal index can be used to decide if cold limit should be set in
1605          * dfll mode.
1606          */
1607         mutex_lock(&dvfs_lock);
1608         if (rail->dfll_mode &&
1609             (rail->therm_floor_idx < rail->therm_mv_floors_num)) {
1610                         int mv = rail->therm_mv_floors[rail->therm_floor_idx];
1611                         ret = dvfs_rail_set_voltage_reg(rail, mv);
1612         }
1613         mutex_unlock(&dvfs_lock);
1614
1615         return ret;
1616 }
1617
1618 /*
1619  * Iterate through all the dvfs regulators, finding the regulator exported
1620  * by the regulator api for each one.  Must be called in late init, after
1621  * all the regulator api's regulators are initialized.
1622  */
1623 int __init tegra_dvfs_late_init(void)
1624 {
1625         bool connected = true;
1626         struct dvfs_rail *rail;
1627
1628         mutex_lock(&dvfs_lock);
1629
1630         list_for_each_entry(rail, &dvfs_rail_list, node)
1631                 if (dvfs_rail_connect_to_regulator(rail))
1632                         connected = false;
1633
1634         list_for_each_entry(rail, &dvfs_rail_list, node)
1635                 if (connected)
1636                         dvfs_rail_update(rail);
1637                 else
1638                         __tegra_dvfs_rail_disable(rail);
1639
1640         mutex_unlock(&dvfs_lock);
1641
1642         if (!connected && tegra_platform_is_silicon()) {
1643                 pr_warn("tegra_dvfs: DVFS regulators connection failed\n"
1644                         "            !!!! voltage scaling is disabled !!!!\n");
1645                 return -ENODEV;
1646         }
1647
1648         register_pm_notifier(&tegra_dvfs_suspend_nb);
1649         register_pm_notifier(&tegra_dvfs_resume_nb);
1650         register_reboot_notifier(&tegra_dvfs_reboot_nb);
1651
1652         list_for_each_entry(rail, &dvfs_rail_list, node) {
1653                         tegra_dvfs_rail_register_vmin_cdev(rail);
1654                         tegra_dvfs_rail_register_vts_cdev(rail);
1655         }
1656
1657         return 0;
1658 }
1659
1660 static int rail_stats_save_to_buf(char *buf, int len)
1661 {
1662         int i;
1663         struct dvfs_rail *rail;
1664         char *str = buf;
1665         char *end = buf + len;
1666
1667         str += scnprintf(str, end - str, "%-12s %-10s\n", "millivolts", "time");
1668
1669         mutex_lock(&dvfs_lock);
1670
1671         list_for_each_entry(rail, &dvfs_rail_list, node) {
1672                 str += scnprintf(str, end - str, "%s (bin: %d.%dmV)\n",
1673                            rail->reg_id,
1674                            rail->stats.bin_uV / 1000,
1675                            (rail->stats.bin_uV / 10) % 100);
1676
1677                 dvfs_rail_stats_update(rail, -1, ktime_get());
1678
1679                 str += scnprintf(str, end - str, "%-12d %-10llu\n", 0,
1680                         cputime64_to_clock_t(msecs_to_jiffies(
1681                                 ktime_to_ms(rail->stats.time_at_mv[0]))));
1682
1683                 for (i = 1; i <= DVFS_RAIL_STATS_TOP_BIN; i++) {
1684                         ktime_t ktime_zero = ktime_set(0, 0);
1685                         if (ktime_equal(rail->stats.time_at_mv[i], ktime_zero))
1686                                 continue;
1687                         str += scnprintf(str, end - str, "%-12d %-10llu\n",
1688                                 rail->min_millivolts +
1689                                 (i - 1) * rail->stats.bin_uV / 1000,
1690                                 cputime64_to_clock_t(msecs_to_jiffies(
1691                                         ktime_to_ms(rail->stats.time_at_mv[i])))
1692                         );
1693                 }
1694         }
1695         mutex_unlock(&dvfs_lock);
1696         return str - buf;
1697 }
1698
1699 #ifdef CONFIG_DEBUG_FS
1700 static int dvfs_tree_sort_cmp(void *p, struct list_head *a, struct list_head *b)
1701 {
1702         struct dvfs *da = list_entry(a, struct dvfs, reg_node);
1703         struct dvfs *db = list_entry(b, struct dvfs, reg_node);
1704         int ret;
1705
1706         ret = strcmp(da->dvfs_rail->reg_id, db->dvfs_rail->reg_id);
1707         if (ret != 0)
1708                 return ret;
1709
1710         if (da->cur_millivolts < db->cur_millivolts)
1711                 return 1;
1712         if (da->cur_millivolts > db->cur_millivolts)
1713                 return -1;
1714
1715         return strcmp(da->clk_name, db->clk_name);
1716 }
1717
1718 static int dvfs_tree_show(struct seq_file *s, void *data)
1719 {
1720         struct dvfs *d;
1721         struct dvfs_rail *rail;
1722         struct dvfs_relationship *rel;
1723
1724         seq_printf(s, "   clock      rate       mV\n");
1725         seq_printf(s, "--------------------------------\n");
1726
1727         mutex_lock(&dvfs_lock);
1728
1729         list_for_each_entry(rail, &dvfs_rail_list, node) {
1730                 int thermal_mv_floor = 0;
1731
1732                 seq_printf(s, "%s %d mV%s:\n", rail->reg_id,
1733                            rail->stats.off ? 0 : rail->millivolts,
1734                            rail->dfll_mode ? " dfll mode" :
1735                                 rail->disabled ? " disabled" : "");
1736                 list_for_each_entry(rel, &rail->relationships_from, from_node) {
1737                         seq_printf(s, "   %-10s %-7d mV %-4d mV\n",
1738                                 rel->from->reg_id, rel->from->millivolts,
1739                                 dvfs_solve_relationship(rel));
1740                 }
1741                 seq_printf(s, "   offset     %-7d mV\n", rail->dbg_mv_offs);
1742
1743                 if (rail->therm_mv_floors) {
1744                         int i = rail->therm_floor_idx;
1745                         if (i < rail->therm_mv_floors_num)
1746                                 thermal_mv_floor = rail->therm_mv_floors[i];
1747                 }
1748                 seq_printf(s, "   thermal    %-7d mV\n", thermal_mv_floor);
1749
1750                 if (rail == tegra_core_rail) {
1751                         seq_printf(s, "   override   %-7d mV [%-4d...%-4d]\n",
1752                                    rail->override_millivolts,
1753                                    rail->min_override_millivolts,
1754                                    rail->nominal_millivolts);
1755                 }
1756
1757                 list_sort(NULL, &rail->dvfs, dvfs_tree_sort_cmp);
1758
1759                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1760                         seq_printf(s, "   %-10s %-10lu %-4d mV\n", d->clk_name,
1761                                 d->cur_rate, d->cur_millivolts);
1762                 }
1763         }
1764
1765         mutex_unlock(&dvfs_lock);
1766
1767         return 0;
1768 }
1769
1770 static int dvfs_tree_open(struct inode *inode, struct file *file)
1771 {
1772         return single_open(file, dvfs_tree_show, inode->i_private);
1773 }
1774
1775 static const struct file_operations dvfs_tree_fops = {
1776         .open           = dvfs_tree_open,
1777         .read           = seq_read,
1778         .llseek         = seq_lseek,
1779         .release        = single_release,
1780 };
1781
1782 static int rail_stats_show(struct seq_file *s, void *data)
1783 {
1784         char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
1785         int size = 0;
1786
1787         if (!buf)
1788                 return -ENOMEM;
1789
1790         size = rail_stats_save_to_buf(buf, PAGE_SIZE);
1791         seq_write(s, buf, size);
1792         kfree(buf);
1793         return 0;
1794 }
1795
1796 static int rail_stats_open(struct inode *inode, struct file *file)
1797 {
1798         return single_open(file, rail_stats_show, inode->i_private);
1799 }
1800
1801 static const struct file_operations rail_stats_fops = {
1802         .open           = rail_stats_open,
1803         .read           = seq_read,
1804         .llseek         = seq_lseek,
1805         .release        = single_release,
1806 };
1807
1808 static int rail_offs_set(struct dvfs_rail *rail, int offs)
1809 {
1810         if (rail) {
1811                 mutex_lock(&dvfs_lock);
1812                 rail->dbg_mv_offs = offs;
1813                 dvfs_rail_update(rail);
1814                 mutex_unlock(&dvfs_lock);
1815                 return 0;
1816         }
1817         return -ENOENT;
1818 }
1819
1820 static int cpu_offs_get(void *data, u64 *val)
1821 {
1822         if (tegra_cpu_rail) {
1823                 *val = (u64)tegra_cpu_rail->dbg_mv_offs;
1824                 return 0;
1825         }
1826         *val = 0;
1827         return -ENOENT;
1828 }
1829 static int cpu_offs_set(void *data, u64 val)
1830 {
1831         return rail_offs_set(tegra_cpu_rail, (int)val);
1832 }
1833 DEFINE_SIMPLE_ATTRIBUTE(cpu_offs_fops, cpu_offs_get, cpu_offs_set, "%lld\n");
1834
1835 static int gpu_offs_get(void *data, u64 *val)
1836 {
1837         if (tegra_gpu_rail) {
1838                 *val = (u64)tegra_gpu_rail->dbg_mv_offs;
1839                 return 0;
1840         }
1841         *val = 0;
1842         return -ENOENT;
1843 }
1844 static int gpu_offs_set(void *data, u64 val)
1845 {
1846         return rail_offs_set(tegra_gpu_rail, (int)val);
1847 }
1848 DEFINE_SIMPLE_ATTRIBUTE(gpu_offs_fops, gpu_offs_get, gpu_offs_set, "%lld\n");
1849
1850 static int core_offs_get(void *data, u64 *val)
1851 {
1852         if (tegra_core_rail) {
1853                 *val = (u64)tegra_core_rail->dbg_mv_offs;
1854                 return 0;
1855         }
1856         *val = 0;
1857         return -ENOENT;
1858 }
1859 static int core_offs_set(void *data, u64 val)
1860 {
1861         return rail_offs_set(tegra_core_rail, (int)val);
1862 }
1863 DEFINE_SIMPLE_ATTRIBUTE(core_offs_fops, core_offs_get, core_offs_set, "%lld\n");
1864
1865 static int core_override_get(void *data, u64 *val)
1866 {
1867         if (tegra_core_rail) {
1868                 *val = (u64)tegra_core_rail->override_millivolts;
1869                 return 0;
1870         }
1871         *val = 0;
1872         return -ENOENT;
1873 }
1874 static int core_override_set(void *data, u64 val)
1875 {
1876         return dvfs_override_core_voltage((int)val);
1877 }
1878 DEFINE_SIMPLE_ATTRIBUTE(core_override_fops,
1879                         core_override_get, core_override_set, "%llu\n");
1880
1881 static int gpu_dvfs_t_show(struct seq_file *s, void *data)
1882 {
1883         int i, j;
1884         int num_ranges = 1;
1885         int *trips = NULL;
1886         struct dvfs *d;
1887         struct dvfs_rail *rail = tegra_gpu_rail;
1888
1889         if (!tegra_gpu_rail) {
1890                 seq_printf(s, "Only supported for T124 or higher\n");
1891                 return -ENOSYS;
1892         }
1893
1894         mutex_lock(&dvfs_lock);
1895
1896         d = list_first_entry(&rail->dvfs, struct dvfs, reg_node);
1897         if (rail->vts_cdev && d->therm_dvfs) {
1898                 num_ranges = rail->vts_cdev->trip_temperatures_num + 1;
1899                 trips = rail->vts_cdev->trip_temperatures;
1900         }
1901
1902         seq_printf(s, "%-11s", "T(C)\\F(kHz)");
1903         for (i = 0; i < d->num_freqs; i++) {
1904                 unsigned int f = d->freqs[i]/100;
1905                 seq_printf(s, " %7u", f);
1906         }
1907         seq_printf(s, "\n");
1908
1909         for (j = 0; j < num_ranges; j++) {
1910                 seq_printf(s, "%s", j == rail->therm_scale_idx ? ">" : " ");
1911
1912                 if (!trips || (num_ranges == 1))
1913                         seq_printf(s, "%4s..%-4s", "", "");
1914                 else if (j == 0)
1915                         seq_printf(s, "%4s..%-4d", "", trips[j]);
1916                 else if (j == num_ranges - 1)
1917                         seq_printf(s, "%4d..%-4s", trips[j], "");
1918                 else
1919                         seq_printf(s, "%4d..%-4d", trips[j-1], trips[j]);
1920
1921                 for (i = 0; i < d->num_freqs; i++) {
1922                         int mv = *(d->millivolts + j * MAX_DVFS_FREQS + i);
1923                         seq_printf(s, " %7d", mv);
1924                 }
1925                 seq_printf(s, " mV\n");
1926         }
1927
1928         seq_printf(s, "%3s%-8s\n", "", "------");
1929         seq_printf(s, "%3s%-8s", "", "max(T)");
1930         for (i = 0; i < d->num_freqs; i++)
1931                 seq_printf(s, " %7d", d->peak_millivolts[i]);
1932         seq_printf(s, " mV\n");
1933
1934         mutex_unlock(&dvfs_lock);
1935
1936         return 0;
1937 }
1938
1939 static int gpu_dvfs_t_open(struct inode *inode, struct file *file)
1940 {
1941         return single_open(file, gpu_dvfs_t_show, NULL);
1942 }
1943
1944 static const struct file_operations gpu_dvfs_t_fops = {
1945         .open           = gpu_dvfs_t_open,
1946         .read           = seq_read,
1947         .llseek         = seq_lseek,
1948         .release        = single_release,
1949 };
1950
1951 static int dvfs_table_show(struct seq_file *s, void *data)
1952 {
1953         int i;
1954         struct dvfs *d;
1955         struct dvfs_rail *rail;
1956         const int *v_pll, *last_v_pll = NULL;
1957         const int *v_dfll, *last_v_dfll = NULL;
1958
1959         seq_printf(s, "DVFS tables: units mV/MHz\n");
1960
1961         mutex_lock(&dvfs_lock);
1962
1963         list_for_each_entry(rail, &dvfs_rail_list, node) {
1964                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1965                         bool mv_done = false;
1966                         v_pll = tegra_dvfs_get_millivolts_pll(d);
1967                         v_dfll = d->dfll_millivolts;
1968
1969                         if (v_pll && (last_v_pll != v_pll)) {
1970                                 if (!mv_done) {
1971                                         seq_printf(s, "\n");
1972                                         mv_done = true;
1973                                 }
1974                                 last_v_pll = v_pll;
1975                                 seq_printf(s, "%-16s", rail->reg_id);
1976                                 for (i = 0; i < d->num_freqs; i++)
1977                                         seq_printf(s, "%7d", v_pll[i]);
1978                                 seq_printf(s, "\n");
1979                         }
1980
1981                         if (v_dfll && (last_v_dfll != v_dfll)) {
1982                                 if (!mv_done) {
1983                                         seq_printf(s, "\n");
1984                                         mv_done = true;
1985                                 }
1986                                 last_v_dfll = v_dfll;
1987                                 seq_printf(s, "%-8s (dfll) ", rail->reg_id);
1988                                 for (i = 0; i < d->num_freqs; i++)
1989                                         seq_printf(s, "%7d", v_dfll[i]);
1990                                 seq_printf(s, "\n");
1991                         }
1992
1993                         seq_printf(s, "%-16s", d->clk_name);
1994                         for (i = 0; i < d->num_freqs; i++) {
1995                                 unsigned int f = d->freqs[i]/100000;
1996                                 seq_printf(s, " %4u.%u", f/10, f%10);
1997                         }
1998                         seq_printf(s, "\n");
1999                 }
2000         }
2001
2002         mutex_unlock(&dvfs_lock);
2003
2004         return 0;
2005 }
2006
2007 static int dvfs_table_open(struct inode *inode, struct file *file)
2008 {
2009         return single_open(file, dvfs_table_show, inode->i_private);
2010 }
2011
2012 static const struct file_operations dvfs_table_fops = {
2013         .open           = dvfs_table_open,
2014         .read           = seq_read,
2015         .llseek         = seq_lseek,
2016         .release        = single_release,
2017 };
2018
2019 int __init dvfs_debugfs_init(struct dentry *clk_debugfs_root)
2020 {
2021         struct dentry *d;
2022
2023         d = debugfs_create_file("dvfs", S_IRUGO, clk_debugfs_root, NULL,
2024                 &dvfs_tree_fops);
2025         if (!d)
2026                 return -ENOMEM;
2027
2028         d = debugfs_create_file("rails", S_IRUGO, clk_debugfs_root, NULL,
2029                 &rail_stats_fops);
2030         if (!d)
2031                 return -ENOMEM;
2032
2033         d = debugfs_create_file("vdd_cpu_offs", S_IRUGO | S_IWUSR,
2034                 clk_debugfs_root, NULL, &cpu_offs_fops);
2035         if (!d)
2036                 return -ENOMEM;
2037
2038         d = debugfs_create_file("vdd_gpu_offs", S_IRUGO | S_IWUSR,
2039                 clk_debugfs_root, NULL, &gpu_offs_fops);
2040         if (!d)
2041                 return -ENOMEM;
2042
2043         d = debugfs_create_file("vdd_core_offs", S_IRUGO | S_IWUSR,
2044                 clk_debugfs_root, NULL, &core_offs_fops);
2045         if (!d)
2046                 return -ENOMEM;
2047
2048         d = debugfs_create_file("vdd_core_override", S_IRUGO | S_IWUSR,
2049                 clk_debugfs_root, NULL, &core_override_fops);
2050         if (!d)
2051                 return -ENOMEM;
2052
2053         d = debugfs_create_file("gpu_dvfs_t", S_IRUGO | S_IWUSR,
2054                 clk_debugfs_root, NULL, &gpu_dvfs_t_fops);
2055         if (!d)
2056                 return -ENOMEM;
2057
2058         d = debugfs_create_file("dvfs_table", S_IRUGO, clk_debugfs_root, NULL,
2059                 &dvfs_table_fops);
2060         if (!d)
2061                 return -ENOMEM;
2062
2063         return 0;
2064 }
2065
2066 #endif
2067
2068 #ifdef CONFIG_PM
2069 static ssize_t tegra_rail_stats_show(struct kobject *kobj,
2070                                         struct kobj_attribute *attr,
2071                                         char *buf)
2072 {
2073         return rail_stats_save_to_buf(buf, PAGE_SIZE);
2074 }
2075
2076 static struct kobj_attribute rail_stats_attr =
2077                 __ATTR_RO(tegra_rail_stats);
2078
2079 static int __init tegra_dvfs_sysfs_stats_init(void)
2080 {
2081         int error;
2082         error = sysfs_create_file(power_kobj, &rail_stats_attr.attr);
2083         return 0;
2084 }
2085 late_initcall(tegra_dvfs_sysfs_stats_init);
2086 #endif