arm: mach-tegra: Export dvfs frequencies
[linux-3.10.git] / arch / arm / mach-tegra / dvfs.c
1 /*
2  *
3  * Copyright (C) 2010 Google, Inc.
4  *
5  * Author:
6  *      Colin Cross <ccross@google.com>
7  *
8  * Copyright (C) 2010-2013 NVIDIA CORPORATION. All rights reserved.
9  *
10  * This software is licensed under the terms of the GNU General Public
11  * License version 2, as published by the Free Software Foundation, and
12  * may be copied, distributed, and modified under those terms.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/clk.h>
23 #include <linux/clkdev.h>
24 #include <linux/debugfs.h>
25 #include <linux/init.h>
26 #include <linux/list.h>
27 #include <linux/list_sort.h>
28 #include <linux/module.h>
29 #include <linux/regulator/consumer.h>
30 #include <linux/seq_file.h>
31 #include <linux/slab.h>
32 #include <linux/suspend.h>
33 #include <linux/delay.h>
34 #include <linux/clk/tegra.h>
35 #include <linux/reboot.h>
36
37 #include <mach/hardware.h>
38
39 #include "board.h"
40 #include "clock.h"
41 #include "dvfs.h"
42
43 #define DVFS_RAIL_STATS_BIN     12500
44
45 struct dvfs_rail *tegra_cpu_rail;
46 struct dvfs_rail *tegra_core_rail;
47 static struct dvfs_rail *tegra_gpu_rail;
48
49 static LIST_HEAD(dvfs_rail_list);
50 static DEFINE_MUTEX(dvfs_lock);
51 static DEFINE_MUTEX(rail_disable_lock);
52
53 static int dvfs_rail_update(struct dvfs_rail *rail);
54
55 void tegra_dvfs_add_relationships(struct dvfs_relationship *rels, int n)
56 {
57         int i;
58         struct dvfs_relationship *rel;
59
60         mutex_lock(&dvfs_lock);
61
62         for (i = 0; i < n; i++) {
63                 rel = &rels[i];
64                 list_add_tail(&rel->from_node, &rel->to->relationships_from);
65                 list_add_tail(&rel->to_node, &rel->from->relationships_to);
66         }
67
68         mutex_unlock(&dvfs_lock);
69 }
70
71 /* Make sure there is a matching cooling device for thermal limit profile. */
72 static void dvfs_validate_cdevs(struct dvfs_rail *rail)
73 {
74         if (!rail->therm_mv_caps != !rail->therm_mv_caps_num) {
75                 rail->therm_mv_caps_num = 0;
76                 rail->therm_mv_caps = NULL;
77                 WARN(1, "%s: not matching thermal caps/num\n", rail->reg_id);
78         }
79
80         if (rail->therm_mv_caps && !rail->vmax_cdev)
81                 WARN(1, "%s: missing vmax cooling device\n", rail->reg_id);
82
83         if (!rail->therm_mv_floors != !rail->therm_mv_floors_num) {
84                 rail->therm_mv_floors_num = 0;
85                 rail->therm_mv_floors = NULL;
86                 WARN(1, "%s: not matching thermal floors/num\n", rail->reg_id);
87         }
88
89         if (rail->therm_mv_floors && !rail->vmin_cdev)
90                 WARN(1, "%s: missing vmin cooling device\n", rail->reg_id);
91
92         /* Limit override range to maximum floor */
93         if (rail->therm_mv_floors)
94                 rail->min_override_millivolts = rail->therm_mv_floors[0];
95 }
96
97 int tegra_dvfs_init_rails(struct dvfs_rail *rails[], int n)
98 {
99         int i;
100
101         mutex_lock(&dvfs_lock);
102
103         for (i = 0; i < n; i++) {
104                 INIT_LIST_HEAD(&rails[i]->dvfs);
105                 INIT_LIST_HEAD(&rails[i]->relationships_from);
106                 INIT_LIST_HEAD(&rails[i]->relationships_to);
107                 rails[i]->millivolts = rails[i]->nominal_millivolts;
108                 rails[i]->new_millivolts = rails[i]->nominal_millivolts;
109                 if (!rails[i]->step)
110                         rails[i]->step = rails[i]->max_millivolts;
111                 if (!rails[i]->step_up)
112                         rails[i]->step_up = rails[i]->step;
113
114                 list_add_tail(&rails[i]->node, &dvfs_rail_list);
115
116                 dvfs_validate_cdevs(rails[i]);
117
118                 if (!strcmp("vdd_cpu", rails[i]->reg_id))
119                         tegra_cpu_rail = rails[i];
120                 else if (!strcmp("vdd_gpu", rails[i]->reg_id))
121                         tegra_gpu_rail = rails[i];
122                 else if (!strcmp("vdd_core", rails[i]->reg_id))
123                         tegra_core_rail = rails[i];
124         }
125
126         mutex_unlock(&dvfs_lock);
127
128         return 0;
129 };
130
131 static int dvfs_solve_relationship(struct dvfs_relationship *rel)
132 {
133         return rel->solve(rel->from, rel->to);
134 }
135
136 /* rail statistic - called during rail init, or under dfs_lock, or with
137    CPU0 only on-line, and interrupts disabled */
138 static void dvfs_rail_stats_init(struct dvfs_rail *rail, int millivolts)
139 {
140         int dvfs_rail_stats_range;
141
142         if (!rail->stats.bin_uV)
143                 rail->stats.bin_uV = DVFS_RAIL_STATS_BIN;
144
145         dvfs_rail_stats_range =
146                 (DVFS_RAIL_STATS_TOP_BIN - 1) * rail->stats.bin_uV / 1000;
147
148         rail->stats.last_update = ktime_get();
149         if (millivolts >= rail->min_millivolts) {
150                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
151                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
152                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
153         }
154
155         if (rail->max_millivolts >
156             rail->min_millivolts + dvfs_rail_stats_range)
157                 pr_warn("tegra_dvfs: %s: stats above %d mV will be squashed\n",
158                         rail->reg_id,
159                         rail->min_millivolts + dvfs_rail_stats_range);
160 }
161
162 static void dvfs_rail_stats_update(
163         struct dvfs_rail *rail, int millivolts, ktime_t now)
164 {
165         rail->stats.time_at_mv[rail->stats.last_index] = ktime_add(
166                 rail->stats.time_at_mv[rail->stats.last_index], ktime_sub(
167                         now, rail->stats.last_update));
168         rail->stats.last_update = now;
169
170         if (rail->stats.off)
171                 return;
172
173         if (millivolts >= rail->min_millivolts) {
174                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
175                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
176                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
177         } else if (millivolts == 0)
178                         rail->stats.last_index = 0;
179 }
180
181 static void dvfs_rail_stats_pause(struct dvfs_rail *rail,
182                                   ktime_t delta, bool on)
183 {
184         int i = on ? rail->stats.last_index : 0;
185         rail->stats.time_at_mv[i] = ktime_add(rail->stats.time_at_mv[i], delta);
186 }
187
188 void tegra_dvfs_rail_off(struct dvfs_rail *rail, ktime_t now)
189 {
190         if (rail) {
191                 dvfs_rail_stats_update(rail, 0, now);
192                 rail->stats.off = true;
193         }
194 }
195
196 void tegra_dvfs_rail_on(struct dvfs_rail *rail, ktime_t now)
197 {
198         if (rail) {
199                 rail->stats.off = false;
200                 dvfs_rail_stats_update(rail, rail->millivolts, now);
201         }
202 }
203
204 void tegra_dvfs_rail_pause(struct dvfs_rail *rail, ktime_t delta, bool on)
205 {
206         if (rail)
207                 dvfs_rail_stats_pause(rail, delta, on);
208 }
209
210 static int dvfs_rail_set_voltage_reg(struct dvfs_rail *rail, int millivolts)
211 {
212         int ret;
213
214         /*
215          * safely return success for low voltage requests on fixed regulator
216          * (higher requests will go through and fail, as they should)
217          */
218         if (rail->fixed_millivolts && (millivolts <= rail->fixed_millivolts))
219                 return 0;
220
221         rail->updating = true;
222         rail->reg_max_millivolts = rail->reg_max_millivolts ==
223                 rail->max_millivolts ?
224                 rail->max_millivolts + 1 : rail->max_millivolts;
225         ret = regulator_set_voltage(rail->reg,
226                 millivolts * 1000,
227                 rail->reg_max_millivolts * 1000);
228         rail->updating = false;
229
230         return ret;
231 }
232
233 /* Sets the voltage on a dvfs rail to a specific value, and updates any
234  * rails that depend on this rail. */
235 static int dvfs_rail_set_voltage(struct dvfs_rail *rail, int millivolts)
236 {
237         int ret = 0;
238         struct dvfs_relationship *rel;
239         int step, offset;
240         int i;
241         int steps;
242         bool jmp_to_zero;
243
244         if (!rail->reg) {
245                 if (millivolts == rail->millivolts)
246                         return 0;
247                 else
248                         return -EINVAL;
249         }
250
251         if (millivolts > rail->millivolts) {
252                 step = rail->step_up;
253                 offset = step;
254         } else {
255                 step = rail->step;
256                 offset = -step;
257         }
258
259         /*
260          * DFLL adjusts rail voltage automatically, but not exactly to the
261          * expected level - update stats, anyway.
262          */
263         if (rail->dfll_mode) {
264                 rail->millivolts = rail->new_millivolts = millivolts;
265                 dvfs_rail_stats_update(rail, millivolts, ktime_get());
266                 return 0;
267         }
268
269         if (rail->disabled)
270                 return 0;
271
272         rail->resolving_to = true;
273         jmp_to_zero = rail->jmp_to_zero &&
274                         ((millivolts == 0) || (rail->millivolts == 0));
275         steps = jmp_to_zero ? 1 :
276                 DIV_ROUND_UP(abs(millivolts - rail->millivolts), step);
277
278         for (i = 0; i < steps; i++) {
279                 if (!jmp_to_zero &&
280                     (abs(millivolts - rail->millivolts) > step))
281                         rail->new_millivolts = rail->millivolts + offset;
282                 else
283                         rail->new_millivolts = millivolts;
284
285                 /* Before changing the voltage, tell each rail that depends
286                  * on this rail that the voltage will change.
287                  * This rail will be the "from" rail in the relationship,
288                  * the rail that depends on this rail will be the "to" rail.
289                  * from->millivolts will be the old voltage
290                  * from->new_millivolts will be the new voltage */
291                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
292                         ret = dvfs_rail_update(rel->to);
293                         if (ret)
294                                 goto out;
295                 }
296
297                 ret = dvfs_rail_set_voltage_reg(rail, rail->new_millivolts);
298                 if (ret) {
299                         pr_err("Failed to set dvfs regulator %s\n", rail->reg_id);
300                         goto out;
301                 }
302
303                 rail->millivolts = rail->new_millivolts;
304                 dvfs_rail_stats_update(rail, rail->millivolts, ktime_get());
305
306                 /* After changing the voltage, tell each rail that depends
307                  * on this rail that the voltage has changed.
308                  * from->millivolts and from->new_millivolts will be the
309                  * new voltage */
310                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
311                         ret = dvfs_rail_update(rel->to);
312                         if (ret)
313                                 goto out;
314                 }
315         }
316
317         if (unlikely(rail->millivolts != millivolts)) {
318                 pr_err("%s: rail didn't reach target %d in %d steps (%d)\n",
319                         __func__, millivolts, steps, rail->millivolts);
320                 ret = -EINVAL;
321         }
322
323 out:
324         rail->resolving_to = false;
325         return ret;
326 }
327
328 /* Determine the minimum valid voltage for a rail, taking into account
329  * the dvfs clocks and any rails that this rail depends on.  Calls
330  * dvfs_rail_set_voltage with the new voltage, which will call
331  * dvfs_rail_update on any rails that depend on this rail. */
332 static inline int dvfs_rail_apply_limits(struct dvfs_rail *rail, int millivolts)
333 {
334         int min_mv = rail->min_millivolts;
335
336         if (rail->therm_mv_floors) {
337                 int i = rail->therm_floor_idx;
338                 if (i < rail->therm_mv_floors_num)
339                         min_mv = rail->therm_mv_floors[i];
340         }
341
342         if (rail->override_millivolts) {
343                 millivolts = rail->override_millivolts;
344         } else {
345                 /* apply offset and clip up to pll mode fixed mv */
346                 millivolts += rail->offs_millivolts;
347                 if (!rail->dfll_mode && rail->fixed_millivolts &&
348                     (millivolts < rail->fixed_millivolts))
349                         millivolts = rail->fixed_millivolts;
350         }
351
352         if (millivolts > rail->max_millivolts)
353                 millivolts = rail->max_millivolts;
354         else if (millivolts < min_mv)
355                 millivolts = min_mv;
356
357         return millivolts;
358 }
359
360 static int dvfs_rail_update(struct dvfs_rail *rail)
361 {
362         int millivolts = 0;
363         struct dvfs *d;
364         struct dvfs_relationship *rel;
365         int ret = 0;
366         int steps;
367
368         /* if dvfs is suspended, return and handle it during resume */
369         if (rail->suspended)
370                 return 0;
371
372         /* if regulators are not connected yet, return and handle it later */
373         if (!rail->reg)
374                 return 0;
375
376         /* if rail update is entered while resolving circular dependencies,
377            abort recursion */
378         if (rail->resolving_to)
379                 return 0;
380
381         /* Find the maximum voltage requested by any clock */
382         list_for_each_entry(d, &rail->dvfs, reg_node)
383                 millivolts = max(d->cur_millivolts, millivolts);
384
385         /* Apply offset and min/max limits if any clock is requesting voltage */
386         if (millivolts)
387                 millivolts = dvfs_rail_apply_limits(rail, millivolts);
388
389         /* retry update if limited by from-relationship to account for
390            circular dependencies */
391         steps = DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
392         for (; steps >= 0; steps--) {
393                 rail->new_millivolts = millivolts;
394
395                 /* Check any rails that this rail depends on */
396                 list_for_each_entry(rel, &rail->relationships_from, from_node)
397                         rail->new_millivolts = dvfs_solve_relationship(rel);
398
399                 if (rail->new_millivolts == rail->millivolts)
400                         break;
401
402                 ret = dvfs_rail_set_voltage(rail, rail->new_millivolts);
403         }
404
405         return ret;
406 }
407
408 /*
409  * This function is called on entry to suspend, or when rail scaling is disabled
410  * - can't do anything in either case if regulsator is fixed in pll mode. Since
411  * the pll mode frequency is already capped according to fixed voltage level, it
412  * is safe to substitute fixed level for nominal, just for stats update.
413  */
414 static int dvfs_rail_set_nominal(struct dvfs_rail *rail)
415 {
416         int mv;
417         if (!rail->dfll_mode && rail->fixed_millivolts)
418                 mv = rail->fixed_millivolts;
419         else
420                 mv = dvfs_rail_apply_limits(rail, rail->nominal_millivolts);
421         return dvfs_rail_set_voltage(rail, mv);
422 }
423
424 static struct regulator *get_fixed_regulator(struct dvfs_rail *rail)
425 {
426         struct regulator *reg;
427         char reg_id[80];
428         struct dvfs *d;
429         int v, i;
430         unsigned long dfll_boost;
431
432         strcpy(reg_id, rail->reg_id);
433         strcat(reg_id, "_fixed");
434         reg = regulator_get(NULL, reg_id);
435         if (IS_ERR(reg))
436                 return reg;
437
438         v = regulator_get_voltage(reg) / 1000;
439         if ((v < rail->min_millivolts) || (v > rail->nominal_millivolts) ||
440             (rail->therm_mv_floors && v < rail->therm_mv_floors[0])) {
441                 pr_err("tegra_dvfs: ivalid fixed %s voltage %d\n",
442                        rail->reg_id, v);
443                 return ERR_PTR(-EINVAL);
444         }
445
446         /*
447          * Only fixed at nominal voltage vdd_core regulator is allowed, same
448          * is true for cpu rail if dfll mode is not supported at all. No thermal
449          * capping can be implemented in this case.
450          */
451         if (!IS_ENABLED(CONFIG_ARCH_TEGRA_HAS_CL_DVFS) ||
452             (rail != tegra_cpu_rail)) {
453                 if (v != rail->nominal_millivolts) {
454                         pr_err("tegra_dvfs: %s fixed below nominal at %d\n",
455                                rail->reg_id, v);
456                         return ERR_PTR(-EINVAL);
457                 }
458                 if (rail->therm_mv_caps) {
459                         pr_err("tegra_dvfs: cannot fix %s with thermal caps\n",
460                                rail->reg_id);
461                         return ERR_PTR(-ENOSYS);
462                 }
463                 return reg;
464         }
465
466         /*
467          * If dfll mode is supported, fixed vdd_cpu regulator may be below
468          * nominal in pll mode - maximum cpu rate in pll mode is limited
469          * respectively. Regulator is required to allow automatic scaling
470          * in dfll mode.
471          *
472          * FIXME: platform data to explicitly identify such "hybrid" regulator?
473          */
474         d = list_first_entry(&rail->dvfs, struct dvfs, reg_node);
475         for (i = 0; i < d->num_freqs; i++) {
476                 if (d->millivolts[i] > v)
477                         break;
478         }
479
480         if (!i) {
481                 pr_err("tegra_dvfs: %s fixed at %d: too low for min rate\n",
482                        rail->reg_id, v);
483                 return ERR_PTR(-EINVAL);
484         }
485
486         dfll_boost = (d->freqs[d->num_freqs - 1] - d->freqs[i - 1]);
487         if (d->dfll_data.max_rate_boost < dfll_boost)
488                 d->dfll_data.max_rate_boost = dfll_boost;
489
490         rail->fixed_millivolts = v;
491         return reg;
492 }
493
494 static int dvfs_rail_connect_to_regulator(struct dvfs_rail *rail)
495 {
496         struct regulator *reg;
497         int v;
498
499         if (!rail->reg) {
500                 reg = regulator_get(NULL, rail->reg_id);
501                 if (IS_ERR(reg)) {
502                         reg = get_fixed_regulator(rail);
503                         if (IS_ERR(reg)) {
504                                 pr_err("tegra_dvfs: failed to connect %s rail\n",
505                                        rail->reg_id);
506                                 return PTR_ERR(reg);
507                         }
508                 }
509                 rail->reg = reg;
510         }
511
512         v = regulator_enable(rail->reg);
513         if (v < 0) {
514                 pr_err("tegra_dvfs: failed on enabling regulator %s\n, err %d",
515                         rail->reg_id, v);
516                 return v;
517         }
518
519         v = regulator_get_voltage(rail->reg);
520         if (v < 0) {
521                 pr_err("tegra_dvfs: failed initial get %s voltage\n",
522                        rail->reg_id);
523                 return v;
524         }
525         rail->millivolts = v / 1000;
526         rail->new_millivolts = rail->millivolts;
527         dvfs_rail_stats_init(rail, rail->millivolts);
528         return 0;
529 }
530
531 static inline unsigned long *dvfs_get_freqs(struct dvfs *d)
532 {
533         return d->alt_freqs ? : &d->freqs[0];
534 }
535
536 static inline const int *dvfs_get_millivolts(struct dvfs *d, unsigned long rate)
537 {
538         if (tegra_dvfs_is_dfll_scale(d, rate))
539                 return d->dfll_millivolts;
540
541         return d->millivolts;
542 }
543
544 static int
545 __tegra_dvfs_set_rate(struct dvfs *d, unsigned long rate)
546 {
547         int i = 0;
548         int ret;
549         unsigned long *freqs = dvfs_get_freqs(d);
550         const int *millivolts = dvfs_get_millivolts(d, rate);
551
552         if (freqs == NULL || millivolts == NULL)
553                 return -ENODEV;
554
555         /* On entry to dfll range limit 1st step to range bottom (full ramp of
556            voltage/rate is completed automatically in dfll mode) */
557         if (tegra_dvfs_is_dfll_range_entry(d, rate))
558                 rate = d->dfll_data.use_dfll_rate_min;
559
560         if (rate > freqs[d->num_freqs - 1]) {
561                 pr_warn("tegra_dvfs: rate %lu too high for dvfs on %s\n", rate,
562                         d->clk_name);
563                 return -EINVAL;
564         }
565
566         if (rate == 0) {
567                 d->cur_millivolts = 0;
568         } else {
569                 while (i < d->num_freqs && rate > freqs[i])
570                         i++;
571
572                 if ((d->max_millivolts) &&
573                     (millivolts[i] > d->max_millivolts)) {
574                         pr_warn("tegra_dvfs: voltage %d too high for dvfs on"
575                                 " %s\n", millivolts[i], d->clk_name);
576                         return -EINVAL;
577                 }
578                 d->cur_millivolts = millivolts[i];
579         }
580
581         d->cur_rate = rate;
582
583         ret = dvfs_rail_update(d->dvfs_rail);
584         if (ret)
585                 pr_err("Failed to set regulator %s for clock %s to %d mV\n",
586                         d->dvfs_rail->reg_id, d->clk_name, d->cur_millivolts);
587
588         return ret;
589 }
590
591 int tegra_dvfs_alt_freqs_set(struct dvfs *d, unsigned long *alt_freqs)
592 {
593         int ret = 0;
594
595         mutex_lock(&dvfs_lock);
596
597         if (d->alt_freqs != alt_freqs) {
598                 d->alt_freqs = alt_freqs;
599                 ret = __tegra_dvfs_set_rate(d, d->cur_rate);
600         }
601
602         mutex_unlock(&dvfs_lock);
603         return ret;
604 }
605
606 static int predict_millivolts(struct clk *c, const int *millivolts,
607                               unsigned long rate)
608 {
609         int i;
610
611         if (!millivolts)
612                 return -ENODEV;
613         /*
614          * Predicted voltage can not be used across the switch to alternative
615          * frequency limits. For now, just fail the call for clock that has
616          * alternative limits initialized.
617          */
618         if (c->dvfs->alt_freqs)
619                 return -ENOSYS;
620
621         for (i = 0; i < c->dvfs->num_freqs; i++) {
622                 if (rate <= c->dvfs->freqs[i])
623                         break;
624         }
625
626         if (i == c->dvfs->num_freqs)
627                 return -EINVAL;
628
629         return millivolts[i];
630 }
631
632 int tegra_dvfs_predict_millivolts(struct clk *c, unsigned long rate)
633 {
634         const int *millivolts;
635
636         if (!rate || !c->dvfs)
637                 return 0;
638
639         millivolts = dvfs_get_millivolts(c->dvfs, rate);
640         return predict_millivolts(c, millivolts, rate);
641 }
642
643 int tegra_dvfs_predict_millivolts_pll(struct clk *c, unsigned long rate)
644 {
645         const int *millivolts;
646
647         if (!rate || !c->dvfs)
648                 return 0;
649
650         millivolts = c->dvfs->millivolts;
651         return predict_millivolts(c, millivolts, rate);
652 }
653
654 int tegra_dvfs_predict_millivolts_dfll(struct clk *c, unsigned long rate)
655 {
656         const int *millivolts;
657
658         if (!rate || !c->dvfs)
659                 return 0;
660
661         millivolts = c->dvfs->dfll_millivolts;
662         return predict_millivolts(c, millivolts, rate);
663 }
664
665 int tegra_dvfs_set_rate(struct clk *c, unsigned long rate)
666 {
667         int ret;
668
669         if (!c->dvfs)
670                 return -EINVAL;
671
672         mutex_lock(&dvfs_lock);
673         ret = __tegra_dvfs_set_rate(c->dvfs, rate);
674         mutex_unlock(&dvfs_lock);
675
676         return ret;
677 }
678 EXPORT_SYMBOL(tegra_dvfs_set_rate);
679
680 int tegra_dvfs_get_freqs(struct clk *c, unsigned long **freqs, int *num_freqs)
681 {
682         if (!c->dvfs)
683                 return -ENOSYS;
684
685         if (c->dvfs->alt_freqs)
686                 return -ENOSYS;
687
688         *num_freqs = c->dvfs->num_freqs;
689         *freqs = c->dvfs->freqs;
690
691         return 0;
692 }
693 EXPORT_SYMBOL(tegra_dvfs_get_freqs);
694
695 #ifdef CONFIG_TEGRA_VDD_CORE_OVERRIDE
696 static DEFINE_MUTEX(rail_override_lock);
697
698 int tegra_dvfs_override_core_voltage(int override_mv)
699 {
700         int ret, floor, ceiling;
701         struct dvfs_rail *rail = tegra_core_rail;
702
703         if (!rail)
704                 return -ENOENT;
705
706         if (rail->fixed_millivolts)
707                 return -ENOSYS;
708
709         floor = rail->min_override_millivolts;
710         ceiling = rail->nominal_millivolts;
711         if (override_mv && ((override_mv < floor) || (override_mv > ceiling))) {
712                 pr_err("%s: override level %d outside the range [%d...%d]\n",
713                        __func__, override_mv, floor, ceiling);
714                 return -EINVAL;
715         }
716
717         mutex_lock(&rail_override_lock);
718
719         if (override_mv == rail->override_millivolts) {
720                 ret = 0;
721                 goto out;
722         }
723
724         if (override_mv) {
725                 ret = tegra_dvfs_core_cap_level_apply(override_mv);
726                 if (ret) {
727                         pr_err("%s: failed to set cap for override level %d\n",
728                                __func__, override_mv);
729                         goto out;
730                 }
731         }
732
733         mutex_lock(&dvfs_lock);
734         if (rail->disabled || rail->suspended) {
735                 pr_err("%s: cannot scale %s rail\n", __func__,
736                        rail->disabled ? "disabled" : "suspended");
737                 ret = -EPERM;
738                 if (!override_mv) {
739                         mutex_unlock(&dvfs_lock);
740                         goto out;
741                 }
742         } else {
743                 rail->override_millivolts = override_mv;
744                 ret = dvfs_rail_update(rail);
745                 if (ret) {
746                         pr_err("%s: failed to set override level %d\n",
747                                __func__, override_mv);
748                         rail->override_millivolts = 0;
749                         dvfs_rail_update(rail);
750                 }
751         }
752         mutex_unlock(&dvfs_lock);
753
754         if (!override_mv || ret)
755                 tegra_dvfs_core_cap_level_apply(0);
756 out:
757         mutex_unlock(&rail_override_lock);
758         return ret;
759 }
760 #else
761 int tegra_dvfs_override_core_voltage(int override_mv)
762 {
763         pr_err("%s: vdd core override is not supported\n", __func__);
764         return -ENOSYS;
765 }
766 #endif
767 EXPORT_SYMBOL(tegra_dvfs_override_core_voltage);
768
769 /* May only be called during clock init, does not take any locks on clock c. */
770 int __init tegra_enable_dvfs_on_clk(struct clk *c, struct dvfs *d)
771 {
772         int i;
773
774         if (c->dvfs) {
775                 pr_err("Error when enabling dvfs on %s for clock %s:\n",
776                         d->dvfs_rail->reg_id, c->name);
777                 pr_err("DVFS already enabled for %s\n",
778                         c->dvfs->dvfs_rail->reg_id);
779                 return -EINVAL;
780         }
781
782         for (i = 0; i < MAX_DVFS_FREQS; i++) {
783                 if (d->millivolts[i] == 0)
784                         break;
785
786                 d->freqs[i] *= d->freqs_mult;
787
788                 /* If final frequencies are 0, pad with previous frequency */
789                 if (d->freqs[i] == 0 && i > 1)
790                         d->freqs[i] = d->freqs[i - 1];
791         }
792         d->num_freqs = i;
793
794         if (d->auto_dvfs) {
795                 c->auto_dvfs = true;
796                 clk_set_cansleep(c);
797         }
798
799         c->dvfs = d;
800
801         /*
802          * Minimum core override level is determined as maximum voltage required
803          * for clocks outside shared buses (shared bus rates can be capped to
804          * safe levels when override limit is set)
805          */
806         if (i && c->ops && !c->ops->shared_bus_update &&
807             !(c->flags & PERIPH_ON_CBUS)) {
808                 int mv = tegra_dvfs_predict_millivolts(c, d->freqs[i-1]);
809                 if (d->dvfs_rail->min_override_millivolts < mv)
810                         d->dvfs_rail->min_override_millivolts = mv;
811         }
812
813         mutex_lock(&dvfs_lock);
814         list_add_tail(&d->reg_node, &d->dvfs_rail->dvfs);
815         mutex_unlock(&dvfs_lock);
816
817         return 0;
818 }
819
820 static bool tegra_dvfs_all_rails_suspended(void)
821 {
822         struct dvfs_rail *rail;
823         bool all_suspended = true;
824
825         list_for_each_entry(rail, &dvfs_rail_list, node)
826                 if (!rail->suspended && !rail->disabled)
827                         all_suspended = false;
828
829         return all_suspended;
830 }
831
832 static bool tegra_dvfs_from_rails_suspended_or_solved(struct dvfs_rail *to)
833 {
834         struct dvfs_relationship *rel;
835         bool all_suspended = true;
836
837         list_for_each_entry(rel, &to->relationships_from, from_node)
838                 if (!rel->from->suspended && !rel->from->disabled &&
839                         !rel->solved_at_nominal)
840                         all_suspended = false;
841
842         return all_suspended;
843 }
844
845 static int tegra_dvfs_suspend_one(void)
846 {
847         struct dvfs_rail *rail;
848         int ret;
849
850         list_for_each_entry(rail, &dvfs_rail_list, node) {
851                 if (!rail->suspended && !rail->disabled &&
852                     tegra_dvfs_from_rails_suspended_or_solved(rail)) {
853                         ret = dvfs_rail_set_nominal(rail);
854                         if (ret)
855                                 return ret;
856                         rail->suspended = true;
857                         return 0;
858                 }
859         }
860
861         return -EINVAL;
862 }
863
864 static void tegra_dvfs_resume(void)
865 {
866         struct dvfs_rail *rail;
867
868         mutex_lock(&dvfs_lock);
869
870         list_for_each_entry(rail, &dvfs_rail_list, node)
871                 rail->suspended = false;
872
873         list_for_each_entry(rail, &dvfs_rail_list, node)
874                 dvfs_rail_update(rail);
875
876         mutex_unlock(&dvfs_lock);
877 }
878
879 static int tegra_dvfs_suspend(void)
880 {
881         int ret = 0;
882
883         mutex_lock(&dvfs_lock);
884
885         while (!tegra_dvfs_all_rails_suspended()) {
886                 ret = tegra_dvfs_suspend_one();
887                 if (ret)
888                         break;
889         }
890
891         mutex_unlock(&dvfs_lock);
892
893         if (ret)
894                 tegra_dvfs_resume();
895
896         return ret;
897 }
898
899 static int tegra_dvfs_pm_notify(struct notifier_block *nb,
900                                 unsigned long event, void *data)
901 {
902         switch (event) {
903         case PM_SUSPEND_PREPARE:
904                 if (tegra_dvfs_suspend())
905                         return NOTIFY_STOP;
906                 break;
907         case PM_POST_SUSPEND:
908                 tegra_dvfs_resume();
909                 break;
910         }
911
912         return NOTIFY_OK;
913 };
914
915 static struct notifier_block tegra_dvfs_nb = {
916         .notifier_call = tegra_dvfs_pm_notify,
917 };
918
919 static int tegra_dvfs_reboot_notify(struct notifier_block *nb,
920                                 unsigned long event, void *data)
921 {
922         switch (event) {
923         case SYS_RESTART:
924         case SYS_HALT:
925         case SYS_POWER_OFF:
926                 tegra_dvfs_suspend();
927                 return NOTIFY_OK;
928         }
929         return NOTIFY_DONE;
930 }
931
932 static struct notifier_block tegra_dvfs_reboot_nb = {
933         .notifier_call = tegra_dvfs_reboot_notify,
934 };
935
936 /* must be called with dvfs lock held */
937 static void __tegra_dvfs_rail_disable(struct dvfs_rail *rail)
938 {
939         int ret;
940
941         /* don't set voltage in DFLL mode - won't work, but break stats */
942         if (rail->dfll_mode) {
943                 rail->disabled = true;
944                 return;
945         }
946
947         ret = dvfs_rail_set_nominal(rail);
948         if (ret) {
949                 pr_info("dvfs: failed to set regulator %s to disable "
950                         "voltage %d\n", rail->reg_id,
951                         rail->nominal_millivolts);
952                 return;
953         }
954         rail->disabled = true;
955 }
956
957 /* must be called with dvfs lock held */
958 static void __tegra_dvfs_rail_enable(struct dvfs_rail *rail)
959 {
960         rail->disabled = false;
961         dvfs_rail_update(rail);
962 }
963
964 void tegra_dvfs_rail_enable(struct dvfs_rail *rail)
965 {
966         if (!rail)
967                 return;
968
969         mutex_lock(&rail_disable_lock);
970
971         if (rail->disabled) {
972                 mutex_lock(&dvfs_lock);
973                 __tegra_dvfs_rail_enable(rail);
974                 mutex_unlock(&dvfs_lock);
975
976                 tegra_dvfs_rail_post_enable(rail);
977         }
978         mutex_unlock(&rail_disable_lock);
979 }
980
981 void tegra_dvfs_rail_disable(struct dvfs_rail *rail)
982 {
983         if (!rail)
984                 return;
985
986         mutex_lock(&rail_disable_lock);
987         if (rail->disabled)
988                 goto out;
989
990         /* rail disable will set it to nominal voltage underneath clock
991            framework - need to re-configure clock rates that are not safe
992            at nominal (yes, unsafe at nominal is ugly, but possible). Rate
993            change must be done outside of dvfs lock. */
994         if (tegra_dvfs_rail_disable_prepare(rail)) {
995                 pr_info("dvfs: failed to prepare regulator %s to disable\n",
996                         rail->reg_id);
997                 goto out;
998         }
999
1000         mutex_lock(&dvfs_lock);
1001         __tegra_dvfs_rail_disable(rail);
1002         mutex_unlock(&dvfs_lock);
1003 out:
1004         mutex_unlock(&rail_disable_lock);
1005 }
1006
1007 int tegra_dvfs_rail_disable_by_name(const char *reg_id)
1008 {
1009         struct dvfs_rail *rail = tegra_dvfs_get_rail_by_name(reg_id);
1010         if (!rail)
1011                 return -EINVAL;
1012
1013         tegra_dvfs_rail_disable(rail);
1014         return 0;
1015 }
1016
1017 struct dvfs_rail *tegra_dvfs_get_rail_by_name(const char *reg_id)
1018 {
1019         struct dvfs_rail *rail;
1020
1021         mutex_lock(&dvfs_lock);
1022         list_for_each_entry(rail, &dvfs_rail_list, node) {
1023                 if (!strcmp(reg_id, rail->reg_id)) {
1024                         mutex_unlock(&dvfs_lock);
1025                         return rail;
1026                 }
1027         }
1028         mutex_unlock(&dvfs_lock);
1029         return NULL;
1030 }
1031
1032 bool tegra_dvfs_rail_updating(struct clk *clk)
1033 {
1034         return (!clk ? false :
1035                 (!clk->dvfs ? false :
1036                  (!clk->dvfs->dvfs_rail ? false :
1037                   (clk->dvfs->dvfs_rail->updating ||
1038                    clk->dvfs->dvfs_rail->dfll_mode_updating))));
1039 }
1040
1041 #ifdef CONFIG_OF
1042 int __init of_tegra_dvfs_init(const struct of_device_id *matches)
1043 {
1044         int ret;
1045         struct device_node *np;
1046
1047         for_each_matching_node(np, matches) {
1048                 const struct of_device_id *match = of_match_node(matches, np);
1049                 of_tegra_dvfs_init_cb_t dvfs_init_cb = match->data;
1050                 ret = dvfs_init_cb(np);
1051                 if (ret) {
1052                         pr_err("dt: Failed to read %s tables from DT\n",
1053                                                         match->compatible);
1054                         return ret;
1055                 }
1056         }
1057         return 0;
1058 }
1059 #endif
1060 int tegra_dvfs_dfll_mode_set(struct dvfs *d, unsigned long rate)
1061 {
1062         mutex_lock(&dvfs_lock);
1063         if (!d->dvfs_rail->dfll_mode) {
1064                 d->dvfs_rail->dfll_mode = true;
1065                 __tegra_dvfs_set_rate(d, rate);
1066         }
1067         mutex_unlock(&dvfs_lock);
1068         return 0;
1069 }
1070
1071 int tegra_dvfs_dfll_mode_clear(struct dvfs *d, unsigned long rate)
1072 {
1073         int ret = 0;
1074
1075         mutex_lock(&dvfs_lock);
1076         if (d->dvfs_rail->dfll_mode) {
1077                 d->dvfs_rail->dfll_mode = false;
1078                 /* avoid false detection of matching target (voltage in dfll
1079                    mode is fluctuating, and recorded level is just estimate) */
1080                 d->dvfs_rail->millivolts--;
1081                 if (d->dvfs_rail->disabled) {
1082                         d->dvfs_rail->disabled = false;
1083                         __tegra_dvfs_rail_disable(d->dvfs_rail);
1084                 }
1085                 ret = __tegra_dvfs_set_rate(d, rate);
1086         }
1087         mutex_unlock(&dvfs_lock);
1088         return ret;
1089 }
1090
1091 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmax_cdev(void)
1092 {
1093         if (tegra_cpu_rail)
1094                 return tegra_cpu_rail->vmax_cdev;
1095         return NULL;
1096 }
1097
1098 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmin_cdev(void)
1099 {
1100         if (tegra_cpu_rail)
1101                 return tegra_cpu_rail->vmin_cdev;
1102         return NULL;
1103 }
1104
1105 struct tegra_cooling_device *tegra_dvfs_get_core_vmin_cdev(void)
1106 {
1107         if (tegra_core_rail)
1108                 return tegra_core_rail->vmin_cdev;
1109         return NULL;
1110 }
1111
1112 #ifdef CONFIG_THERMAL
1113 /* Cooling device limits minimum rail voltage at cold temperature in pll mode */
1114 static int tegra_dvfs_rail_get_vmin_cdev_max_state(
1115         struct thermal_cooling_device *cdev, unsigned long *max_state)
1116 {
1117         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1118         *max_state = rail->vmin_cdev->trip_temperatures_num;
1119         return 0;
1120 }
1121
1122 static int tegra_dvfs_rail_get_vmin_cdev_cur_state(
1123         struct thermal_cooling_device *cdev, unsigned long *cur_state)
1124 {
1125         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1126         *cur_state = rail->therm_floor_idx;
1127         return 0;
1128 }
1129
1130 static int tegra_dvfs_rail_set_vmin_cdev_state(
1131         struct thermal_cooling_device *cdev, unsigned long cur_state)
1132 {
1133         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1134
1135         mutex_lock(&dvfs_lock);
1136         if (rail->therm_floor_idx != cur_state) {
1137                 rail->therm_floor_idx = cur_state;
1138                 dvfs_rail_update(rail);
1139         }
1140         mutex_unlock(&dvfs_lock);
1141         return 0;
1142 }
1143
1144 static struct thermal_cooling_device_ops tegra_dvfs_rail_cooling_ops = {
1145         .get_max_state = tegra_dvfs_rail_get_vmin_cdev_max_state,
1146         .get_cur_state = tegra_dvfs_rail_get_vmin_cdev_cur_state,
1147         .set_cur_state = tegra_dvfs_rail_set_vmin_cdev_state,
1148 };
1149
1150 static void tegra_dvfs_rail_register_vmin_cdev(struct dvfs_rail *rail)
1151 {
1152         if (!rail->vmin_cdev)
1153                 return;
1154
1155         /* just report error - initialized for cold temperature, anyway */
1156         if (IS_ERR_OR_NULL(thermal_cooling_device_register(
1157                 rail->vmin_cdev->cdev_type, (void *)rail,
1158                 &tegra_dvfs_rail_cooling_ops)))
1159                 pr_err("tegra cooling device %s failed to register\n",
1160                        rail->vmin_cdev->cdev_type);
1161 }
1162 #else
1163 #define tegra_dvfs_rail_register_vmin_cdev(rail)
1164 #endif
1165
1166 /* Directly set cold temperature limit in dfll mode */
1167 int tegra_dvfs_rail_dfll_mode_set_cold(struct dvfs_rail *rail)
1168 {
1169         int ret = 0;
1170
1171         /* No thermal floors - nothing to do */
1172         if (!rail || !rail->therm_mv_floors)
1173                 return ret;
1174
1175         /*
1176          * Since cooling thresholds are the same in pll and dfll modes, pll mode
1177          * thermal index can be used to decide if cold limit should be set in
1178          * dfll mode.
1179          */
1180         mutex_lock(&dvfs_lock);
1181         if (rail->dfll_mode &&
1182             (rail->therm_floor_idx < rail->therm_mv_floors_num)) {
1183                         int mv = rail->therm_mv_floors[rail->therm_floor_idx];
1184                         ret = dvfs_rail_set_voltage_reg(rail, mv);
1185         }
1186         mutex_unlock(&dvfs_lock);
1187
1188         return ret;
1189 }
1190
1191 /*
1192  * Iterate through all the dvfs regulators, finding the regulator exported
1193  * by the regulator api for each one.  Must be called in late init, after
1194  * all the regulator api's regulators are initialized.
1195  */
1196 int __init tegra_dvfs_late_init(void)
1197 {
1198         bool connected = true;
1199         struct dvfs_rail *rail;
1200
1201         mutex_lock(&dvfs_lock);
1202
1203         list_for_each_entry(rail, &dvfs_rail_list, node)
1204                 if (dvfs_rail_connect_to_regulator(rail))
1205                         connected = false;
1206
1207         list_for_each_entry(rail, &dvfs_rail_list, node)
1208                 if (connected)
1209                         dvfs_rail_update(rail);
1210                 else
1211                         __tegra_dvfs_rail_disable(rail);
1212
1213         mutex_unlock(&dvfs_lock);
1214
1215         if (!connected && tegra_platform_is_silicon())
1216                 return -ENODEV;
1217
1218         register_pm_notifier(&tegra_dvfs_nb);
1219         register_reboot_notifier(&tegra_dvfs_reboot_nb);
1220
1221         list_for_each_entry(rail, &dvfs_rail_list, node)
1222                 tegra_dvfs_rail_register_vmin_cdev(rail);
1223
1224         return 0;
1225 }
1226
1227 static int rail_stats_save_to_buf(char *buf, int len)
1228 {
1229         int i;
1230         struct dvfs_rail *rail;
1231         char *str = buf;
1232         char *end = buf + len;
1233
1234         str += scnprintf(str, end - str, "%-12s %-10s\n", "millivolts", "time");
1235
1236         mutex_lock(&dvfs_lock);
1237
1238         list_for_each_entry(rail, &dvfs_rail_list, node) {
1239                 str += scnprintf(str, end - str, "%s (bin: %d.%dmV)\n",
1240                            rail->reg_id,
1241                            rail->stats.bin_uV / 1000,
1242                            (rail->stats.bin_uV / 10) % 100);
1243
1244                 dvfs_rail_stats_update(rail, -1, ktime_get());
1245
1246                 str += scnprintf(str, end - str, "%-12d %-10llu\n", 0,
1247                         cputime64_to_clock_t(msecs_to_jiffies(
1248                                 ktime_to_ms(rail->stats.time_at_mv[0]))));
1249
1250                 for (i = 1; i <= DVFS_RAIL_STATS_TOP_BIN; i++) {
1251                         ktime_t ktime_zero = ktime_set(0, 0);
1252                         if (ktime_equal(rail->stats.time_at_mv[i], ktime_zero))
1253                                 continue;
1254                         str += scnprintf(str, end - str, "%-12d %-10llu\n",
1255                                 rail->min_millivolts +
1256                                 (i - 1) * rail->stats.bin_uV / 1000,
1257                                 cputime64_to_clock_t(msecs_to_jiffies(
1258                                         ktime_to_ms(rail->stats.time_at_mv[i])))
1259                         );
1260                 }
1261         }
1262         mutex_unlock(&dvfs_lock);
1263         return str - buf;
1264 }
1265
1266 #ifdef CONFIG_DEBUG_FS
1267 static int dvfs_tree_sort_cmp(void *p, struct list_head *a, struct list_head *b)
1268 {
1269         struct dvfs *da = list_entry(a, struct dvfs, reg_node);
1270         struct dvfs *db = list_entry(b, struct dvfs, reg_node);
1271         int ret;
1272
1273         ret = strcmp(da->dvfs_rail->reg_id, db->dvfs_rail->reg_id);
1274         if (ret != 0)
1275                 return ret;
1276
1277         if (da->cur_millivolts < db->cur_millivolts)
1278                 return 1;
1279         if (da->cur_millivolts > db->cur_millivolts)
1280                 return -1;
1281
1282         return strcmp(da->clk_name, db->clk_name);
1283 }
1284
1285 static int dvfs_tree_show(struct seq_file *s, void *data)
1286 {
1287         struct dvfs *d;
1288         struct dvfs_rail *rail;
1289         struct dvfs_relationship *rel;
1290
1291         seq_printf(s, "   clock      rate       mV\n");
1292         seq_printf(s, "--------------------------------\n");
1293
1294         mutex_lock(&dvfs_lock);
1295
1296         list_for_each_entry(rail, &dvfs_rail_list, node) {
1297                 int thermal_mv_floor = 0;
1298
1299                 seq_printf(s, "%s %d mV%s:\n", rail->reg_id, rail->millivolts,
1300                            rail->dfll_mode ? " dfll mode" :
1301                                 rail->disabled ? " disabled" : "");
1302                 list_for_each_entry(rel, &rail->relationships_from, from_node) {
1303                         seq_printf(s, "   %-10s %-7d mV %-4d mV\n",
1304                                 rel->from->reg_id, rel->from->millivolts,
1305                                 dvfs_solve_relationship(rel));
1306                 }
1307                 seq_printf(s, "   offset     %-7d mV\n", rail->offs_millivolts);
1308
1309                 if (rail->therm_mv_floors) {
1310                         int i = rail->therm_floor_idx;
1311                         if (i < rail->therm_mv_floors_num)
1312                                 thermal_mv_floor = rail->therm_mv_floors[i];
1313                 }
1314                 seq_printf(s, "   thermal    %-7d mV\n", thermal_mv_floor);
1315
1316                 if (rail == tegra_core_rail) {
1317                         seq_printf(s, "   override   %-7d mV [%-4d...%-4d]\n",
1318                                    rail->override_millivolts,
1319                                    rail->min_override_millivolts,
1320                                    rail->nominal_millivolts);
1321                 }
1322
1323                 list_sort(NULL, &rail->dvfs, dvfs_tree_sort_cmp);
1324
1325                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1326                         seq_printf(s, "   %-10s %-10lu %-4d mV\n", d->clk_name,
1327                                 d->cur_rate, d->cur_millivolts);
1328                 }
1329         }
1330
1331         mutex_unlock(&dvfs_lock);
1332
1333         return 0;
1334 }
1335
1336 static int dvfs_tree_open(struct inode *inode, struct file *file)
1337 {
1338         return single_open(file, dvfs_tree_show, inode->i_private);
1339 }
1340
1341 static const struct file_operations dvfs_tree_fops = {
1342         .open           = dvfs_tree_open,
1343         .read           = seq_read,
1344         .llseek         = seq_lseek,
1345         .release        = single_release,
1346 };
1347
1348 static int rail_stats_show(struct seq_file *s, void *data)
1349 {
1350         char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
1351         int size = 0;
1352
1353         if (!buf)
1354                 return -ENOMEM;
1355
1356         size = rail_stats_save_to_buf(buf, PAGE_SIZE);
1357         seq_write(s, buf, size);
1358         kfree(buf);
1359         return 0;
1360 }
1361
1362 static int rail_stats_open(struct inode *inode, struct file *file)
1363 {
1364         return single_open(file, rail_stats_show, inode->i_private);
1365 }
1366
1367 static const struct file_operations rail_stats_fops = {
1368         .open           = rail_stats_open,
1369         .read           = seq_read,
1370         .llseek         = seq_lseek,
1371         .release        = single_release,
1372 };
1373
1374 static int gpu_dvfs_show(struct seq_file *s, void *data)
1375 {
1376         int idx;
1377         int *millivolts;
1378         unsigned long *freqs;
1379
1380         if (read_gpu_dvfs_table(&millivolts, &freqs)) {
1381                 seq_printf(s, "Only supported for T124 or higher\n");
1382                 return 0;
1383         }
1384
1385         seq_printf(s, "millivolts \t \t frequency\n");
1386         seq_printf(s, "=====================================\n");
1387
1388         for (idx = 0; millivolts[idx]; idx++)
1389                 seq_printf(s, "%d mV \t \t %lu Hz\n", millivolts[idx],
1390                                 freqs[idx]);
1391
1392         return 0;
1393 }
1394
1395 static int gpu_dvfs_open(struct inode *inode, struct file *file)
1396 {
1397         return single_open(file, gpu_dvfs_show, NULL);
1398 }
1399
1400 static const struct file_operations gpu_dvfs_fops = {
1401         .open           = gpu_dvfs_open,
1402         .read           = seq_read,
1403         .llseek         = seq_lseek,
1404         .release        = single_release,
1405 };
1406
1407 static int rail_offs_set(struct dvfs_rail *rail, int offs)
1408 {
1409         if (rail) {
1410                 mutex_lock(&dvfs_lock);
1411                 rail->offs_millivolts = offs;
1412                 dvfs_rail_update(rail);
1413                 mutex_unlock(&dvfs_lock);
1414                 return 0;
1415         }
1416         return -ENOENT;
1417 }
1418
1419 static int cpu_offs_get(void *data, u64 *val)
1420 {
1421         if (tegra_cpu_rail) {
1422                 *val = (u64)tegra_cpu_rail->offs_millivolts;
1423                 return 0;
1424         }
1425         *val = 0;
1426         return -ENOENT;
1427 }
1428 static int cpu_offs_set(void *data, u64 val)
1429 {
1430         return rail_offs_set(tegra_cpu_rail, (int)val);
1431 }
1432 DEFINE_SIMPLE_ATTRIBUTE(cpu_offs_fops, cpu_offs_get, cpu_offs_set, "%lld\n");
1433
1434 static int gpu_offs_get(void *data, u64 *val)
1435 {
1436         if (tegra_gpu_rail) {
1437                 *val = (u64)tegra_gpu_rail->offs_millivolts;
1438                 return 0;
1439         }
1440         *val = 0;
1441         return -ENOENT;
1442 }
1443 static int gpu_offs_set(void *data, u64 val)
1444 {
1445         return rail_offs_set(tegra_gpu_rail, (int)val);
1446 }
1447 DEFINE_SIMPLE_ATTRIBUTE(gpu_offs_fops, gpu_offs_get, gpu_offs_set, "%lld\n");
1448
1449 static int core_offs_get(void *data, u64 *val)
1450 {
1451         if (tegra_core_rail) {
1452                 *val = (u64)tegra_core_rail->offs_millivolts;
1453                 return 0;
1454         }
1455         *val = 0;
1456         return -ENOENT;
1457 }
1458 static int core_offs_set(void *data, u64 val)
1459 {
1460         return rail_offs_set(tegra_core_rail, (int)val);
1461 }
1462 DEFINE_SIMPLE_ATTRIBUTE(core_offs_fops, core_offs_get, core_offs_set, "%lld\n");
1463
1464 static int core_override_get(void *data, u64 *val)
1465 {
1466         if (tegra_core_rail) {
1467                 *val = (u64)tegra_core_rail->override_millivolts;
1468                 return 0;
1469         }
1470         *val = 0;
1471         return -ENOENT;
1472 }
1473 static int core_override_set(void *data, u64 val)
1474 {
1475         return tegra_dvfs_override_core_voltage((int)val);
1476 }
1477 DEFINE_SIMPLE_ATTRIBUTE(core_override_fops,
1478                         core_override_get, core_override_set, "%llu\n");
1479
1480 static int dvfs_table_show(struct seq_file *s, void *data)
1481 {
1482         int i;
1483         struct dvfs *d;
1484         struct dvfs_rail *rail;
1485
1486         seq_printf(s, "DVFS tables: units mV/MHz\n\n");
1487
1488         mutex_lock(&dvfs_lock);
1489
1490         list_for_each_entry(rail, &dvfs_rail_list, node) {
1491                 bool mv_done = false;
1492                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1493                         if (!mv_done) {
1494                                 mv_done = true;
1495                                 seq_printf(s, "%-16s", rail->reg_id);
1496                                 for (i = 0; i < d->num_freqs; i++) {
1497                                         int mv = d->millivolts[i];
1498                                         seq_printf(s, "%7d", mv);
1499                                 }
1500                                 seq_printf(s, "\n");
1501                                 if (d->dfll_millivolts) {
1502                                         seq_printf(s, "%-8s (dfll) ",
1503                                                    rail->reg_id);
1504                                         for (i = 0; i < d->num_freqs; i++) {
1505                                                 int mv = d->dfll_millivolts[i];
1506                                                 seq_printf(s, "%7d", mv);
1507                                         }
1508                                         seq_printf(s, "\n");
1509                                 }
1510                         }
1511
1512                         seq_printf(s, "%-16s", d->clk_name);
1513                         for (i = 0; i < d->num_freqs; i++) {
1514                                 unsigned int f = d->freqs[i]/100000;
1515                                 seq_printf(s, " %4u.%u", f/10, f%10);
1516                         }
1517                         seq_printf(s, "\n");
1518                 }
1519                 seq_printf(s, "\n");
1520         }
1521
1522         mutex_unlock(&dvfs_lock);
1523
1524         return 0;
1525 }
1526
1527 static int dvfs_table_open(struct inode *inode, struct file *file)
1528 {
1529         return single_open(file, dvfs_table_show, inode->i_private);
1530 }
1531
1532 static const struct file_operations dvfs_table_fops = {
1533         .open           = dvfs_table_open,
1534         .read           = seq_read,
1535         .llseek         = seq_lseek,
1536         .release        = single_release,
1537 };
1538
1539 int __init dvfs_debugfs_init(struct dentry *clk_debugfs_root)
1540 {
1541         struct dentry *d;
1542
1543         d = debugfs_create_file("dvfs", S_IRUGO, clk_debugfs_root, NULL,
1544                 &dvfs_tree_fops);
1545         if (!d)
1546                 return -ENOMEM;
1547
1548         d = debugfs_create_file("rails", S_IRUGO, clk_debugfs_root, NULL,
1549                 &rail_stats_fops);
1550         if (!d)
1551                 return -ENOMEM;
1552
1553         d = debugfs_create_file("vdd_cpu_offs", S_IRUGO | S_IWUSR,
1554                 clk_debugfs_root, NULL, &cpu_offs_fops);
1555         if (!d)
1556                 return -ENOMEM;
1557
1558         d = debugfs_create_file("vdd_gpu_offs", S_IRUGO | S_IWUSR,
1559                 clk_debugfs_root, NULL, &gpu_offs_fops);
1560         if (!d)
1561                 return -ENOMEM;
1562
1563         d = debugfs_create_file("vdd_core_offs", S_IRUGO | S_IWUSR,
1564                 clk_debugfs_root, NULL, &core_offs_fops);
1565         if (!d)
1566                 return -ENOMEM;
1567
1568         d = debugfs_create_file("vdd_core_override", S_IRUGO | S_IWUSR,
1569                 clk_debugfs_root, NULL, &core_override_fops);
1570         if (!d)
1571                 return -ENOMEM;
1572
1573         d = debugfs_create_file("gpu_dvfs", S_IRUGO | S_IWUSR,
1574                 clk_debugfs_root, NULL, &gpu_dvfs_fops);
1575         if (!d)
1576                 return -ENOMEM;
1577
1578         d = debugfs_create_file("dvfs_table", S_IRUGO, clk_debugfs_root, NULL,
1579                 &dvfs_table_fops);
1580         if (!d)
1581                 return -ENOMEM;
1582
1583         return 0;
1584 }
1585
1586 #endif
1587
1588 #ifdef CONFIG_PM
1589 static ssize_t tegra_rail_stats_show(struct kobject *kobj,
1590                                         struct kobj_attribute *attr,
1591                                         char *buf)
1592 {
1593         return rail_stats_save_to_buf(buf, PAGE_SIZE);
1594 }
1595
1596 static struct kobj_attribute rail_stats_attr =
1597                 __ATTR_RO(tegra_rail_stats);
1598
1599 static int __init tegra_dvfs_sysfs_stats_init(void)
1600 {
1601         int error;
1602         error = sysfs_create_file(power_kobj, &rail_stats_attr.attr);
1603         return 0;
1604 }
1605 late_initcall(tegra_dvfs_sysfs_stats_init);
1606 #endif