ARM: tegra: dvfs: Add interface to set fmax at vmin
[linux-3.10.git] / arch / arm / mach-tegra / dvfs.c
1 /*
2  *
3  * Copyright (C) 2010 Google, Inc.
4  *
5  * Author:
6  *      Colin Cross <ccross@google.com>
7  *
8  * Copyright (C) 2010-2013 NVIDIA CORPORATION. All rights reserved.
9  *
10  * This software is licensed under the terms of the GNU General Public
11  * License version 2, as published by the Free Software Foundation, and
12  * may be copied, distributed, and modified under those terms.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/clk.h>
23 #include <linux/clkdev.h>
24 #include <linux/debugfs.h>
25 #include <linux/init.h>
26 #include <linux/list.h>
27 #include <linux/list_sort.h>
28 #include <linux/module.h>
29 #include <linux/regulator/consumer.h>
30 #include <linux/seq_file.h>
31 #include <linux/slab.h>
32 #include <linux/suspend.h>
33 #include <linux/delay.h>
34 #include <linux/clk/tegra.h>
35 #include <linux/reboot.h>
36 #include <linux/clk/tegra.h>
37 #include <linux/tegra-soc.h>
38
39 #include "board.h"
40 #include "clock.h"
41 #include "dvfs.h"
42
43 #define DVFS_RAIL_STATS_BIN     12500
44
45 struct dvfs_rail *tegra_cpu_rail;
46 struct dvfs_rail *tegra_core_rail;
47 struct dvfs_rail *tegra_gpu_rail;
48
49 static LIST_HEAD(dvfs_rail_list);
50 static DEFINE_MUTEX(dvfs_lock);
51 static DEFINE_MUTEX(rail_disable_lock);
52
53 static int dvfs_rail_update(struct dvfs_rail *rail);
54
55 static inline int tegra_dvfs_rail_get_disable_level(struct dvfs_rail *rail)
56 {
57         return rail->disable_millivolts ? : rail->nominal_millivolts;
58 }
59
60 static inline int tegra_dvfs_rail_get_suspend_level(struct dvfs_rail *rail)
61 {
62         return rail->suspend_millivolts ? : rail->nominal_millivolts;
63 }
64
65 void tegra_dvfs_add_relationships(struct dvfs_relationship *rels, int n)
66 {
67         int i;
68         struct dvfs_relationship *rel;
69
70         mutex_lock(&dvfs_lock);
71
72         for (i = 0; i < n; i++) {
73                 rel = &rels[i];
74                 list_add_tail(&rel->from_node, &rel->to->relationships_from);
75                 list_add_tail(&rel->to_node, &rel->from->relationships_to);
76         }
77
78         mutex_unlock(&dvfs_lock);
79 }
80
81 /* Make sure there is a matching cooling device for thermal limit profile. */
82 static void dvfs_validate_cdevs(struct dvfs_rail *rail)
83 {
84         if (!rail->therm_mv_caps != !rail->therm_mv_caps_num) {
85                 rail->therm_mv_caps_num = 0;
86                 rail->therm_mv_caps = NULL;
87                 WARN(1, "%s: not matching thermal caps/num\n", rail->reg_id);
88         }
89
90         if (rail->therm_mv_caps && !rail->vmax_cdev)
91                 WARN(1, "%s: missing vmax cooling device\n", rail->reg_id);
92
93         if (!rail->therm_mv_floors != !rail->therm_mv_floors_num) {
94                 rail->therm_mv_floors_num = 0;
95                 rail->therm_mv_floors = NULL;
96                 WARN(1, "%s: not matching thermal floors/num\n", rail->reg_id);
97         }
98
99         if (rail->therm_mv_floors && !rail->vmin_cdev)
100                 WARN(1, "%s: missing vmin cooling device\n", rail->reg_id);
101
102         /* Limit override range to maximum floor */
103         if (rail->therm_mv_floors)
104                 rail->min_override_millivolts = rail->therm_mv_floors[0];
105
106         /* Only GPU thermal dvfs is supported */
107         if (rail->vts_cdev && (rail != tegra_gpu_rail)) {
108                 rail->vts_cdev = NULL;
109                 WARN(1, "%s: thermal dvfs is not supported\n", rail->reg_id);
110         }
111
112         if (!rail->simon_vmin_offsets != !rail->simon_vmin_offs_num) {
113                 rail->simon_vmin_offs_num = 0;
114                 rail->simon_vmin_offsets = NULL;
115                 WARN(1, "%s: not matching simon offsets/num\n", rail->reg_id);
116         }
117 }
118
119 int tegra_dvfs_init_rails(struct dvfs_rail *rails[], int n)
120 {
121         int i, mv;
122
123         mutex_lock(&dvfs_lock);
124
125         for (i = 0; i < n; i++) {
126                 INIT_LIST_HEAD(&rails[i]->dvfs);
127                 INIT_LIST_HEAD(&rails[i]->relationships_from);
128                 INIT_LIST_HEAD(&rails[i]->relationships_to);
129
130                 mv = rails[i]->nominal_millivolts;
131                 if (rails[i]->boot_millivolts > mv)
132                         WARN(1, "%s: boot voltage %d above nominal %d\n",
133                              rails[i]->reg_id, rails[i]->boot_millivolts, mv);
134                 if (rails[i]->disable_millivolts > mv)
135                         rails[i]->disable_millivolts = mv;
136                 if (rails[i]->suspend_millivolts > mv)
137                         rails[i]->suspend_millivolts = mv;
138
139                 mv = tegra_dvfs_rail_get_boot_level(rails[i]);
140                 rails[i]->millivolts = mv;
141                 rails[i]->new_millivolts = mv;
142                 if (!rails[i]->step)
143                         rails[i]->step = rails[i]->max_millivolts;
144                 if (!rails[i]->step_up)
145                         rails[i]->step_up = rails[i]->step;
146
147                 list_add_tail(&rails[i]->node, &dvfs_rail_list);
148
149                 if (!strcmp("vdd_cpu", rails[i]->reg_id))
150                         tegra_cpu_rail = rails[i];
151                 else if (!strcmp("vdd_gpu", rails[i]->reg_id))
152                         tegra_gpu_rail = rails[i];
153                 else if (!strcmp("vdd_core", rails[i]->reg_id))
154                         tegra_core_rail = rails[i];
155
156                 dvfs_validate_cdevs(rails[i]);
157         }
158
159         mutex_unlock(&dvfs_lock);
160
161         return 0;
162 };
163
164 static int dvfs_solve_relationship(struct dvfs_relationship *rel)
165 {
166         return rel->solve(rel->from, rel->to);
167 }
168
169 /* rail statistic - called during rail init, or under dfs_lock, or with
170    CPU0 only on-line, and interrupts disabled */
171 static void dvfs_rail_stats_init(struct dvfs_rail *rail, int millivolts)
172 {
173         int dvfs_rail_stats_range;
174
175         if (!rail->stats.bin_uV)
176                 rail->stats.bin_uV = DVFS_RAIL_STATS_BIN;
177
178         dvfs_rail_stats_range =
179                 (DVFS_RAIL_STATS_TOP_BIN - 1) * rail->stats.bin_uV / 1000;
180
181         rail->stats.last_update = ktime_get();
182         if (millivolts >= rail->min_millivolts) {
183                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
184                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
185                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
186         }
187
188         if (rail->max_millivolts >
189             rail->min_millivolts + dvfs_rail_stats_range)
190                 pr_warn("tegra_dvfs: %s: stats above %d mV will be squashed\n",
191                         rail->reg_id,
192                         rail->min_millivolts + dvfs_rail_stats_range);
193 }
194
195 static void dvfs_rail_stats_update(
196         struct dvfs_rail *rail, int millivolts, ktime_t now)
197 {
198         rail->stats.time_at_mv[rail->stats.last_index] = ktime_add(
199                 rail->stats.time_at_mv[rail->stats.last_index], ktime_sub(
200                         now, rail->stats.last_update));
201         rail->stats.last_update = now;
202
203         if (rail->stats.off)
204                 return;
205
206         if (millivolts >= rail->min_millivolts) {
207                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
208                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
209                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
210         } else if (millivolts == 0)
211                         rail->stats.last_index = 0;
212 }
213
214 static void dvfs_rail_stats_pause(struct dvfs_rail *rail,
215                                   ktime_t delta, bool on)
216 {
217         int i = on ? rail->stats.last_index : 0;
218         rail->stats.time_at_mv[i] = ktime_add(rail->stats.time_at_mv[i], delta);
219 }
220
221 void tegra_dvfs_rail_off(struct dvfs_rail *rail, ktime_t now)
222 {
223         if (rail) {
224                 dvfs_rail_stats_update(rail, 0, now);
225                 rail->stats.off = true;
226         }
227 }
228
229 void tegra_dvfs_rail_on(struct dvfs_rail *rail, ktime_t now)
230 {
231         if (rail) {
232                 rail->stats.off = false;
233                 dvfs_rail_stats_update(rail, rail->millivolts, now);
234         }
235 }
236
237 void tegra_dvfs_rail_pause(struct dvfs_rail *rail, ktime_t delta, bool on)
238 {
239         if (rail)
240                 dvfs_rail_stats_pause(rail, delta, on);
241 }
242
243 static int dvfs_rail_set_voltage_reg(struct dvfs_rail *rail, int millivolts)
244 {
245         int ret;
246
247         /*
248          * safely return success for low voltage requests on fixed regulator
249          * (higher requests will go through and fail, as they should)
250          */
251         if (rail->fixed_millivolts && (millivolts <= rail->fixed_millivolts))
252                 return 0;
253
254         rail->updating = true;
255         rail->reg_max_millivolts = rail->reg_max_millivolts ==
256                 rail->max_millivolts ?
257                 rail->max_millivolts + 1 : rail->max_millivolts;
258         ret = regulator_set_voltage(rail->reg,
259                 millivolts * 1000,
260                 rail->reg_max_millivolts * 1000);
261         rail->updating = false;
262
263         return ret;
264 }
265
266 /* Sets the voltage on a dvfs rail to a specific value, and updates any
267  * rails that depend on this rail. */
268 static int dvfs_rail_set_voltage(struct dvfs_rail *rail, int millivolts)
269 {
270         int ret = 0;
271         struct dvfs_relationship *rel;
272         int step, offset;
273         int i;
274         int steps;
275         bool jmp_to_zero;
276
277         if (!rail->reg) {
278                 if (millivolts == rail->millivolts)
279                         return 0;
280                 else
281                         return -EINVAL;
282         }
283
284         if (millivolts > rail->millivolts) {
285                 step = rail->step_up;
286                 offset = step;
287         } else {
288                 step = rail->step;
289                 offset = -step;
290         }
291
292         /*
293          * DFLL adjusts rail voltage automatically, but not exactly to the
294          * expected level - update stats, anyway.
295          */
296         if (rail->dfll_mode) {
297                 rail->millivolts = rail->new_millivolts = millivolts;
298                 dvfs_rail_stats_update(rail, millivolts, ktime_get());
299                 return 0;
300         }
301
302         if (rail->disabled)
303                 return 0;
304
305         rail->resolving_to = true;
306         jmp_to_zero = rail->jmp_to_zero &&
307                         ((millivolts == 0) || (rail->millivolts == 0));
308         steps = jmp_to_zero ? 1 :
309                 DIV_ROUND_UP(abs(millivolts - rail->millivolts), step);
310
311         for (i = 0; i < steps; i++) {
312                 if (!jmp_to_zero &&
313                     (abs(millivolts - rail->millivolts) > step))
314                         rail->new_millivolts = rail->millivolts + offset;
315                 else
316                         rail->new_millivolts = millivolts;
317
318                 /* Before changing the voltage, tell each rail that depends
319                  * on this rail that the voltage will change.
320                  * This rail will be the "from" rail in the relationship,
321                  * the rail that depends on this rail will be the "to" rail.
322                  * from->millivolts will be the old voltage
323                  * from->new_millivolts will be the new voltage */
324                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
325                         ret = dvfs_rail_update(rel->to);
326                         if (ret)
327                                 goto out;
328                 }
329
330                 ret = dvfs_rail_set_voltage_reg(rail, rail->new_millivolts);
331                 if (ret) {
332                         pr_err("Failed to set dvfs regulator %s\n", rail->reg_id);
333                         goto out;
334                 }
335
336                 rail->millivolts = rail->new_millivolts;
337                 dvfs_rail_stats_update(rail, rail->millivolts, ktime_get());
338
339                 /* After changing the voltage, tell each rail that depends
340                  * on this rail that the voltage has changed.
341                  * from->millivolts and from->new_millivolts will be the
342                  * new voltage */
343                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
344                         ret = dvfs_rail_update(rel->to);
345                         if (ret)
346                                 goto out;
347                 }
348         }
349
350         if (unlikely(rail->millivolts != millivolts)) {
351                 pr_err("%s: rail didn't reach target %d in %d steps (%d)\n",
352                         __func__, millivolts, steps, rail->millivolts);
353                 ret = -EINVAL;
354         }
355
356 out:
357         rail->resolving_to = false;
358         return ret;
359 }
360
361 /* Determine the minimum valid voltage for a rail, taking into account
362  * the dvfs clocks and any rails that this rail depends on.  Calls
363  * dvfs_rail_set_voltage with the new voltage, which will call
364  * dvfs_rail_update on any rails that depend on this rail. */
365 static inline int dvfs_rail_apply_limits(struct dvfs_rail *rail, int millivolts)
366 {
367         int min_mv = rail->min_millivolts;
368
369         if (rail->therm_mv_floors) {
370                 int i = rail->therm_floor_idx;
371                 if (i < rail->therm_mv_floors_num)
372                         min_mv = rail->therm_mv_floors[i];
373         }
374
375         if (rail->override_millivolts) {
376                 millivolts = rail->override_millivolts;
377         } else {
378                 /* apply offset and clip up to pll mode fixed mv */
379                 millivolts += rail->dbg_mv_offs;
380                 if (!rail->dfll_mode && rail->fixed_millivolts &&
381                     (millivolts < rail->fixed_millivolts))
382                         millivolts = rail->fixed_millivolts;
383         }
384
385         if (millivolts < min_mv)
386                 millivolts = min_mv;
387
388         return millivolts;
389 }
390
391 static int dvfs_rail_update(struct dvfs_rail *rail)
392 {
393         int millivolts = 0;
394         struct dvfs *d;
395         struct dvfs_relationship *rel;
396         int ret = 0;
397         int steps;
398
399         /* if dvfs is suspended, return and handle it during resume */
400         if (rail->suspended)
401                 return 0;
402
403         /* if regulators are not connected yet, return and handle it later */
404         if (!rail->reg)
405                 return 0;
406
407         /* if no clock has requested voltage since boot, defer update */
408         if (!rail->rate_set)
409                 return 0;
410
411         /* if rail update is entered while resolving circular dependencies,
412            abort recursion */
413         if (rail->resolving_to)
414                 return 0;
415
416         /* Find the maximum voltage requested by any clock */
417         list_for_each_entry(d, &rail->dvfs, reg_node)
418                 millivolts = max(d->cur_millivolts, millivolts);
419
420         /* Apply offset and min/max limits if any clock is requesting voltage */
421         if (millivolts)
422                 millivolts = dvfs_rail_apply_limits(rail, millivolts);
423         /* Keep current voltage if regulator is to be disabled via explicitly */
424         else if (rail->in_band_pm)
425                 return 0;
426         /* Keep current voltage if regulator must not be disabled at run time */
427         else if (!rail->jmp_to_zero) {
428                 WARN(1, "%s cannot be turned off by dvfs\n", rail->reg_id);
429                 return 0;
430         }
431         /* else: fall thru if regulator is turned off by side band signaling */
432
433         /* retry update if limited by from-relationship to account for
434            circular dependencies */
435         steps = DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
436         for (; steps >= 0; steps--) {
437                 rail->new_millivolts = millivolts;
438
439                 /* Check any rails that this rail depends on */
440                 list_for_each_entry(rel, &rail->relationships_from, from_node)
441                         rail->new_millivolts = dvfs_solve_relationship(rel);
442
443                 if (rail->new_millivolts == rail->millivolts)
444                         break;
445
446                 ret = dvfs_rail_set_voltage(rail, rail->new_millivolts);
447         }
448
449         return ret;
450 }
451
452 static struct regulator *get_fixed_regulator(struct dvfs_rail *rail)
453 {
454         struct regulator *reg;
455         char reg_id[80];
456         struct dvfs *d;
457         int v, i;
458         unsigned long dfll_boost;
459
460         strcpy(reg_id, rail->reg_id);
461         strcat(reg_id, "_fixed");
462         reg = regulator_get(NULL, reg_id);
463         if (IS_ERR(reg))
464                 return reg;
465
466         v = regulator_get_voltage(reg) / 1000;
467         if ((v < rail->min_millivolts) || (v > rail->nominal_millivolts) ||
468             (rail->therm_mv_floors && v < rail->therm_mv_floors[0])) {
469                 pr_err("tegra_dvfs: ivalid fixed %s voltage %d\n",
470                        rail->reg_id, v);
471                 return ERR_PTR(-EINVAL);
472         }
473
474         /*
475          * Only fixed at nominal voltage vdd_core regulator is allowed, same
476          * is true for cpu rail if dfll mode is not supported at all. No thermal
477          * capping can be implemented in this case.
478          */
479         if (!IS_ENABLED(CONFIG_ARCH_TEGRA_HAS_CL_DVFS) ||
480             (rail != tegra_cpu_rail)) {
481                 if (v != rail->nominal_millivolts) {
482                         pr_err("tegra_dvfs: %s fixed below nominal at %d\n",
483                                rail->reg_id, v);
484                         return ERR_PTR(-EINVAL);
485                 }
486                 if (rail->therm_mv_caps) {
487                         pr_err("tegra_dvfs: cannot fix %s with thermal caps\n",
488                                rail->reg_id);
489                         return ERR_PTR(-ENOSYS);
490                 }
491                 return reg;
492         }
493
494         /*
495          * If dfll mode is supported, fixed vdd_cpu regulator may be below
496          * nominal in pll mode - maximum cpu rate in pll mode is limited
497          * respectively. Regulator is required to allow automatic scaling
498          * in dfll mode.
499          *
500          * FIXME: platform data to explicitly identify such "hybrid" regulator?
501          */
502         d = list_first_entry(&rail->dvfs, struct dvfs, reg_node);
503         for (i = 0; i < d->num_freqs; i++) {
504                 if (d->millivolts[i] > v)
505                         break;
506         }
507
508         if (!i) {
509                 pr_err("tegra_dvfs: %s fixed at %d: too low for min rate\n",
510                        rail->reg_id, v);
511                 return ERR_PTR(-EINVAL);
512         }
513
514         dfll_boost = (d->freqs[d->num_freqs - 1] - d->freqs[i - 1]);
515         if (d->dfll_data.max_rate_boost < dfll_boost)
516                 d->dfll_data.max_rate_boost = dfll_boost;
517
518         rail->fixed_millivolts = v;
519         return reg;
520 }
521
522 static int dvfs_rail_connect_to_regulator(struct dvfs_rail *rail)
523 {
524         struct regulator *reg;
525         int v;
526
527         if (!rail->reg) {
528                 reg = regulator_get(NULL, rail->reg_id);
529                 if (IS_ERR(reg)) {
530                         reg = get_fixed_regulator(rail);
531                         if (IS_ERR(reg)) {
532                                 pr_err("tegra_dvfs: failed to connect %s rail\n",
533                                        rail->reg_id);
534                                 return PTR_ERR(reg);
535                         }
536                 }
537                 rail->reg = reg;
538         }
539
540         v = regulator_enable(rail->reg);
541         if (v < 0) {
542                 pr_err("tegra_dvfs: failed on enabling regulator %s\n, err %d",
543                         rail->reg_id, v);
544                 return v;
545         }
546
547         v = regulator_get_voltage(rail->reg);
548         if (v < 0) {
549                 pr_err("tegra_dvfs: failed initial get %s voltage\n",
550                        rail->reg_id);
551                 return v;
552         }
553         rail->millivolts = v / 1000;
554         rail->new_millivolts = rail->millivolts;
555         dvfs_rail_stats_init(rail, rail->millivolts);
556
557         if (rail->boot_millivolts &&
558             (rail->boot_millivolts != rail->millivolts)) {
559                 WARN(1, "%s boot voltage %d does not match expected %d\n",
560                      rail->reg_id, rail->millivolts, rail->boot_millivolts);
561                 rail->boot_millivolts = rail->millivolts;
562         }
563         return 0;
564 }
565
566 static inline unsigned long *dvfs_get_freqs(struct dvfs *d)
567 {
568         return d->alt_freqs && d->use_alt_freqs ? d->alt_freqs : &d->freqs[0];
569 }
570
571 static inline const int *dvfs_get_millivolts(struct dvfs *d, unsigned long rate)
572 {
573         if (tegra_dvfs_is_dfll_scale(d, rate))
574                 return d->dfll_millivolts;
575
576         return tegra_dvfs_get_millivolts_pll(d);
577 }
578
579 static int
580 __tegra_dvfs_set_rate(struct dvfs *d, unsigned long rate)
581 {
582         int i = 0;
583         int ret, mv, detach_mv;
584         unsigned long *freqs = dvfs_get_freqs(d);
585         const int *millivolts = dvfs_get_millivolts(d, rate);
586
587         if (freqs == NULL || millivolts == NULL)
588                 return -ENODEV;
589
590         /* On entry to dfll range limit 1st step to range bottom (full ramp of
591            voltage/rate is completed automatically in dfll mode) */
592         if (tegra_dvfs_is_dfll_range_entry(d, rate))
593                 rate = d->dfll_data.use_dfll_rate_min;
594
595         if (rate > freqs[d->num_freqs - 1]) {
596                 pr_warn("tegra_dvfs: rate %lu too high for dvfs on %s\n", rate,
597                         d->clk_name);
598                 return -EINVAL;
599         }
600
601         if (rate == 0) {
602                 d->cur_millivolts = 0;
603         } else {
604                 while (i < d->num_freqs && rate > freqs[i])
605                         i++;
606
607                 mv = millivolts[i];
608
609                 if ((d->max_millivolts) && (mv > d->max_millivolts)) {
610                         pr_warn("tegra_dvfs: voltage %d too high for dvfs on %s\n",
611                                 mv, d->clk_name);
612                         return -EINVAL;
613                 }
614
615                 detach_mv = tegra_dvfs_rail_get_boot_level(d->dvfs_rail);
616                 if (!d->dvfs_rail->reg && (mv > detach_mv)) {
617                         pr_warn("%s: %s: voltage %d above boot limit %d\n",
618                                 __func__, d->clk_name, mv, detach_mv);
619                         return -EINVAL;
620                 }
621
622                 detach_mv = tegra_dvfs_rail_get_disable_level(d->dvfs_rail);
623                 if (d->dvfs_rail->disabled && (mv > detach_mv)) {
624                         pr_warn("%s: %s: voltage %d above disable limit %d\n",
625                                 __func__, d->clk_name, mv, detach_mv);
626                         return -EINVAL;
627                 }
628
629                 detach_mv = tegra_dvfs_rail_get_suspend_level(d->dvfs_rail);
630                 if (d->dvfs_rail->suspended && (mv > detach_mv)) {
631                         pr_warn("%s: %s: voltage %d above disable limit %d\n",
632                                 __func__, d->clk_name, mv, detach_mv);
633                         return -EINVAL;
634                 }
635
636                 detach_mv = d->dvfs_rail->override_millivolts;
637                 if (detach_mv && (mv > detach_mv)) {
638                         pr_warn("%s: %s: voltage %d above override level %d\n",
639                                 __func__, d->clk_name, mv, detach_mv);
640                         return -EINVAL;
641                 }
642                 d->cur_millivolts = mv;
643         }
644
645         d->cur_rate = rate;
646
647         d->dvfs_rail->rate_set = true;
648         ret = dvfs_rail_update(d->dvfs_rail);
649         if (ret)
650                 pr_err("Failed to set regulator %s for clock %s to %d mV\n",
651                         d->dvfs_rail->reg_id, d->clk_name, d->cur_millivolts);
652
653         return ret;
654 }
655
656 /*
657  * Some clocks may have alternative frequency ladder that provides lower minimum
658  * voltage at the same rate (or complimentary: higher maximum rate at the same
659  * voltage). Interfaces below allows dvfs clients to install such ladder, and
660  * switch between primary and alternative frequencies in flight.
661  */
662 static int alt_freqs_validate(struct dvfs *d, unsigned long *alt_freqs)
663 {
664         int i;
665
666         if (alt_freqs) {
667                 for (i = 0; i < d->num_freqs; i++) {
668                         if (d->freqs[i] > alt_freqs[i]) {
669                                 pr_err("%s: Invalid alt freqs for %s\n",
670                                        __func__, d->clk_name);
671                                 return -EINVAL;
672                         }
673                 }
674         }
675         return 0;
676 }
677
678 int tegra_dvfs_alt_freqs_install(struct dvfs *d, unsigned long *alt_freqs)
679 {
680         int ret = 0;
681
682         mutex_lock(&dvfs_lock);
683
684         ret = alt_freqs_validate(d, alt_freqs);
685         if (!ret)
686                 d->alt_freqs = alt_freqs;
687
688         mutex_unlock(&dvfs_lock);
689         return ret;
690 }
691
692 int tegra_dvfs_use_alt_freqs_on_clk(struct clk *c, bool use_alt_freq)
693 {
694         int ret = -ENOENT;
695         struct dvfs *d = c->dvfs;
696
697         mutex_lock(&dvfs_lock);
698
699         if (d && d->alt_freqs) {
700                 ret = 0;
701                 if (d->use_alt_freqs != use_alt_freq) {
702                         d->use_alt_freqs = use_alt_freq;
703                         ret = __tegra_dvfs_set_rate(d, d->cur_rate);
704                 }
705         }
706
707         mutex_unlock(&dvfs_lock);
708         return ret;
709 }
710
711 int tegra_dvfs_alt_freqs_set(struct dvfs *d, unsigned long *alt_freqs)
712 {
713         int ret = 0;
714
715         mutex_lock(&dvfs_lock);
716
717         if (d->alt_freqs != alt_freqs) {
718                 ret = alt_freqs_validate(d, alt_freqs);
719                 if (!ret) {
720                         d->use_alt_freqs = !!alt_freqs;
721                         d->alt_freqs = alt_freqs;
722                         ret = __tegra_dvfs_set_rate(d, d->cur_rate);
723                 }
724         }
725
726         mutex_unlock(&dvfs_lock);
727         return ret;
728 }
729
730 /*
731  * Some clocks may need run-time voltage ladder replacement. Allow it only if
732  * peak voltages across all possible ladders are specified, and new voltages
733  * do not violate peaks.
734  */
735 static int new_voltages_validate(struct dvfs *d, const int *new_millivolts,
736                                  int freqs_num, int ranges_num)
737 {
738         const int *millivolts;
739         int freq_idx, therm_idx;
740
741         for (therm_idx = 0; therm_idx < ranges_num; therm_idx++) {
742                 millivolts = new_millivolts + therm_idx * MAX_DVFS_FREQS;
743                 for (freq_idx = 0; freq_idx < freqs_num; freq_idx++) {
744                         if (millivolts[freq_idx] >
745                             d->peak_millivolts[freq_idx]) {
746                                 pr_err("%s: Invalid new voltages for %s\n",
747                                        __func__, d->clk_name);
748                                 return -EINVAL;
749                         }
750                 }
751         }
752         return 0;
753 }
754
755 int tegra_dvfs_replace_voltage_table(struct dvfs *d, const int *new_millivolts)
756 {
757         int ret = 0;
758         int ranges_num = 1;
759
760         mutex_lock(&dvfs_lock);
761
762         if (!d->peak_millivolts) {
763                 ret = -EINVAL;
764                 goto out;
765         }
766
767         if (d->therm_dvfs && d->dvfs_rail->vts_cdev)
768                 ranges_num += d->dvfs_rail->vts_cdev->trip_temperatures_num;
769
770         if (new_voltages_validate(d, new_millivolts,
771                                   d->num_freqs, ranges_num)) {
772                 ret = -EINVAL;
773                 goto out;
774         }
775
776         d->millivolts = new_millivolts;
777         if (__tegra_dvfs_set_rate(d, d->cur_rate))
778                 ret = -EAGAIN;
779 out:
780         mutex_unlock(&dvfs_lock);
781         return ret;
782 }
783
784 /*
785  *  Using non alt frequencies always results in peak voltage
786  * (enforced by alt_freqs_validate())
787  */
788 static int predict_non_alt_millivolts(struct clk *c, const int *millivolts,
789                                       unsigned long rate)
790 {
791         int i;
792
793         if (!millivolts)
794                 return -ENODEV;
795
796         for (i = 0; i < c->dvfs->num_freqs; i++) {
797                 if (rate <= c->dvfs->freqs[i])
798                         break;
799         }
800
801         if (i == c->dvfs->num_freqs)
802                 i--;
803
804         return millivolts[i];
805 }
806
807 static int predict_millivolts(struct clk *c, const int *millivolts,
808                               unsigned long rate)
809 {
810         /*
811          * Predicted voltage can not be used across the switch to alternative
812          * frequency limits. For now, just fail the call for clock that has
813          * alternative limits initialized.
814          */
815         if (c->dvfs->alt_freqs)
816                 return -ENOSYS;
817
818         return predict_non_alt_millivolts(c, millivolts, rate);
819 }
820
821 int tegra_dvfs_predict_millivolts(struct clk *c, unsigned long rate)
822 {
823         const int *millivolts;
824
825         if (!rate || !c->dvfs)
826                 return 0;
827
828         millivolts = tegra_dvfs_is_dfll_range(c->dvfs, rate) ?
829                 c->dvfs->dfll_millivolts :
830                 tegra_dvfs_get_millivolts_pll(c->dvfs);
831         return predict_millivolts(c, millivolts, rate);
832 }
833
834 int tegra_dvfs_predict_peak_millivolts(struct clk *c, unsigned long rate)
835 {
836         int mv;
837         const int *millivolts;
838
839         if (!rate || !c->dvfs)
840                 return 0;
841
842         millivolts = tegra_dvfs_is_dfll_range(c->dvfs, rate) ?
843                         c->dvfs->dfll_millivolts : c->dvfs->peak_millivolts ? :
844                         tegra_dvfs_get_millivolts_pll(c->dvfs);
845
846         mv = predict_non_alt_millivolts(c, millivolts, rate);
847         if (mv < 0)
848                 return mv;
849
850         if (c->dvfs->dvfs_rail->therm_mv_floors)
851                 mv = max(mv, c->dvfs->dvfs_rail->therm_mv_floors[0]);
852         return mv;
853 }
854
855 const int *tegra_dvfs_get_millivolts_pll(struct dvfs *d)
856 {
857         if (d->therm_dvfs) {
858                 int therm_idx = d->dvfs_rail->therm_scale_idx;
859                 return d->millivolts + therm_idx * MAX_DVFS_FREQS;
860         }
861         return d->millivolts;
862 }
863
864 int tegra_dvfs_set_rate(struct clk *c, unsigned long rate)
865 {
866         int ret;
867
868         if (!c->dvfs)
869                 return -EINVAL;
870
871         mutex_lock(&dvfs_lock);
872         ret = __tegra_dvfs_set_rate(c->dvfs, rate);
873         mutex_unlock(&dvfs_lock);
874
875         return ret;
876 }
877 EXPORT_SYMBOL(tegra_dvfs_set_rate);
878
879 int tegra_dvfs_get_freqs(struct clk *c, unsigned long **freqs, int *num_freqs)
880 {
881         if (!c->dvfs)
882                 return -ENOSYS;
883
884         if (c->dvfs->alt_freqs)
885                 return -ENOSYS;
886
887         *num_freqs = c->dvfs->num_freqs;
888         *freqs = c->dvfs->freqs;
889
890         return 0;
891 }
892 EXPORT_SYMBOL(tegra_dvfs_get_freqs);
893
894 static inline int dvfs_rail_get_override_floor(struct dvfs_rail *rail)
895 {
896         return rail->override_unresolved ? rail->nominal_millivolts :
897                 rail->min_override_millivolts;
898 }
899
900 #ifdef CONFIG_TEGRA_VDD_CORE_OVERRIDE
901 static DEFINE_MUTEX(rail_override_lock);
902
903 static int dvfs_override_core_voltage(int override_mv)
904 {
905         int ret, floor, ceiling;
906         struct dvfs_rail *rail = tegra_core_rail;
907
908         if (!rail)
909                 return -ENOENT;
910
911         if (rail->fixed_millivolts)
912                 return -ENOSYS;
913
914         mutex_lock(&rail_override_lock);
915
916         floor = dvfs_rail_get_override_floor(rail);
917         ceiling = rail->nominal_millivolts;
918         if (override_mv && ((override_mv < floor) || (override_mv > ceiling))) {
919                 pr_err("%s: override level %d outside the range [%d...%d]\n",
920                        __func__, override_mv, floor, ceiling);
921                 mutex_unlock(&rail_override_lock);
922                 return -EINVAL;
923         }
924
925         if (override_mv == rail->override_millivolts) {
926                 ret = 0;
927                 goto out;
928         }
929
930         if (override_mv) {
931                 ret = tegra_dvfs_override_core_cap_apply(override_mv);
932                 if (ret) {
933                         pr_err("%s: failed to set cap for override level %d\n",
934                                __func__, override_mv);
935                         goto out;
936                 }
937         }
938
939         mutex_lock(&dvfs_lock);
940         if (rail->disabled || rail->suspended) {
941                 pr_err("%s: cannot scale %s rail\n", __func__,
942                        rail->disabled ? "disabled" : "suspended");
943                 ret = -EPERM;
944                 if (!override_mv) {
945                         mutex_unlock(&dvfs_lock);
946                         goto out;
947                 }
948         } else {
949                 rail->override_millivolts = override_mv;
950                 ret = dvfs_rail_update(rail);
951                 if (ret) {
952                         pr_err("%s: failed to set override level %d\n",
953                                __func__, override_mv);
954                         rail->override_millivolts = 0;
955                         dvfs_rail_update(rail);
956                 }
957         }
958         mutex_unlock(&dvfs_lock);
959
960         if (!override_mv || ret)
961                 tegra_dvfs_override_core_cap_apply(0);
962 out:
963         mutex_unlock(&rail_override_lock);
964         return ret;
965 }
966
967 int tegra_dvfs_resolve_override(struct clk *c, unsigned long max_rate)
968 {
969         int mv;
970         struct dvfs *d = c->dvfs;
971         struct dvfs_rail *rail;
972
973         if (!d)
974                 return 0;
975         rail = d->dvfs_rail;
976
977         mutex_lock(&rail_override_lock);
978         mutex_lock(&dvfs_lock);
979
980         if (d->defer_override && rail->override_unresolved) {
981                 d->defer_override = false;
982
983                 mv = tegra_dvfs_predict_peak_millivolts(c, max_rate);
984                 if (rail->min_override_millivolts < mv)
985                         rail->min_override_millivolts = mv;
986
987                 rail->override_unresolved--;
988                 if (!rail->override_unresolved && rail->resolve_override)
989                         rail->resolve_override(rail->min_override_millivolts);
990         }
991         mutex_unlock(&dvfs_lock);
992         mutex_unlock(&rail_override_lock);
993         return 0;
994 }
995
996 int tegra_dvfs_rail_get_override_floor(struct dvfs_rail *rail)
997 {
998         if (rail) {
999                 int mv;
1000                 mutex_lock(&rail_override_lock);
1001                 mv = dvfs_rail_get_override_floor(rail);
1002                 mutex_unlock(&rail_override_lock);
1003                 return mv;
1004         }
1005         return -ENOENT;
1006 }
1007
1008 static int dvfs_set_fmax_at_vmin(struct clk *c, unsigned long f_max, int v_min)
1009 {
1010         int i, ret = 0;
1011         struct dvfs *d = c->dvfs;
1012         unsigned long f_min = 1000;     /* 1kHz min rate in DVFS tables */
1013
1014         mutex_lock(&rail_override_lock);
1015         mutex_lock(&dvfs_lock);
1016
1017         if (v_min > d->dvfs_rail->override_millivolts) {
1018                 pr_err("%s: new %s vmin %dmV is above override voltage %dmV\n",
1019                        __func__, c->name, v_min,
1020                        d->dvfs_rail->override_millivolts);
1021                 ret = -EPERM;
1022                 goto out;
1023         }
1024
1025         if (v_min >= d->max_millivolts) {
1026                 pr_err("%s: new %s vmin %dmV is at/above max voltage %dmV\n",
1027                        __func__, c->name, v_min, d->max_millivolts);
1028                 ret = -EINVAL;
1029                 goto out;
1030         }
1031
1032         /*
1033          * dvfs table update:
1034          * - for voltages below new v_min the respective frequencies are shifted
1035          * below new f_max to the levels already present in the table; if the
1036          * 1st table entry has frequency above new fmax, all entries below v_min
1037          * are filled in with 1kHz (min rate used in DVFS tables).
1038          * - for voltages above new v_min, the respective frequencies are
1039          * increased to at least new f_max
1040          * - if new v_min is already in the table set the respective frequency
1041          * to new f_max
1042          */
1043         for (i = 0; i < d->num_freqs; i++) {
1044                 int mv = d->millivolts[i];
1045                 unsigned long f = d->freqs[i];
1046
1047                 if (mv < v_min) {
1048                         if (d->freqs[i] >= f_max)
1049                                 d->freqs[i] = i ? d->freqs[i-1] : f_min;
1050                 } else if (mv > v_min) {
1051                         d->freqs[i] = max(f, f_max);
1052                 } else {
1053                         d->freqs[i] = f_max;
1054                 }
1055                 ret = __tegra_dvfs_set_rate(d, d->cur_rate);
1056         }
1057 out:
1058         mutex_unlock(&dvfs_lock);
1059         mutex_unlock(&rail_override_lock);
1060
1061         return ret;
1062 }
1063 #else
1064 static int dvfs_override_core_voltage(int override_mv)
1065 {
1066         pr_err("%s: vdd core override is not supported\n", __func__);
1067         return -ENOSYS;
1068 }
1069
1070 static int dvfs_set_fmax_at_vmin(struct clk *c, unsigned long f_max, int v_min)
1071 {
1072         pr_err("%s: vdd core override is not supported\n", __func__);
1073         return -ENOSYS;
1074 }
1075 #endif
1076
1077 int tegra_dvfs_override_core_voltage(struct clk *c, int override_mv)
1078 {
1079         if (!c->dvfs || !c->dvfs->can_override) {
1080                 pr_err("%s: %s cannot override vdd core\n", __func__, c->name);
1081                 return -EPERM;
1082         }
1083         return dvfs_override_core_voltage(override_mv);
1084 }
1085 EXPORT_SYMBOL(tegra_dvfs_override_core_voltage);
1086
1087 int tegra_dvfs_set_fmax_at_vmin(struct clk *c, unsigned long f_max, int v_min)
1088 {
1089         if (!c->dvfs || !c->dvfs->can_override) {
1090                 pr_err("%s: %s cannot set fmax_at_vmin)\n", __func__, c->name);
1091                 return -EPERM;
1092         }
1093         return dvfs_set_fmax_at_vmin(c, f_max, v_min);
1094 }
1095 EXPORT_SYMBOL(tegra_dvfs_set_fmax_at_vmin);
1096
1097 /* May only be called during clock init, does not take any locks on clock c. */
1098 int __init tegra_enable_dvfs_on_clk(struct clk *c, struct dvfs *d)
1099 {
1100         int i;
1101
1102         if (c->dvfs) {
1103                 pr_err("Error when enabling dvfs on %s for clock %s:\n",
1104                         d->dvfs_rail->reg_id, c->name);
1105                 pr_err("DVFS already enabled for %s\n",
1106                         c->dvfs->dvfs_rail->reg_id);
1107                 return -EINVAL;
1108         }
1109
1110         for (i = 0; i < MAX_DVFS_FREQS; i++) {
1111                 if (d->millivolts[i] == 0)
1112                         break;
1113
1114                 d->freqs[i] *= d->freqs_mult;
1115
1116                 /* If final frequencies are 0, pad with previous frequency */
1117                 if (d->freqs[i] == 0 && i > 1)
1118                         d->freqs[i] = d->freqs[i - 1];
1119         }
1120         d->num_freqs = i;
1121
1122         if (d->auto_dvfs) {
1123                 c->auto_dvfs = true;
1124                 clk_set_cansleep(c);
1125         }
1126
1127         c->dvfs = d;
1128
1129         /*
1130          * Minimum core override level is determined as maximum voltage required
1131          * for clocks outside shared buses (shared bus rates can be capped to
1132          * safe levels when override limit is set)
1133          */
1134         if (i && c->ops && !c->ops->shared_bus_update &&
1135             !(c->flags & PERIPH_ON_CBUS) && !d->can_override) {
1136                 int mv = tegra_dvfs_predict_peak_millivolts(c, d->freqs[i-1]);
1137                 struct dvfs_rail *rail = d->dvfs_rail;
1138                 if (d->defer_override)
1139                         rail->override_unresolved++;
1140                 else if (rail->min_override_millivolts < mv)
1141                         rail->min_override_millivolts =
1142                                 min(mv, rail->nominal_millivolts);
1143         }
1144
1145         mutex_lock(&dvfs_lock);
1146         list_add_tail(&d->reg_node, &d->dvfs_rail->dvfs);
1147         mutex_unlock(&dvfs_lock);
1148
1149         return 0;
1150 }
1151
1152 static bool tegra_dvfs_all_rails_suspended(void)
1153 {
1154         struct dvfs_rail *rail;
1155         bool all_suspended = true;
1156
1157         list_for_each_entry(rail, &dvfs_rail_list, node)
1158                 if (!rail->suspended && !rail->disabled)
1159                         all_suspended = false;
1160
1161         return all_suspended;
1162 }
1163
1164 static bool tegra_dvfs_from_rails_suspended_or_solved(struct dvfs_rail *to)
1165 {
1166         struct dvfs_relationship *rel;
1167         bool all_suspended = true;
1168
1169         list_for_each_entry(rel, &to->relationships_from, from_node)
1170                 if (!rel->from->suspended && !rel->from->disabled &&
1171                         !rel->solved_at_nominal)
1172                         all_suspended = false;
1173
1174         return all_suspended;
1175 }
1176
1177 static int tegra_dvfs_suspend_one(void)
1178 {
1179         struct dvfs_rail *rail;
1180         int ret, mv;
1181
1182         list_for_each_entry(rail, &dvfs_rail_list, node) {
1183                 if (!rail->suspended && !rail->disabled &&
1184                     tegra_dvfs_from_rails_suspended_or_solved(rail)) {
1185                         /* Safe, as pll mode rate is capped to fixed level */
1186                         if (!rail->dfll_mode && rail->fixed_millivolts) {
1187                                 mv = rail->fixed_millivolts;
1188                         } else {
1189                                 mv = tegra_dvfs_rail_get_suspend_level(rail);
1190                                 mv = dvfs_rail_apply_limits(rail, mv);
1191                         }
1192
1193                         /* apply suspend limit only if it is above current mv */
1194                         ret = -EPERM;
1195                         if (mv >= rail->millivolts)
1196                                 ret = dvfs_rail_set_voltage(rail, mv);
1197                         if (ret) {
1198                                 pr_err("tegra_dvfs: failed %s suspend at %d\n",
1199                                        rail->reg_id, rail->millivolts);
1200                                 return ret;
1201                         }
1202
1203                         rail->suspended = true;
1204                         return 0;
1205                 }
1206         }
1207
1208         return -EINVAL;
1209 }
1210
1211 static void tegra_dvfs_resume(void)
1212 {
1213         struct dvfs_rail *rail;
1214
1215         mutex_lock(&dvfs_lock);
1216
1217         list_for_each_entry(rail, &dvfs_rail_list, node)
1218                 rail->suspended = false;
1219
1220         list_for_each_entry(rail, &dvfs_rail_list, node)
1221                 dvfs_rail_update(rail);
1222
1223         mutex_unlock(&dvfs_lock);
1224 }
1225
1226 static int tegra_dvfs_suspend(void)
1227 {
1228         int ret = 0;
1229
1230         mutex_lock(&dvfs_lock);
1231
1232         while (!tegra_dvfs_all_rails_suspended()) {
1233                 ret = tegra_dvfs_suspend_one();
1234                 if (ret)
1235                         break;
1236         }
1237
1238         mutex_unlock(&dvfs_lock);
1239
1240         if (ret)
1241                 tegra_dvfs_resume();
1242
1243         return ret;
1244 }
1245
1246 static int tegra_dvfs_pm_suspend(struct notifier_block *nb,
1247                                  unsigned long event, void *data)
1248 {
1249         if (event == PM_SUSPEND_PREPARE) {
1250                 if (tegra_dvfs_suspend())
1251                         return NOTIFY_STOP;
1252                 pr_info("tegra_dvfs: suspended\n");
1253         }
1254         return NOTIFY_OK;
1255 };
1256
1257 static int tegra_dvfs_pm_resume(struct notifier_block *nb,
1258                                 unsigned long event, void *data)
1259 {
1260         if (event == PM_POST_SUSPEND) {
1261                 tegra_dvfs_resume();
1262                 pr_info("tegra_dvfs: resumed\n");
1263         }
1264         return NOTIFY_OK;
1265 };
1266
1267 static struct notifier_block tegra_dvfs_suspend_nb = {
1268         .notifier_call = tegra_dvfs_pm_suspend,
1269         .priority = -1,
1270 };
1271
1272 static struct notifier_block tegra_dvfs_resume_nb = {
1273         .notifier_call = tegra_dvfs_pm_resume,
1274         .priority = 1,
1275 };
1276
1277 static int tegra_dvfs_reboot_notify(struct notifier_block *nb,
1278                                 unsigned long event, void *data)
1279 {
1280         switch (event) {
1281         case SYS_RESTART:
1282         case SYS_HALT:
1283         case SYS_POWER_OFF:
1284                 tegra_dvfs_suspend();
1285                 return NOTIFY_OK;
1286         }
1287         return NOTIFY_DONE;
1288 }
1289
1290 static struct notifier_block tegra_dvfs_reboot_nb = {
1291         .notifier_call = tegra_dvfs_reboot_notify,
1292 };
1293
1294 /* must be called with dvfs lock held */
1295 static void __tegra_dvfs_rail_disable(struct dvfs_rail *rail)
1296 {
1297         int ret = -EPERM;
1298         int mv;
1299
1300         /* don't set voltage in DFLL mode - won't work, but break stats */
1301         if (rail->dfll_mode) {
1302                 rail->disabled = true;
1303                 return;
1304         }
1305
1306         /* Safe, as pll mode rate is capped to fixed level */
1307         if (!rail->dfll_mode && rail->fixed_millivolts) {
1308                 mv = rail->fixed_millivolts;
1309         } else {
1310                 mv = tegra_dvfs_rail_get_disable_level(rail);
1311                 mv = dvfs_rail_apply_limits(rail, mv);
1312         }
1313
1314         /* apply detach mode limit provided it is above current volatge */
1315         if (mv >= rail->millivolts)
1316                 ret = dvfs_rail_set_voltage(rail, mv);
1317         if (ret) {
1318                 pr_err("tegra_dvfs: failed to disable %s at %d\n",
1319                        rail->reg_id, rail->millivolts);
1320                 return;
1321         }
1322         rail->disabled = true;
1323 }
1324
1325 /* must be called with dvfs lock held */
1326 static void __tegra_dvfs_rail_enable(struct dvfs_rail *rail)
1327 {
1328         rail->disabled = false;
1329         dvfs_rail_update(rail);
1330 }
1331
1332 void tegra_dvfs_rail_enable(struct dvfs_rail *rail)
1333 {
1334         if (!rail)
1335                 return;
1336
1337         mutex_lock(&rail_disable_lock);
1338
1339         if (rail->disabled) {
1340                 mutex_lock(&dvfs_lock);
1341                 __tegra_dvfs_rail_enable(rail);
1342                 mutex_unlock(&dvfs_lock);
1343
1344                 tegra_dvfs_rail_post_enable(rail);
1345         }
1346         mutex_unlock(&rail_disable_lock);
1347 }
1348
1349 void tegra_dvfs_rail_disable(struct dvfs_rail *rail)
1350 {
1351         if (!rail)
1352                 return;
1353
1354         mutex_lock(&rail_disable_lock);
1355         if (rail->disabled)
1356                 goto out;
1357
1358         /* rail disable will set it to nominal voltage underneath clock
1359            framework - need to re-configure clock rates that are not safe
1360            at nominal (yes, unsafe at nominal is ugly, but possible). Rate
1361            change must be done outside of dvfs lock. */
1362         if (tegra_dvfs_rail_disable_prepare(rail)) {
1363                 pr_info("dvfs: failed to prepare regulator %s to disable\n",
1364                         rail->reg_id);
1365                 goto out;
1366         }
1367
1368         mutex_lock(&dvfs_lock);
1369         __tegra_dvfs_rail_disable(rail);
1370         mutex_unlock(&dvfs_lock);
1371 out:
1372         mutex_unlock(&rail_disable_lock);
1373 }
1374
1375 int tegra_dvfs_rail_disable_by_name(const char *reg_id)
1376 {
1377         struct dvfs_rail *rail = tegra_dvfs_get_rail_by_name(reg_id);
1378         if (!rail)
1379                 return -EINVAL;
1380
1381         tegra_dvfs_rail_disable(rail);
1382         return 0;
1383 }
1384
1385 struct dvfs_rail *tegra_dvfs_get_rail_by_name(const char *reg_id)
1386 {
1387         struct dvfs_rail *rail;
1388
1389         mutex_lock(&dvfs_lock);
1390         list_for_each_entry(rail, &dvfs_rail_list, node) {
1391                 if (!strcmp(reg_id, rail->reg_id)) {
1392                         mutex_unlock(&dvfs_lock);
1393                         return rail;
1394                 }
1395         }
1396         mutex_unlock(&dvfs_lock);
1397         return NULL;
1398 }
1399
1400 int tegra_dvfs_rail_power_up(struct dvfs_rail *rail)
1401 {
1402         int ret = -ENOENT;
1403
1404         if (!rail || !rail->in_band_pm)
1405                 return -ENOSYS;
1406
1407         mutex_lock(&dvfs_lock);
1408         if (rail->reg) {
1409                 ret = regulator_enable(rail->reg);
1410                 if (!ret && !timekeeping_suspended)
1411                         tegra_dvfs_rail_on(rail, ktime_get());
1412         }
1413         mutex_unlock(&dvfs_lock);
1414         return ret;
1415 }
1416
1417 int tegra_dvfs_rail_power_down(struct dvfs_rail *rail)
1418 {
1419         int ret = -ENOENT;
1420
1421         if (!rail || !rail->in_band_pm)
1422                 return -ENOSYS;
1423
1424         mutex_lock(&dvfs_lock);
1425         if (rail->reg) {
1426                 ret = regulator_disable(rail->reg);
1427                 if (!ret && !timekeeping_suspended)
1428                         tegra_dvfs_rail_off(rail, ktime_get());
1429         }
1430         mutex_unlock(&dvfs_lock);
1431         return ret;
1432 }
1433
1434 bool tegra_dvfs_is_rail_up(struct dvfs_rail *rail)
1435 {
1436         bool ret = false;
1437
1438         if (!rail)
1439                 return false;
1440
1441         if (!rail->in_band_pm)
1442                 return true;
1443
1444         mutex_lock(&dvfs_lock);
1445         if (rail->reg)
1446                 ret = regulator_is_enabled(rail->reg) > 0;
1447         mutex_unlock(&dvfs_lock);
1448         return ret;
1449 }
1450
1451 int tegra_dvfs_rail_set_mode(struct dvfs_rail *rail, unsigned int mode)
1452 {
1453         int ret = -ENOENT;
1454
1455         if (!rail)
1456                 return ret;
1457
1458         pr_debug("%s: updating %s mode from %u to %u\n", __func__,
1459                 rail->reg_id, regulator_get_mode(rail->reg), mode);
1460
1461         if (rail->reg)
1462                 ret = regulator_set_mode(rail->reg, mode);
1463
1464         if (ret)
1465                 pr_err("Failed to set dvfs regulator %s mode %u\n",
1466                         rail->reg_id, mode);
1467         return ret;
1468 }
1469
1470 bool tegra_dvfs_rail_updating(struct clk *clk)
1471 {
1472         return (!clk ? false :
1473                 (!clk->dvfs ? false :
1474                  (!clk->dvfs->dvfs_rail ? false :
1475                   (clk->dvfs->dvfs_rail->updating ||
1476                    clk->dvfs->dvfs_rail->dfll_mode_updating))));
1477 }
1478
1479 #ifdef CONFIG_OF
1480 int __init of_tegra_dvfs_init(const struct of_device_id *matches)
1481 {
1482         int ret;
1483         struct device_node *np;
1484
1485         for_each_matching_node(np, matches) {
1486                 const struct of_device_id *match = of_match_node(matches, np);
1487                 of_tegra_dvfs_init_cb_t dvfs_init_cb = match->data;
1488                 ret = dvfs_init_cb(np);
1489                 if (ret) {
1490                         pr_err("dt: Failed to read %s tables from DT\n",
1491                                                         match->compatible);
1492                         return ret;
1493                 }
1494         }
1495         return 0;
1496 }
1497 #endif
1498 int tegra_dvfs_dfll_mode_set(struct dvfs *d, unsigned long rate)
1499 {
1500         mutex_lock(&dvfs_lock);
1501         if (!d->dvfs_rail->dfll_mode) {
1502                 d->dvfs_rail->dfll_mode = true;
1503                 __tegra_dvfs_set_rate(d, rate);
1504         }
1505         mutex_unlock(&dvfs_lock);
1506         return 0;
1507 }
1508
1509 int tegra_dvfs_dfll_mode_clear(struct dvfs *d, unsigned long rate)
1510 {
1511         int ret = 0;
1512
1513         mutex_lock(&dvfs_lock);
1514         if (d->dvfs_rail->dfll_mode) {
1515                 d->dvfs_rail->dfll_mode = false;
1516                 /* avoid false detection of matching target (voltage in dfll
1517                    mode is fluctuating, and recorded level is just estimate) */
1518                 d->dvfs_rail->millivolts--;
1519                 if (d->dvfs_rail->disabled) {
1520                         d->dvfs_rail->disabled = false;
1521                         __tegra_dvfs_rail_disable(d->dvfs_rail);
1522                 }
1523                 ret = __tegra_dvfs_set_rate(d, rate);
1524         }
1525         mutex_unlock(&dvfs_lock);
1526         return ret;
1527 }
1528
1529 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmax_cdev(void)
1530 {
1531         if (tegra_cpu_rail)
1532                 return tegra_cpu_rail->vmax_cdev;
1533         return NULL;
1534 }
1535
1536 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmin_cdev(void)
1537 {
1538         if (tegra_cpu_rail)
1539                 return tegra_cpu_rail->vmin_cdev;
1540         return NULL;
1541 }
1542
1543 struct tegra_cooling_device *tegra_dvfs_get_core_vmax_cdev(void)
1544 {
1545         if (tegra_core_rail)
1546                 return tegra_core_rail->vmax_cdev;
1547         return NULL;
1548 }
1549
1550 struct tegra_cooling_device *tegra_dvfs_get_core_vmin_cdev(void)
1551 {
1552         if (tegra_core_rail)
1553                 return tegra_core_rail->vmin_cdev;
1554         return NULL;
1555 }
1556
1557 struct tegra_cooling_device *tegra_dvfs_get_gpu_vmin_cdev(void)
1558 {
1559         if (tegra_gpu_rail)
1560                 return tegra_gpu_rail->vmin_cdev;
1561         return NULL;
1562 }
1563
1564 struct tegra_cooling_device *tegra_dvfs_get_gpu_vts_cdev(void)
1565 {
1566         if (tegra_gpu_rail)
1567                 return tegra_gpu_rail->vts_cdev;
1568         return NULL;
1569 }
1570
1571 static void make_safe_thermal_dvfs(struct dvfs_rail *rail)
1572 {
1573         struct dvfs *d;
1574
1575         mutex_lock(&dvfs_lock);
1576         list_for_each_entry(d, &rail->dvfs, reg_node) {
1577                 if (d->therm_dvfs) {
1578                         BUG_ON(!d->peak_millivolts);
1579                         d->millivolts = d->peak_millivolts;
1580                         d->therm_dvfs = false;
1581                 }
1582         }
1583         mutex_unlock(&dvfs_lock);
1584 }
1585
1586 #ifdef CONFIG_THERMAL
1587 /* Cooling device limits minimum rail voltage at cold temperature in pll mode */
1588 static int tegra_dvfs_rail_get_vmin_cdev_max_state(
1589         struct thermal_cooling_device *cdev, unsigned long *max_state)
1590 {
1591         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1592         *max_state = rail->vmin_cdev->trip_temperatures_num;
1593         return 0;
1594 }
1595
1596 static int tegra_dvfs_rail_get_vmin_cdev_cur_state(
1597         struct thermal_cooling_device *cdev, unsigned long *cur_state)
1598 {
1599         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1600         *cur_state = rail->therm_floor_idx;
1601         return 0;
1602 }
1603
1604 static int tegra_dvfs_rail_set_vmin_cdev_state(
1605         struct thermal_cooling_device *cdev, unsigned long cur_state)
1606 {
1607         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1608
1609         mutex_lock(&dvfs_lock);
1610         if (rail->therm_floor_idx != cur_state) {
1611                 rail->therm_floor_idx = cur_state;
1612                 dvfs_rail_update(rail);
1613         }
1614         mutex_unlock(&dvfs_lock);
1615         return 0;
1616 }
1617
1618 static struct thermal_cooling_device_ops tegra_dvfs_vmin_cooling_ops = {
1619         .get_max_state = tegra_dvfs_rail_get_vmin_cdev_max_state,
1620         .get_cur_state = tegra_dvfs_rail_get_vmin_cdev_cur_state,
1621         .set_cur_state = tegra_dvfs_rail_set_vmin_cdev_state,
1622 };
1623
1624 static void tegra_dvfs_rail_register_vmin_cdev(struct dvfs_rail *rail)
1625 {
1626         if (!rail->vmin_cdev)
1627                 return;
1628
1629         /* just report error - initialized for cold temperature, anyway */
1630         if (IS_ERR_OR_NULL(thermal_cooling_device_register(
1631                 rail->vmin_cdev->cdev_type, (void *)rail,
1632                 &tegra_dvfs_vmin_cooling_ops)))
1633                 pr_err("tegra cooling device %s failed to register\n",
1634                        rail->vmin_cdev->cdev_type);
1635 }
1636
1637 /*
1638  * Cooling device limits frequencies of the clocks in pll mode based on rail
1639  * vmax thermal profile. Supported for core rail only, and applied only to
1640  * shared buses selected by platform specific code.
1641  */
1642 static int tegra_dvfs_rail_get_vmax_cdev_max_state(
1643         struct thermal_cooling_device *cdev, unsigned long *max_state)
1644 {
1645         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1646         *max_state = rail->vmax_cdev->trip_temperatures_num;
1647         return 0;
1648 }
1649
1650 static int tegra_dvfs_rail_get_vmax_cdev_cur_state(
1651         struct thermal_cooling_device *cdev, unsigned long *cur_state)
1652 {
1653         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1654         *cur_state = rail->therm_cap_idx;
1655         return 0;
1656 }
1657
1658 static int tegra_dvfs_rail_set_vmax_cdev_state(
1659         struct thermal_cooling_device *cdev, unsigned long cur_state)
1660 {
1661         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1662         int cur_cap = cur_state ? rail->therm_mv_caps[cur_state - 1] : 0;
1663
1664         return tegra_dvfs_therm_vmax_core_cap_apply(&rail->therm_cap_idx,
1665                                                     cur_state, cur_cap);
1666 }
1667
1668 static struct thermal_cooling_device_ops tegra_dvfs_vmax_cooling_ops = {
1669         .get_max_state = tegra_dvfs_rail_get_vmax_cdev_max_state,
1670         .get_cur_state = tegra_dvfs_rail_get_vmax_cdev_cur_state,
1671         .set_cur_state = tegra_dvfs_rail_set_vmax_cdev_state,
1672 };
1673
1674 void tegra_dvfs_rail_register_vmax_cdev(struct dvfs_rail *rail)
1675 {
1676         struct thermal_cooling_device *dev;
1677
1678         if (!rail || !rail->vmax_cdev || (rail != tegra_core_rail))
1679                 return;
1680
1681         dev = thermal_cooling_device_register(rail->vmax_cdev->cdev_type,
1682                 (void *)rail, &tegra_dvfs_vmax_cooling_ops);
1683
1684         if (IS_ERR_OR_NULL(dev) || list_empty(&dev->thermal_instances)) {
1685                 /* report error & set the most agressive caps */
1686                 int cur_state = rail->vmax_cdev->trip_temperatures_num;
1687                 int cur_cap = rail->therm_mv_caps[cur_state - 1];
1688                 tegra_dvfs_therm_vmax_core_cap_apply(&rail->therm_cap_idx,
1689                                                      cur_state, cur_cap);
1690                 pr_err("tegra cooling device %s failed to register\n",
1691                        rail->vmax_cdev->cdev_type);
1692         }
1693 }
1694
1695 /* Cooling device to scale voltage with temperature in pll mode */
1696 static int tegra_dvfs_rail_get_vts_cdev_max_state(
1697         struct thermal_cooling_device *cdev, unsigned long *max_state)
1698 {
1699         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1700         *max_state = rail->vts_cdev->trip_temperatures_num;
1701         return 0;
1702 }
1703
1704 static int tegra_dvfs_rail_get_vts_cdev_cur_state(
1705         struct thermal_cooling_device *cdev, unsigned long *cur_state)
1706 {
1707         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1708         *cur_state = rail->therm_scale_idx;
1709         return 0;
1710 }
1711
1712 static int tegra_dvfs_rail_set_vts_cdev_state(
1713         struct thermal_cooling_device *cdev, unsigned long cur_state)
1714 {
1715         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1716         struct dvfs *d;
1717
1718         mutex_lock(&dvfs_lock);
1719         if (rail->therm_scale_idx != cur_state) {
1720                 rail->therm_scale_idx = cur_state;
1721                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1722                         if (d->therm_dvfs)
1723                                 __tegra_dvfs_set_rate(d, d->cur_rate);
1724                 }
1725         }
1726         mutex_unlock(&dvfs_lock);
1727         return 0;
1728 }
1729
1730 static struct thermal_cooling_device_ops tegra_dvfs_vts_cooling_ops = {
1731         .get_max_state = tegra_dvfs_rail_get_vts_cdev_max_state,
1732         .get_cur_state = tegra_dvfs_rail_get_vts_cdev_cur_state,
1733         .set_cur_state = tegra_dvfs_rail_set_vts_cdev_state,
1734 };
1735
1736 static void tegra_dvfs_rail_register_vts_cdev(struct dvfs_rail *rail)
1737 {
1738         struct thermal_cooling_device *dev;
1739
1740         if (!rail->vts_cdev)
1741                 return;
1742
1743         dev = thermal_cooling_device_register(rail->vts_cdev->cdev_type,
1744                 (void *)rail, &tegra_dvfs_vts_cooling_ops);
1745         /* report error & set max limits across thermal ranges as safe dvfs */
1746         if (IS_ERR_OR_NULL(dev) || list_empty(&dev->thermal_instances)) {
1747                 pr_err("tegra cooling device %s failed to register\n",
1748                        rail->vts_cdev->cdev_type);
1749                 make_safe_thermal_dvfs(rail);
1750         }
1751 }
1752
1753 #else
1754 #define tegra_dvfs_rail_register_vmin_cdev(rail)
1755 void tegra_dvfs_rail_register_vmax_cdev(struct dvfs_rail *rail)
1756 { }
1757 static inline void tegra_dvfs_rail_register_vts_cdev(struct dvfs_rail *rail)
1758 {
1759         make_safe_thermal_dvfs(rail);
1760 }
1761 #endif
1762
1763 /*
1764  * Validate rail SiMon Vmin offsets. Valid offsets should be negative,
1765  * descending, starting from zero.
1766  */
1767 void __init tegra_dvfs_rail_init_simon_vmin_offsets(
1768         int *offsets, int offs_num, struct dvfs_rail *rail)
1769 {
1770         int i;
1771
1772         if (!offsets || !offs_num || offsets[0]) {
1773                 WARN(1, "%s: invalid initial SiMon offset\n", rail->reg_id);
1774                 return;
1775         }
1776
1777         for (i = 0; i < offs_num - 1; i++) {
1778                 if (offsets[i] < offsets[i+1]) {
1779                         WARN(1, "%s: SiMon offsets are not ordered\n",
1780                              rail->reg_id);
1781                         return;
1782                 }
1783         }
1784         rail->simon_vmin_offsets = offsets;
1785         rail->simon_vmin_offs_num = offs_num;
1786 }
1787
1788 /*
1789  * Validate rail thermal profile, and get its size. Valid profile:
1790  * - voltage limits are descending with temperature increasing
1791  * - the lowest limit is above rail minimum voltage in pll and
1792  *   in dfll mode (if applicable)
1793  * - the highest limit is below rail nominal voltage (required only
1794  *   for Vmin profile)
1795  */
1796 static int __init get_thermal_profile_size(
1797         int *trips_table, int *limits_table,
1798         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1799 {
1800         int i, min_mv;
1801
1802         for (i = 0; i < MAX_THERMAL_LIMITS - 1; i++) {
1803                 if (!limits_table[i+1])
1804                         break;
1805
1806                 if ((trips_table[i] >= trips_table[i+1]) ||
1807                     (limits_table[i] < limits_table[i+1])) {
1808                         pr_warn("%s: not ordered profile\n", rail->reg_id);
1809                         return -EINVAL;
1810                 }
1811         }
1812
1813         min_mv = max(rail->min_millivolts, d ? d->min_millivolts : 0);
1814         if (limits_table[i] < min_mv) {
1815                 pr_warn("%s: thermal profile below Vmin\n", rail->reg_id);
1816                 return -EINVAL;
1817         }
1818
1819         return i + 1;
1820 }
1821
1822 void __init tegra_dvfs_rail_init_vmax_thermal_profile(
1823         int *therm_trips_table, int *therm_caps_table,
1824         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1825 {
1826         int i = get_thermal_profile_size(therm_trips_table,
1827                                          therm_caps_table, rail, d);
1828         if (i <= 0) {
1829                 rail->vmax_cdev = NULL;
1830                 WARN(1, "%s: invalid Vmax thermal profile\n", rail->reg_id);
1831                 return;
1832         }
1833
1834         /* Install validated thermal caps */
1835         rail->therm_mv_caps = therm_caps_table;
1836         rail->therm_mv_caps_num = i;
1837
1838         /* Setup trip-points if applicable */
1839         if (rail->vmax_cdev) {
1840                 rail->vmax_cdev->trip_temperatures_num = i;
1841                 rail->vmax_cdev->trip_temperatures = therm_trips_table;
1842         }
1843 }
1844
1845 void __init tegra_dvfs_rail_init_vmin_thermal_profile(
1846         int *therm_trips_table, int *therm_floors_table,
1847         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1848 {
1849         int i = get_thermal_profile_size(therm_trips_table,
1850                                          therm_floors_table, rail, d);
1851
1852         if (i <= 0 || therm_floors_table[0] > rail->nominal_millivolts) {
1853                 rail->vmin_cdev = NULL;
1854                 WARN(1, "%s: invalid Vmin thermal profile\n", rail->reg_id);
1855                 return;
1856         }
1857
1858         /* Install validated thermal floors */
1859         rail->therm_mv_floors = therm_floors_table;
1860         rail->therm_mv_floors_num = i;
1861
1862         /* Setup trip-points if applicable */
1863         if (rail->vmin_cdev) {
1864                 rail->vmin_cdev->trip_temperatures_num = i;
1865                 rail->vmin_cdev->trip_temperatures = therm_trips_table;
1866         }
1867 }
1868
1869 /*
1870  * Validate thermal dvfs settings:
1871  * - trip-points are montonically increasing
1872  * - voltages in any temperature range are montonically increasing with
1873  *   frequency (can go up/down across ranges at iso frequency)
1874  * - voltage for any frequency/thermal range combination must be within
1875  *   rail minimum/maximum limits
1876  */
1877 int __init tegra_dvfs_rail_init_thermal_dvfs_trips(
1878         int *therm_trips_table, struct dvfs_rail *rail)
1879 {
1880         int i;
1881
1882         if (!rail->vts_cdev) {
1883                 WARN(1, "%s: missing thermal dvfs cooling device\n",
1884                      rail->reg_id);
1885                 return -ENOENT;
1886         }
1887
1888         for (i = 0; i < MAX_THERMAL_LIMITS - 1; i++) {
1889                 if (therm_trips_table[i] >= therm_trips_table[i+1])
1890                         break;
1891         }
1892
1893         rail->vts_cdev->trip_temperatures_num = i + 1;
1894         rail->vts_cdev->trip_temperatures = therm_trips_table;
1895         return 0;
1896 }
1897
1898 int __init tegra_dvfs_init_thermal_dvfs_voltages(int *therm_voltages,
1899         int *peak_voltages, int freqs_num, int ranges_num, struct dvfs *d)
1900 {
1901         int *millivolts;
1902         int freq_idx, therm_idx;
1903
1904         for (therm_idx = 0; therm_idx < ranges_num; therm_idx++) {
1905                 millivolts = therm_voltages + therm_idx * MAX_DVFS_FREQS;
1906                 for (freq_idx = 0; freq_idx < freqs_num; freq_idx++) {
1907                         int mv = millivolts[freq_idx];
1908                         if ((mv > d->dvfs_rail->max_millivolts) ||
1909                             (mv < d->dvfs_rail->min_millivolts) ||
1910                             (freq_idx && (mv < millivolts[freq_idx - 1]))) {
1911                                 WARN(1, "%s: invalid thermal dvfs entry %d(%d, %d)\n",
1912                                      d->clk_name, mv, freq_idx, therm_idx);
1913                                 return -EINVAL;
1914                         }
1915                         if (mv > peak_voltages[freq_idx])
1916                                 peak_voltages[freq_idx] = mv;
1917                 }
1918         }
1919
1920         d->millivolts = therm_voltages;
1921         d->peak_millivolts = peak_voltages;
1922         d->therm_dvfs = true;
1923         return 0;
1924 }
1925
1926 /* Directly set cold temperature limit in dfll mode */
1927 int tegra_dvfs_rail_dfll_mode_set_cold(struct dvfs_rail *rail)
1928 {
1929         int ret = 0;
1930
1931         /* No thermal floors - nothing to do */
1932         if (!rail || !rail->therm_mv_floors)
1933                 return ret;
1934
1935         /*
1936          * Since cooling thresholds are the same in pll and dfll modes, pll mode
1937          * thermal index can be used to decide if cold limit should be set in
1938          * dfll mode.
1939          */
1940         mutex_lock(&dvfs_lock);
1941         if (rail->dfll_mode &&
1942             (rail->therm_floor_idx < rail->therm_mv_floors_num)) {
1943                         int mv = rail->therm_mv_floors[rail->therm_floor_idx];
1944                         ret = dvfs_rail_set_voltage_reg(rail, mv);
1945         }
1946         mutex_unlock(&dvfs_lock);
1947
1948         return ret;
1949 }
1950
1951 /*
1952  * Iterate through all the dvfs regulators, finding the regulator exported
1953  * by the regulator api for each one.  Must be called in late init, after
1954  * all the regulator api's regulators are initialized.
1955  */
1956 int __init tegra_dvfs_rail_connect_regulators(void)
1957 {
1958         bool connected = true;
1959         struct dvfs_rail *rail;
1960
1961         mutex_lock(&dvfs_lock);
1962
1963         list_for_each_entry(rail, &dvfs_rail_list, node)
1964                 if (dvfs_rail_connect_to_regulator(rail))
1965                         connected = false;
1966
1967         list_for_each_entry(rail, &dvfs_rail_list, node) {
1968                 if (connected) {
1969                         dvfs_rail_update(rail);
1970                         if (!rail->disabled)
1971                                 continue;
1972                         /* Don't rely on boot level - force disabled voltage */
1973                         rail->disabled = false;
1974                 }
1975                 __tegra_dvfs_rail_disable(rail);
1976         }
1977         mutex_unlock(&dvfs_lock);
1978
1979         if (!connected && tegra_platform_is_silicon()) {
1980                 pr_warn("tegra_dvfs: DVFS regulators connection failed\n"
1981                         "            !!!! voltage scaling is disabled !!!!\n");
1982                 return -ENODEV;
1983         }
1984
1985         return 0;
1986 }
1987
1988 int __init tegra_dvfs_rail_register_notifiers(void)
1989 {
1990         struct dvfs_rail *rail;
1991
1992         register_pm_notifier(&tegra_dvfs_suspend_nb);
1993         register_pm_notifier(&tegra_dvfs_resume_nb);
1994         register_reboot_notifier(&tegra_dvfs_reboot_nb);
1995
1996         list_for_each_entry(rail, &dvfs_rail_list, node) {
1997                         tegra_dvfs_rail_register_vmin_cdev(rail);
1998                         tegra_dvfs_rail_register_vts_cdev(rail);
1999         }
2000
2001         return 0;
2002 }
2003
2004 static int rail_stats_save_to_buf(char *buf, int len)
2005 {
2006         int i;
2007         struct dvfs_rail *rail;
2008         char *str = buf;
2009         char *end = buf + len;
2010
2011         str += scnprintf(str, end - str, "%-12s %-10s\n", "millivolts", "time");
2012
2013         mutex_lock(&dvfs_lock);
2014
2015         list_for_each_entry(rail, &dvfs_rail_list, node) {
2016                 str += scnprintf(str, end - str, "%s (bin: %d.%dmV)\n",
2017                            rail->reg_id,
2018                            rail->stats.bin_uV / 1000,
2019                            (rail->stats.bin_uV / 10) % 100);
2020
2021                 dvfs_rail_stats_update(rail, -1, ktime_get());
2022
2023                 str += scnprintf(str, end - str, "%-12d %-10llu\n", 0,
2024                         cputime64_to_clock_t(msecs_to_jiffies(
2025                                 ktime_to_ms(rail->stats.time_at_mv[0]))));
2026
2027                 for (i = 1; i <= DVFS_RAIL_STATS_TOP_BIN; i++) {
2028                         ktime_t ktime_zero = ktime_set(0, 0);
2029                         if (ktime_equal(rail->stats.time_at_mv[i], ktime_zero))
2030                                 continue;
2031                         str += scnprintf(str, end - str, "%-12d %-10llu\n",
2032                                 rail->min_millivolts +
2033                                 (i - 1) * rail->stats.bin_uV / 1000,
2034                                 cputime64_to_clock_t(msecs_to_jiffies(
2035                                         ktime_to_ms(rail->stats.time_at_mv[i])))
2036                         );
2037                 }
2038         }
2039         mutex_unlock(&dvfs_lock);
2040         return str - buf;
2041 }
2042
2043 #ifdef CONFIG_DEBUG_FS
2044 static int dvfs_tree_sort_cmp(void *p, struct list_head *a, struct list_head *b)
2045 {
2046         struct dvfs *da = list_entry(a, struct dvfs, reg_node);
2047         struct dvfs *db = list_entry(b, struct dvfs, reg_node);
2048         int ret;
2049
2050         ret = strcmp(da->dvfs_rail->reg_id, db->dvfs_rail->reg_id);
2051         if (ret != 0)
2052                 return ret;
2053
2054         if (da->cur_millivolts < db->cur_millivolts)
2055                 return 1;
2056         if (da->cur_millivolts > db->cur_millivolts)
2057                 return -1;
2058
2059         return strcmp(da->clk_name, db->clk_name);
2060 }
2061
2062 static int dvfs_tree_show(struct seq_file *s, void *data)
2063 {
2064         struct dvfs *d;
2065         struct dvfs_rail *rail;
2066         struct dvfs_relationship *rel;
2067
2068         seq_printf(s, "   clock      rate       mV\n");
2069         seq_printf(s, "--------------------------------\n");
2070
2071         mutex_lock(&dvfs_lock);
2072
2073         list_for_each_entry(rail, &dvfs_rail_list, node) {
2074                 int thermal_mv_floor = 0;
2075
2076                 seq_printf(s, "%s %d mV%s:\n", rail->reg_id,
2077                            rail->stats.off ? 0 : rail->millivolts,
2078                            rail->dfll_mode ? " dfll mode" :
2079                                 rail->disabled ? " disabled" : "");
2080                 list_for_each_entry(rel, &rail->relationships_from, from_node) {
2081                         seq_printf(s, "   %-10s %-7d mV %-4d mV\n",
2082                                 rel->from->reg_id, rel->from->millivolts,
2083                                 dvfs_solve_relationship(rel));
2084                 }
2085                 seq_printf(s, "   nominal    %-7d mV\n",
2086                            rail->nominal_millivolts);
2087                 seq_printf(s, "   offset     %-7d mV\n", rail->dbg_mv_offs);
2088
2089                 if (rail->therm_mv_floors) {
2090                         int i = rail->therm_floor_idx;
2091                         if (i < rail->therm_mv_floors_num)
2092                                 thermal_mv_floor = rail->therm_mv_floors[i];
2093                 }
2094                 seq_printf(s, "   thermal    %-7d mV\n", thermal_mv_floor);
2095
2096                 if (rail == tegra_core_rail) {
2097                         seq_printf(s, "   override   %-7d mV [%-4d...%-4d]",
2098                                    rail->override_millivolts,
2099                                    dvfs_rail_get_override_floor(rail),
2100                                    rail->nominal_millivolts);
2101                         if (rail->override_unresolved)
2102                                 seq_printf(s, " unresolved %d",
2103                                            rail->override_unresolved);
2104                         seq_putc(s, '\n');
2105                 }
2106
2107                 list_sort(NULL, &rail->dvfs, dvfs_tree_sort_cmp);
2108
2109                 list_for_each_entry(d, &rail->dvfs, reg_node) {
2110                         seq_printf(s, "   %-10s %-10lu %-4d mV\n", d->clk_name,
2111                                 d->cur_rate, d->cur_millivolts);
2112                 }
2113         }
2114
2115         mutex_unlock(&dvfs_lock);
2116
2117         return 0;
2118 }
2119
2120 static int dvfs_tree_open(struct inode *inode, struct file *file)
2121 {
2122         return single_open(file, dvfs_tree_show, inode->i_private);
2123 }
2124
2125 static const struct file_operations dvfs_tree_fops = {
2126         .open           = dvfs_tree_open,
2127         .read           = seq_read,
2128         .llseek         = seq_lseek,
2129         .release        = single_release,
2130 };
2131
2132 static int rail_stats_show(struct seq_file *s, void *data)
2133 {
2134         char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
2135         int size = 0;
2136
2137         if (!buf)
2138                 return -ENOMEM;
2139
2140         size = rail_stats_save_to_buf(buf, PAGE_SIZE);
2141         seq_write(s, buf, size);
2142         kfree(buf);
2143         return 0;
2144 }
2145
2146 static int rail_stats_open(struct inode *inode, struct file *file)
2147 {
2148         return single_open(file, rail_stats_show, inode->i_private);
2149 }
2150
2151 static const struct file_operations rail_stats_fops = {
2152         .open           = rail_stats_open,
2153         .read           = seq_read,
2154         .llseek         = seq_lseek,
2155         .release        = single_release,
2156 };
2157
2158 static int rail_offs_set(struct dvfs_rail *rail, int offs)
2159 {
2160         if (rail) {
2161                 mutex_lock(&dvfs_lock);
2162                 rail->dbg_mv_offs = offs;
2163                 dvfs_rail_update(rail);
2164                 mutex_unlock(&dvfs_lock);
2165                 return 0;
2166         }
2167         return -ENOENT;
2168 }
2169
2170 static int cpu_offs_get(void *data, u64 *val)
2171 {
2172         if (tegra_cpu_rail) {
2173                 *val = (u64)tegra_cpu_rail->dbg_mv_offs;
2174                 return 0;
2175         }
2176         *val = 0;
2177         return -ENOENT;
2178 }
2179 static int cpu_offs_set(void *data, u64 val)
2180 {
2181         return rail_offs_set(tegra_cpu_rail, (int)val);
2182 }
2183 DEFINE_SIMPLE_ATTRIBUTE(cpu_offs_fops, cpu_offs_get, cpu_offs_set, "%lld\n");
2184
2185 static int gpu_offs_get(void *data, u64 *val)
2186 {
2187         if (tegra_gpu_rail) {
2188                 *val = (u64)tegra_gpu_rail->dbg_mv_offs;
2189                 return 0;
2190         }
2191         *val = 0;
2192         return -ENOENT;
2193 }
2194 static int gpu_offs_set(void *data, u64 val)
2195 {
2196         return rail_offs_set(tegra_gpu_rail, (int)val);
2197 }
2198 DEFINE_SIMPLE_ATTRIBUTE(gpu_offs_fops, gpu_offs_get, gpu_offs_set, "%lld\n");
2199
2200 static int core_offs_get(void *data, u64 *val)
2201 {
2202         if (tegra_core_rail) {
2203                 *val = (u64)tegra_core_rail->dbg_mv_offs;
2204                 return 0;
2205         }
2206         *val = 0;
2207         return -ENOENT;
2208 }
2209 static int core_offs_set(void *data, u64 val)
2210 {
2211         return rail_offs_set(tegra_core_rail, (int)val);
2212 }
2213 DEFINE_SIMPLE_ATTRIBUTE(core_offs_fops, core_offs_get, core_offs_set, "%lld\n");
2214
2215 static int core_override_get(void *data, u64 *val)
2216 {
2217         if (tegra_core_rail) {
2218                 *val = (u64)tegra_core_rail->override_millivolts;
2219                 return 0;
2220         }
2221         *val = 0;
2222         return -ENOENT;
2223 }
2224 static int core_override_set(void *data, u64 val)
2225 {
2226         return dvfs_override_core_voltage((int)val);
2227 }
2228 DEFINE_SIMPLE_ATTRIBUTE(core_override_fops,
2229                         core_override_get, core_override_set, "%llu\n");
2230
2231 static int gpu_dvfs_t_show(struct seq_file *s, void *data)
2232 {
2233         int i, j;
2234         int num_ranges = 1;
2235         int *trips = NULL;
2236         struct dvfs *d;
2237         struct dvfs_rail *rail = tegra_gpu_rail;
2238         int max_mv[MAX_DVFS_FREQS] = {};
2239
2240         if (!tegra_gpu_rail) {
2241                 seq_printf(s, "Only supported for T124 or higher\n");
2242                 return -ENOSYS;
2243         }
2244
2245         mutex_lock(&dvfs_lock);
2246
2247         d = list_first_entry(&rail->dvfs, struct dvfs, reg_node);
2248         if (rail->vts_cdev && d->therm_dvfs) {
2249                 num_ranges = rail->vts_cdev->trip_temperatures_num + 1;
2250                 trips = rail->vts_cdev->trip_temperatures;
2251         }
2252
2253         seq_printf(s, "%-11s", "T(C)\\F(kHz)");
2254         for (i = 0; i < d->num_freqs; i++) {
2255                 unsigned int f = d->freqs[i]/1000;
2256                 seq_printf(s, " %7u", f);
2257         }
2258         seq_printf(s, "\n");
2259
2260         for (j = 0; j < num_ranges; j++) {
2261                 seq_printf(s, "%s", j == rail->therm_scale_idx ? ">" : " ");
2262
2263                 if (!trips || (num_ranges == 1))
2264                         seq_printf(s, "%4s..%-4s", "", "");
2265                 else if (j == 0)
2266                         seq_printf(s, "%4s..%-4d", "", trips[j]);
2267                 else if (j == num_ranges - 1)
2268                         seq_printf(s, "%4d..%-4s", trips[j], "");
2269                 else
2270                         seq_printf(s, "%4d..%-4d", trips[j-1], trips[j]);
2271
2272                 for (i = 0; i < d->num_freqs; i++) {
2273                         int mv = *(d->millivolts + j * MAX_DVFS_FREQS + i);
2274                         seq_printf(s, " %7d", mv);
2275                         max_mv[i] = max(max_mv[i], mv);
2276                 }
2277                 seq_printf(s, " mV\n");
2278         }
2279
2280         seq_printf(s, "%3s%-8s\n", "", "------");
2281         seq_printf(s, "%3s%-8s", "", "max(T)");
2282         for (i = 0; i < d->num_freqs; i++)
2283                 seq_printf(s, " %7d", max_mv[i]);
2284         seq_printf(s, " mV\n");
2285
2286         mutex_unlock(&dvfs_lock);
2287
2288         return 0;
2289 }
2290
2291 static int gpu_dvfs_t_open(struct inode *inode, struct file *file)
2292 {
2293         return single_open(file, gpu_dvfs_t_show, NULL);
2294 }
2295
2296 static const struct file_operations gpu_dvfs_t_fops = {
2297         .open           = gpu_dvfs_t_open,
2298         .read           = seq_read,
2299         .llseek         = seq_lseek,
2300         .release        = single_release,
2301 };
2302
2303 static int dvfs_table_show(struct seq_file *s, void *data)
2304 {
2305         int i;
2306         struct dvfs *d;
2307         struct dvfs_rail *rail;
2308         const int *v_pll, *last_v_pll = NULL;
2309         const int *v_dfll, *last_v_dfll = NULL;
2310
2311         seq_printf(s, "DVFS tables: units mV/MHz\n");
2312
2313         mutex_lock(&dvfs_lock);
2314
2315         list_for_each_entry(rail, &dvfs_rail_list, node) {
2316                 list_for_each_entry(d, &rail->dvfs, reg_node) {
2317                         bool mv_done = false;
2318                         v_pll = tegra_dvfs_get_millivolts_pll(d);
2319                         v_dfll = d->dfll_millivolts;
2320
2321                         if (v_pll && (last_v_pll != v_pll)) {
2322                                 if (!mv_done) {
2323                                         seq_printf(s, "\n");
2324                                         mv_done = true;
2325                                 }
2326                                 last_v_pll = v_pll;
2327                                 seq_printf(s, "%-16s", rail->reg_id);
2328                                 for (i = 0; i < d->num_freqs; i++)
2329                                         seq_printf(s, "%7d", v_pll[i]);
2330                                 seq_printf(s, "\n");
2331                         }
2332
2333                         if (v_dfll && (last_v_dfll != v_dfll)) {
2334                                 if (!mv_done) {
2335                                         seq_printf(s, "\n");
2336                                         mv_done = true;
2337                                 }
2338                                 last_v_dfll = v_dfll;
2339                                 seq_printf(s, "%-8s (dfll) ", rail->reg_id);
2340                                 for (i = 0; i < d->num_freqs; i++)
2341                                         seq_printf(s, "%7d", v_dfll[i]);
2342                                 seq_printf(s, "\n");
2343                         }
2344
2345                         seq_printf(s, "%-16s", d->clk_name);
2346                         for (i = 0; i < d->num_freqs; i++) {
2347                                 unsigned long *freqs = dvfs_get_freqs(d);
2348                                 unsigned int f = freqs[i]/100000;
2349                                 seq_printf(s, " %4u.%u", f/10, f%10);
2350                         }
2351                         seq_printf(s, "\n");
2352                 }
2353         }
2354
2355         mutex_unlock(&dvfs_lock);
2356
2357         return 0;
2358 }
2359
2360 static int dvfs_table_open(struct inode *inode, struct file *file)
2361 {
2362         return single_open(file, dvfs_table_show, inode->i_private);
2363 }
2364
2365 static const struct file_operations dvfs_table_fops = {
2366         .open           = dvfs_table_open,
2367         .read           = seq_read,
2368         .llseek         = seq_lseek,
2369         .release        = single_release,
2370 };
2371
2372 int __init dvfs_debugfs_init(struct dentry *clk_debugfs_root)
2373 {
2374         struct dentry *d;
2375
2376         d = debugfs_create_file("dvfs", S_IRUGO, clk_debugfs_root, NULL,
2377                 &dvfs_tree_fops);
2378         if (!d)
2379                 return -ENOMEM;
2380
2381         d = debugfs_create_file("rails", S_IRUGO, clk_debugfs_root, NULL,
2382                 &rail_stats_fops);
2383         if (!d)
2384                 return -ENOMEM;
2385
2386         d = debugfs_create_file("vdd_cpu_offs", S_IRUGO | S_IWUSR,
2387                 clk_debugfs_root, NULL, &cpu_offs_fops);
2388         if (!d)
2389                 return -ENOMEM;
2390
2391         d = debugfs_create_file("vdd_gpu_offs", S_IRUGO | S_IWUSR,
2392                 clk_debugfs_root, NULL, &gpu_offs_fops);
2393         if (!d)
2394                 return -ENOMEM;
2395
2396         d = debugfs_create_file("vdd_core_offs", S_IRUGO | S_IWUSR,
2397                 clk_debugfs_root, NULL, &core_offs_fops);
2398         if (!d)
2399                 return -ENOMEM;
2400
2401         d = debugfs_create_file("vdd_core_override", S_IRUGO | S_IWUSR,
2402                 clk_debugfs_root, NULL, &core_override_fops);
2403         if (!d)
2404                 return -ENOMEM;
2405
2406         d = debugfs_create_file("gpu_dvfs_t", S_IRUGO | S_IWUSR,
2407                 clk_debugfs_root, NULL, &gpu_dvfs_t_fops);
2408         if (!d)
2409                 return -ENOMEM;
2410
2411         d = debugfs_create_file("dvfs_table", S_IRUGO, clk_debugfs_root, NULL,
2412                 &dvfs_table_fops);
2413         if (!d)
2414                 return -ENOMEM;
2415
2416         return 0;
2417 }
2418
2419 #endif
2420
2421 #ifdef CONFIG_PM
2422 static ssize_t tegra_rail_stats_show(struct kobject *kobj,
2423                                         struct kobj_attribute *attr,
2424                                         char *buf)
2425 {
2426         return rail_stats_save_to_buf(buf, PAGE_SIZE);
2427 }
2428
2429 static struct kobj_attribute rail_stats_attr =
2430                 __ATTR_RO(tegra_rail_stats);
2431
2432 static int __init tegra_dvfs_sysfs_stats_init(void)
2433 {
2434         int error;
2435         error = sysfs_create_file(power_kobj, &rail_stats_attr.attr);
2436         return 0;
2437 }
2438 late_initcall(tegra_dvfs_sysfs_stats_init);
2439 #endif