ARM: tegra: dvfs: Add VDD_CORE override mechanism
[linux-3.10.git] / arch / arm / mach-tegra / dvfs.c
1 /*
2  *
3  * Copyright (C) 2010 Google, Inc.
4  *
5  * Author:
6  *      Colin Cross <ccross@google.com>
7  *
8  * Copyright (C) 2010-2013 NVIDIA CORPORATION. All rights reserved.
9  *
10  * This software is licensed under the terms of the GNU General Public
11  * License version 2, as published by the Free Software Foundation, and
12  * may be copied, distributed, and modified under those terms.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/clk.h>
23 #include <linux/clkdev.h>
24 #include <linux/debugfs.h>
25 #include <linux/init.h>
26 #include <linux/list.h>
27 #include <linux/list_sort.h>
28 #include <linux/module.h>
29 #include <linux/regulator/consumer.h>
30 #include <linux/seq_file.h>
31 #include <linux/slab.h>
32 #include <linux/suspend.h>
33 #include <linux/delay.h>
34 #include <linux/clk/tegra.h>
35 #include <linux/reboot.h>
36
37 #include "board.h"
38 #include "clock.h"
39 #include "dvfs.h"
40
41 #define DVFS_RAIL_STATS_BIN     12500
42
43 struct dvfs_rail *tegra_cpu_rail;
44 struct dvfs_rail *tegra_core_rail;
45
46 static LIST_HEAD(dvfs_rail_list);
47 static DEFINE_MUTEX(dvfs_lock);
48 static DEFINE_MUTEX(rail_disable_lock);
49
50 static int dvfs_rail_update(struct dvfs_rail *rail);
51
52 void tegra_dvfs_add_relationships(struct dvfs_relationship *rels, int n)
53 {
54         int i;
55         struct dvfs_relationship *rel;
56
57         mutex_lock(&dvfs_lock);
58
59         for (i = 0; i < n; i++) {
60                 rel = &rels[i];
61                 list_add_tail(&rel->from_node, &rel->to->relationships_from);
62                 list_add_tail(&rel->to_node, &rel->from->relationships_to);
63         }
64
65         mutex_unlock(&dvfs_lock);
66 }
67
68 /* Make sure there is a matching cooling device for thermal limit profile. */
69 static void dvfs_validate_cdevs(struct dvfs_rail *rail)
70 {
71         if (!rail->therm_mv_caps != !rail->therm_mv_caps_num) {
72                 rail->therm_mv_caps_num = 0;
73                 rail->therm_mv_caps = NULL;
74                 WARN(1, "%s: not matching thermal caps/num\n", rail->reg_id);
75         }
76
77         if (rail->therm_mv_caps && !rail->vmax_cdev)
78                 WARN(1, "%s: missing vmax cooling device\n", rail->reg_id);
79
80         if (!rail->therm_mv_floors != !rail->therm_mv_floors_num) {
81                 rail->therm_mv_floors_num = 0;
82                 rail->therm_mv_floors = NULL;
83                 WARN(1, "%s: not matching thermal floors/num\n", rail->reg_id);
84         }
85
86         if (rail->therm_mv_floors && !rail->vmin_cdev)
87                 WARN(1, "%s: missing vmin cooling device\n", rail->reg_id);
88
89         /* Limit override range to maximum floor */
90         if (rail->therm_mv_floors)
91                 rail->min_override_millivolts = rail->therm_mv_floors[0];
92 }
93
94 int tegra_dvfs_init_rails(struct dvfs_rail *rails[], int n)
95 {
96         int i;
97
98         mutex_lock(&dvfs_lock);
99
100         for (i = 0; i < n; i++) {
101                 INIT_LIST_HEAD(&rails[i]->dvfs);
102                 INIT_LIST_HEAD(&rails[i]->relationships_from);
103                 INIT_LIST_HEAD(&rails[i]->relationships_to);
104                 rails[i]->millivolts = rails[i]->nominal_millivolts;
105                 rails[i]->new_millivolts = rails[i]->nominal_millivolts;
106                 if (!rails[i]->step)
107                         rails[i]->step = rails[i]->max_millivolts;
108
109                 list_add_tail(&rails[i]->node, &dvfs_rail_list);
110
111                 dvfs_validate_cdevs(rails[i]);
112
113                 if (!strcmp("vdd_cpu", rails[i]->reg_id))
114                         tegra_cpu_rail = rails[i];
115                 else if (!strcmp("vdd_core", rails[i]->reg_id))
116                         tegra_core_rail = rails[i];
117         }
118
119         mutex_unlock(&dvfs_lock);
120
121         return 0;
122 };
123
124 static int dvfs_solve_relationship(struct dvfs_relationship *rel)
125 {
126         return rel->solve(rel->from, rel->to);
127 }
128
129 /* rail statistic - called during rail init, or under dfs_lock, or with
130    CPU0 only on-line, and interrupts disabled */
131 static void dvfs_rail_stats_init(struct dvfs_rail *rail, int millivolts)
132 {
133         int dvfs_rail_stats_range;
134
135         if (!rail->stats.bin_uV)
136                 rail->stats.bin_uV = DVFS_RAIL_STATS_BIN;
137
138         dvfs_rail_stats_range =
139                 (DVFS_RAIL_STATS_TOP_BIN - 1) * rail->stats.bin_uV / 1000;
140
141         rail->stats.last_update = ktime_get();
142         if (millivolts >= rail->min_millivolts) {
143                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
144                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
145                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
146         }
147
148         if (rail->max_millivolts >
149             rail->min_millivolts + dvfs_rail_stats_range)
150                 pr_warn("tegra_dvfs: %s: stats above %d mV will be squashed\n",
151                         rail->reg_id,
152                         rail->min_millivolts + dvfs_rail_stats_range);
153 }
154
155 static void dvfs_rail_stats_update(
156         struct dvfs_rail *rail, int millivolts, ktime_t now)
157 {
158         rail->stats.time_at_mv[rail->stats.last_index] = ktime_add(
159                 rail->stats.time_at_mv[rail->stats.last_index], ktime_sub(
160                         now, rail->stats.last_update));
161         rail->stats.last_update = now;
162
163         if (rail->stats.off)
164                 return;
165
166         if (millivolts >= rail->min_millivolts) {
167                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
168                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
169                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
170         } else if (millivolts == 0)
171                         rail->stats.last_index = 0;
172 }
173
174 static void dvfs_rail_stats_pause(struct dvfs_rail *rail,
175                                   ktime_t delta, bool on)
176 {
177         int i = on ? rail->stats.last_index : 0;
178         rail->stats.time_at_mv[i] = ktime_add(rail->stats.time_at_mv[i], delta);
179 }
180
181 void tegra_dvfs_rail_off(struct dvfs_rail *rail, ktime_t now)
182 {
183         if (rail) {
184                 dvfs_rail_stats_update(rail, 0, now);
185                 rail->stats.off = true;
186         }
187 }
188
189 void tegra_dvfs_rail_on(struct dvfs_rail *rail, ktime_t now)
190 {
191         if (rail) {
192                 rail->stats.off = false;
193                 dvfs_rail_stats_update(rail, rail->millivolts, now);
194         }
195 }
196
197 void tegra_dvfs_rail_pause(struct dvfs_rail *rail, ktime_t delta, bool on)
198 {
199         if (rail)
200                 dvfs_rail_stats_pause(rail, delta, on);
201 }
202
203 static int dvfs_rail_set_voltage_reg(struct dvfs_rail *rail, int millivolts)
204 {
205         int ret;
206
207         rail->updating = true;
208         rail->reg_max_millivolts = rail->reg_max_millivolts ==
209                 rail->max_millivolts ?
210                 rail->max_millivolts + 1 : rail->max_millivolts;
211         ret = regulator_set_voltage(rail->reg,
212                 millivolts * 1000,
213                 rail->reg_max_millivolts * 1000);
214         rail->updating = false;
215
216         return ret;
217 }
218
219 /* Sets the voltage on a dvfs rail to a specific value, and updates any
220  * rails that depend on this rail. */
221 static int dvfs_rail_set_voltage(struct dvfs_rail *rail, int millivolts)
222 {
223         int ret = 0;
224         struct dvfs_relationship *rel;
225         int step = (millivolts > rail->millivolts) ? rail->step : -rail->step;
226         int i;
227         int steps;
228         bool jmp_to_zero;
229
230         if (!rail->reg) {
231                 if (millivolts == rail->millivolts)
232                         return 0;
233                 else
234                         return -EINVAL;
235         }
236
237         /*
238          * DFLL adjusts rail voltage automatically, but not exactly to the
239          * expected level - update stats, anyway.
240          */
241         if (rail->dfll_mode) {
242                 rail->millivolts = rail->new_millivolts = millivolts;
243                 dvfs_rail_stats_update(rail, millivolts, ktime_get());
244                 return 0;
245         }
246
247         if (rail->disabled)
248                 return 0;
249
250         rail->resolving_to = true;
251         jmp_to_zero = rail->jmp_to_zero &&
252                         ((millivolts == 0) || (rail->millivolts == 0));
253         steps = jmp_to_zero ? 1 :
254                 DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
255
256         for (i = 0; i < steps; i++) {
257                 if (!jmp_to_zero &&
258                     (abs(millivolts - rail->millivolts) > rail->step))
259                         rail->new_millivolts = rail->millivolts + step;
260                 else
261                         rail->new_millivolts = millivolts;
262
263                 /* Before changing the voltage, tell each rail that depends
264                  * on this rail that the voltage will change.
265                  * This rail will be the "from" rail in the relationship,
266                  * the rail that depends on this rail will be the "to" rail.
267                  * from->millivolts will be the old voltage
268                  * from->new_millivolts will be the new voltage */
269                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
270                         ret = dvfs_rail_update(rel->to);
271                         if (ret)
272                                 goto out;
273                 }
274
275                 ret = dvfs_rail_set_voltage_reg(rail, rail->new_millivolts);
276                 if (ret) {
277                         pr_err("Failed to set dvfs regulator %s\n", rail->reg_id);
278                         goto out;
279                 }
280
281                 rail->millivolts = rail->new_millivolts;
282                 dvfs_rail_stats_update(rail, rail->millivolts, ktime_get());
283
284                 /* After changing the voltage, tell each rail that depends
285                  * on this rail that the voltage has changed.
286                  * from->millivolts and from->new_millivolts will be the
287                  * new voltage */
288                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
289                         ret = dvfs_rail_update(rel->to);
290                         if (ret)
291                                 goto out;
292                 }
293         }
294
295         if (unlikely(rail->millivolts != millivolts)) {
296                 pr_err("%s: rail didn't reach target %d in %d steps (%d)\n",
297                         __func__, millivolts, steps, rail->millivolts);
298                 ret = -EINVAL;
299         }
300
301 out:
302         rail->resolving_to = false;
303         return ret;
304 }
305
306 /* Determine the minimum valid voltage for a rail, taking into account
307  * the dvfs clocks and any rails that this rail depends on.  Calls
308  * dvfs_rail_set_voltage with the new voltage, which will call
309  * dvfs_rail_update on any rails that depend on this rail. */
310 static inline int dvfs_rail_apply_limits(struct dvfs_rail *rail, int millivolts)
311 {
312         int min_mv = rail->min_millivolts;
313
314         if (rail->therm_mv_floors) {
315                 int i = rail->therm_floor_idx;
316                 if (i < rail->therm_mv_floors_num)
317                         min_mv = rail->therm_mv_floors[i];
318         }
319
320         if (rail->override_millivolts)
321                 millivolts = rail->override_millivolts;
322         else
323                 millivolts += rail->offs_millivolts;
324         if (millivolts > rail->max_millivolts)
325                 millivolts = rail->max_millivolts;
326         else if (millivolts < min_mv)
327                 millivolts = min_mv;
328
329         return millivolts;
330 }
331
332 static int dvfs_rail_update(struct dvfs_rail *rail)
333 {
334         int millivolts = 0;
335         struct dvfs *d;
336         struct dvfs_relationship *rel;
337         int ret = 0;
338         int steps;
339
340         /* if dvfs is suspended, return and handle it during resume */
341         if (rail->suspended)
342                 return 0;
343
344         /* if regulators are not connected yet, return and handle it later */
345         if (!rail->reg)
346                 return 0;
347
348         /* if rail update is entered while resolving circular dependencies,
349            abort recursion */
350         if (rail->resolving_to)
351                 return 0;
352
353         /* Find the maximum voltage requested by any clock */
354         list_for_each_entry(d, &rail->dvfs, reg_node)
355                 millivolts = max(d->cur_millivolts, millivolts);
356
357         /* Apply offset and min/max limits if any clock is requesting voltage */
358         if (millivolts)
359                 millivolts = dvfs_rail_apply_limits(rail, millivolts);
360
361         /* retry update if limited by from-relationship to account for
362            circular dependencies */
363         steps = DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
364         for (; steps >= 0; steps--) {
365                 rail->new_millivolts = millivolts;
366
367                 /* Check any rails that this rail depends on */
368                 list_for_each_entry(rel, &rail->relationships_from, from_node)
369                         rail->new_millivolts = dvfs_solve_relationship(rel);
370
371                 if (rail->new_millivolts == rail->millivolts)
372                         break;
373
374                 ret = dvfs_rail_set_voltage(rail, rail->new_millivolts);
375         }
376
377         return ret;
378 }
379
380 static int dvfs_rail_connect_to_regulator(struct dvfs_rail *rail)
381 {
382         struct regulator *reg;
383         int v;
384
385         if (!rail->reg) {
386                 reg = regulator_get(NULL, rail->reg_id);
387                 if (IS_ERR(reg)) {
388                         pr_err("tegra_dvfs: failed to connect %s rail\n",
389                                rail->reg_id);
390                         return -EINVAL;
391                 }
392                 rail->reg = reg;
393         }
394
395         v = regulator_enable(rail->reg);
396         if (v < 0) {
397                 pr_err("tegra_dvfs: failed on enabling regulator %s\n, err %d",
398                         rail->reg_id, v);
399                 return v;
400         }
401
402         v = regulator_get_voltage(rail->reg);
403         if (v < 0) {
404                 pr_err("tegra_dvfs: failed initial get %s voltage\n",
405                        rail->reg_id);
406                 return v;
407         }
408         rail->millivolts = v / 1000;
409         rail->new_millivolts = rail->millivolts;
410         dvfs_rail_stats_init(rail, rail->millivolts);
411         return 0;
412 }
413
414 static inline unsigned long *dvfs_get_freqs(struct dvfs *d)
415 {
416         return d->alt_freqs ? : &d->freqs[0];
417 }
418
419 static inline const int *dvfs_get_millivolts(struct dvfs *d, unsigned long rate)
420 {
421         if (tegra_dvfs_is_dfll_scale(d, rate))
422                 return d->dfll_millivolts;
423
424         return d->millivolts;
425 }
426
427 static int
428 __tegra_dvfs_set_rate(struct dvfs *d, unsigned long rate)
429 {
430         int i = 0;
431         int ret;
432         unsigned long *freqs = dvfs_get_freqs(d);
433         const int *millivolts = dvfs_get_millivolts(d, rate);
434
435         if (freqs == NULL || millivolts == NULL)
436                 return -ENODEV;
437
438         /* On entry to dfll range limit 1st step to range bottom (full ramp of
439            voltage/rate is completed automatically in dfll mode) */
440         if (tegra_dvfs_is_dfll_range_entry(d, rate))
441                 rate = d->dfll_data.use_dfll_rate_min;
442
443         if (rate > freqs[d->num_freqs - 1]) {
444                 pr_warn("tegra_dvfs: rate %lu too high for dvfs on %s\n", rate,
445                         d->clk_name);
446                 return -EINVAL;
447         }
448
449         if (rate == 0) {
450                 d->cur_millivolts = 0;
451         } else {
452                 while (i < d->num_freqs && rate > freqs[i])
453                         i++;
454
455                 if ((d->max_millivolts) &&
456                     (millivolts[i] > d->max_millivolts)) {
457                         pr_warn("tegra_dvfs: voltage %d too high for dvfs on"
458                                 " %s\n", millivolts[i], d->clk_name);
459                         return -EINVAL;
460                 }
461                 d->cur_millivolts = millivolts[i];
462         }
463
464         d->cur_rate = rate;
465
466         ret = dvfs_rail_update(d->dvfs_rail);
467         if (ret)
468                 pr_err("Failed to set regulator %s for clock %s to %d mV\n",
469                         d->dvfs_rail->reg_id, d->clk_name, d->cur_millivolts);
470
471         return ret;
472 }
473
474 int tegra_dvfs_alt_freqs_set(struct dvfs *d, unsigned long *alt_freqs)
475 {
476         int ret = 0;
477
478         mutex_lock(&dvfs_lock);
479
480         if (d->alt_freqs != alt_freqs) {
481                 d->alt_freqs = alt_freqs;
482                 ret = __tegra_dvfs_set_rate(d, d->cur_rate);
483         }
484
485         mutex_unlock(&dvfs_lock);
486         return ret;
487 }
488
489 static int predict_millivolts(struct clk *c, const int *millivolts,
490                               unsigned long rate)
491 {
492         int i;
493
494         if (!millivolts)
495                 return -ENODEV;
496         /*
497          * Predicted voltage can not be used across the switch to alternative
498          * frequency limits. For now, just fail the call for clock that has
499          * alternative limits initialized.
500          */
501         if (c->dvfs->alt_freqs)
502                 return -ENOSYS;
503
504         for (i = 0; i < c->dvfs->num_freqs; i++) {
505                 if (rate <= c->dvfs->freqs[i])
506                         break;
507         }
508
509         if (i == c->dvfs->num_freqs)
510                 return -EINVAL;
511
512         return millivolts[i];
513 }
514
515 int tegra_dvfs_predict_millivolts(struct clk *c, unsigned long rate)
516 {
517         const int *millivolts;
518
519         if (!rate || !c->dvfs)
520                 return 0;
521
522         millivolts = dvfs_get_millivolts(c->dvfs, rate);
523         return predict_millivolts(c, millivolts, rate);
524 }
525
526 int tegra_dvfs_predict_millivolts_pll(struct clk *c, unsigned long rate)
527 {
528         const int *millivolts;
529
530         if (!rate || !c->dvfs)
531                 return 0;
532
533         millivolts = c->dvfs->millivolts;
534         return predict_millivolts(c, millivolts, rate);
535 }
536
537 int tegra_dvfs_predict_millivolts_dfll(struct clk *c, unsigned long rate)
538 {
539         const int *millivolts;
540
541         if (!rate || !c->dvfs)
542                 return 0;
543
544         millivolts = c->dvfs->dfll_millivolts;
545         return predict_millivolts(c, millivolts, rate);
546 }
547
548 int tegra_dvfs_set_rate(struct clk *c, unsigned long rate)
549 {
550         int ret;
551
552         if (!c->dvfs)
553                 return -EINVAL;
554
555         mutex_lock(&dvfs_lock);
556         ret = __tegra_dvfs_set_rate(c->dvfs, rate);
557         mutex_unlock(&dvfs_lock);
558
559         return ret;
560 }
561 EXPORT_SYMBOL(tegra_dvfs_set_rate);
562
563 #ifdef CONFIG_TEGRA_VDD_CORE_OVERRIDE
564 static DEFINE_MUTEX(rail_override_lock);
565
566 int tegra_dvfs_override_core_voltage(int override_mv)
567 {
568         int ret, floor, ceiling;
569         struct dvfs_rail *rail = tegra_core_rail;
570
571         if (!rail)
572                 return -ENOENT;
573
574         floor = rail->min_override_millivolts;
575         ceiling = rail->nominal_millivolts;
576         if (override_mv && ((override_mv < floor) || (override_mv > ceiling))) {
577                 pr_err("%s: override level %d outside the range [%d...%d]\n",
578                        __func__, override_mv, floor, ceiling);
579                 return -EINVAL;
580         }
581
582         mutex_lock(&rail_override_lock);
583
584         if (override_mv == rail->override_millivolts) {
585                 ret = 0;
586                 goto out;
587         }
588
589         if (override_mv) {
590                 ret = tegra_dvfs_core_cap_level_apply(override_mv);
591                 if (ret) {
592                         pr_err("%s: failed to set cap for override level %d\n",
593                                __func__, override_mv);
594                         goto out;
595                 }
596         }
597
598         mutex_lock(&dvfs_lock);
599         if (rail->disabled || rail->suspended) {
600                 pr_err("%s: cannot scale %s rail\n", __func__,
601                        rail->disabled ? "disabled" : "suspended");
602                 ret = -EPERM;
603                 if (!override_mv) {
604                         mutex_unlock(&dvfs_lock);
605                         goto out;
606                 }
607         } else {
608                 rail->override_millivolts = override_mv;
609                 ret = dvfs_rail_update(rail);
610                 if (ret) {
611                         pr_err("%s: failed to set override level %d\n",
612                                __func__, override_mv);
613                         rail->override_millivolts = 0;
614                         dvfs_rail_update(rail);
615                 }
616         }
617         mutex_unlock(&dvfs_lock);
618
619         if (!override_mv || ret)
620                 tegra_dvfs_core_cap_level_apply(0);
621 out:
622         mutex_unlock(&rail_override_lock);
623         return ret;
624 }
625 #else
626 int tegra_dvfs_override_core_voltage(int override_mv)
627 {
628         pr_err("%s: vdd core override is not supported\n", __func__);
629         return -ENOSYS;
630 }
631 #endif
632 EXPORT_SYMBOL(tegra_dvfs_override_core_voltage);
633
634 /* May only be called during clock init, does not take any locks on clock c. */
635 int __init tegra_enable_dvfs_on_clk(struct clk *c, struct dvfs *d)
636 {
637         int i;
638
639         if (c->dvfs) {
640                 pr_err("Error when enabling dvfs on %s for clock %s:\n",
641                         d->dvfs_rail->reg_id, c->name);
642                 pr_err("DVFS already enabled for %s\n",
643                         c->dvfs->dvfs_rail->reg_id);
644                 return -EINVAL;
645         }
646
647         for (i = 0; i < MAX_DVFS_FREQS; i++) {
648                 if (d->millivolts[i] == 0)
649                         break;
650
651                 d->freqs[i] *= d->freqs_mult;
652
653                 /* If final frequencies are 0, pad with previous frequency */
654                 if (d->freqs[i] == 0 && i > 1)
655                         d->freqs[i] = d->freqs[i - 1];
656         }
657         d->num_freqs = i;
658
659         if (d->auto_dvfs) {
660                 c->auto_dvfs = true;
661                 clk_set_cansleep(c);
662         }
663
664         c->dvfs = d;
665
666         /*
667          * Minimum core override level is determined as maximum voltage required
668          * for clocks outside shared buses (shared bus rates can be capped to
669          * safe levels when override limit is set)
670          */
671         if (i && c->ops && !c->ops->shared_bus_update &&
672             !(c->flags & PERIPH_ON_CBUS)) {
673                 int mv = tegra_dvfs_predict_millivolts(c, d->freqs[i-1]);
674                 if (d->dvfs_rail->min_override_millivolts < mv)
675                         d->dvfs_rail->min_override_millivolts = mv;
676         }
677
678         mutex_lock(&dvfs_lock);
679         list_add_tail(&d->reg_node, &d->dvfs_rail->dvfs);
680         mutex_unlock(&dvfs_lock);
681
682         return 0;
683 }
684
685 static bool tegra_dvfs_all_rails_suspended(void)
686 {
687         struct dvfs_rail *rail;
688         bool all_suspended = true;
689
690         list_for_each_entry(rail, &dvfs_rail_list, node)
691                 if (!rail->suspended && !rail->disabled)
692                         all_suspended = false;
693
694         return all_suspended;
695 }
696
697 static bool tegra_dvfs_from_rails_suspended_or_solved(struct dvfs_rail *to)
698 {
699         struct dvfs_relationship *rel;
700         bool all_suspended = true;
701
702         list_for_each_entry(rel, &to->relationships_from, from_node)
703                 if (!rel->from->suspended && !rel->from->disabled &&
704                         !rel->solved_at_nominal)
705                         all_suspended = false;
706
707         return all_suspended;
708 }
709
710 static int tegra_dvfs_suspend_one(void)
711 {
712         struct dvfs_rail *rail;
713         int ret;
714
715         list_for_each_entry(rail, &dvfs_rail_list, node) {
716                 if (!rail->suspended && !rail->disabled &&
717                     tegra_dvfs_from_rails_suspended_or_solved(rail)) {
718                         int mv = dvfs_rail_apply_limits(
719                                 rail, rail->nominal_millivolts);
720                         ret = dvfs_rail_set_voltage(rail, mv);
721                         if (ret)
722                                 return ret;
723                         rail->suspended = true;
724                         return 0;
725                 }
726         }
727
728         return -EINVAL;
729 }
730
731 static void tegra_dvfs_resume(void)
732 {
733         struct dvfs_rail *rail;
734
735         mutex_lock(&dvfs_lock);
736
737         list_for_each_entry(rail, &dvfs_rail_list, node)
738                 rail->suspended = false;
739
740         list_for_each_entry(rail, &dvfs_rail_list, node)
741                 dvfs_rail_update(rail);
742
743         mutex_unlock(&dvfs_lock);
744 }
745
746 static int tegra_dvfs_suspend(void)
747 {
748         int ret = 0;
749
750         mutex_lock(&dvfs_lock);
751
752         while (!tegra_dvfs_all_rails_suspended()) {
753                 ret = tegra_dvfs_suspend_one();
754                 if (ret)
755                         break;
756         }
757
758         mutex_unlock(&dvfs_lock);
759
760         if (ret)
761                 tegra_dvfs_resume();
762
763         return ret;
764 }
765
766 static int tegra_dvfs_pm_notify(struct notifier_block *nb,
767                                 unsigned long event, void *data)
768 {
769         switch (event) {
770         case PM_SUSPEND_PREPARE:
771                 if (tegra_dvfs_suspend())
772                         return NOTIFY_STOP;
773                 break;
774         case PM_POST_SUSPEND:
775                 tegra_dvfs_resume();
776                 break;
777         }
778
779         return NOTIFY_OK;
780 };
781
782 static struct notifier_block tegra_dvfs_nb = {
783         .notifier_call = tegra_dvfs_pm_notify,
784 };
785
786 static int tegra_dvfs_reboot_notify(struct notifier_block *nb,
787                                 unsigned long event, void *data)
788 {
789         switch (event) {
790         case SYS_RESTART:
791         case SYS_HALT:
792         case SYS_POWER_OFF:
793                 tegra_dvfs_suspend();
794                 return NOTIFY_OK;
795         }
796         return NOTIFY_DONE;
797 }
798
799 static struct notifier_block tegra_dvfs_reboot_nb = {
800         .notifier_call = tegra_dvfs_reboot_notify,
801 };
802
803 /* must be called with dvfs lock held */
804 static void __tegra_dvfs_rail_disable(struct dvfs_rail *rail)
805 {
806         int ret;
807
808         /* don't set voltage in DFLL mode - won't work, but break stats */
809         if (rail->dfll_mode) {
810                 rail->disabled = true;
811                 return;
812         }
813
814         ret = dvfs_rail_set_voltage(rail,
815                 dvfs_rail_apply_limits(rail, rail->nominal_millivolts));
816         if (ret) {
817                 pr_info("dvfs: failed to set regulator %s to disable "
818                         "voltage %d\n", rail->reg_id,
819                         rail->nominal_millivolts);
820                 return;
821         }
822         rail->disabled = true;
823 }
824
825 /* must be called with dvfs lock held */
826 static void __tegra_dvfs_rail_enable(struct dvfs_rail *rail)
827 {
828         rail->disabled = false;
829         dvfs_rail_update(rail);
830 }
831
832 void tegra_dvfs_rail_enable(struct dvfs_rail *rail)
833 {
834         if (!rail)
835                 return;
836
837         mutex_lock(&rail_disable_lock);
838
839         if (rail->disabled) {
840                 mutex_lock(&dvfs_lock);
841                 __tegra_dvfs_rail_enable(rail);
842                 mutex_unlock(&dvfs_lock);
843
844                 tegra_dvfs_rail_post_enable(rail);
845         }
846         mutex_unlock(&rail_disable_lock);
847 }
848
849 void tegra_dvfs_rail_disable(struct dvfs_rail *rail)
850 {
851         if (!rail)
852                 return;
853
854         mutex_lock(&rail_disable_lock);
855         if (rail->disabled)
856                 goto out;
857
858         /* rail disable will set it to nominal voltage underneath clock
859            framework - need to re-configure clock rates that are not safe
860            at nominal (yes, unsafe at nominal is ugly, but possible). Rate
861            change must be done outside of dvfs lock. */
862         if (tegra_dvfs_rail_disable_prepare(rail)) {
863                 pr_info("dvfs: failed to prepare regulator %s to disable\n",
864                         rail->reg_id);
865                 goto out;
866         }
867
868         mutex_lock(&dvfs_lock);
869         __tegra_dvfs_rail_disable(rail);
870         mutex_unlock(&dvfs_lock);
871 out:
872         mutex_unlock(&rail_disable_lock);
873 }
874
875 int tegra_dvfs_rail_disable_by_name(const char *reg_id)
876 {
877         struct dvfs_rail *rail = tegra_dvfs_get_rail_by_name(reg_id);
878         if (!rail)
879                 return -EINVAL;
880
881         tegra_dvfs_rail_disable(rail);
882         return 0;
883 }
884
885 struct dvfs_rail *tegra_dvfs_get_rail_by_name(const char *reg_id)
886 {
887         struct dvfs_rail *rail;
888
889         mutex_lock(&dvfs_lock);
890         list_for_each_entry(rail, &dvfs_rail_list, node) {
891                 if (!strcmp(reg_id, rail->reg_id)) {
892                         mutex_unlock(&dvfs_lock);
893                         return rail;
894                 }
895         }
896         mutex_unlock(&dvfs_lock);
897         return NULL;
898 }
899
900 bool tegra_dvfs_rail_updating(struct clk *clk)
901 {
902         return (!clk ? false :
903                 (!clk->dvfs ? false :
904                  (!clk->dvfs->dvfs_rail ? false :
905                   (clk->dvfs->dvfs_rail->updating ||
906                    clk->dvfs->dvfs_rail->dfll_mode_updating))));
907 }
908
909 #ifdef CONFIG_OF
910 int __init of_tegra_dvfs_init(const struct of_device_id *matches)
911 {
912         int ret;
913         struct device_node *np;
914
915         for_each_matching_node(np, matches) {
916                 const struct of_device_id *match = of_match_node(matches, np);
917                 of_tegra_dvfs_init_cb_t dvfs_init_cb = match->data;
918                 ret = dvfs_init_cb(np);
919                 if (ret) {
920                         pr_err("dt: Failed to read %s tables from DT\n",
921                                                         match->compatible);
922                         return ret;
923                 }
924         }
925         return 0;
926 }
927 #endif
928 int tegra_dvfs_dfll_mode_set(struct dvfs *d, unsigned long rate)
929 {
930         mutex_lock(&dvfs_lock);
931         if (!d->dvfs_rail->dfll_mode) {
932                 d->dvfs_rail->dfll_mode = true;
933                 __tegra_dvfs_set_rate(d, rate);
934         }
935         mutex_unlock(&dvfs_lock);
936         return 0;
937 }
938
939 int tegra_dvfs_dfll_mode_clear(struct dvfs *d, unsigned long rate)
940 {
941         int ret = 0;
942
943         mutex_lock(&dvfs_lock);
944         if (d->dvfs_rail->dfll_mode) {
945                 d->dvfs_rail->dfll_mode = false;
946                 /* avoid false detection of matching target (voltage in dfll
947                    mode is fluctuating, and recorded level is just estimate) */
948                 d->dvfs_rail->millivolts--;
949                 if (d->dvfs_rail->disabled) {
950                         d->dvfs_rail->disabled = false;
951                         __tegra_dvfs_rail_disable(d->dvfs_rail);
952                 }
953                 ret = __tegra_dvfs_set_rate(d, rate);
954         }
955         mutex_unlock(&dvfs_lock);
956         return ret;
957 }
958
959 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmax_cdev(void)
960 {
961         if (tegra_cpu_rail)
962                 return tegra_cpu_rail->vmax_cdev;
963         return NULL;
964 }
965
966 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmin_cdev(void)
967 {
968         if (tegra_cpu_rail)
969                 return tegra_cpu_rail->vmin_cdev;
970         return NULL;
971 }
972
973 struct tegra_cooling_device *tegra_dvfs_get_core_vmin_cdev(void)
974 {
975         if (tegra_core_rail)
976                 return tegra_core_rail->vmin_cdev;
977         return NULL;
978 }
979
980 #ifdef CONFIG_THERMAL
981 /* Cooling device limits minimum rail voltage at cold temperature in pll mode */
982 static int tegra_dvfs_rail_get_vmin_cdev_max_state(
983         struct thermal_cooling_device *cdev, unsigned long *max_state)
984 {
985         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
986         *max_state = rail->vmin_cdev->trip_temperatures_num;
987         return 0;
988 }
989
990 static int tegra_dvfs_rail_get_vmin_cdev_cur_state(
991         struct thermal_cooling_device *cdev, unsigned long *cur_state)
992 {
993         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
994         *cur_state = rail->therm_floor_idx;
995         return 0;
996 }
997
998 static int tegra_dvfs_rail_set_vmin_cdev_state(
999         struct thermal_cooling_device *cdev, unsigned long cur_state)
1000 {
1001         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1002
1003         mutex_lock(&dvfs_lock);
1004         if (rail->therm_floor_idx != cur_state) {
1005                 rail->therm_floor_idx = cur_state;
1006                 dvfs_rail_update(rail);
1007         }
1008         mutex_unlock(&dvfs_lock);
1009         return 0;
1010 }
1011
1012 static struct thermal_cooling_device_ops tegra_dvfs_rail_cooling_ops = {
1013         .get_max_state = tegra_dvfs_rail_get_vmin_cdev_max_state,
1014         .get_cur_state = tegra_dvfs_rail_get_vmin_cdev_cur_state,
1015         .set_cur_state = tegra_dvfs_rail_set_vmin_cdev_state,
1016 };
1017
1018 static void tegra_dvfs_rail_register_vmin_cdev(struct dvfs_rail *rail)
1019 {
1020         if (!rail->vmin_cdev)
1021                 return;
1022
1023         /* just report error - initialized for cold temperature, anyway */
1024         if (IS_ERR_OR_NULL(thermal_cooling_device_register(
1025                 rail->vmin_cdev->cdev_type, (void *)rail,
1026                 &tegra_dvfs_rail_cooling_ops)))
1027                 pr_err("tegra cooling device %s failed to register\n",
1028                        rail->vmin_cdev->cdev_type);
1029 }
1030 #else
1031 #define tegra_dvfs_rail_register_vmin_cdev(rail)
1032 #endif
1033
1034 /* Directly set cold temperature limit in dfll mode */
1035 int tegra_dvfs_rail_dfll_mode_set_cold(struct dvfs_rail *rail)
1036 {
1037         int ret = 0;
1038
1039         /* No thermal floors - nothing to do */
1040         if (!rail || !rail->therm_mv_floors)
1041                 return ret;
1042
1043         /*
1044          * Since cooling thresholds are the same in pll and dfll modes, pll mode
1045          * thermal index can be used to decide if cold limit should be set in
1046          * dfll mode.
1047          */
1048         mutex_lock(&dvfs_lock);
1049         if (rail->dfll_mode &&
1050             (rail->therm_floor_idx < rail->therm_mv_floors_num)) {
1051                         int mv = rail->therm_mv_floors[rail->therm_floor_idx];
1052                         ret = dvfs_rail_set_voltage_reg(rail, mv);
1053         }
1054         mutex_unlock(&dvfs_lock);
1055
1056         return ret;
1057 }
1058
1059 /*
1060  * Iterate through all the dvfs regulators, finding the regulator exported
1061  * by the regulator api for each one.  Must be called in late init, after
1062  * all the regulator api's regulators are initialized.
1063  */
1064 int __init tegra_dvfs_late_init(void)
1065 {
1066         bool connected = true;
1067         struct dvfs_rail *rail;
1068
1069         mutex_lock(&dvfs_lock);
1070
1071         list_for_each_entry(rail, &dvfs_rail_list, node)
1072                 if (dvfs_rail_connect_to_regulator(rail))
1073                         connected = false;
1074
1075         list_for_each_entry(rail, &dvfs_rail_list, node)
1076                 if (connected)
1077                         dvfs_rail_update(rail);
1078                 else
1079                         __tegra_dvfs_rail_disable(rail);
1080
1081         mutex_unlock(&dvfs_lock);
1082
1083 #ifdef CONFIG_TEGRA_SILICON_PLATFORM
1084         if (!connected)
1085                 return -ENODEV;
1086 #endif
1087         register_pm_notifier(&tegra_dvfs_nb);
1088         register_reboot_notifier(&tegra_dvfs_reboot_nb);
1089
1090         list_for_each_entry(rail, &dvfs_rail_list, node)
1091                 tegra_dvfs_rail_register_vmin_cdev(rail);
1092
1093         return 0;
1094 }
1095
1096 static int rail_stats_save_to_buf(char *buf, int len)
1097 {
1098         int i;
1099         struct dvfs_rail *rail;
1100         char *str = buf;
1101         char *end = buf + len;
1102
1103         str += scnprintf(str, end - str, "%-12s %-10s\n", "millivolts", "time");
1104
1105         mutex_lock(&dvfs_lock);
1106
1107         list_for_each_entry(rail, &dvfs_rail_list, node) {
1108                 str += scnprintf(str, end - str, "%s (bin: %d.%dmV)\n",
1109                            rail->reg_id,
1110                            rail->stats.bin_uV / 1000,
1111                            (rail->stats.bin_uV / 10) % 100);
1112
1113                 dvfs_rail_stats_update(rail, -1, ktime_get());
1114
1115                 str += scnprintf(str, end - str, "%-12d %-10llu\n", 0,
1116                         cputime64_to_clock_t(msecs_to_jiffies(
1117                                 ktime_to_ms(rail->stats.time_at_mv[0]))));
1118
1119                 for (i = 1; i <= DVFS_RAIL_STATS_TOP_BIN; i++) {
1120                         ktime_t ktime_zero = ktime_set(0, 0);
1121                         if (ktime_equal(rail->stats.time_at_mv[i], ktime_zero))
1122                                 continue;
1123                         str += scnprintf(str, end - str, "%-12d %-10llu\n",
1124                                 rail->min_millivolts +
1125                                 (i - 1) * rail->stats.bin_uV / 1000,
1126                                 cputime64_to_clock_t(msecs_to_jiffies(
1127                                         ktime_to_ms(rail->stats.time_at_mv[i])))
1128                         );
1129                 }
1130         }
1131         mutex_unlock(&dvfs_lock);
1132         return str - buf;
1133 }
1134
1135 #ifdef CONFIG_DEBUG_FS
1136 static int dvfs_tree_sort_cmp(void *p, struct list_head *a, struct list_head *b)
1137 {
1138         struct dvfs *da = list_entry(a, struct dvfs, reg_node);
1139         struct dvfs *db = list_entry(b, struct dvfs, reg_node);
1140         int ret;
1141
1142         ret = strcmp(da->dvfs_rail->reg_id, db->dvfs_rail->reg_id);
1143         if (ret != 0)
1144                 return ret;
1145
1146         if (da->cur_millivolts < db->cur_millivolts)
1147                 return 1;
1148         if (da->cur_millivolts > db->cur_millivolts)
1149                 return -1;
1150
1151         return strcmp(da->clk_name, db->clk_name);
1152 }
1153
1154 static int dvfs_tree_show(struct seq_file *s, void *data)
1155 {
1156         struct dvfs *d;
1157         struct dvfs_rail *rail;
1158         struct dvfs_relationship *rel;
1159
1160         seq_printf(s, "   clock      rate       mV\n");
1161         seq_printf(s, "--------------------------------\n");
1162
1163         mutex_lock(&dvfs_lock);
1164
1165         list_for_each_entry(rail, &dvfs_rail_list, node) {
1166                 int thermal_mv_floor = 0;
1167
1168                 seq_printf(s, "%s %d mV%s:\n", rail->reg_id, rail->millivolts,
1169                            rail->dfll_mode ? " dfll mode" :
1170                                 rail->disabled ? " disabled" : "");
1171                 list_for_each_entry(rel, &rail->relationships_from, from_node) {
1172                         seq_printf(s, "   %-10s %-7d mV %-4d mV\n",
1173                                 rel->from->reg_id, rel->from->millivolts,
1174                                 dvfs_solve_relationship(rel));
1175                 }
1176                 seq_printf(s, "   offset     %-7d mV\n", rail->offs_millivolts);
1177
1178                 if (rail->therm_mv_floors) {
1179                         int i = rail->therm_floor_idx;
1180                         if (i < rail->therm_mv_floors_num)
1181                                 thermal_mv_floor = rail->therm_mv_floors[i];
1182                 }
1183                 seq_printf(s, "   thermal    %-7d mV\n", thermal_mv_floor);
1184
1185                 if (rail == tegra_core_rail) {
1186                         seq_printf(s, "   override   %-7d mV [%-4d...%-4d]\n",
1187                                    rail->override_millivolts,
1188                                    rail->min_override_millivolts,
1189                                    rail->nominal_millivolts);
1190                 }
1191
1192                 list_sort(NULL, &rail->dvfs, dvfs_tree_sort_cmp);
1193
1194                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1195                         seq_printf(s, "   %-10s %-10lu %-4d mV\n", d->clk_name,
1196                                 d->cur_rate, d->cur_millivolts);
1197                 }
1198         }
1199
1200         mutex_unlock(&dvfs_lock);
1201
1202         return 0;
1203 }
1204
1205 static int dvfs_tree_open(struct inode *inode, struct file *file)
1206 {
1207         return single_open(file, dvfs_tree_show, inode->i_private);
1208 }
1209
1210 static const struct file_operations dvfs_tree_fops = {
1211         .open           = dvfs_tree_open,
1212         .read           = seq_read,
1213         .llseek         = seq_lseek,
1214         .release        = single_release,
1215 };
1216
1217 static int rail_stats_show(struct seq_file *s, void *data)
1218 {
1219         char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
1220         int size = 0;
1221
1222         if (!buf)
1223                 return -ENOMEM;
1224
1225         size = rail_stats_save_to_buf(buf, PAGE_SIZE);
1226         seq_write(s, buf, size);
1227         kfree(buf);
1228         return 0;
1229 }
1230
1231 static int rail_stats_open(struct inode *inode, struct file *file)
1232 {
1233         return single_open(file, rail_stats_show, inode->i_private);
1234 }
1235
1236 static const struct file_operations rail_stats_fops = {
1237         .open           = rail_stats_open,
1238         .read           = seq_read,
1239         .llseek         = seq_lseek,
1240         .release        = single_release,
1241 };
1242
1243 static int cpu_offs_get(void *data, u64 *val)
1244 {
1245         if (tegra_cpu_rail) {
1246                 *val = (u64)tegra_cpu_rail->offs_millivolts;
1247                 return 0;
1248         }
1249         *val = 0;
1250         return -ENOENT;
1251 }
1252 static int cpu_offs_set(void *data, u64 val)
1253 {
1254         if (tegra_cpu_rail) {
1255                 mutex_lock(&dvfs_lock);
1256                 tegra_cpu_rail->offs_millivolts = (int)val;
1257                 dvfs_rail_update(tegra_cpu_rail);
1258                 mutex_unlock(&dvfs_lock);
1259                 return 0;
1260         }
1261         return -ENOENT;
1262 }
1263 DEFINE_SIMPLE_ATTRIBUTE(cpu_offs_fops, cpu_offs_get, cpu_offs_set, "%lld\n");
1264
1265 static int core_offs_get(void *data, u64 *val)
1266 {
1267         if (tegra_core_rail) {
1268                 *val = (u64)tegra_core_rail->offs_millivolts;
1269                 return 0;
1270         }
1271         *val = 0;
1272         return -ENOENT;
1273 }
1274 static int core_offs_set(void *data, u64 val)
1275 {
1276         if (tegra_core_rail) {
1277                 mutex_lock(&dvfs_lock);
1278                 tegra_core_rail->offs_millivolts = (int)val;
1279                 dvfs_rail_update(tegra_core_rail);
1280                 mutex_unlock(&dvfs_lock);
1281                 return 0;
1282         }
1283         return -ENOENT;
1284 }
1285 DEFINE_SIMPLE_ATTRIBUTE(core_offs_fops, core_offs_get, core_offs_set, "%lld\n");
1286
1287 static int core_override_get(void *data, u64 *val)
1288 {
1289         if (tegra_core_rail) {
1290                 *val = (u64)tegra_core_rail->override_millivolts;
1291                 return 0;
1292         }
1293         *val = 0;
1294         return -ENOENT;
1295 }
1296 static int core_override_set(void *data, u64 val)
1297 {
1298         return tegra_dvfs_override_core_voltage((int)val);
1299 }
1300 DEFINE_SIMPLE_ATTRIBUTE(core_override_fops,
1301                         core_override_get, core_override_set, "%llu\n");
1302
1303 int __init dvfs_debugfs_init(struct dentry *clk_debugfs_root)
1304 {
1305         struct dentry *d;
1306
1307         d = debugfs_create_file("dvfs", S_IRUGO, clk_debugfs_root, NULL,
1308                 &dvfs_tree_fops);
1309         if (!d)
1310                 return -ENOMEM;
1311
1312         d = debugfs_create_file("rails", S_IRUGO, clk_debugfs_root, NULL,
1313                 &rail_stats_fops);
1314         if (!d)
1315                 return -ENOMEM;
1316
1317         d = debugfs_create_file("vdd_cpu_offs", S_IRUGO | S_IWUSR,
1318                 clk_debugfs_root, NULL, &cpu_offs_fops);
1319         if (!d)
1320                 return -ENOMEM;
1321
1322         d = debugfs_create_file("vdd_core_offs", S_IRUGO | S_IWUSR,
1323                 clk_debugfs_root, NULL, &core_offs_fops);
1324         if (!d)
1325                 return -ENOMEM;
1326
1327         d = debugfs_create_file("vdd_core_override", S_IRUGO | S_IWUSR,
1328                 clk_debugfs_root, NULL, &core_override_fops);
1329         if (!d)
1330                 return -ENOMEM;
1331
1332         return 0;
1333 }
1334
1335 #endif
1336
1337 #ifdef CONFIG_PM
1338 static ssize_t tegra_rail_stats_show(struct kobject *kobj,
1339                                         struct kobj_attribute *attr,
1340                                         char *buf)
1341 {
1342         return rail_stats_save_to_buf(buf, PAGE_SIZE);
1343 }
1344
1345 static struct kobj_attribute rail_stats_attr =
1346                 __ATTR_RO(tegra_rail_stats);
1347
1348 static int __init tegra_dvfs_sysfs_stats_init(void)
1349 {
1350         int error;
1351         error = sysfs_create_file(power_kobj, &rail_stats_attr.attr);
1352         return 0;
1353 }
1354 late_initcall(tegra_dvfs_sysfs_stats_init);
1355 #endif