ARM: tegra: dvfs: Add entries for rail Vmax thermal control
[linux-3.10.git] / arch / arm / mach-tegra / dvfs.c
1 /*
2  *
3  * Copyright (C) 2010 Google, Inc.
4  *
5  * Author:
6  *      Colin Cross <ccross@google.com>
7  *
8  * Copyright (C) 2010-2013 NVIDIA CORPORATION. All rights reserved.
9  *
10  * This software is licensed under the terms of the GNU General Public
11  * License version 2, as published by the Free Software Foundation, and
12  * may be copied, distributed, and modified under those terms.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/clk.h>
23 #include <linux/clkdev.h>
24 #include <linux/debugfs.h>
25 #include <linux/init.h>
26 #include <linux/list.h>
27 #include <linux/list_sort.h>
28 #include <linux/module.h>
29 #include <linux/regulator/consumer.h>
30 #include <linux/seq_file.h>
31 #include <linux/slab.h>
32 #include <linux/suspend.h>
33 #include <linux/delay.h>
34 #include <linux/clk/tegra.h>
35 #include <linux/reboot.h>
36
37 #include "board.h"
38 #include "clock.h"
39 #include "dvfs.h"
40
41 #define DVFS_RAIL_STATS_BIN     12500
42
43 struct dvfs_rail *tegra_cpu_rail;
44 struct dvfs_rail *tegra_core_rail;
45
46 static LIST_HEAD(dvfs_rail_list);
47 static DEFINE_MUTEX(dvfs_lock);
48 static DEFINE_MUTEX(rail_disable_lock);
49
50 static int dvfs_rail_update(struct dvfs_rail *rail);
51
52 void tegra_dvfs_add_relationships(struct dvfs_relationship *rels, int n)
53 {
54         int i;
55         struct dvfs_relationship *rel;
56
57         mutex_lock(&dvfs_lock);
58
59         for (i = 0; i < n; i++) {
60                 rel = &rels[i];
61                 list_add_tail(&rel->from_node, &rel->to->relationships_from);
62                 list_add_tail(&rel->to_node, &rel->from->relationships_to);
63         }
64
65         mutex_unlock(&dvfs_lock);
66 }
67
68 /* Make sure there is a matching cooling device for thermal limit profile. */
69 static void dvfs_validate_cdevs(struct dvfs_rail *rail)
70 {
71         if (!rail->therm_mv_caps != !rail->therm_mv_caps_num) {
72                 rail->therm_mv_caps_num = 0;
73                 rail->therm_mv_caps = NULL;
74                 WARN(1, "%s: not matching thermal caps/num\n", rail->reg_id);
75         }
76
77         if (rail->therm_mv_caps && !rail->vmax_cdev)
78                 WARN(1, "%s: missing vmax cooling device\n", rail->reg_id);
79
80         if (!rail->therm_mv_floors != !rail->therm_mv_floors_num) {
81                 rail->therm_mv_floors_num = 0;
82                 rail->therm_mv_floors = NULL;
83                 WARN(1, "%s: not matching thermal floors/num\n", rail->reg_id);
84         }
85
86         if (rail->therm_mv_floors && !rail->vmin_cdev)
87                 WARN(1, "%s: missing vmin cooling device\n", rail->reg_id);
88 }
89
90 int tegra_dvfs_init_rails(struct dvfs_rail *rails[], int n)
91 {
92         int i;
93
94         mutex_lock(&dvfs_lock);
95
96         for (i = 0; i < n; i++) {
97                 INIT_LIST_HEAD(&rails[i]->dvfs);
98                 INIT_LIST_HEAD(&rails[i]->relationships_from);
99                 INIT_LIST_HEAD(&rails[i]->relationships_to);
100                 rails[i]->millivolts = rails[i]->nominal_millivolts;
101                 rails[i]->new_millivolts = rails[i]->nominal_millivolts;
102                 if (!rails[i]->step)
103                         rails[i]->step = rails[i]->max_millivolts;
104
105                 list_add_tail(&rails[i]->node, &dvfs_rail_list);
106
107                 dvfs_validate_cdevs(rails[i]);
108
109                 if (!strcmp("vdd_cpu", rails[i]->reg_id))
110                         tegra_cpu_rail = rails[i];
111                 else if (!strcmp("vdd_core", rails[i]->reg_id))
112                         tegra_core_rail = rails[i];
113         }
114
115         mutex_unlock(&dvfs_lock);
116
117         return 0;
118 };
119
120 static int dvfs_solve_relationship(struct dvfs_relationship *rel)
121 {
122         return rel->solve(rel->from, rel->to);
123 }
124
125 /* rail statistic - called during rail init, or under dfs_lock, or with
126    CPU0 only on-line, and interrupts disabled */
127 static void dvfs_rail_stats_init(struct dvfs_rail *rail, int millivolts)
128 {
129         int dvfs_rail_stats_range;
130
131         if (!rail->stats.bin_uV)
132                 rail->stats.bin_uV = DVFS_RAIL_STATS_BIN;
133
134         dvfs_rail_stats_range =
135                 (DVFS_RAIL_STATS_TOP_BIN - 1) * rail->stats.bin_uV / 1000;
136
137         rail->stats.last_update = ktime_get();
138         if (millivolts >= rail->min_millivolts) {
139                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
140                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
141                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
142         }
143
144         if (rail->max_millivolts >
145             rail->min_millivolts + dvfs_rail_stats_range)
146                 pr_warn("tegra_dvfs: %s: stats above %d mV will be squashed\n",
147                         rail->reg_id,
148                         rail->min_millivolts + dvfs_rail_stats_range);
149 }
150
151 static void dvfs_rail_stats_update(
152         struct dvfs_rail *rail, int millivolts, ktime_t now)
153 {
154         rail->stats.time_at_mv[rail->stats.last_index] = ktime_add(
155                 rail->stats.time_at_mv[rail->stats.last_index], ktime_sub(
156                         now, rail->stats.last_update));
157         rail->stats.last_update = now;
158
159         if (rail->stats.off)
160                 return;
161
162         if (millivolts >= rail->min_millivolts) {
163                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
164                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
165                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
166         } else if (millivolts == 0)
167                         rail->stats.last_index = 0;
168 }
169
170 static void dvfs_rail_stats_pause(struct dvfs_rail *rail,
171                                   ktime_t delta, bool on)
172 {
173         int i = on ? rail->stats.last_index : 0;
174         rail->stats.time_at_mv[i] = ktime_add(rail->stats.time_at_mv[i], delta);
175 }
176
177 void tegra_dvfs_rail_off(struct dvfs_rail *rail, ktime_t now)
178 {
179         if (rail) {
180                 dvfs_rail_stats_update(rail, 0, now);
181                 rail->stats.off = true;
182         }
183 }
184
185 void tegra_dvfs_rail_on(struct dvfs_rail *rail, ktime_t now)
186 {
187         if (rail) {
188                 rail->stats.off = false;
189                 dvfs_rail_stats_update(rail, rail->millivolts, now);
190         }
191 }
192
193 void tegra_dvfs_rail_pause(struct dvfs_rail *rail, ktime_t delta, bool on)
194 {
195         if (rail)
196                 dvfs_rail_stats_pause(rail, delta, on);
197 }
198
199 static int dvfs_rail_set_voltage_reg(struct dvfs_rail *rail, int millivolts)
200 {
201         int ret;
202
203         rail->updating = true;
204         rail->reg_max_millivolts = rail->reg_max_millivolts ==
205                 rail->max_millivolts ?
206                 rail->max_millivolts + 1 : rail->max_millivolts;
207         ret = regulator_set_voltage(rail->reg,
208                 millivolts * 1000,
209                 rail->reg_max_millivolts * 1000);
210         rail->updating = false;
211
212         return ret;
213 }
214
215 /* Sets the voltage on a dvfs rail to a specific value, and updates any
216  * rails that depend on this rail. */
217 static int dvfs_rail_set_voltage(struct dvfs_rail *rail, int millivolts)
218 {
219         int ret = 0;
220         struct dvfs_relationship *rel;
221         int step = (millivolts > rail->millivolts) ? rail->step : -rail->step;
222         int i;
223         int steps;
224         bool jmp_to_zero;
225
226         if (!rail->reg) {
227                 if (millivolts == rail->millivolts)
228                         return 0;
229                 else
230                         return -EINVAL;
231         }
232
233         /*
234          * DFLL adjusts rail voltage automatically, but not exactly to the
235          * expected level - update stats, anyway.
236          */
237         if (rail->dfll_mode) {
238                 rail->millivolts = rail->new_millivolts = millivolts;
239                 dvfs_rail_stats_update(rail, millivolts, ktime_get());
240                 return 0;
241         }
242
243         if (rail->disabled)
244                 return 0;
245
246         rail->resolving_to = true;
247         jmp_to_zero = rail->jmp_to_zero &&
248                         ((millivolts == 0) || (rail->millivolts == 0));
249         steps = jmp_to_zero ? 1 :
250                 DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
251
252         for (i = 0; i < steps; i++) {
253                 if (!jmp_to_zero &&
254                     (abs(millivolts - rail->millivolts) > rail->step))
255                         rail->new_millivolts = rail->millivolts + step;
256                 else
257                         rail->new_millivolts = millivolts;
258
259                 /* Before changing the voltage, tell each rail that depends
260                  * on this rail that the voltage will change.
261                  * This rail will be the "from" rail in the relationship,
262                  * the rail that depends on this rail will be the "to" rail.
263                  * from->millivolts will be the old voltage
264                  * from->new_millivolts will be the new voltage */
265                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
266                         ret = dvfs_rail_update(rel->to);
267                         if (ret)
268                                 goto out;
269                 }
270
271                 ret = dvfs_rail_set_voltage_reg(rail, rail->new_millivolts);
272                 if (ret) {
273                         pr_err("Failed to set dvfs regulator %s\n", rail->reg_id);
274                         goto out;
275                 }
276
277                 rail->millivolts = rail->new_millivolts;
278                 dvfs_rail_stats_update(rail, rail->millivolts, ktime_get());
279
280                 /* After changing the voltage, tell each rail that depends
281                  * on this rail that the voltage has changed.
282                  * from->millivolts and from->new_millivolts will be the
283                  * new voltage */
284                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
285                         ret = dvfs_rail_update(rel->to);
286                         if (ret)
287                                 goto out;
288                 }
289         }
290
291         if (unlikely(rail->millivolts != millivolts)) {
292                 pr_err("%s: rail didn't reach target %d in %d steps (%d)\n",
293                         __func__, millivolts, steps, rail->millivolts);
294                 ret = -EINVAL;
295         }
296
297 out:
298         rail->resolving_to = false;
299         return ret;
300 }
301
302 /* Determine the minimum valid voltage for a rail, taking into account
303  * the dvfs clocks and any rails that this rail depends on.  Calls
304  * dvfs_rail_set_voltage with the new voltage, which will call
305  * dvfs_rail_update on any rails that depend on this rail. */
306 static inline int dvfs_rail_apply_limits(struct dvfs_rail *rail, int millivolts)
307 {
308         int min_mv = rail->min_millivolts;
309
310         if (rail->therm_mv_floors) {
311                 int i = rail->therm_floor_idx;
312                 if (i < rail->therm_mv_floors_num)
313                         min_mv = rail->therm_mv_floors[i];
314         }
315
316         millivolts += rail->offs_millivolts;
317         if (millivolts > rail->max_millivolts)
318                 millivolts = rail->max_millivolts;
319         else if (millivolts < min_mv)
320                 millivolts = min_mv;
321
322         return millivolts;
323 }
324
325 static int dvfs_rail_update(struct dvfs_rail *rail)
326 {
327         int millivolts = 0;
328         struct dvfs *d;
329         struct dvfs_relationship *rel;
330         int ret = 0;
331         int steps;
332
333         /* if dvfs is suspended, return and handle it during resume */
334         if (rail->suspended)
335                 return 0;
336
337         /* if regulators are not connected yet, return and handle it later */
338         if (!rail->reg)
339                 return 0;
340
341         /* if rail update is entered while resolving circular dependencies,
342            abort recursion */
343         if (rail->resolving_to)
344                 return 0;
345
346         /* Find the maximum voltage requested by any clock */
347         list_for_each_entry(d, &rail->dvfs, reg_node)
348                 millivolts = max(d->cur_millivolts, millivolts);
349
350         /* Apply offset and min/max limits if any clock is requesting voltage */
351         if (millivolts)
352                 millivolts = dvfs_rail_apply_limits(rail, millivolts);
353
354         /* retry update if limited by from-relationship to account for
355            circular dependencies */
356         steps = DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
357         for (; steps >= 0; steps--) {
358                 rail->new_millivolts = millivolts;
359
360                 /* Check any rails that this rail depends on */
361                 list_for_each_entry(rel, &rail->relationships_from, from_node)
362                         rail->new_millivolts = dvfs_solve_relationship(rel);
363
364                 if (rail->new_millivolts == rail->millivolts)
365                         break;
366
367                 ret = dvfs_rail_set_voltage(rail, rail->new_millivolts);
368         }
369
370         return ret;
371 }
372
373 static int dvfs_rail_connect_to_regulator(struct dvfs_rail *rail)
374 {
375         struct regulator *reg;
376         int v;
377
378         if (!rail->reg) {
379                 reg = regulator_get(NULL, rail->reg_id);
380                 if (IS_ERR(reg)) {
381                         pr_err("tegra_dvfs: failed to connect %s rail\n",
382                                rail->reg_id);
383                         return -EINVAL;
384                 }
385                 rail->reg = reg;
386         }
387
388         v = regulator_enable(rail->reg);
389         if (v < 0) {
390                 pr_err("tegra_dvfs: failed on enabling regulator %s\n, err %d",
391                         rail->reg_id, v);
392                 return v;
393         }
394
395         v = regulator_get_voltage(rail->reg);
396         if (v < 0) {
397                 pr_err("tegra_dvfs: failed initial get %s voltage\n",
398                        rail->reg_id);
399                 return v;
400         }
401         rail->millivolts = v / 1000;
402         rail->new_millivolts = rail->millivolts;
403         dvfs_rail_stats_init(rail, rail->millivolts);
404         return 0;
405 }
406
407 static inline unsigned long *dvfs_get_freqs(struct dvfs *d)
408 {
409         return d->alt_freqs ? : &d->freqs[0];
410 }
411
412 static inline const int *dvfs_get_millivolts(struct dvfs *d, unsigned long rate)
413 {
414         if (tegra_dvfs_is_dfll_scale(d, rate))
415                 return d->dfll_millivolts;
416
417         return d->millivolts;
418 }
419
420 static int
421 __tegra_dvfs_set_rate(struct dvfs *d, unsigned long rate)
422 {
423         int i = 0;
424         int ret;
425         unsigned long *freqs = dvfs_get_freqs(d);
426         const int *millivolts = dvfs_get_millivolts(d, rate);
427
428         if (freqs == NULL || millivolts == NULL)
429                 return -ENODEV;
430
431         /* On entry to dfll range limit 1st step to range bottom (full ramp of
432            voltage/rate is completed automatically in dfll mode) */
433         if (tegra_dvfs_is_dfll_range_entry(d, rate))
434                 rate = d->dfll_data.use_dfll_rate_min;
435
436         if (rate > freqs[d->num_freqs - 1]) {
437                 pr_warn("tegra_dvfs: rate %lu too high for dvfs on %s\n", rate,
438                         d->clk_name);
439                 return -EINVAL;
440         }
441
442         if (rate == 0) {
443                 d->cur_millivolts = 0;
444         } else {
445                 while (i < d->num_freqs && rate > freqs[i])
446                         i++;
447
448                 if ((d->max_millivolts) &&
449                     (millivolts[i] > d->max_millivolts)) {
450                         pr_warn("tegra_dvfs: voltage %d too high for dvfs on"
451                                 " %s\n", millivolts[i], d->clk_name);
452                         return -EINVAL;
453                 }
454                 d->cur_millivolts = millivolts[i];
455         }
456
457         d->cur_rate = rate;
458
459         ret = dvfs_rail_update(d->dvfs_rail);
460         if (ret)
461                 pr_err("Failed to set regulator %s for clock %s to %d mV\n",
462                         d->dvfs_rail->reg_id, d->clk_name, d->cur_millivolts);
463
464         return ret;
465 }
466
467 int tegra_dvfs_alt_freqs_set(struct dvfs *d, unsigned long *alt_freqs)
468 {
469         int ret = 0;
470
471         mutex_lock(&dvfs_lock);
472
473         if (d->alt_freqs != alt_freqs) {
474                 d->alt_freqs = alt_freqs;
475                 ret = __tegra_dvfs_set_rate(d, d->cur_rate);
476         }
477
478         mutex_unlock(&dvfs_lock);
479         return ret;
480 }
481
482 static int predict_millivolts(struct clk *c, const int *millivolts,
483                               unsigned long rate)
484 {
485         int i;
486
487         if (!millivolts)
488                 return -ENODEV;
489         /*
490          * Predicted voltage can not be used across the switch to alternative
491          * frequency limits. For now, just fail the call for clock that has
492          * alternative limits initialized.
493          */
494         if (c->dvfs->alt_freqs)
495                 return -ENOSYS;
496
497         for (i = 0; i < c->dvfs->num_freqs; i++) {
498                 if (rate <= c->dvfs->freqs[i])
499                         break;
500         }
501
502         if (i == c->dvfs->num_freqs)
503                 return -EINVAL;
504
505         return millivolts[i];
506 }
507
508 int tegra_dvfs_predict_millivolts(struct clk *c, unsigned long rate)
509 {
510         const int *millivolts;
511
512         if (!rate || !c->dvfs)
513                 return 0;
514
515         millivolts = dvfs_get_millivolts(c->dvfs, rate);
516         return predict_millivolts(c, millivolts, rate);
517 }
518
519 int tegra_dvfs_predict_millivolts_pll(struct clk *c, unsigned long rate)
520 {
521         const int *millivolts;
522
523         if (!rate || !c->dvfs)
524                 return 0;
525
526         millivolts = c->dvfs->millivolts;
527         return predict_millivolts(c, millivolts, rate);
528 }
529
530 int tegra_dvfs_predict_millivolts_dfll(struct clk *c, unsigned long rate)
531 {
532         const int *millivolts;
533
534         if (!rate || !c->dvfs)
535                 return 0;
536
537         millivolts = c->dvfs->dfll_millivolts;
538         return predict_millivolts(c, millivolts, rate);
539 }
540
541 int tegra_dvfs_set_rate(struct clk *c, unsigned long rate)
542 {
543         int ret;
544
545         if (!c->dvfs)
546                 return -EINVAL;
547
548         mutex_lock(&dvfs_lock);
549         ret = __tegra_dvfs_set_rate(c->dvfs, rate);
550         mutex_unlock(&dvfs_lock);
551
552         return ret;
553 }
554 EXPORT_SYMBOL(tegra_dvfs_set_rate);
555
556 /* May only be called during clock init, does not take any locks on clock c. */
557 int __init tegra_enable_dvfs_on_clk(struct clk *c, struct dvfs *d)
558 {
559         int i;
560
561         if (c->dvfs) {
562                 pr_err("Error when enabling dvfs on %s for clock %s:\n",
563                         d->dvfs_rail->reg_id, c->name);
564                 pr_err("DVFS already enabled for %s\n",
565                         c->dvfs->dvfs_rail->reg_id);
566                 return -EINVAL;
567         }
568
569         for (i = 0; i < MAX_DVFS_FREQS; i++) {
570                 if (d->millivolts[i] == 0)
571                         break;
572
573                 d->freqs[i] *= d->freqs_mult;
574
575                 /* If final frequencies are 0, pad with previous frequency */
576                 if (d->freqs[i] == 0 && i > 1)
577                         d->freqs[i] = d->freqs[i - 1];
578         }
579         d->num_freqs = i;
580
581         if (d->auto_dvfs) {
582                 c->auto_dvfs = true;
583                 clk_set_cansleep(c);
584         }
585
586         c->dvfs = d;
587
588         mutex_lock(&dvfs_lock);
589         list_add_tail(&d->reg_node, &d->dvfs_rail->dvfs);
590         mutex_unlock(&dvfs_lock);
591
592         return 0;
593 }
594
595 static bool tegra_dvfs_all_rails_suspended(void)
596 {
597         struct dvfs_rail *rail;
598         bool all_suspended = true;
599
600         list_for_each_entry(rail, &dvfs_rail_list, node)
601                 if (!rail->suspended && !rail->disabled)
602                         all_suspended = false;
603
604         return all_suspended;
605 }
606
607 static bool tegra_dvfs_from_rails_suspended_or_solved(struct dvfs_rail *to)
608 {
609         struct dvfs_relationship *rel;
610         bool all_suspended = true;
611
612         list_for_each_entry(rel, &to->relationships_from, from_node)
613                 if (!rel->from->suspended && !rel->from->disabled &&
614                         !rel->solved_at_nominal)
615                         all_suspended = false;
616
617         return all_suspended;
618 }
619
620 static int tegra_dvfs_suspend_one(void)
621 {
622         struct dvfs_rail *rail;
623         int ret;
624
625         list_for_each_entry(rail, &dvfs_rail_list, node) {
626                 if (!rail->suspended && !rail->disabled &&
627                     tegra_dvfs_from_rails_suspended_or_solved(rail)) {
628                         int mv = dvfs_rail_apply_limits(
629                                 rail, rail->nominal_millivolts);
630                         ret = dvfs_rail_set_voltage(rail, mv);
631                         if (ret)
632                                 return ret;
633                         rail->suspended = true;
634                         return 0;
635                 }
636         }
637
638         return -EINVAL;
639 }
640
641 static void tegra_dvfs_resume(void)
642 {
643         struct dvfs_rail *rail;
644
645         mutex_lock(&dvfs_lock);
646
647         list_for_each_entry(rail, &dvfs_rail_list, node)
648                 rail->suspended = false;
649
650         list_for_each_entry(rail, &dvfs_rail_list, node)
651                 dvfs_rail_update(rail);
652
653         mutex_unlock(&dvfs_lock);
654 }
655
656 static int tegra_dvfs_suspend(void)
657 {
658         int ret = 0;
659
660         mutex_lock(&dvfs_lock);
661
662         while (!tegra_dvfs_all_rails_suspended()) {
663                 ret = tegra_dvfs_suspend_one();
664                 if (ret)
665                         break;
666         }
667
668         mutex_unlock(&dvfs_lock);
669
670         if (ret)
671                 tegra_dvfs_resume();
672
673         return ret;
674 }
675
676 static int tegra_dvfs_pm_notify(struct notifier_block *nb,
677                                 unsigned long event, void *data)
678 {
679         switch (event) {
680         case PM_SUSPEND_PREPARE:
681                 if (tegra_dvfs_suspend())
682                         return NOTIFY_STOP;
683                 break;
684         case PM_POST_SUSPEND:
685                 tegra_dvfs_resume();
686                 break;
687         }
688
689         return NOTIFY_OK;
690 };
691
692 static struct notifier_block tegra_dvfs_nb = {
693         .notifier_call = tegra_dvfs_pm_notify,
694 };
695
696 static int tegra_dvfs_reboot_notify(struct notifier_block *nb,
697                                 unsigned long event, void *data)
698 {
699         switch (event) {
700         case SYS_RESTART:
701         case SYS_HALT:
702         case SYS_POWER_OFF:
703                 tegra_dvfs_suspend();
704                 return NOTIFY_OK;
705         }
706         return NOTIFY_DONE;
707 }
708
709 static struct notifier_block tegra_dvfs_reboot_nb = {
710         .notifier_call = tegra_dvfs_reboot_notify,
711 };
712
713 /* must be called with dvfs lock held */
714 static void __tegra_dvfs_rail_disable(struct dvfs_rail *rail)
715 {
716         int ret;
717
718         /* don't set voltage in DFLL mode - won't work, but break stats */
719         if (rail->dfll_mode) {
720                 rail->disabled = true;
721                 return;
722         }
723
724         ret = dvfs_rail_set_voltage(rail,
725                 dvfs_rail_apply_limits(rail, rail->nominal_millivolts));
726         if (ret) {
727                 pr_info("dvfs: failed to set regulator %s to disable "
728                         "voltage %d\n", rail->reg_id,
729                         rail->nominal_millivolts);
730                 return;
731         }
732         rail->disabled = true;
733 }
734
735 /* must be called with dvfs lock held */
736 static void __tegra_dvfs_rail_enable(struct dvfs_rail *rail)
737 {
738         rail->disabled = false;
739         dvfs_rail_update(rail);
740 }
741
742 void tegra_dvfs_rail_enable(struct dvfs_rail *rail)
743 {
744         if (!rail)
745                 return;
746
747         mutex_lock(&rail_disable_lock);
748
749         if (rail->disabled) {
750                 mutex_lock(&dvfs_lock);
751                 __tegra_dvfs_rail_enable(rail);
752                 mutex_unlock(&dvfs_lock);
753
754                 tegra_dvfs_rail_post_enable(rail);
755         }
756         mutex_unlock(&rail_disable_lock);
757 }
758
759 void tegra_dvfs_rail_disable(struct dvfs_rail *rail)
760 {
761         if (!rail)
762                 return;
763
764         mutex_lock(&rail_disable_lock);
765         if (rail->disabled)
766                 goto out;
767
768         /* rail disable will set it to nominal voltage underneath clock
769            framework - need to re-configure clock rates that are not safe
770            at nominal (yes, unsafe at nominal is ugly, but possible). Rate
771            change must be done outside of dvfs lock. */
772         if (tegra_dvfs_rail_disable_prepare(rail)) {
773                 pr_info("dvfs: failed to prepare regulator %s to disable\n",
774                         rail->reg_id);
775                 goto out;
776         }
777
778         mutex_lock(&dvfs_lock);
779         __tegra_dvfs_rail_disable(rail);
780         mutex_unlock(&dvfs_lock);
781 out:
782         mutex_unlock(&rail_disable_lock);
783 }
784
785 int tegra_dvfs_rail_disable_by_name(const char *reg_id)
786 {
787         struct dvfs_rail *rail = tegra_dvfs_get_rail_by_name(reg_id);
788         if (!rail)
789                 return -EINVAL;
790
791         tegra_dvfs_rail_disable(rail);
792         return 0;
793 }
794
795 struct dvfs_rail *tegra_dvfs_get_rail_by_name(const char *reg_id)
796 {
797         struct dvfs_rail *rail;
798
799         mutex_lock(&dvfs_lock);
800         list_for_each_entry(rail, &dvfs_rail_list, node) {
801                 if (!strcmp(reg_id, rail->reg_id)) {
802                         mutex_unlock(&dvfs_lock);
803                         return rail;
804                 }
805         }
806         mutex_unlock(&dvfs_lock);
807         return NULL;
808 }
809
810 bool tegra_dvfs_rail_updating(struct clk *clk)
811 {
812         return (!clk ? false :
813                 (!clk->dvfs ? false :
814                  (!clk->dvfs->dvfs_rail ? false :
815                   (clk->dvfs->dvfs_rail->updating ||
816                    clk->dvfs->dvfs_rail->dfll_mode_updating))));
817 }
818
819 #ifdef CONFIG_OF
820 int __init of_tegra_dvfs_init(const struct of_device_id *matches)
821 {
822         int ret;
823         struct device_node *np;
824
825         for_each_matching_node(np, matches) {
826                 const struct of_device_id *match = of_match_node(matches, np);
827                 of_tegra_dvfs_init_cb_t dvfs_init_cb = match->data;
828                 ret = dvfs_init_cb(np);
829                 if (ret) {
830                         pr_err("dt: Failed to read %s tables from DT\n",
831                                                         match->compatible);
832                         return ret;
833                 }
834         }
835         return 0;
836 }
837 #endif
838 int tegra_dvfs_dfll_mode_set(struct dvfs *d, unsigned long rate)
839 {
840         mutex_lock(&dvfs_lock);
841         if (!d->dvfs_rail->dfll_mode) {
842                 d->dvfs_rail->dfll_mode = true;
843                 __tegra_dvfs_set_rate(d, rate);
844         }
845         mutex_unlock(&dvfs_lock);
846         return 0;
847 }
848
849 int tegra_dvfs_dfll_mode_clear(struct dvfs *d, unsigned long rate)
850 {
851         int ret = 0;
852
853         mutex_lock(&dvfs_lock);
854         if (d->dvfs_rail->dfll_mode) {
855                 d->dvfs_rail->dfll_mode = false;
856                 /* avoid false detection of matching target (voltage in dfll
857                    mode is fluctuating, and recorded level is just estimate) */
858                 d->dvfs_rail->millivolts--;
859                 if (d->dvfs_rail->disabled) {
860                         d->dvfs_rail->disabled = false;
861                         __tegra_dvfs_rail_disable(d->dvfs_rail);
862                 }
863                 ret = __tegra_dvfs_set_rate(d, rate);
864         }
865         mutex_unlock(&dvfs_lock);
866         return ret;
867 }
868
869 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmax_cdev(void)
870 {
871         if (tegra_cpu_rail)
872                 return tegra_cpu_rail->vmax_cdev;
873         return NULL;
874 }
875
876 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmin_cdev(void)
877 {
878         if (tegra_cpu_rail)
879                 return tegra_cpu_rail->vmin_cdev;
880         return NULL;
881 }
882
883 struct tegra_cooling_device *tegra_dvfs_get_core_vmin_cdev(void)
884 {
885         if (tegra_core_rail)
886                 return tegra_core_rail->vmin_cdev;
887         return NULL;
888 }
889
890 #ifdef CONFIG_THERMAL
891 /* Cooling device limits minimum rail voltage at cold temperature in pll mode */
892 static int tegra_dvfs_rail_get_vmin_cdev_max_state(
893         struct thermal_cooling_device *cdev, unsigned long *max_state)
894 {
895         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
896         *max_state = rail->vmin_cdev->trip_temperatures_num;
897         return 0;
898 }
899
900 static int tegra_dvfs_rail_get_vmin_cdev_cur_state(
901         struct thermal_cooling_device *cdev, unsigned long *cur_state)
902 {
903         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
904         *cur_state = rail->therm_floor_idx;
905         return 0;
906 }
907
908 static int tegra_dvfs_rail_set_vmin_cdev_state(
909         struct thermal_cooling_device *cdev, unsigned long cur_state)
910 {
911         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
912
913         mutex_lock(&dvfs_lock);
914         if (rail->therm_floor_idx != cur_state) {
915                 rail->therm_floor_idx = cur_state;
916                 dvfs_rail_update(rail);
917         }
918         mutex_unlock(&dvfs_lock);
919         return 0;
920 }
921
922 static struct thermal_cooling_device_ops tegra_dvfs_rail_cooling_ops = {
923         .get_max_state = tegra_dvfs_rail_get_vmin_cdev_max_state,
924         .get_cur_state = tegra_dvfs_rail_get_vmin_cdev_cur_state,
925         .set_cur_state = tegra_dvfs_rail_set_vmin_cdev_state,
926 };
927
928 static void tegra_dvfs_rail_register_vmin_cdev(struct dvfs_rail *rail)
929 {
930         if (!rail->vmin_cdev)
931                 return;
932
933         /* just report error - initialized for cold temperature, anyway */
934         if (IS_ERR_OR_NULL(thermal_cooling_device_register(
935                 rail->vmin_cdev->cdev_type, (void *)rail,
936                 &tegra_dvfs_rail_cooling_ops)))
937                 pr_err("tegra cooling device %s failed to register\n",
938                        rail->vmin_cdev->cdev_type);
939 }
940 #else
941 #define tegra_dvfs_rail_register_vmin_cdev(rail)
942 #endif
943
944 /* Directly set cold temperature limit in dfll mode */
945 int tegra_dvfs_rail_dfll_mode_set_cold(struct dvfs_rail *rail)
946 {
947         int ret = 0;
948
949         /* No thermal floors - nothing to do */
950         if (!rail || !rail->therm_mv_floors)
951                 return ret;
952
953         /*
954          * Since cooling thresholds are the same in pll and dfll modes, pll mode
955          * thermal index can be used to decide if cold limit should be set in
956          * dfll mode.
957          */
958         mutex_lock(&dvfs_lock);
959         if (rail->dfll_mode &&
960             (rail->therm_floor_idx < rail->therm_mv_floors_num)) {
961                         int mv = rail->therm_mv_floors[rail->therm_floor_idx];
962                         ret = dvfs_rail_set_voltage_reg(rail, mv);
963         }
964         mutex_unlock(&dvfs_lock);
965
966         return ret;
967 }
968
969 /*
970  * Iterate through all the dvfs regulators, finding the regulator exported
971  * by the regulator api for each one.  Must be called in late init, after
972  * all the regulator api's regulators are initialized.
973  */
974 int __init tegra_dvfs_late_init(void)
975 {
976         bool connected = true;
977         struct dvfs_rail *rail;
978
979         mutex_lock(&dvfs_lock);
980
981         list_for_each_entry(rail, &dvfs_rail_list, node)
982                 if (dvfs_rail_connect_to_regulator(rail))
983                         connected = false;
984
985         list_for_each_entry(rail, &dvfs_rail_list, node)
986                 if (connected)
987                         dvfs_rail_update(rail);
988                 else
989                         __tegra_dvfs_rail_disable(rail);
990
991         mutex_unlock(&dvfs_lock);
992
993 #ifdef CONFIG_TEGRA_SILICON_PLATFORM
994         if (!connected)
995                 return -ENODEV;
996 #endif
997         register_pm_notifier(&tegra_dvfs_nb);
998         register_reboot_notifier(&tegra_dvfs_reboot_nb);
999
1000         list_for_each_entry(rail, &dvfs_rail_list, node)
1001                 tegra_dvfs_rail_register_vmin_cdev(rail);
1002
1003         return 0;
1004 }
1005
1006 static int rail_stats_save_to_buf(char *buf, int len)
1007 {
1008         int i;
1009         struct dvfs_rail *rail;
1010         char *str = buf;
1011         char *end = buf + len;
1012
1013         str += scnprintf(str, end - str, "%-12s %-10s\n", "millivolts", "time");
1014
1015         mutex_lock(&dvfs_lock);
1016
1017         list_for_each_entry(rail, &dvfs_rail_list, node) {
1018                 str += scnprintf(str, end - str, "%s (bin: %d.%dmV)\n",
1019                            rail->reg_id,
1020                            rail->stats.bin_uV / 1000,
1021                            (rail->stats.bin_uV / 10) % 100);
1022
1023                 dvfs_rail_stats_update(rail, -1, ktime_get());
1024
1025                 str += scnprintf(str, end - str, "%-12d %-10llu\n", 0,
1026                         cputime64_to_clock_t(msecs_to_jiffies(
1027                                 ktime_to_ms(rail->stats.time_at_mv[0]))));
1028
1029                 for (i = 1; i <= DVFS_RAIL_STATS_TOP_BIN; i++) {
1030                         ktime_t ktime_zero = ktime_set(0, 0);
1031                         if (ktime_equal(rail->stats.time_at_mv[i], ktime_zero))
1032                                 continue;
1033                         str += scnprintf(str, end - str, "%-12d %-10llu\n",
1034                                 rail->min_millivolts +
1035                                 (i - 1) * rail->stats.bin_uV / 1000,
1036                                 cputime64_to_clock_t(msecs_to_jiffies(
1037                                         ktime_to_ms(rail->stats.time_at_mv[i])))
1038                         );
1039                 }
1040         }
1041         mutex_unlock(&dvfs_lock);
1042         return str - buf;
1043 }
1044
1045 #ifdef CONFIG_DEBUG_FS
1046 static int dvfs_tree_sort_cmp(void *p, struct list_head *a, struct list_head *b)
1047 {
1048         struct dvfs *da = list_entry(a, struct dvfs, reg_node);
1049         struct dvfs *db = list_entry(b, struct dvfs, reg_node);
1050         int ret;
1051
1052         ret = strcmp(da->dvfs_rail->reg_id, db->dvfs_rail->reg_id);
1053         if (ret != 0)
1054                 return ret;
1055
1056         if (da->cur_millivolts < db->cur_millivolts)
1057                 return 1;
1058         if (da->cur_millivolts > db->cur_millivolts)
1059                 return -1;
1060
1061         return strcmp(da->clk_name, db->clk_name);
1062 }
1063
1064 static int dvfs_tree_show(struct seq_file *s, void *data)
1065 {
1066         struct dvfs *d;
1067         struct dvfs_rail *rail;
1068         struct dvfs_relationship *rel;
1069
1070         seq_printf(s, "   clock      rate       mV\n");
1071         seq_printf(s, "--------------------------------\n");
1072
1073         mutex_lock(&dvfs_lock);
1074
1075         list_for_each_entry(rail, &dvfs_rail_list, node) {
1076                 int thermal_mv_floor = 0;
1077
1078                 seq_printf(s, "%s %d mV%s:\n", rail->reg_id, rail->millivolts,
1079                            rail->dfll_mode ? " dfll mode" :
1080                                 rail->disabled ? " disabled" : "");
1081                 list_for_each_entry(rel, &rail->relationships_from, from_node) {
1082                         seq_printf(s, "   %-10s %-7d mV %-4d mV\n",
1083                                 rel->from->reg_id, rel->from->millivolts,
1084                                 dvfs_solve_relationship(rel));
1085                 }
1086                 seq_printf(s, "   offset     %-7d mV\n", rail->offs_millivolts);
1087
1088                 if (rail->therm_mv_floors) {
1089                         int i = rail->therm_floor_idx;
1090                         if (i < rail->therm_mv_floors_num)
1091                                 thermal_mv_floor = rail->therm_mv_floors[i];
1092                 }
1093                 seq_printf(s, "   thermal    %-7d mV\n", thermal_mv_floor);
1094
1095                 list_sort(NULL, &rail->dvfs, dvfs_tree_sort_cmp);
1096
1097                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1098                         seq_printf(s, "   %-10s %-10lu %-4d mV\n", d->clk_name,
1099                                 d->cur_rate, d->cur_millivolts);
1100                 }
1101         }
1102
1103         mutex_unlock(&dvfs_lock);
1104
1105         return 0;
1106 }
1107
1108 static int dvfs_tree_open(struct inode *inode, struct file *file)
1109 {
1110         return single_open(file, dvfs_tree_show, inode->i_private);
1111 }
1112
1113 static const struct file_operations dvfs_tree_fops = {
1114         .open           = dvfs_tree_open,
1115         .read           = seq_read,
1116         .llseek         = seq_lseek,
1117         .release        = single_release,
1118 };
1119
1120 static int rail_stats_show(struct seq_file *s, void *data)
1121 {
1122         char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
1123         int size = 0;
1124
1125         if (!buf)
1126                 return -ENOMEM;
1127
1128         size = rail_stats_save_to_buf(buf, PAGE_SIZE);
1129         seq_write(s, buf, size);
1130         kfree(buf);
1131         return 0;
1132 }
1133
1134 static int rail_stats_open(struct inode *inode, struct file *file)
1135 {
1136         return single_open(file, rail_stats_show, inode->i_private);
1137 }
1138
1139 static const struct file_operations rail_stats_fops = {
1140         .open           = rail_stats_open,
1141         .read           = seq_read,
1142         .llseek         = seq_lseek,
1143         .release        = single_release,
1144 };
1145
1146 static int cpu_offs_get(void *data, u64 *val)
1147 {
1148         if (tegra_cpu_rail) {
1149                 *val = (u64)tegra_cpu_rail->offs_millivolts;
1150                 return 0;
1151         }
1152         *val = 0;
1153         return -ENOENT;
1154 }
1155 static int cpu_offs_set(void *data, u64 val)
1156 {
1157         if (tegra_cpu_rail) {
1158                 mutex_lock(&dvfs_lock);
1159                 tegra_cpu_rail->offs_millivolts = (int)val;
1160                 dvfs_rail_update(tegra_cpu_rail);
1161                 mutex_unlock(&dvfs_lock);
1162                 return 0;
1163         }
1164         return -ENOENT;
1165 }
1166 DEFINE_SIMPLE_ATTRIBUTE(cpu_offs_fops, cpu_offs_get, cpu_offs_set, "%lld\n");
1167
1168 static int core_offs_get(void *data, u64 *val)
1169 {
1170         if (tegra_core_rail) {
1171                 *val = (u64)tegra_core_rail->offs_millivolts;
1172                 return 0;
1173         }
1174         *val = 0;
1175         return -ENOENT;
1176 }
1177 static int core_offs_set(void *data, u64 val)
1178 {
1179         if (tegra_core_rail) {
1180                 mutex_lock(&dvfs_lock);
1181                 tegra_core_rail->offs_millivolts = (int)val;
1182                 dvfs_rail_update(tegra_core_rail);
1183                 mutex_unlock(&dvfs_lock);
1184                 return 0;
1185         }
1186         return -ENOENT;
1187 }
1188 DEFINE_SIMPLE_ATTRIBUTE(core_offs_fops, core_offs_get, core_offs_set, "%lld\n");
1189
1190 int __init dvfs_debugfs_init(struct dentry *clk_debugfs_root)
1191 {
1192         struct dentry *d;
1193
1194         d = debugfs_create_file("dvfs", S_IRUGO, clk_debugfs_root, NULL,
1195                 &dvfs_tree_fops);
1196         if (!d)
1197                 return -ENOMEM;
1198
1199         d = debugfs_create_file("rails", S_IRUGO, clk_debugfs_root, NULL,
1200                 &rail_stats_fops);
1201         if (!d)
1202                 return -ENOMEM;
1203
1204         d = debugfs_create_file("vdd_cpu_offs", S_IRUGO | S_IWUSR,
1205                 clk_debugfs_root, NULL, &cpu_offs_fops);
1206         if (!d)
1207                 return -ENOMEM;
1208
1209         d = debugfs_create_file("vdd_core_offs", S_IRUGO | S_IWUSR,
1210                 clk_debugfs_root, NULL, &core_offs_fops);
1211         if (!d)
1212                 return -ENOMEM;
1213
1214         return 0;
1215 }
1216
1217 #endif
1218
1219 #ifdef CONFIG_PM
1220 static ssize_t tegra_rail_stats_show(struct kobject *kobj,
1221                                         struct kobj_attribute *attr,
1222                                         char *buf)
1223 {
1224         return rail_stats_save_to_buf(buf, PAGE_SIZE);
1225 }
1226
1227 static struct kobj_attribute rail_stats_attr =
1228                 __ATTR_RO(tegra_rail_stats);
1229
1230 static int __init tegra_dvfs_sysfs_stats_init(void)
1231 {
1232         int error;
1233         error = sysfs_create_file(power_kobj, &rail_stats_attr.attr);
1234         return 0;
1235 }
1236 late_initcall(tegra_dvfs_sysfs_stats_init);
1237 #endif