ARM: tegra11: dvfs: Add core rail cooling device
[linux-3.10.git] / arch / arm / mach-tegra / dvfs.c
1 /*
2  *
3  * Copyright (C) 2010 Google, Inc.
4  *
5  * Author:
6  *      Colin Cross <ccross@google.com>
7  *
8  * Copyright (C) 2010-2011 NVIDIA Corporation.
9  *
10  * This software is licensed under the terms of the GNU General Public
11  * License version 2, as published by the Free Software Foundation, and
12  * may be copied, distributed, and modified under those terms.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/clk.h>
23 #include <linux/clkdev.h>
24 #include <linux/debugfs.h>
25 #include <linux/init.h>
26 #include <linux/list.h>
27 #include <linux/list_sort.h>
28 #include <linux/module.h>
29 #include <linux/regulator/consumer.h>
30 #include <linux/seq_file.h>
31 #include <linux/slab.h>
32 #include <linux/suspend.h>
33 #include <linux/delay.h>
34 #include <linux/clk/tegra.h>
35 #include <linux/reboot.h>
36
37 #include "board.h"
38 #include "clock.h"
39 #include "dvfs.h"
40
41 #define DVFS_RAIL_STATS_BIN     12500
42
43 struct dvfs_rail *tegra_cpu_rail;
44 struct dvfs_rail *tegra_core_rail;
45
46 static LIST_HEAD(dvfs_rail_list);
47 static DEFINE_MUTEX(dvfs_lock);
48 static DEFINE_MUTEX(rail_disable_lock);
49
50 static int dvfs_rail_update(struct dvfs_rail *rail);
51
52 void tegra_dvfs_add_relationships(struct dvfs_relationship *rels, int n)
53 {
54         int i;
55         struct dvfs_relationship *rel;
56
57         mutex_lock(&dvfs_lock);
58
59         for (i = 0; i < n; i++) {
60                 rel = &rels[i];
61                 list_add_tail(&rel->from_node, &rel->to->relationships_from);
62                 list_add_tail(&rel->to_node, &rel->from->relationships_to);
63         }
64
65         mutex_unlock(&dvfs_lock);
66 }
67
68 int tegra_dvfs_init_rails(struct dvfs_rail *rails[], int n)
69 {
70         int i;
71
72         mutex_lock(&dvfs_lock);
73
74         for (i = 0; i < n; i++) {
75                 INIT_LIST_HEAD(&rails[i]->dvfs);
76                 INIT_LIST_HEAD(&rails[i]->relationships_from);
77                 INIT_LIST_HEAD(&rails[i]->relationships_to);
78                 rails[i]->millivolts = rails[i]->nominal_millivolts;
79                 rails[i]->new_millivolts = rails[i]->nominal_millivolts;
80                 if (!rails[i]->step)
81                         rails[i]->step = rails[i]->max_millivolts;
82
83                 list_add_tail(&rails[i]->node, &dvfs_rail_list);
84
85                 if (!strcmp("vdd_cpu", rails[i]->reg_id))
86                         tegra_cpu_rail = rails[i];
87                 else if (!strcmp("vdd_core", rails[i]->reg_id))
88                         tegra_core_rail = rails[i];
89         }
90
91         mutex_unlock(&dvfs_lock);
92
93         return 0;
94 };
95
96 static int dvfs_solve_relationship(struct dvfs_relationship *rel)
97 {
98         return rel->solve(rel->from, rel->to);
99 }
100
101 /* rail statistic - called during rail init, or under dfs_lock, or with
102    CPU0 only on-line, and interrupts disabled */
103 static void dvfs_rail_stats_init(struct dvfs_rail *rail, int millivolts)
104 {
105         int dvfs_rail_stats_range;
106
107         if (!rail->stats.bin_uV)
108                 rail->stats.bin_uV = DVFS_RAIL_STATS_BIN;
109
110         dvfs_rail_stats_range =
111                 (DVFS_RAIL_STATS_TOP_BIN - 1) * rail->stats.bin_uV / 1000;
112
113         rail->stats.last_update = ktime_get();
114         if (millivolts >= rail->min_millivolts) {
115                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
116                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
117                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
118         }
119
120         if (rail->max_millivolts >
121             rail->min_millivolts + dvfs_rail_stats_range)
122                 pr_warn("tegra_dvfs: %s: stats above %d mV will be squashed\n",
123                         rail->reg_id,
124                         rail->min_millivolts + dvfs_rail_stats_range);
125 }
126
127 static void dvfs_rail_stats_update(
128         struct dvfs_rail *rail, int millivolts, ktime_t now)
129 {
130         rail->stats.time_at_mv[rail->stats.last_index] = ktime_add(
131                 rail->stats.time_at_mv[rail->stats.last_index], ktime_sub(
132                         now, rail->stats.last_update));
133         rail->stats.last_update = now;
134
135         if (rail->stats.off)
136                 return;
137
138         if (millivolts >= rail->min_millivolts) {
139                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
140                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
141                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
142         } else if (millivolts == 0)
143                         rail->stats.last_index = 0;
144 }
145
146 static void dvfs_rail_stats_pause(struct dvfs_rail *rail,
147                                   ktime_t delta, bool on)
148 {
149         int i = on ? rail->stats.last_index : 0;
150         rail->stats.time_at_mv[i] = ktime_add(rail->stats.time_at_mv[i], delta);
151 }
152
153 void tegra_dvfs_rail_off(struct dvfs_rail *rail, ktime_t now)
154 {
155         if (rail) {
156                 dvfs_rail_stats_update(rail, 0, now);
157                 rail->stats.off = true;
158         }
159 }
160
161 void tegra_dvfs_rail_on(struct dvfs_rail *rail, ktime_t now)
162 {
163         if (rail) {
164                 rail->stats.off = false;
165                 dvfs_rail_stats_update(rail, rail->millivolts, now);
166         }
167 }
168
169 void tegra_dvfs_rail_pause(struct dvfs_rail *rail, ktime_t delta, bool on)
170 {
171         if (rail)
172                 dvfs_rail_stats_pause(rail, delta, on);
173 }
174
175 /* Sets the voltage on a dvfs rail to a specific value, and updates any
176  * rails that depend on this rail. */
177 static int dvfs_rail_set_voltage(struct dvfs_rail *rail, int millivolts)
178 {
179         int ret = 0;
180         struct dvfs_relationship *rel;
181         int step = (millivolts > rail->millivolts) ? rail->step : -rail->step;
182         int i;
183         int steps;
184         bool jmp_to_zero;
185
186         if (!rail->reg) {
187                 if (millivolts == rail->millivolts)
188                         return 0;
189                 else
190                         return -EINVAL;
191         }
192
193         /*
194          * DFLL adjusts rail voltage automatically, but not exactly to the
195          * expected level - update stats, anyway, and made sure that recorded
196          * level will not match any target that can be requested when/if we
197          * switch back from DFLL to s/w control
198          */
199         if (rail->dfll_mode) {
200                 rail->millivolts = rail->new_millivolts = millivolts - 1;
201                 dvfs_rail_stats_update(rail, millivolts, ktime_get());
202                 return 0;
203         }
204
205         if (rail->disabled)
206                 return 0;
207
208         rail->resolving_to = true;
209         jmp_to_zero = rail->jmp_to_zero &&
210                         ((millivolts == 0) || (rail->millivolts == 0));
211         steps = jmp_to_zero ? 1 :
212                 DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
213
214         for (i = 0; i < steps; i++) {
215                 if (!jmp_to_zero &&
216                     (abs(millivolts - rail->millivolts) > rail->step))
217                         rail->new_millivolts = rail->millivolts + step;
218                 else
219                         rail->new_millivolts = millivolts;
220
221                 /* Before changing the voltage, tell each rail that depends
222                  * on this rail that the voltage will change.
223                  * This rail will be the "from" rail in the relationship,
224                  * the rail that depends on this rail will be the "to" rail.
225                  * from->millivolts will be the old voltage
226                  * from->new_millivolts will be the new voltage */
227                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
228                         ret = dvfs_rail_update(rel->to);
229                         if (ret)
230                                 goto out;
231                 }
232
233                 if (!rail->disabled) {
234                         rail->updating = true;
235                         rail->reg_max_millivolts = rail->reg_max_millivolts ==
236                                 rail->max_millivolts ?
237                                 rail->max_millivolts + 1 : rail->max_millivolts;
238                         ret = regulator_set_voltage(rail->reg,
239                                 rail->new_millivolts * 1000,
240                                 rail->reg_max_millivolts * 1000);
241                         rail->updating = false;
242                 }
243                 if (ret) {
244                         pr_err("Failed to set dvfs regulator %s\n", rail->reg_id);
245                         goto out;
246                 }
247
248                 rail->millivolts = rail->new_millivolts;
249                 dvfs_rail_stats_update(rail, rail->millivolts, ktime_get());
250
251                 /* After changing the voltage, tell each rail that depends
252                  * on this rail that the voltage has changed.
253                  * from->millivolts and from->new_millivolts will be the
254                  * new voltage */
255                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
256                         ret = dvfs_rail_update(rel->to);
257                         if (ret)
258                                 goto out;
259                 }
260         }
261
262         if (unlikely(rail->millivolts != millivolts)) {
263                 pr_err("%s: rail didn't reach target %d in %d steps (%d)\n",
264                         __func__, millivolts, steps, rail->millivolts);
265                 ret = -EINVAL;
266         }
267
268 out:
269         rail->resolving_to = false;
270         return ret;
271 }
272
273 /* Determine the minimum valid voltage for a rail, taking into account
274  * the dvfs clocks and any rails that this rail depends on.  Calls
275  * dvfs_rail_set_voltage with the new voltage, which will call
276  * dvfs_rail_update on any rails that depend on this rail. */
277 static int dvfs_rail_update(struct dvfs_rail *rail)
278 {
279         int millivolts = 0;
280         struct dvfs *d;
281         struct dvfs_relationship *rel;
282         int ret = 0;
283         int steps;
284
285         /* if dvfs is suspended, return and handle it during resume */
286         if (rail->suspended)
287                 return 0;
288
289         /* if regulators are not connected yet, return and handle it later */
290         if (!rail->reg)
291                 return 0;
292
293         /* if rail update is entered while resolving circular dependencies,
294            abort recursion */
295         if (rail->resolving_to)
296                 return 0;
297
298         /* Find the maximum voltage requested by any clock */
299         list_for_each_entry(d, &rail->dvfs, reg_node)
300                 millivolts = max(d->cur_millivolts, millivolts);
301
302         /* Apply offset if any clock is requesting voltage */
303         if (millivolts) {
304                 int min_mv = rail->min_millivolts;
305                 if (rail->pll_mode_cdev)
306                         min_mv = max(min_mv, rail->thermal_idx ?
307                                      0 : rail->min_millivolts_cold);
308
309                 millivolts += rail->offs_millivolts;
310                 if (millivolts > rail->max_millivolts)
311                         millivolts = rail->max_millivolts;
312                 else if (millivolts < min_mv)
313                         millivolts = min_mv;
314         }
315
316         /* retry update if limited by from-relationship to account for
317            circular dependencies */
318         steps = DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
319         for (; steps >= 0; steps--) {
320                 rail->new_millivolts = millivolts;
321
322                 /* Check any rails that this rail depends on */
323                 list_for_each_entry(rel, &rail->relationships_from, from_node)
324                         rail->new_millivolts = dvfs_solve_relationship(rel);
325
326                 if (rail->new_millivolts == rail->millivolts)
327                         break;
328
329                 ret = dvfs_rail_set_voltage(rail, rail->new_millivolts);
330         }
331
332         return ret;
333 }
334
335 static int dvfs_rail_connect_to_regulator(struct dvfs_rail *rail)
336 {
337         struct regulator *reg;
338         int v;
339
340         if (!rail->reg) {
341                 reg = regulator_get(NULL, rail->reg_id);
342                 if (IS_ERR(reg)) {
343                         pr_err("tegra_dvfs: failed to connect %s rail\n",
344                                rail->reg_id);
345                         return -EINVAL;
346                 }
347                 rail->reg = reg;
348         }
349
350         v = regulator_enable(rail->reg);
351         if (v < 0) {
352                 pr_err("tegra_dvfs: failed on enabling regulator %s\n, err %d",
353                         rail->reg_id, v);
354                 return v;
355         }
356
357         v = regulator_get_voltage(rail->reg);
358         if (v < 0) {
359                 pr_err("tegra_dvfs: failed initial get %s voltage\n",
360                        rail->reg_id);
361                 return v;
362         }
363         rail->millivolts = v / 1000;
364         rail->new_millivolts = rail->millivolts;
365         dvfs_rail_stats_init(rail, rail->millivolts);
366         return 0;
367 }
368
369 static inline unsigned long *dvfs_get_freqs(struct dvfs *d)
370 {
371         return d->alt_freqs ? : &d->freqs[0];
372 }
373
374 static inline const int *dvfs_get_millivolts(struct dvfs *d, unsigned long rate)
375 {
376         if (tegra_dvfs_is_dfll_scale(d, rate))
377                 return d->dfll_millivolts;
378
379         return d->millivolts;
380 }
381
382 static int
383 __tegra_dvfs_set_rate(struct dvfs *d, unsigned long rate)
384 {
385         int i = 0;
386         int ret;
387         unsigned long *freqs = dvfs_get_freqs(d);
388         const int *millivolts = dvfs_get_millivolts(d, rate);
389
390         if (freqs == NULL || millivolts == NULL)
391                 return -ENODEV;
392
393         if (rate > freqs[d->num_freqs - 1]) {
394                 pr_warn("tegra_dvfs: rate %lu too high for dvfs on %s\n", rate,
395                         d->clk_name);
396                 return -EINVAL;
397         }
398
399         if (rate == 0) {
400                 d->cur_millivolts = 0;
401         } else {
402                 while (i < d->num_freqs && rate > freqs[i])
403                         i++;
404
405                 if ((d->max_millivolts) &&
406                     (millivolts[i] > d->max_millivolts)) {
407                         pr_warn("tegra_dvfs: voltage %d too high for dvfs on"
408                                 " %s\n", millivolts[i], d->clk_name);
409                         return -EINVAL;
410                 }
411                 d->cur_millivolts = millivolts[i];
412         }
413
414         d->cur_rate = rate;
415
416         ret = dvfs_rail_update(d->dvfs_rail);
417         if (ret)
418                 pr_err("Failed to set regulator %s for clock %s to %d mV\n",
419                         d->dvfs_rail->reg_id, d->clk_name, d->cur_millivolts);
420
421         return ret;
422 }
423
424 int tegra_dvfs_alt_freqs_set(struct dvfs *d, unsigned long *alt_freqs)
425 {
426         int ret = 0;
427
428         mutex_lock(&dvfs_lock);
429
430         if (d->alt_freqs != alt_freqs) {
431                 d->alt_freqs = alt_freqs;
432                 ret = __tegra_dvfs_set_rate(d, d->cur_rate);
433         }
434
435         mutex_unlock(&dvfs_lock);
436         return ret;
437 }
438
439 static int predict_millivolts(struct clk *c, const int *millivolts,
440                               unsigned long rate)
441 {
442         int i;
443
444         if (!millivolts)
445                 return -ENODEV;
446         /*
447          * Predicted voltage can not be used across the switch to alternative
448          * frequency limits. For now, just fail the call for clock that has
449          * alternative limits initialized.
450          */
451         if (c->dvfs->alt_freqs)
452                 return -ENOSYS;
453
454         for (i = 0; i < c->dvfs->num_freqs; i++) {
455                 if (rate <= c->dvfs->freqs[i])
456                         break;
457         }
458
459         if (i == c->dvfs->num_freqs)
460                 return -EINVAL;
461
462         return millivolts[i];
463 }
464
465 int tegra_dvfs_predict_millivolts(struct clk *c, unsigned long rate)
466 {
467         const int *millivolts;
468
469         if (!rate || !c->dvfs)
470                 return 0;
471
472         millivolts = dvfs_get_millivolts(c->dvfs, rate);
473         return predict_millivolts(c, millivolts, rate);
474 }
475
476 int tegra_dvfs_predict_millivolts_pll(struct clk *c, unsigned long rate)
477 {
478         const int *millivolts;
479
480         if (!rate || !c->dvfs)
481                 return 0;
482
483         millivolts = c->dvfs->millivolts;
484         return predict_millivolts(c, millivolts, rate);
485 }
486
487 int tegra_dvfs_predict_millivolts_dfll(struct clk *c, unsigned long rate)
488 {
489         const int *millivolts;
490
491         if (!rate || !c->dvfs)
492                 return 0;
493
494         millivolts = c->dvfs->dfll_millivolts;
495         return predict_millivolts(c, millivolts, rate);
496 }
497
498 int tegra_dvfs_set_rate(struct clk *c, unsigned long rate)
499 {
500         int ret;
501
502         if (!c->dvfs)
503                 return -EINVAL;
504
505         mutex_lock(&dvfs_lock);
506         ret = __tegra_dvfs_set_rate(c->dvfs, rate);
507         mutex_unlock(&dvfs_lock);
508
509         return ret;
510 }
511 EXPORT_SYMBOL(tegra_dvfs_set_rate);
512
513 /* May only be called during clock init, does not take any locks on clock c. */
514 int __init tegra_enable_dvfs_on_clk(struct clk *c, struct dvfs *d)
515 {
516         int i;
517
518         if (c->dvfs) {
519                 pr_err("Error when enabling dvfs on %s for clock %s:\n",
520                         d->dvfs_rail->reg_id, c->name);
521                 pr_err("DVFS already enabled for %s\n",
522                         c->dvfs->dvfs_rail->reg_id);
523                 return -EINVAL;
524         }
525
526         for (i = 0; i < MAX_DVFS_FREQS; i++) {
527                 if (d->millivolts[i] == 0)
528                         break;
529
530                 d->freqs[i] *= d->freqs_mult;
531
532                 /* If final frequencies are 0, pad with previous frequency */
533                 if (d->freqs[i] == 0 && i > 1)
534                         d->freqs[i] = d->freqs[i - 1];
535         }
536         d->num_freqs = i;
537
538         if (d->auto_dvfs) {
539                 c->auto_dvfs = true;
540                 clk_set_cansleep(c);
541         }
542
543         c->dvfs = d;
544
545         mutex_lock(&dvfs_lock);
546         list_add_tail(&d->reg_node, &d->dvfs_rail->dvfs);
547         mutex_unlock(&dvfs_lock);
548
549         return 0;
550 }
551
552 static bool tegra_dvfs_all_rails_suspended(void)
553 {
554         struct dvfs_rail *rail;
555         bool all_suspended = true;
556
557         list_for_each_entry(rail, &dvfs_rail_list, node)
558                 if (!rail->suspended && !rail->disabled)
559                         all_suspended = false;
560
561         return all_suspended;
562 }
563
564 static bool tegra_dvfs_from_rails_suspended_or_solved(struct dvfs_rail *to)
565 {
566         struct dvfs_relationship *rel;
567         bool all_suspended = true;
568
569         list_for_each_entry(rel, &to->relationships_from, from_node)
570                 if (!rel->from->suspended && !rel->from->disabled &&
571                         !rel->solved_at_nominal)
572                         all_suspended = false;
573
574         return all_suspended;
575 }
576
577 static int tegra_dvfs_suspend_one(void)
578 {
579         struct dvfs_rail *rail;
580         int ret;
581
582         list_for_each_entry(rail, &dvfs_rail_list, node) {
583                 if (!rail->suspended && !rail->disabled &&
584                     tegra_dvfs_from_rails_suspended_or_solved(rail)) {
585                         ret = dvfs_rail_set_voltage(rail,
586                                 rail->nominal_millivolts);
587                         if (ret)
588                                 return ret;
589                         rail->suspended = true;
590                         return 0;
591                 }
592         }
593
594         return -EINVAL;
595 }
596
597 static void tegra_dvfs_resume(void)
598 {
599         struct dvfs_rail *rail;
600
601         mutex_lock(&dvfs_lock);
602
603         list_for_each_entry(rail, &dvfs_rail_list, node)
604                 rail->suspended = false;
605
606         list_for_each_entry(rail, &dvfs_rail_list, node)
607                 dvfs_rail_update(rail);
608
609         mutex_unlock(&dvfs_lock);
610 }
611
612 static int tegra_dvfs_suspend(void)
613 {
614         int ret = 0;
615
616         mutex_lock(&dvfs_lock);
617
618         while (!tegra_dvfs_all_rails_suspended()) {
619                 ret = tegra_dvfs_suspend_one();
620                 if (ret)
621                         break;
622         }
623
624         mutex_unlock(&dvfs_lock);
625
626         if (ret)
627                 tegra_dvfs_resume();
628
629         return ret;
630 }
631
632 static int tegra_dvfs_pm_notify(struct notifier_block *nb,
633                                 unsigned long event, void *data)
634 {
635         switch (event) {
636         case PM_SUSPEND_PREPARE:
637                 if (tegra_dvfs_suspend())
638                         return NOTIFY_STOP;
639                 break;
640         case PM_POST_SUSPEND:
641                 tegra_dvfs_resume();
642                 break;
643         }
644
645         return NOTIFY_OK;
646 };
647
648 static struct notifier_block tegra_dvfs_nb = {
649         .notifier_call = tegra_dvfs_pm_notify,
650 };
651
652 static int tegra_dvfs_reboot_notify(struct notifier_block *nb,
653                                 unsigned long event, void *data)
654 {
655         switch (event) {
656         case SYS_RESTART:
657         case SYS_HALT:
658         case SYS_POWER_OFF:
659                 tegra_dvfs_suspend();
660                 return NOTIFY_OK;
661         }
662         return NOTIFY_DONE;
663 }
664
665 static struct notifier_block tegra_dvfs_reboot_nb = {
666         .notifier_call = tegra_dvfs_reboot_notify,
667 };
668
669 /* must be called with dvfs lock held */
670 static void __tegra_dvfs_rail_disable(struct dvfs_rail *rail)
671 {
672         int ret;
673
674         /* don't set voltage in DFLL mode - won't work, but break stats */
675         if (rail->dfll_mode) {
676                 rail->disabled = true;
677                 return;
678         }
679
680         ret = dvfs_rail_set_voltage(rail, rail->nominal_millivolts);
681         if (ret) {
682                 pr_info("dvfs: failed to set regulator %s to disable "
683                         "voltage %d\n", rail->reg_id,
684                         rail->nominal_millivolts);
685                 return;
686         }
687         rail->disabled = true;
688 }
689
690 /* must be called with dvfs lock held */
691 static void __tegra_dvfs_rail_enable(struct dvfs_rail *rail)
692 {
693         rail->disabled = false;
694         dvfs_rail_update(rail);
695 }
696
697 void tegra_dvfs_rail_enable(struct dvfs_rail *rail)
698 {
699         mutex_lock(&rail_disable_lock);
700
701         if (rail->disabled) {
702                 mutex_lock(&dvfs_lock);
703                 __tegra_dvfs_rail_enable(rail);
704                 mutex_unlock(&dvfs_lock);
705
706                 tegra_dvfs_rail_post_enable(rail);
707         }
708         mutex_unlock(&rail_disable_lock);
709
710 }
711
712 void tegra_dvfs_rail_disable(struct dvfs_rail *rail)
713 {
714         mutex_lock(&rail_disable_lock);
715         if (rail->disabled)
716                 goto out;
717
718         /* rail disable will set it to nominal voltage underneath clock
719            framework - need to re-configure clock rates that are not safe
720            at nominal (yes, unsafe at nominal is ugly, but possible). Rate
721            change must be done outside of dvfs lock. */
722         if (tegra_dvfs_rail_disable_prepare(rail)) {
723                 pr_info("dvfs: failed to prepare regulator %s to disable\n",
724                         rail->reg_id);
725                 goto out;
726         }
727
728         mutex_lock(&dvfs_lock);
729         __tegra_dvfs_rail_disable(rail);
730         mutex_unlock(&dvfs_lock);
731 out:
732         mutex_unlock(&rail_disable_lock);
733 }
734
735 int tegra_dvfs_rail_disable_by_name(const char *reg_id)
736 {
737         struct dvfs_rail *rail = tegra_dvfs_get_rail_by_name(reg_id);
738         if (!rail)
739                 return -EINVAL;
740
741         tegra_dvfs_rail_disable(rail);
742         return 0;
743 }
744
745 struct dvfs_rail *tegra_dvfs_get_rail_by_name(const char *reg_id)
746 {
747         struct dvfs_rail *rail;
748
749         mutex_lock(&dvfs_lock);
750         list_for_each_entry(rail, &dvfs_rail_list, node) {
751                 if (!strcmp(reg_id, rail->reg_id)) {
752                         mutex_unlock(&dvfs_lock);
753                         return rail;
754                 }
755         }
756         mutex_unlock(&dvfs_lock);
757         return NULL;
758 }
759
760 bool tegra_dvfs_rail_updating(struct clk *clk)
761 {
762         return (!clk ? false :
763                 (!clk->dvfs ? false :
764                  (!clk->dvfs->dvfs_rail ? false :
765                   (clk->dvfs->dvfs_rail->updating))));
766 }
767
768 #ifdef CONFIG_OF
769 int __init of_tegra_dvfs_init(const struct of_device_id *matches)
770 {
771         int ret;
772         struct device_node *np;
773
774         for_each_matching_node(np, matches) {
775                 const struct of_device_id *match = of_match_node(matches, np);
776                 of_tegra_dvfs_init_cb_t dvfs_init_cb = match->data;
777                 ret = dvfs_init_cb(np);
778                 if (ret) {
779                         pr_err("dt: Failed to read %s tables from DT\n",
780                                                         match->compatible);
781                         return ret;
782                 }
783         }
784         return 0;
785 }
786 #endif
787 int tegra_dvfs_dfll_mode_set(struct dvfs *d, unsigned long rate)
788 {
789         mutex_lock(&dvfs_lock);
790         if (!d->dvfs_rail->dfll_mode) {
791                 d->dvfs_rail->dfll_mode = true;
792                 __tegra_dvfs_set_rate(d, rate);
793         }
794         mutex_unlock(&dvfs_lock);
795         return 0;
796 }
797
798 int tegra_dvfs_dfll_mode_clear(struct dvfs *d, unsigned long rate)
799 {
800         int ret = 0;
801
802         mutex_lock(&dvfs_lock);
803         if (d->dvfs_rail->dfll_mode) {
804                 d->dvfs_rail->dfll_mode = false;
805                 if (d->dvfs_rail->disabled) {
806                         d->dvfs_rail->disabled = false;
807                         __tegra_dvfs_rail_disable(d->dvfs_rail);
808                 }
809                 ret = __tegra_dvfs_set_rate(d, rate);
810         }
811         mutex_unlock(&dvfs_lock);
812         return ret;
813 }
814
815 struct tegra_cooling_device *tegra_dvfs_get_cpu_dfll_cdev(void)
816 {
817         if (tegra_cpu_rail)
818                 return tegra_cpu_rail->dfll_mode_cdev;
819         return NULL;
820 }
821
822 struct tegra_cooling_device *tegra_dvfs_get_core_cdev(void)
823 {
824         if (tegra_core_rail)
825                 return tegra_core_rail->pll_mode_cdev;
826         return NULL;
827 }
828
829 #ifdef CONFIG_THERMAL
830 /* Cooling device limits minimum rail voltage at cold temperature in pll mode */
831 static int tegra_dvfs_rail_get_cdev_max_state(
832         struct thermal_cooling_device *cdev, unsigned long *max_state)
833 {
834         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
835         *max_state = rail->pll_mode_cdev->trip_temperatures_num;
836         return 0;
837 }
838
839 static int tegra_dvfs_rail_get_cdev_cur_state(
840         struct thermal_cooling_device *cdev, unsigned long *cur_state)
841 {
842         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
843         *cur_state = rail->thermal_idx;
844         return 0;
845 }
846
847 static int tegra_dvfs_rail_set_cdev_state(
848         struct thermal_cooling_device *cdev, unsigned long cur_state)
849 {
850         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
851
852         mutex_lock(&dvfs_lock);
853         if (rail->thermal_idx != cur_state) {
854                 rail->thermal_idx = cur_state;
855                 dvfs_rail_update(rail);
856         }
857         mutex_unlock(&dvfs_lock);
858         return 0;
859 }
860
861 static struct thermal_cooling_device_ops tegra_dvfs_rail_cooling_ops = {
862         .get_max_state = tegra_dvfs_rail_get_cdev_max_state,
863         .get_cur_state = tegra_dvfs_rail_get_cdev_cur_state,
864         .set_cur_state = tegra_dvfs_rail_set_cdev_state,
865 };
866
867 static void tegra_dvfs_rail_register_pll_mode_cdev(struct dvfs_rail *rail)
868 {
869         if (!rail->pll_mode_cdev)
870                 return;
871
872         /* just report error - initialized for cold temperature, anyway */
873         if (IS_ERR_OR_NULL(thermal_cooling_device_register(
874                 rail->pll_mode_cdev->cdev_type, (void *)rail,
875                 &tegra_dvfs_rail_cooling_ops)))
876                 pr_err("tegra cooling device %s failed to register\n",
877                        rail->pll_mode_cdev->cdev_type);
878 }
879 #else
880 #define tegra_dvfs_rail_register_pll_mode_cdev(rail)
881 #endif
882
883 /*
884  * Iterate through all the dvfs regulators, finding the regulator exported
885  * by the regulator api for each one.  Must be called in late init, after
886  * all the regulator api's regulators are initialized.
887  */
888 int __init tegra_dvfs_late_init(void)
889 {
890         bool connected = true;
891         struct dvfs_rail *rail;
892
893         mutex_lock(&dvfs_lock);
894
895         list_for_each_entry(rail, &dvfs_rail_list, node)
896                 if (dvfs_rail_connect_to_regulator(rail))
897                         connected = false;
898
899         list_for_each_entry(rail, &dvfs_rail_list, node)
900                 if (connected)
901                         dvfs_rail_update(rail);
902                 else
903                         __tegra_dvfs_rail_disable(rail);
904
905         mutex_unlock(&dvfs_lock);
906
907 #ifdef CONFIG_TEGRA_SILICON_PLATFORM
908         if (!connected)
909                 return -ENODEV;
910 #endif
911         register_pm_notifier(&tegra_dvfs_nb);
912         register_reboot_notifier(&tegra_dvfs_reboot_nb);
913
914         list_for_each_entry(rail, &dvfs_rail_list, node)
915                 tegra_dvfs_rail_register_pll_mode_cdev(rail);
916
917         return 0;
918 }
919
920 #ifdef CONFIG_DEBUG_FS
921 static int dvfs_tree_sort_cmp(void *p, struct list_head *a, struct list_head *b)
922 {
923         struct dvfs *da = list_entry(a, struct dvfs, reg_node);
924         struct dvfs *db = list_entry(b, struct dvfs, reg_node);
925         int ret;
926
927         ret = strcmp(da->dvfs_rail->reg_id, db->dvfs_rail->reg_id);
928         if (ret != 0)
929                 return ret;
930
931         if (da->cur_millivolts < db->cur_millivolts)
932                 return 1;
933         if (da->cur_millivolts > db->cur_millivolts)
934                 return -1;
935
936         return strcmp(da->clk_name, db->clk_name);
937 }
938
939 static int dvfs_tree_show(struct seq_file *s, void *data)
940 {
941         struct dvfs *d;
942         struct dvfs_rail *rail;
943         struct dvfs_relationship *rel;
944
945         seq_printf(s, "   clock      rate       mV\n");
946         seq_printf(s, "--------------------------------\n");
947
948         mutex_lock(&dvfs_lock);
949
950         list_for_each_entry(rail, &dvfs_rail_list, node) {
951                 seq_printf(s, "%s %d mV%s:\n", rail->reg_id,
952                            rail->millivolts + (rail->dfll_mode ? 1 : 0),
953                            rail->dfll_mode ? " dfll mode" :
954                                 rail->disabled ? " disabled" : "");
955                 list_for_each_entry(rel, &rail->relationships_from, from_node) {
956                         seq_printf(s, "   %-10s %-7d mV %-4d mV\n",
957                                 rel->from->reg_id,
958                                 rel->from->millivolts +
959                                    (rel->from->dfll_mode ? 1 : 0),
960                                 dvfs_solve_relationship(rel));
961                 }
962                 seq_printf(s, "   offset     %-7d mV\n", rail->offs_millivolts);
963
964                 list_sort(NULL, &rail->dvfs, dvfs_tree_sort_cmp);
965
966                 list_for_each_entry(d, &rail->dvfs, reg_node) {
967                         seq_printf(s, "   %-10s %-10lu %-4d mV\n", d->clk_name,
968                                 d->cur_rate, d->cur_millivolts);
969                 }
970         }
971
972         mutex_unlock(&dvfs_lock);
973
974         return 0;
975 }
976
977 static int dvfs_tree_open(struct inode *inode, struct file *file)
978 {
979         return single_open(file, dvfs_tree_show, inode->i_private);
980 }
981
982 static const struct file_operations dvfs_tree_fops = {
983         .open           = dvfs_tree_open,
984         .read           = seq_read,
985         .llseek         = seq_lseek,
986         .release        = single_release,
987 };
988
989 static int rail_stats_show(struct seq_file *s, void *data)
990 {
991         int i;
992         struct dvfs_rail *rail;
993
994         seq_printf(s, "%-12s %-10s\n", "millivolts", "time");
995
996         mutex_lock(&dvfs_lock);
997
998         list_for_each_entry(rail, &dvfs_rail_list, node) {
999                 seq_printf(s, "%s (bin: %d.%dmV)\n", rail->reg_id,
1000                            rail->stats.bin_uV / 1000,
1001                            (rail->stats.bin_uV / 10) % 100);
1002
1003                 dvfs_rail_stats_update(rail, -1, ktime_get());
1004
1005                 seq_printf(s, "%-12d %-10llu\n", 0,
1006                         cputime64_to_clock_t(msecs_to_jiffies(
1007                                 ktime_to_ms(rail->stats.time_at_mv[0]))));
1008
1009                 for (i = 1; i <= DVFS_RAIL_STATS_TOP_BIN; i++) {
1010                         ktime_t ktime_zero = ktime_set(0, 0);
1011                         if (ktime_equal(rail->stats.time_at_mv[i], ktime_zero))
1012                                 continue;
1013                         seq_printf(s, "%-12d %-10llu\n", rail->min_millivolts +
1014                                 (i - 1) * rail->stats.bin_uV / 1000,
1015                                 cputime64_to_clock_t(msecs_to_jiffies(
1016                                         ktime_to_ms(rail->stats.time_at_mv[i])))
1017                         );
1018                 }
1019         }
1020         mutex_unlock(&dvfs_lock);
1021         return 0;
1022 }
1023
1024 static int rail_stats_open(struct inode *inode, struct file *file)
1025 {
1026         return single_open(file, rail_stats_show, inode->i_private);
1027 }
1028
1029 static const struct file_operations rail_stats_fops = {
1030         .open           = rail_stats_open,
1031         .read           = seq_read,
1032         .llseek         = seq_lseek,
1033         .release        = single_release,
1034 };
1035
1036 static int cpu_offs_get(void *data, u64 *val)
1037 {
1038         if (tegra_cpu_rail) {
1039                 *val = (u64)tegra_cpu_rail->offs_millivolts;
1040                 return 0;
1041         }
1042         *val = 0;
1043         return -ENOENT;
1044 }
1045 static int cpu_offs_set(void *data, u64 val)
1046 {
1047         if (tegra_cpu_rail) {
1048                 mutex_lock(&dvfs_lock);
1049                 tegra_cpu_rail->offs_millivolts = (int)val;
1050                 dvfs_rail_update(tegra_cpu_rail);
1051                 mutex_unlock(&dvfs_lock);
1052                 return 0;
1053         }
1054         return -ENOENT;
1055 }
1056 DEFINE_SIMPLE_ATTRIBUTE(cpu_offs_fops, cpu_offs_get, cpu_offs_set, "%lld\n");
1057
1058 static int core_offs_get(void *data, u64 *val)
1059 {
1060         if (tegra_core_rail) {
1061                 *val = (u64)tegra_core_rail->offs_millivolts;
1062                 return 0;
1063         }
1064         *val = 0;
1065         return -ENOENT;
1066 }
1067 static int core_offs_set(void *data, u64 val)
1068 {
1069         if (tegra_core_rail) {
1070                 mutex_lock(&dvfs_lock);
1071                 tegra_core_rail->offs_millivolts = (int)val;
1072                 dvfs_rail_update(tegra_core_rail);
1073                 mutex_unlock(&dvfs_lock);
1074                 return 0;
1075         }
1076         return -ENOENT;
1077 }
1078 DEFINE_SIMPLE_ATTRIBUTE(core_offs_fops, core_offs_get, core_offs_set, "%lld\n");
1079
1080 int __init dvfs_debugfs_init(struct dentry *clk_debugfs_root)
1081 {
1082         struct dentry *d;
1083
1084         d = debugfs_create_file("dvfs", S_IRUGO, clk_debugfs_root, NULL,
1085                 &dvfs_tree_fops);
1086         if (!d)
1087                 return -ENOMEM;
1088
1089         d = debugfs_create_file("rails", S_IRUGO, clk_debugfs_root, NULL,
1090                 &rail_stats_fops);
1091         if (!d)
1092                 return -ENOMEM;
1093
1094         d = debugfs_create_file("vdd_cpu_offs", S_IRUGO | S_IWUSR,
1095                 clk_debugfs_root, NULL, &cpu_offs_fops);
1096         if (!d)
1097                 return -ENOMEM;
1098
1099         d = debugfs_create_file("vdd_core_offs", S_IRUGO | S_IWUSR,
1100                 clk_debugfs_root, NULL, &core_offs_fops);
1101         if (!d)
1102                 return -ENOMEM;
1103
1104         return 0;
1105 }
1106
1107 #endif