ARM: tegra11: dvfs: Update switch between DFLL and PLL
[linux-3.10.git] / arch / arm / mach-tegra / dvfs.c
1 /*
2  *
3  * Copyright (C) 2010 Google, Inc.
4  *
5  * Author:
6  *      Colin Cross <ccross@google.com>
7  *
8  * Copyright (C) 2010-2011 NVIDIA Corporation.
9  *
10  * This software is licensed under the terms of the GNU General Public
11  * License version 2, as published by the Free Software Foundation, and
12  * may be copied, distributed, and modified under those terms.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/clk.h>
23 #include <linux/clkdev.h>
24 #include <linux/debugfs.h>
25 #include <linux/init.h>
26 #include <linux/list.h>
27 #include <linux/list_sort.h>
28 #include <linux/module.h>
29 #include <linux/regulator/consumer.h>
30 #include <linux/seq_file.h>
31 #include <linux/slab.h>
32 #include <linux/suspend.h>
33 #include <linux/delay.h>
34 #include <linux/clk/tegra.h>
35 #include <linux/reboot.h>
36
37 #include "board.h"
38 #include "clock.h"
39 #include "dvfs.h"
40
41 #define DVFS_RAIL_STATS_BIN     12500
42
43 struct dvfs_rail *tegra_cpu_rail;
44 struct dvfs_rail *tegra_core_rail;
45
46 static LIST_HEAD(dvfs_rail_list);
47 static DEFINE_MUTEX(dvfs_lock);
48 static DEFINE_MUTEX(rail_disable_lock);
49
50 static int dvfs_rail_update(struct dvfs_rail *rail);
51
52 void tegra_dvfs_add_relationships(struct dvfs_relationship *rels, int n)
53 {
54         int i;
55         struct dvfs_relationship *rel;
56
57         mutex_lock(&dvfs_lock);
58
59         for (i = 0; i < n; i++) {
60                 rel = &rels[i];
61                 list_add_tail(&rel->from_node, &rel->to->relationships_from);
62                 list_add_tail(&rel->to_node, &rel->from->relationships_to);
63         }
64
65         mutex_unlock(&dvfs_lock);
66 }
67
68 int tegra_dvfs_init_rails(struct dvfs_rail *rails[], int n)
69 {
70         int i;
71
72         mutex_lock(&dvfs_lock);
73
74         for (i = 0; i < n; i++) {
75                 INIT_LIST_HEAD(&rails[i]->dvfs);
76                 INIT_LIST_HEAD(&rails[i]->relationships_from);
77                 INIT_LIST_HEAD(&rails[i]->relationships_to);
78                 rails[i]->millivolts = rails[i]->nominal_millivolts;
79                 rails[i]->new_millivolts = rails[i]->nominal_millivolts;
80                 if (!rails[i]->step)
81                         rails[i]->step = rails[i]->max_millivolts;
82
83                 list_add_tail(&rails[i]->node, &dvfs_rail_list);
84
85                 if (!strcmp("vdd_cpu", rails[i]->reg_id))
86                         tegra_cpu_rail = rails[i];
87                 else if (!strcmp("vdd_core", rails[i]->reg_id))
88                         tegra_core_rail = rails[i];
89         }
90
91         mutex_unlock(&dvfs_lock);
92
93         return 0;
94 };
95
96 static int dvfs_solve_relationship(struct dvfs_relationship *rel)
97 {
98         return rel->solve(rel->from, rel->to);
99 }
100
101 /* rail statistic - called during rail init, or under dfs_lock, or with
102    CPU0 only on-line, and interrupts disabled */
103 static void dvfs_rail_stats_init(struct dvfs_rail *rail, int millivolts)
104 {
105         int dvfs_rail_stats_range;
106
107         if (!rail->stats.bin_uV)
108                 rail->stats.bin_uV = DVFS_RAIL_STATS_BIN;
109
110         dvfs_rail_stats_range =
111                 (DVFS_RAIL_STATS_TOP_BIN - 1) * rail->stats.bin_uV / 1000;
112
113         rail->stats.last_update = ktime_get();
114         if (millivolts >= rail->min_millivolts) {
115                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
116                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
117                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
118         }
119
120         if (rail->max_millivolts >
121             rail->min_millivolts + dvfs_rail_stats_range)
122                 pr_warn("tegra_dvfs: %s: stats above %d mV will be squashed\n",
123                         rail->reg_id,
124                         rail->min_millivolts + dvfs_rail_stats_range);
125 }
126
127 static void dvfs_rail_stats_update(
128         struct dvfs_rail *rail, int millivolts, ktime_t now)
129 {
130         rail->stats.time_at_mv[rail->stats.last_index] = ktime_add(
131                 rail->stats.time_at_mv[rail->stats.last_index], ktime_sub(
132                         now, rail->stats.last_update));
133         rail->stats.last_update = now;
134
135         if (rail->stats.off)
136                 return;
137
138         if (millivolts >= rail->min_millivolts) {
139                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
140                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
141                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
142         } else if (millivolts == 0)
143                         rail->stats.last_index = 0;
144 }
145
146 static void dvfs_rail_stats_pause(struct dvfs_rail *rail,
147                                   ktime_t delta, bool on)
148 {
149         int i = on ? rail->stats.last_index : 0;
150         rail->stats.time_at_mv[i] = ktime_add(rail->stats.time_at_mv[i], delta);
151 }
152
153 void tegra_dvfs_rail_off(struct dvfs_rail *rail, ktime_t now)
154 {
155         if (rail) {
156                 dvfs_rail_stats_update(rail, 0, now);
157                 rail->stats.off = true;
158         }
159 }
160
161 void tegra_dvfs_rail_on(struct dvfs_rail *rail, ktime_t now)
162 {
163         if (rail) {
164                 rail->stats.off = false;
165                 dvfs_rail_stats_update(rail, rail->millivolts, now);
166         }
167 }
168
169 void tegra_dvfs_rail_pause(struct dvfs_rail *rail, ktime_t delta, bool on)
170 {
171         if (rail)
172                 dvfs_rail_stats_pause(rail, delta, on);
173 }
174
175 static int dvfs_rail_set_voltage_reg(struct dvfs_rail *rail, int millivolts)
176 {
177         int ret;
178
179         rail->updating = true;
180         rail->reg_max_millivolts = rail->reg_max_millivolts ==
181                 rail->max_millivolts ?
182                 rail->max_millivolts + 1 : rail->max_millivolts;
183         ret = regulator_set_voltage(rail->reg,
184                 millivolts * 1000,
185                 rail->reg_max_millivolts * 1000);
186         rail->updating = false;
187
188         return ret;
189 }
190
191 /* Sets the voltage on a dvfs rail to a specific value, and updates any
192  * rails that depend on this rail. */
193 static int dvfs_rail_set_voltage(struct dvfs_rail *rail, int millivolts)
194 {
195         int ret = 0;
196         struct dvfs_relationship *rel;
197         int step = (millivolts > rail->millivolts) ? rail->step : -rail->step;
198         int i;
199         int steps;
200         bool jmp_to_zero;
201
202         if (!rail->reg) {
203                 if (millivolts == rail->millivolts)
204                         return 0;
205                 else
206                         return -EINVAL;
207         }
208
209         /*
210          * DFLL adjusts rail voltage automatically, but not exactly to the
211          * expected level - update stats, anyway.
212          */
213         if (rail->dfll_mode) {
214                 rail->millivolts = rail->new_millivolts = millivolts;
215                 dvfs_rail_stats_update(rail, millivolts, ktime_get());
216                 return 0;
217         }
218
219         if (rail->disabled)
220                 return 0;
221
222         rail->resolving_to = true;
223         jmp_to_zero = rail->jmp_to_zero &&
224                         ((millivolts == 0) || (rail->millivolts == 0));
225         steps = jmp_to_zero ? 1 :
226                 DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
227
228         for (i = 0; i < steps; i++) {
229                 if (!jmp_to_zero &&
230                     (abs(millivolts - rail->millivolts) > rail->step))
231                         rail->new_millivolts = rail->millivolts + step;
232                 else
233                         rail->new_millivolts = millivolts;
234
235                 /* Before changing the voltage, tell each rail that depends
236                  * on this rail that the voltage will change.
237                  * This rail will be the "from" rail in the relationship,
238                  * the rail that depends on this rail will be the "to" rail.
239                  * from->millivolts will be the old voltage
240                  * from->new_millivolts will be the new voltage */
241                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
242                         ret = dvfs_rail_update(rel->to);
243                         if (ret)
244                                 goto out;
245                 }
246
247                 ret = dvfs_rail_set_voltage_reg(rail, rail->new_millivolts);
248                 if (ret) {
249                         pr_err("Failed to set dvfs regulator %s\n", rail->reg_id);
250                         goto out;
251                 }
252
253                 rail->millivolts = rail->new_millivolts;
254                 dvfs_rail_stats_update(rail, rail->millivolts, ktime_get());
255
256                 /* After changing the voltage, tell each rail that depends
257                  * on this rail that the voltage has changed.
258                  * from->millivolts and from->new_millivolts will be the
259                  * new voltage */
260                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
261                         ret = dvfs_rail_update(rel->to);
262                         if (ret)
263                                 goto out;
264                 }
265         }
266
267         if (unlikely(rail->millivolts != millivolts)) {
268                 pr_err("%s: rail didn't reach target %d in %d steps (%d)\n",
269                         __func__, millivolts, steps, rail->millivolts);
270                 ret = -EINVAL;
271         }
272
273 out:
274         rail->resolving_to = false;
275         return ret;
276 }
277
278 /* Determine the minimum valid voltage for a rail, taking into account
279  * the dvfs clocks and any rails that this rail depends on.  Calls
280  * dvfs_rail_set_voltage with the new voltage, which will call
281  * dvfs_rail_update on any rails that depend on this rail. */
282 static inline int dvfs_rail_apply_limits(struct dvfs_rail *rail, int millivolts)
283 {
284         int min_mv = rail->min_millivolts;
285
286         if (rail->pll_mode_cdev)
287                 min_mv = max(min_mv, rail->thermal_idx ?
288                              0 : rail->min_millivolts_cold);
289
290         millivolts += rail->offs_millivolts;
291         if (millivolts > rail->max_millivolts)
292                 millivolts = rail->max_millivolts;
293         else if (millivolts < min_mv)
294                 millivolts = min_mv;
295
296         return millivolts;
297 }
298
299 static int dvfs_rail_update(struct dvfs_rail *rail)
300 {
301         int millivolts = 0;
302         struct dvfs *d;
303         struct dvfs_relationship *rel;
304         int ret = 0;
305         int steps;
306
307         /* if dvfs is suspended, return and handle it during resume */
308         if (rail->suspended)
309                 return 0;
310
311         /* if regulators are not connected yet, return and handle it later */
312         if (!rail->reg)
313                 return 0;
314
315         /* if rail update is entered while resolving circular dependencies,
316            abort recursion */
317         if (rail->resolving_to)
318                 return 0;
319
320         /* Find the maximum voltage requested by any clock */
321         list_for_each_entry(d, &rail->dvfs, reg_node)
322                 millivolts = max(d->cur_millivolts, millivolts);
323
324         /* Apply offset and min/max limits if any clock is requesting voltage */
325         if (millivolts)
326                 millivolts = dvfs_rail_apply_limits(rail, millivolts);
327
328         /* retry update if limited by from-relationship to account for
329            circular dependencies */
330         steps = DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
331         for (; steps >= 0; steps--) {
332                 rail->new_millivolts = millivolts;
333
334                 /* Check any rails that this rail depends on */
335                 list_for_each_entry(rel, &rail->relationships_from, from_node)
336                         rail->new_millivolts = dvfs_solve_relationship(rel);
337
338                 if (rail->new_millivolts == rail->millivolts)
339                         break;
340
341                 ret = dvfs_rail_set_voltage(rail, rail->new_millivolts);
342         }
343
344         return ret;
345 }
346
347 static int dvfs_rail_connect_to_regulator(struct dvfs_rail *rail)
348 {
349         struct regulator *reg;
350         int v;
351
352         if (!rail->reg) {
353                 reg = regulator_get(NULL, rail->reg_id);
354                 if (IS_ERR(reg)) {
355                         pr_err("tegra_dvfs: failed to connect %s rail\n",
356                                rail->reg_id);
357                         return -EINVAL;
358                 }
359                 rail->reg = reg;
360         }
361
362         v = regulator_enable(rail->reg);
363         if (v < 0) {
364                 pr_err("tegra_dvfs: failed on enabling regulator %s\n, err %d",
365                         rail->reg_id, v);
366                 return v;
367         }
368
369         v = regulator_get_voltage(rail->reg);
370         if (v < 0) {
371                 pr_err("tegra_dvfs: failed initial get %s voltage\n",
372                        rail->reg_id);
373                 return v;
374         }
375         rail->millivolts = v / 1000;
376         rail->new_millivolts = rail->millivolts;
377         dvfs_rail_stats_init(rail, rail->millivolts);
378         return 0;
379 }
380
381 static inline unsigned long *dvfs_get_freqs(struct dvfs *d)
382 {
383         return d->alt_freqs ? : &d->freqs[0];
384 }
385
386 static inline const int *dvfs_get_millivolts(struct dvfs *d, unsigned long rate)
387 {
388         if (tegra_dvfs_is_dfll_scale(d, rate))
389                 return d->dfll_millivolts;
390
391         return d->millivolts;
392 }
393
394 static int
395 __tegra_dvfs_set_rate(struct dvfs *d, unsigned long rate)
396 {
397         int i = 0;
398         int ret;
399         unsigned long *freqs = dvfs_get_freqs(d);
400         const int *millivolts = dvfs_get_millivolts(d, rate);
401
402         if (freqs == NULL || millivolts == NULL)
403                 return -ENODEV;
404
405         /* On entry to dfll range limit 1st step to range bottom (full ramp of
406            voltage/rate is completed automatically in dfll mode) */
407         if (tegra_dvfs_is_dfll_range_entry(d, rate))
408                 rate = d->dfll_data.use_dfll_rate_min;
409
410         if (rate > freqs[d->num_freqs - 1]) {
411                 pr_warn("tegra_dvfs: rate %lu too high for dvfs on %s\n", rate,
412                         d->clk_name);
413                 return -EINVAL;
414         }
415
416         if (rate == 0) {
417                 d->cur_millivolts = 0;
418         } else {
419                 while (i < d->num_freqs && rate > freqs[i])
420                         i++;
421
422                 if ((d->max_millivolts) &&
423                     (millivolts[i] > d->max_millivolts)) {
424                         pr_warn("tegra_dvfs: voltage %d too high for dvfs on"
425                                 " %s\n", millivolts[i], d->clk_name);
426                         return -EINVAL;
427                 }
428                 d->cur_millivolts = millivolts[i];
429         }
430
431         d->cur_rate = rate;
432
433         ret = dvfs_rail_update(d->dvfs_rail);
434         if (ret)
435                 pr_err("Failed to set regulator %s for clock %s to %d mV\n",
436                         d->dvfs_rail->reg_id, d->clk_name, d->cur_millivolts);
437
438         return ret;
439 }
440
441 int tegra_dvfs_alt_freqs_set(struct dvfs *d, unsigned long *alt_freqs)
442 {
443         int ret = 0;
444
445         mutex_lock(&dvfs_lock);
446
447         if (d->alt_freqs != alt_freqs) {
448                 d->alt_freqs = alt_freqs;
449                 ret = __tegra_dvfs_set_rate(d, d->cur_rate);
450         }
451
452         mutex_unlock(&dvfs_lock);
453         return ret;
454 }
455
456 static int predict_millivolts(struct clk *c, const int *millivolts,
457                               unsigned long rate)
458 {
459         int i;
460
461         if (!millivolts)
462                 return -ENODEV;
463         /*
464          * Predicted voltage can not be used across the switch to alternative
465          * frequency limits. For now, just fail the call for clock that has
466          * alternative limits initialized.
467          */
468         if (c->dvfs->alt_freqs)
469                 return -ENOSYS;
470
471         for (i = 0; i < c->dvfs->num_freqs; i++) {
472                 if (rate <= c->dvfs->freqs[i])
473                         break;
474         }
475
476         if (i == c->dvfs->num_freqs)
477                 return -EINVAL;
478
479         return millivolts[i];
480 }
481
482 int tegra_dvfs_predict_millivolts(struct clk *c, unsigned long rate)
483 {
484         const int *millivolts;
485
486         if (!rate || !c->dvfs)
487                 return 0;
488
489         millivolts = dvfs_get_millivolts(c->dvfs, rate);
490         return predict_millivolts(c, millivolts, rate);
491 }
492
493 int tegra_dvfs_predict_millivolts_pll(struct clk *c, unsigned long rate)
494 {
495         const int *millivolts;
496
497         if (!rate || !c->dvfs)
498                 return 0;
499
500         millivolts = c->dvfs->millivolts;
501         return predict_millivolts(c, millivolts, rate);
502 }
503
504 int tegra_dvfs_predict_millivolts_dfll(struct clk *c, unsigned long rate)
505 {
506         const int *millivolts;
507
508         if (!rate || !c->dvfs)
509                 return 0;
510
511         millivolts = c->dvfs->dfll_millivolts;
512         return predict_millivolts(c, millivolts, rate);
513 }
514
515 int tegra_dvfs_set_rate(struct clk *c, unsigned long rate)
516 {
517         int ret;
518
519         if (!c->dvfs)
520                 return -EINVAL;
521
522         mutex_lock(&dvfs_lock);
523         ret = __tegra_dvfs_set_rate(c->dvfs, rate);
524         mutex_unlock(&dvfs_lock);
525
526         return ret;
527 }
528 EXPORT_SYMBOL(tegra_dvfs_set_rate);
529
530 /* May only be called during clock init, does not take any locks on clock c. */
531 int __init tegra_enable_dvfs_on_clk(struct clk *c, struct dvfs *d)
532 {
533         int i;
534
535         if (c->dvfs) {
536                 pr_err("Error when enabling dvfs on %s for clock %s:\n",
537                         d->dvfs_rail->reg_id, c->name);
538                 pr_err("DVFS already enabled for %s\n",
539                         c->dvfs->dvfs_rail->reg_id);
540                 return -EINVAL;
541         }
542
543         for (i = 0; i < MAX_DVFS_FREQS; i++) {
544                 if (d->millivolts[i] == 0)
545                         break;
546
547                 d->freqs[i] *= d->freqs_mult;
548
549                 /* If final frequencies are 0, pad with previous frequency */
550                 if (d->freqs[i] == 0 && i > 1)
551                         d->freqs[i] = d->freqs[i - 1];
552         }
553         d->num_freqs = i;
554
555         if (d->auto_dvfs) {
556                 c->auto_dvfs = true;
557                 clk_set_cansleep(c);
558         }
559
560         c->dvfs = d;
561
562         mutex_lock(&dvfs_lock);
563         list_add_tail(&d->reg_node, &d->dvfs_rail->dvfs);
564         mutex_unlock(&dvfs_lock);
565
566         return 0;
567 }
568
569 static bool tegra_dvfs_all_rails_suspended(void)
570 {
571         struct dvfs_rail *rail;
572         bool all_suspended = true;
573
574         list_for_each_entry(rail, &dvfs_rail_list, node)
575                 if (!rail->suspended && !rail->disabled)
576                         all_suspended = false;
577
578         return all_suspended;
579 }
580
581 static bool tegra_dvfs_from_rails_suspended_or_solved(struct dvfs_rail *to)
582 {
583         struct dvfs_relationship *rel;
584         bool all_suspended = true;
585
586         list_for_each_entry(rel, &to->relationships_from, from_node)
587                 if (!rel->from->suspended && !rel->from->disabled &&
588                         !rel->solved_at_nominal)
589                         all_suspended = false;
590
591         return all_suspended;
592 }
593
594 static int tegra_dvfs_suspend_one(void)
595 {
596         struct dvfs_rail *rail;
597         int ret;
598
599         list_for_each_entry(rail, &dvfs_rail_list, node) {
600                 if (!rail->suspended && !rail->disabled &&
601                     tegra_dvfs_from_rails_suspended_or_solved(rail)) {
602                         int mv = dvfs_rail_apply_limits(
603                                 rail, rail->nominal_millivolts);
604                         ret = dvfs_rail_set_voltage(rail, mv);
605                         if (ret)
606                                 return ret;
607                         rail->suspended = true;
608                         return 0;
609                 }
610         }
611
612         return -EINVAL;
613 }
614
615 static void tegra_dvfs_resume(void)
616 {
617         struct dvfs_rail *rail;
618
619         mutex_lock(&dvfs_lock);
620
621         list_for_each_entry(rail, &dvfs_rail_list, node)
622                 rail->suspended = false;
623
624         list_for_each_entry(rail, &dvfs_rail_list, node)
625                 dvfs_rail_update(rail);
626
627         mutex_unlock(&dvfs_lock);
628 }
629
630 static int tegra_dvfs_suspend(void)
631 {
632         int ret = 0;
633
634         mutex_lock(&dvfs_lock);
635
636         while (!tegra_dvfs_all_rails_suspended()) {
637                 ret = tegra_dvfs_suspend_one();
638                 if (ret)
639                         break;
640         }
641
642         mutex_unlock(&dvfs_lock);
643
644         if (ret)
645                 tegra_dvfs_resume();
646
647         return ret;
648 }
649
650 static int tegra_dvfs_pm_notify(struct notifier_block *nb,
651                                 unsigned long event, void *data)
652 {
653         switch (event) {
654         case PM_SUSPEND_PREPARE:
655                 if (tegra_dvfs_suspend())
656                         return NOTIFY_STOP;
657                 break;
658         case PM_POST_SUSPEND:
659                 tegra_dvfs_resume();
660                 break;
661         }
662
663         return NOTIFY_OK;
664 };
665
666 static struct notifier_block tegra_dvfs_nb = {
667         .notifier_call = tegra_dvfs_pm_notify,
668 };
669
670 static int tegra_dvfs_reboot_notify(struct notifier_block *nb,
671                                 unsigned long event, void *data)
672 {
673         switch (event) {
674         case SYS_RESTART:
675         case SYS_HALT:
676         case SYS_POWER_OFF:
677                 tegra_dvfs_suspend();
678                 return NOTIFY_OK;
679         }
680         return NOTIFY_DONE;
681 }
682
683 static struct notifier_block tegra_dvfs_reboot_nb = {
684         .notifier_call = tegra_dvfs_reboot_notify,
685 };
686
687 /* must be called with dvfs lock held */
688 static void __tegra_dvfs_rail_disable(struct dvfs_rail *rail)
689 {
690         int ret;
691
692         /* don't set voltage in DFLL mode - won't work, but break stats */
693         if (rail->dfll_mode) {
694                 rail->disabled = true;
695                 return;
696         }
697
698         ret = dvfs_rail_set_voltage(rail,
699                 dvfs_rail_apply_limits(rail, rail->nominal_millivolts));
700         if (ret) {
701                 pr_info("dvfs: failed to set regulator %s to disable "
702                         "voltage %d\n", rail->reg_id,
703                         rail->nominal_millivolts);
704                 return;
705         }
706         rail->disabled = true;
707 }
708
709 /* must be called with dvfs lock held */
710 static void __tegra_dvfs_rail_enable(struct dvfs_rail *rail)
711 {
712         rail->disabled = false;
713         dvfs_rail_update(rail);
714 }
715
716 void tegra_dvfs_rail_enable(struct dvfs_rail *rail)
717 {
718         if (!rail)
719                 return;
720
721         mutex_lock(&rail_disable_lock);
722
723         if (rail->disabled) {
724                 mutex_lock(&dvfs_lock);
725                 __tegra_dvfs_rail_enable(rail);
726                 mutex_unlock(&dvfs_lock);
727
728                 tegra_dvfs_rail_post_enable(rail);
729         }
730         mutex_unlock(&rail_disable_lock);
731 }
732
733 void tegra_dvfs_rail_disable(struct dvfs_rail *rail)
734 {
735         if (!rail)
736                 return;
737
738         mutex_lock(&rail_disable_lock);
739         if (rail->disabled)
740                 goto out;
741
742         /* rail disable will set it to nominal voltage underneath clock
743            framework - need to re-configure clock rates that are not safe
744            at nominal (yes, unsafe at nominal is ugly, but possible). Rate
745            change must be done outside of dvfs lock. */
746         if (tegra_dvfs_rail_disable_prepare(rail)) {
747                 pr_info("dvfs: failed to prepare regulator %s to disable\n",
748                         rail->reg_id);
749                 goto out;
750         }
751
752         mutex_lock(&dvfs_lock);
753         __tegra_dvfs_rail_disable(rail);
754         mutex_unlock(&dvfs_lock);
755 out:
756         mutex_unlock(&rail_disable_lock);
757 }
758
759 int tegra_dvfs_rail_disable_by_name(const char *reg_id)
760 {
761         struct dvfs_rail *rail = tegra_dvfs_get_rail_by_name(reg_id);
762         if (!rail)
763                 return -EINVAL;
764
765         tegra_dvfs_rail_disable(rail);
766         return 0;
767 }
768
769 struct dvfs_rail *tegra_dvfs_get_rail_by_name(const char *reg_id)
770 {
771         struct dvfs_rail *rail;
772
773         mutex_lock(&dvfs_lock);
774         list_for_each_entry(rail, &dvfs_rail_list, node) {
775                 if (!strcmp(reg_id, rail->reg_id)) {
776                         mutex_unlock(&dvfs_lock);
777                         return rail;
778                 }
779         }
780         mutex_unlock(&dvfs_lock);
781         return NULL;
782 }
783
784 bool tegra_dvfs_rail_updating(struct clk *clk)
785 {
786         return (!clk ? false :
787                 (!clk->dvfs ? false :
788                  (!clk->dvfs->dvfs_rail ? false :
789                   (clk->dvfs->dvfs_rail->updating ||
790                    clk->dvfs->dvfs_rail->dfll_mode_updating))));
791 }
792
793 #ifdef CONFIG_OF
794 int __init of_tegra_dvfs_init(const struct of_device_id *matches)
795 {
796         int ret;
797         struct device_node *np;
798
799         for_each_matching_node(np, matches) {
800                 const struct of_device_id *match = of_match_node(matches, np);
801                 of_tegra_dvfs_init_cb_t dvfs_init_cb = match->data;
802                 ret = dvfs_init_cb(np);
803                 if (ret) {
804                         pr_err("dt: Failed to read %s tables from DT\n",
805                                                         match->compatible);
806                         return ret;
807                 }
808         }
809         return 0;
810 }
811 #endif
812 int tegra_dvfs_dfll_mode_set(struct dvfs *d, unsigned long rate)
813 {
814         mutex_lock(&dvfs_lock);
815         if (!d->dvfs_rail->dfll_mode) {
816                 d->dvfs_rail->dfll_mode = true;
817                 __tegra_dvfs_set_rate(d, rate);
818         }
819         mutex_unlock(&dvfs_lock);
820         return 0;
821 }
822
823 int tegra_dvfs_dfll_mode_clear(struct dvfs *d, unsigned long rate)
824 {
825         int ret = 0;
826
827         mutex_lock(&dvfs_lock);
828         if (d->dvfs_rail->dfll_mode) {
829                 d->dvfs_rail->dfll_mode = false;
830                 /* avoid false detection of matching target (voltage in dfll
831                    mode is fluctuating, and recorded level is just estimate) */
832                 d->dvfs_rail->millivolts--;
833                 if (d->dvfs_rail->disabled) {
834                         d->dvfs_rail->disabled = false;
835                         __tegra_dvfs_rail_disable(d->dvfs_rail);
836                 }
837                 ret = __tegra_dvfs_set_rate(d, rate);
838         }
839         mutex_unlock(&dvfs_lock);
840         return ret;
841 }
842
843 struct tegra_cooling_device *tegra_dvfs_get_cpu_dfll_cdev(void)
844 {
845         if (tegra_cpu_rail)
846                 return tegra_cpu_rail->dfll_mode_cdev;
847         return NULL;
848 }
849
850 struct tegra_cooling_device *tegra_dvfs_get_cpu_pll_cdev(void)
851 {
852         if (tegra_cpu_rail)
853                 return tegra_cpu_rail->pll_mode_cdev;
854         return NULL;
855 }
856
857 struct tegra_cooling_device *tegra_dvfs_get_core_cdev(void)
858 {
859         if (tegra_core_rail)
860                 return tegra_core_rail->pll_mode_cdev;
861         return NULL;
862 }
863
864 #ifdef CONFIG_THERMAL
865 /* Cooling device limits minimum rail voltage at cold temperature in pll mode */
866 static int tegra_dvfs_rail_get_cdev_max_state(
867         struct thermal_cooling_device *cdev, unsigned long *max_state)
868 {
869         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
870         *max_state = rail->pll_mode_cdev->trip_temperatures_num;
871         return 0;
872 }
873
874 static int tegra_dvfs_rail_get_cdev_cur_state(
875         struct thermal_cooling_device *cdev, unsigned long *cur_state)
876 {
877         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
878         *cur_state = rail->thermal_idx;
879         return 0;
880 }
881
882 static int tegra_dvfs_rail_set_cdev_state(
883         struct thermal_cooling_device *cdev, unsigned long cur_state)
884 {
885         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
886
887         mutex_lock(&dvfs_lock);
888         if (rail->thermal_idx != cur_state) {
889                 rail->thermal_idx = cur_state;
890                 dvfs_rail_update(rail);
891         }
892         mutex_unlock(&dvfs_lock);
893         return 0;
894 }
895
896 static struct thermal_cooling_device_ops tegra_dvfs_rail_cooling_ops = {
897         .get_max_state = tegra_dvfs_rail_get_cdev_max_state,
898         .get_cur_state = tegra_dvfs_rail_get_cdev_cur_state,
899         .set_cur_state = tegra_dvfs_rail_set_cdev_state,
900 };
901
902 static void tegra_dvfs_rail_register_pll_mode_cdev(struct dvfs_rail *rail)
903 {
904         if (!rail->pll_mode_cdev)
905                 return;
906
907         /* just report error - initialized for cold temperature, anyway */
908         if (IS_ERR_OR_NULL(thermal_cooling_device_register(
909                 rail->pll_mode_cdev->cdev_type, (void *)rail,
910                 &tegra_dvfs_rail_cooling_ops)))
911                 pr_err("tegra cooling device %s failed to register\n",
912                        rail->pll_mode_cdev->cdev_type);
913 }
914 #else
915 #define tegra_dvfs_rail_register_pll_mode_cdev(rail)
916 #endif
917
918 /* Directly set cold temperature limit in dfll mode */
919 int tegra_dvfs_rail_dfll_mode_set_cold(struct dvfs_rail *rail)
920 {
921         int ret = 0;
922
923 #ifdef CONFIG_THERMAL
924         if (!rail || !rail->dfll_mode_cdev || !rail->min_millivolts_cold)
925                 return ret;
926
927         mutex_lock(&dvfs_lock);
928         if (rail->dfll_mode)
929                 ret = dvfs_rail_set_voltage_reg(
930                         rail, rail->min_millivolts_cold);
931         mutex_unlock(&dvfs_lock);
932 #endif
933         return ret;
934 }
935
936 /*
937  * Iterate through all the dvfs regulators, finding the regulator exported
938  * by the regulator api for each one.  Must be called in late init, after
939  * all the regulator api's regulators are initialized.
940  */
941 int __init tegra_dvfs_late_init(void)
942 {
943         bool connected = true;
944         struct dvfs_rail *rail;
945
946         mutex_lock(&dvfs_lock);
947
948         list_for_each_entry(rail, &dvfs_rail_list, node)
949                 if (dvfs_rail_connect_to_regulator(rail))
950                         connected = false;
951
952         list_for_each_entry(rail, &dvfs_rail_list, node)
953                 if (connected)
954                         dvfs_rail_update(rail);
955                 else
956                         __tegra_dvfs_rail_disable(rail);
957
958         mutex_unlock(&dvfs_lock);
959
960 #ifdef CONFIG_TEGRA_SILICON_PLATFORM
961         if (!connected)
962                 return -ENODEV;
963 #endif
964         register_pm_notifier(&tegra_dvfs_nb);
965         register_reboot_notifier(&tegra_dvfs_reboot_nb);
966
967         list_for_each_entry(rail, &dvfs_rail_list, node)
968                 tegra_dvfs_rail_register_pll_mode_cdev(rail);
969
970         return 0;
971 }
972
973 static int rail_stats_save_to_buf(char *buf, int len)
974 {
975         int i;
976         struct dvfs_rail *rail;
977         char *str = buf;
978         char *end = buf + len;
979
980         str += scnprintf(str, end - str, "%-12s %-10s\n", "millivolts", "time");
981
982         mutex_lock(&dvfs_lock);
983
984         list_for_each_entry(rail, &dvfs_rail_list, node) {
985                 str += scnprintf(str, end - str, "%s (bin: %d.%dmV)\n",
986                            rail->reg_id,
987                            rail->stats.bin_uV / 1000,
988                            (rail->stats.bin_uV / 10) % 100);
989
990                 dvfs_rail_stats_update(rail, -1, ktime_get());
991
992                 str += scnprintf(str, end - str, "%-12d %-10llu\n", 0,
993                         cputime64_to_clock_t(msecs_to_jiffies(
994                                 ktime_to_ms(rail->stats.time_at_mv[0]))));
995
996                 for (i = 1; i <= DVFS_RAIL_STATS_TOP_BIN; i++) {
997                         ktime_t ktime_zero = ktime_set(0, 0);
998                         if (ktime_equal(rail->stats.time_at_mv[i], ktime_zero))
999                                 continue;
1000                         str += scnprintf(str, end - str, "%-12d %-10llu\n",
1001                                 rail->min_millivolts +
1002                                 (i - 1) * rail->stats.bin_uV / 1000,
1003                                 cputime64_to_clock_t(msecs_to_jiffies(
1004                                         ktime_to_ms(rail->stats.time_at_mv[i])))
1005                         );
1006                 }
1007         }
1008         mutex_unlock(&dvfs_lock);
1009         return str - buf;
1010 }
1011
1012 #ifdef CONFIG_DEBUG_FS
1013 static int dvfs_tree_sort_cmp(void *p, struct list_head *a, struct list_head *b)
1014 {
1015         struct dvfs *da = list_entry(a, struct dvfs, reg_node);
1016         struct dvfs *db = list_entry(b, struct dvfs, reg_node);
1017         int ret;
1018
1019         ret = strcmp(da->dvfs_rail->reg_id, db->dvfs_rail->reg_id);
1020         if (ret != 0)
1021                 return ret;
1022
1023         if (da->cur_millivolts < db->cur_millivolts)
1024                 return 1;
1025         if (da->cur_millivolts > db->cur_millivolts)
1026                 return -1;
1027
1028         return strcmp(da->clk_name, db->clk_name);
1029 }
1030
1031 static int dvfs_tree_show(struct seq_file *s, void *data)
1032 {
1033         struct dvfs *d;
1034         struct dvfs_rail *rail;
1035         struct dvfs_relationship *rel;
1036
1037         seq_printf(s, "   clock      rate       mV\n");
1038         seq_printf(s, "--------------------------------\n");
1039
1040         mutex_lock(&dvfs_lock);
1041
1042         list_for_each_entry(rail, &dvfs_rail_list, node) {
1043                 seq_printf(s, "%s %d mV%s:\n", rail->reg_id, rail->millivolts,
1044                            rail->dfll_mode ? " dfll mode" :
1045                                 rail->disabled ? " disabled" : "");
1046                 list_for_each_entry(rel, &rail->relationships_from, from_node) {
1047                         seq_printf(s, "   %-10s %-7d mV %-4d mV\n",
1048                                 rel->from->reg_id, rel->from->millivolts,
1049                                 dvfs_solve_relationship(rel));
1050                 }
1051                 seq_printf(s, "   offset     %-7d mV\n", rail->offs_millivolts);
1052
1053                 list_sort(NULL, &rail->dvfs, dvfs_tree_sort_cmp);
1054
1055                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1056                         seq_printf(s, "   %-10s %-10lu %-4d mV\n", d->clk_name,
1057                                 d->cur_rate, d->cur_millivolts);
1058                 }
1059         }
1060
1061         mutex_unlock(&dvfs_lock);
1062
1063         return 0;
1064 }
1065
1066 static int dvfs_tree_open(struct inode *inode, struct file *file)
1067 {
1068         return single_open(file, dvfs_tree_show, inode->i_private);
1069 }
1070
1071 static const struct file_operations dvfs_tree_fops = {
1072         .open           = dvfs_tree_open,
1073         .read           = seq_read,
1074         .llseek         = seq_lseek,
1075         .release        = single_release,
1076 };
1077
1078 static int rail_stats_show(struct seq_file *s, void *data)
1079 {
1080         char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
1081         int size = 0;
1082
1083         if (!buf)
1084                 return -ENOMEM;
1085
1086         size = rail_stats_save_to_buf(buf, PAGE_SIZE);
1087         seq_write(s, buf, size);
1088         kfree(buf);
1089         return 0;
1090 }
1091
1092 static int rail_stats_open(struct inode *inode, struct file *file)
1093 {
1094         return single_open(file, rail_stats_show, inode->i_private);
1095 }
1096
1097 static const struct file_operations rail_stats_fops = {
1098         .open           = rail_stats_open,
1099         .read           = seq_read,
1100         .llseek         = seq_lseek,
1101         .release        = single_release,
1102 };
1103
1104 static int cpu_offs_get(void *data, u64 *val)
1105 {
1106         if (tegra_cpu_rail) {
1107                 *val = (u64)tegra_cpu_rail->offs_millivolts;
1108                 return 0;
1109         }
1110         *val = 0;
1111         return -ENOENT;
1112 }
1113 static int cpu_offs_set(void *data, u64 val)
1114 {
1115         if (tegra_cpu_rail) {
1116                 mutex_lock(&dvfs_lock);
1117                 tegra_cpu_rail->offs_millivolts = (int)val;
1118                 dvfs_rail_update(tegra_cpu_rail);
1119                 mutex_unlock(&dvfs_lock);
1120                 return 0;
1121         }
1122         return -ENOENT;
1123 }
1124 DEFINE_SIMPLE_ATTRIBUTE(cpu_offs_fops, cpu_offs_get, cpu_offs_set, "%lld\n");
1125
1126 static int core_offs_get(void *data, u64 *val)
1127 {
1128         if (tegra_core_rail) {
1129                 *val = (u64)tegra_core_rail->offs_millivolts;
1130                 return 0;
1131         }
1132         *val = 0;
1133         return -ENOENT;
1134 }
1135 static int core_offs_set(void *data, u64 val)
1136 {
1137         if (tegra_core_rail) {
1138                 mutex_lock(&dvfs_lock);
1139                 tegra_core_rail->offs_millivolts = (int)val;
1140                 dvfs_rail_update(tegra_core_rail);
1141                 mutex_unlock(&dvfs_lock);
1142                 return 0;
1143         }
1144         return -ENOENT;
1145 }
1146 DEFINE_SIMPLE_ATTRIBUTE(core_offs_fops, core_offs_get, core_offs_set, "%lld\n");
1147
1148 int __init dvfs_debugfs_init(struct dentry *clk_debugfs_root)
1149 {
1150         struct dentry *d;
1151
1152         d = debugfs_create_file("dvfs", S_IRUGO, clk_debugfs_root, NULL,
1153                 &dvfs_tree_fops);
1154         if (!d)
1155                 return -ENOMEM;
1156
1157         d = debugfs_create_file("rails", S_IRUGO, clk_debugfs_root, NULL,
1158                 &rail_stats_fops);
1159         if (!d)
1160                 return -ENOMEM;
1161
1162         d = debugfs_create_file("vdd_cpu_offs", S_IRUGO | S_IWUSR,
1163                 clk_debugfs_root, NULL, &cpu_offs_fops);
1164         if (!d)
1165                 return -ENOMEM;
1166
1167         d = debugfs_create_file("vdd_core_offs", S_IRUGO | S_IWUSR,
1168                 clk_debugfs_root, NULL, &core_offs_fops);
1169         if (!d)
1170                 return -ENOMEM;
1171
1172         return 0;
1173 }
1174
1175 #endif
1176
1177 #ifdef CONFIG_PM
1178 static ssize_t tegra_rail_stats_show(struct kobject *kobj,
1179                                         struct kobj_attribute *attr,
1180                                         char *buf)
1181 {
1182         return rail_stats_save_to_buf(buf, PAGE_SIZE);
1183 }
1184
1185 static struct kobj_attribute rail_stats_attr =
1186                 __ATTR_RO(tegra_rail_stats);
1187
1188 static int __init tegra_dvfs_sysfs_stats_init(void)
1189 {
1190         int error;
1191         error = sysfs_create_file(power_kobj, &rail_stats_attr.attr);
1192         return 0;
1193 }
1194 late_initcall(tegra_dvfs_sysfs_stats_init);
1195 #endif