ARM: tegra11: clock: Combine DFLL usage controls
[linux-3.10.git] / arch / arm / mach-tegra / dvfs.c
1 /*
2  *
3  * Copyright (C) 2010 Google, Inc.
4  *
5  * Author:
6  *      Colin Cross <ccross@google.com>
7  *
8  * Copyright (C) 2010-2011 NVIDIA Corporation.
9  *
10  * This software is licensed under the terms of the GNU General Public
11  * License version 2, as published by the Free Software Foundation, and
12  * may be copied, distributed, and modified under those terms.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/clk.h>
23 #include <linux/clkdev.h>
24 #include <linux/debugfs.h>
25 #include <linux/init.h>
26 #include <linux/list.h>
27 #include <linux/list_sort.h>
28 #include <linux/module.h>
29 #include <linux/regulator/consumer.h>
30 #include <linux/seq_file.h>
31 #include <linux/slab.h>
32 #include <linux/suspend.h>
33 #include <linux/delay.h>
34 #include <linux/clk/tegra.h>
35 #include <linux/reboot.h>
36
37 #include "board.h"
38 #include "clock.h"
39 #include "dvfs.h"
40
41 #define DVFS_RAIL_STATS_BIN     25
42 #define DVFS_RAIL_STATS_SCALE   2
43 #define DVFS_RAIL_STATS_RANGE   ((DVFS_RAIL_STATS_TOP_BIN - 1) * \
44                                  DVFS_RAIL_STATS_BIN / DVFS_RAIL_STATS_SCALE)
45
46 struct dvfs_rail *tegra_cpu_rail;
47 struct dvfs_rail *tegra_core_rail;
48
49 static LIST_HEAD(dvfs_rail_list);
50 static DEFINE_MUTEX(dvfs_lock);
51 static DEFINE_MUTEX(rail_disable_lock);
52
53 static int dvfs_rail_update(struct dvfs_rail *rail);
54
55 void tegra_dvfs_add_relationships(struct dvfs_relationship *rels, int n)
56 {
57         int i;
58         struct dvfs_relationship *rel;
59
60         mutex_lock(&dvfs_lock);
61
62         for (i = 0; i < n; i++) {
63                 rel = &rels[i];
64                 list_add_tail(&rel->from_node, &rel->to->relationships_from);
65                 list_add_tail(&rel->to_node, &rel->from->relationships_to);
66         }
67
68         mutex_unlock(&dvfs_lock);
69 }
70
71 int tegra_dvfs_init_rails(struct dvfs_rail *rails[], int n)
72 {
73         int i;
74
75         mutex_lock(&dvfs_lock);
76
77         for (i = 0; i < n; i++) {
78                 INIT_LIST_HEAD(&rails[i]->dvfs);
79                 INIT_LIST_HEAD(&rails[i]->relationships_from);
80                 INIT_LIST_HEAD(&rails[i]->relationships_to);
81                 rails[i]->millivolts = rails[i]->nominal_millivolts;
82                 rails[i]->new_millivolts = rails[i]->nominal_millivolts;
83                 if (!rails[i]->step)
84                         rails[i]->step = rails[i]->max_millivolts;
85
86                 list_add_tail(&rails[i]->node, &dvfs_rail_list);
87
88                 if (!strcmp("vdd_cpu", rails[i]->reg_id))
89                         tegra_cpu_rail = rails[i];
90                 else if (!strcmp("vdd_core", rails[i]->reg_id))
91                         tegra_core_rail = rails[i];
92         }
93
94         mutex_unlock(&dvfs_lock);
95
96         return 0;
97 };
98
99 static int dvfs_solve_relationship(struct dvfs_relationship *rel)
100 {
101         return rel->solve(rel->from, rel->to);
102 }
103
104 /* rail statistic - called during rail init, or under dfs_lock, or with
105    CPU0 only on-line, and interrupts disabled */
106 static void dvfs_rail_stats_init(struct dvfs_rail *rail, int millivolts)
107 {
108         rail->stats.last_update = ktime_get();
109         if (millivolts >= rail->min_millivolts) {
110                 int i = 1 + (2 * (millivolts - rail->min_millivolts) *
111                         DVFS_RAIL_STATS_SCALE + DVFS_RAIL_STATS_BIN) /
112                         (2 * DVFS_RAIL_STATS_BIN);
113                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
114         }
115
116         if (rail->max_millivolts >
117             rail->min_millivolts + DVFS_RAIL_STATS_RANGE)
118                 pr_warn("tegra_dvfs: %s: stats above %d mV will be squashed\n",
119                         rail->reg_id,
120                         rail->min_millivolts + DVFS_RAIL_STATS_RANGE);
121 }
122
123 static void dvfs_rail_stats_update(
124         struct dvfs_rail *rail, int millivolts, ktime_t now)
125 {
126         rail->stats.time_at_mv[rail->stats.last_index] = ktime_add(
127                 rail->stats.time_at_mv[rail->stats.last_index], ktime_sub(
128                         now, rail->stats.last_update));
129         rail->stats.last_update = now;
130
131         if (rail->stats.off)
132                 return;
133
134         if (millivolts >= rail->min_millivolts) {
135                 int i = 1 + (2 * (millivolts - rail->min_millivolts) *
136                         DVFS_RAIL_STATS_SCALE + DVFS_RAIL_STATS_BIN) /
137                         (2 * DVFS_RAIL_STATS_BIN);
138                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
139         } else if (millivolts == 0)
140                         rail->stats.last_index = 0;
141 }
142
143 static void dvfs_rail_stats_pause(struct dvfs_rail *rail,
144                                   ktime_t delta, bool on)
145 {
146         int i = on ? rail->stats.last_index : 0;
147         rail->stats.time_at_mv[i] = ktime_add(rail->stats.time_at_mv[i], delta);
148 }
149
150 void tegra_dvfs_rail_off(struct dvfs_rail *rail, ktime_t now)
151 {
152         if (rail) {
153                 dvfs_rail_stats_update(rail, 0, now);
154                 rail->stats.off = true;
155         }
156 }
157
158 void tegra_dvfs_rail_on(struct dvfs_rail *rail, ktime_t now)
159 {
160         if (rail) {
161                 rail->stats.off = false;
162                 dvfs_rail_stats_update(rail, rail->millivolts, now);
163         }
164 }
165
166 void tegra_dvfs_rail_pause(struct dvfs_rail *rail, ktime_t delta, bool on)
167 {
168         if (rail)
169                 dvfs_rail_stats_pause(rail, delta, on);
170 }
171
172 /* Sets the voltage on a dvfs rail to a specific value, and updates any
173  * rails that depend on this rail. */
174 static int dvfs_rail_set_voltage(struct dvfs_rail *rail, int millivolts)
175 {
176         int ret = 0;
177         struct dvfs_relationship *rel;
178         int step = (millivolts > rail->millivolts) ? rail->step : -rail->step;
179         int i;
180         int steps;
181         bool jmp_to_zero;
182
183         if (!rail->reg) {
184                 if (millivolts == rail->millivolts)
185                         return 0;
186                 else
187                         return -EINVAL;
188         }
189
190         /*
191          * DFLL adjusts rail voltage automatically, but not exactly to the
192          * expected level - update stats, anyway, and made sure that recorded
193          * level will not match any target that can be requested when/if we
194          * switch back from DFLL to s/w control
195          */
196         if (rail->dfll_mode) {
197                 rail->millivolts = rail->new_millivolts = millivolts - 1;
198                 dvfs_rail_stats_update(rail, millivolts, ktime_get());
199                 return 0;
200         }
201
202         if (rail->disabled)
203                 return 0;
204
205         rail->resolving_to = true;
206         jmp_to_zero = rail->jmp_to_zero &&
207                         ((millivolts == 0) || (rail->millivolts == 0));
208         steps = jmp_to_zero ? 1 :
209                 DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
210
211         for (i = 0; i < steps; i++) {
212                 if (!jmp_to_zero &&
213                     (abs(millivolts - rail->millivolts) > rail->step))
214                         rail->new_millivolts = rail->millivolts + step;
215                 else
216                         rail->new_millivolts = millivolts;
217
218                 /* Before changing the voltage, tell each rail that depends
219                  * on this rail that the voltage will change.
220                  * This rail will be the "from" rail in the relationship,
221                  * the rail that depends on this rail will be the "to" rail.
222                  * from->millivolts will be the old voltage
223                  * from->new_millivolts will be the new voltage */
224                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
225                         ret = dvfs_rail_update(rel->to);
226                         if (ret)
227                                 goto out;
228                 }
229
230                 if (!rail->disabled) {
231                         rail->updating = true;
232                         rail->reg_max_millivolts = rail->reg_max_millivolts ==
233                                 rail->max_millivolts ?
234                                 rail->max_millivolts + 1 : rail->max_millivolts;
235                         ret = regulator_set_voltage(rail->reg,
236                                 rail->new_millivolts * 1000,
237                                 rail->reg_max_millivolts * 1000);
238                         rail->updating = false;
239                 }
240                 if (ret) {
241                         pr_err("Failed to set dvfs regulator %s\n", rail->reg_id);
242                         goto out;
243                 }
244
245                 rail->millivolts = rail->new_millivolts;
246                 dvfs_rail_stats_update(rail, rail->millivolts, ktime_get());
247
248                 /* After changing the voltage, tell each rail that depends
249                  * on this rail that the voltage has changed.
250                  * from->millivolts and from->new_millivolts will be the
251                  * new voltage */
252                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
253                         ret = dvfs_rail_update(rel->to);
254                         if (ret)
255                                 goto out;
256                 }
257         }
258
259         if (unlikely(rail->millivolts != millivolts)) {
260                 pr_err("%s: rail didn't reach target %d in %d steps (%d)\n",
261                         __func__, millivolts, steps, rail->millivolts);
262                 ret = -EINVAL;
263         }
264
265 out:
266         rail->resolving_to = false;
267         return ret;
268 }
269
270 /* Determine the minimum valid voltage for a rail, taking into account
271  * the dvfs clocks and any rails that this rail depends on.  Calls
272  * dvfs_rail_set_voltage with the new voltage, which will call
273  * dvfs_rail_update on any rails that depend on this rail. */
274 static int dvfs_rail_update(struct dvfs_rail *rail)
275 {
276         int millivolts = 0;
277         struct dvfs *d;
278         struct dvfs_relationship *rel;
279         int ret = 0;
280         int steps;
281
282         /* if dvfs is suspended, return and handle it during resume */
283         if (rail->suspended)
284                 return 0;
285
286         /* if regulators are not connected yet, return and handle it later */
287         if (!rail->reg)
288                 return 0;
289
290         /* if rail update is entered while resolving circular dependencies,
291            abort recursion */
292         if (rail->resolving_to)
293                 return 0;
294
295         /* Find the maximum voltage requested by any clock */
296         list_for_each_entry(d, &rail->dvfs, reg_node)
297                 millivolts = max(d->cur_millivolts, millivolts);
298
299         /* Apply offset if any clock is requesting voltage */
300         if (millivolts) {
301                 millivolts += rail->offs_millivolts;
302                 if (millivolts > rail->max_millivolts)
303                         millivolts = rail->max_millivolts;
304                 else if (millivolts < rail->min_millivolts)
305                         millivolts = rail->min_millivolts;
306         }
307
308         /* retry update if limited by from-relationship to account for
309            circular dependencies */
310         steps = DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
311         for (; steps >= 0; steps--) {
312                 rail->new_millivolts = millivolts;
313
314                 /* Check any rails that this rail depends on */
315                 list_for_each_entry(rel, &rail->relationships_from, from_node)
316                         rail->new_millivolts = dvfs_solve_relationship(rel);
317
318                 if (rail->new_millivolts == rail->millivolts)
319                         break;
320
321                 ret = dvfs_rail_set_voltage(rail, rail->new_millivolts);
322         }
323
324         return ret;
325 }
326
327 static int dvfs_rail_connect_to_regulator(struct dvfs_rail *rail)
328 {
329         struct regulator *reg;
330         int v;
331
332         if (!rail->reg) {
333                 reg = regulator_get(NULL, rail->reg_id);
334                 if (IS_ERR(reg)) {
335                         pr_err("tegra_dvfs: failed to connect %s rail\n",
336                                rail->reg_id);
337                         return -EINVAL;
338                 }
339                 rail->reg = reg;
340         }
341
342         v = regulator_enable(rail->reg);
343         if (v < 0) {
344                 pr_err("tegra_dvfs: failed on enabling regulator %s\n, err %d",
345                         rail->reg_id, v);
346                 return v;
347         }
348
349         v = regulator_get_voltage(rail->reg);
350         if (v < 0) {
351                 pr_err("tegra_dvfs: failed initial get %s voltage\n",
352                        rail->reg_id);
353                 return v;
354         }
355         rail->millivolts = v / 1000;
356         rail->new_millivolts = rail->millivolts;
357         dvfs_rail_stats_init(rail, rail->millivolts);
358         return 0;
359 }
360
361 static inline unsigned long *dvfs_get_freqs(struct dvfs *d)
362 {
363         return d->alt_freqs ? : &d->freqs[0];
364 }
365
366 static inline const int *dvfs_get_millivolts(struct dvfs *d, unsigned long rate)
367 {
368         if (tegra_dvfs_is_dfll_scale(d, rate))
369                 return d->dfll_millivolts;
370
371         return d->millivolts;
372 }
373
374 static int
375 __tegra_dvfs_set_rate(struct dvfs *d, unsigned long rate)
376 {
377         int i = 0;
378         int ret;
379         unsigned long *freqs = dvfs_get_freqs(d);
380         const int *millivolts = dvfs_get_millivolts(d, rate);
381
382         if (freqs == NULL || millivolts == NULL)
383                 return -ENODEV;
384
385         if (rate > freqs[d->num_freqs - 1]) {
386                 pr_warn("tegra_dvfs: rate %lu too high for dvfs on %s\n", rate,
387                         d->clk_name);
388                 return -EINVAL;
389         }
390
391         if (rate == 0) {
392                 d->cur_millivolts = 0;
393         } else {
394                 while (i < d->num_freqs && rate > freqs[i])
395                         i++;
396
397                 if ((d->max_millivolts) &&
398                     (millivolts[i] > d->max_millivolts)) {
399                         pr_warn("tegra_dvfs: voltage %d too high for dvfs on"
400                                 " %s\n", millivolts[i], d->clk_name);
401                         return -EINVAL;
402                 }
403                 d->cur_millivolts = millivolts[i];
404         }
405
406         d->cur_rate = rate;
407
408         ret = dvfs_rail_update(d->dvfs_rail);
409         if (ret)
410                 pr_err("Failed to set regulator %s for clock %s to %d mV\n",
411                         d->dvfs_rail->reg_id, d->clk_name, d->cur_millivolts);
412
413         return ret;
414 }
415
416 int tegra_dvfs_alt_freqs_set(struct dvfs *d, unsigned long *alt_freqs)
417 {
418         int ret = 0;
419
420         mutex_lock(&dvfs_lock);
421
422         if (d->alt_freqs != alt_freqs) {
423                 d->alt_freqs = alt_freqs;
424                 ret = __tegra_dvfs_set_rate(d, d->cur_rate);
425         }
426
427         mutex_unlock(&dvfs_lock);
428         return ret;
429 }
430
431 int tegra_dvfs_predict_millivolts(struct clk *c, unsigned long rate)
432 {
433         int i;
434         const int *millivolts;
435
436         if (!rate || !c->dvfs)
437                 return 0;
438
439         millivolts = dvfs_get_millivolts(c->dvfs, rate);
440         if (!millivolts)
441                 return -ENODEV;
442
443         /*
444          * Predicted voltage can not be used across the switch to alternative
445          * frequency limits. For now, just fail the call for clock that has
446          * alternative limits initialized.
447          */
448         if (c->dvfs->alt_freqs)
449                 return -ENOSYS;
450
451         for (i = 0; i < c->dvfs->num_freqs; i++) {
452                 if (rate <= c->dvfs->freqs[i])
453                         break;
454         }
455
456         if (i == c->dvfs->num_freqs)
457                 return -EINVAL;
458
459         return millivolts[i];
460 }
461
462 int tegra_dvfs_set_rate(struct clk *c, unsigned long rate)
463 {
464         int ret;
465
466         if (!c->dvfs)
467                 return -EINVAL;
468
469         mutex_lock(&dvfs_lock);
470         ret = __tegra_dvfs_set_rate(c->dvfs, rate);
471         mutex_unlock(&dvfs_lock);
472
473         return ret;
474 }
475 EXPORT_SYMBOL(tegra_dvfs_set_rate);
476
477 /* May only be called during clock init, does not take any locks on clock c. */
478 int __init tegra_enable_dvfs_on_clk(struct clk *c, struct dvfs *d)
479 {
480         int i;
481
482         if (c->dvfs) {
483                 pr_err("Error when enabling dvfs on %s for clock %s:\n",
484                         d->dvfs_rail->reg_id, c->name);
485                 pr_err("DVFS already enabled for %s\n",
486                         c->dvfs->dvfs_rail->reg_id);
487                 return -EINVAL;
488         }
489
490         for (i = 0; i < MAX_DVFS_FREQS; i++) {
491                 if (d->millivolts[i] == 0)
492                         break;
493
494                 d->freqs[i] *= d->freqs_mult;
495
496                 /* If final frequencies are 0, pad with previous frequency */
497                 if (d->freqs[i] == 0 && i > 1)
498                         d->freqs[i] = d->freqs[i - 1];
499         }
500         d->num_freqs = i;
501
502         if (d->auto_dvfs) {
503                 c->auto_dvfs = true;
504                 clk_set_cansleep(c);
505         }
506
507         c->dvfs = d;
508
509         mutex_lock(&dvfs_lock);
510         list_add_tail(&d->reg_node, &d->dvfs_rail->dvfs);
511         mutex_unlock(&dvfs_lock);
512
513         return 0;
514 }
515
516 static bool tegra_dvfs_all_rails_suspended(void)
517 {
518         struct dvfs_rail *rail;
519         bool all_suspended = true;
520
521         list_for_each_entry(rail, &dvfs_rail_list, node)
522                 if (!rail->suspended && !rail->disabled)
523                         all_suspended = false;
524
525         return all_suspended;
526 }
527
528 static bool tegra_dvfs_from_rails_suspended_or_solved(struct dvfs_rail *to)
529 {
530         struct dvfs_relationship *rel;
531         bool all_suspended = true;
532
533         list_for_each_entry(rel, &to->relationships_from, from_node)
534                 if (!rel->from->suspended && !rel->from->disabled &&
535                         !rel->solved_at_nominal)
536                         all_suspended = false;
537
538         return all_suspended;
539 }
540
541 static int tegra_dvfs_suspend_one(void)
542 {
543         struct dvfs_rail *rail;
544         int ret;
545
546         list_for_each_entry(rail, &dvfs_rail_list, node) {
547                 if (!rail->suspended && !rail->disabled &&
548                     tegra_dvfs_from_rails_suspended_or_solved(rail)) {
549                         ret = dvfs_rail_set_voltage(rail,
550                                 rail->nominal_millivolts);
551                         if (ret)
552                                 return ret;
553                         rail->suspended = true;
554                         return 0;
555                 }
556         }
557
558         return -EINVAL;
559 }
560
561 static void tegra_dvfs_resume(void)
562 {
563         struct dvfs_rail *rail;
564
565         mutex_lock(&dvfs_lock);
566
567         list_for_each_entry(rail, &dvfs_rail_list, node)
568                 rail->suspended = false;
569
570         list_for_each_entry(rail, &dvfs_rail_list, node)
571                 dvfs_rail_update(rail);
572
573         mutex_unlock(&dvfs_lock);
574 }
575
576 static int tegra_dvfs_suspend(void)
577 {
578         int ret = 0;
579
580         mutex_lock(&dvfs_lock);
581
582         while (!tegra_dvfs_all_rails_suspended()) {
583                 ret = tegra_dvfs_suspend_one();
584                 if (ret)
585                         break;
586         }
587
588         mutex_unlock(&dvfs_lock);
589
590         if (ret)
591                 tegra_dvfs_resume();
592
593         return ret;
594 }
595
596 static int tegra_dvfs_pm_notify(struct notifier_block *nb,
597                                 unsigned long event, void *data)
598 {
599         switch (event) {
600         case PM_SUSPEND_PREPARE:
601                 if (tegra_dvfs_suspend())
602                         return NOTIFY_STOP;
603                 break;
604         case PM_POST_SUSPEND:
605                 tegra_dvfs_resume();
606                 break;
607         }
608
609         return NOTIFY_OK;
610 };
611
612 static struct notifier_block tegra_dvfs_nb = {
613         .notifier_call = tegra_dvfs_pm_notify,
614 };
615
616 static int tegra_dvfs_reboot_notify(struct notifier_block *nb,
617                                 unsigned long event, void *data)
618 {
619         switch (event) {
620         case SYS_RESTART:
621         case SYS_HALT:
622         case SYS_POWER_OFF:
623                 tegra_dvfs_suspend();
624                 return NOTIFY_OK;
625         }
626         return NOTIFY_DONE;
627 }
628
629 static struct notifier_block tegra_dvfs_reboot_nb = {
630         .notifier_call = tegra_dvfs_reboot_notify,
631 };
632
633 /* must be called with dvfs lock held */
634 static void __tegra_dvfs_rail_disable(struct dvfs_rail *rail)
635 {
636         int ret;
637
638         /* don't set voltage in DFLL mode - won't work, but break stats */
639         if (rail->dfll_mode) {
640                 rail->disabled = true;
641                 return;
642         }
643
644         ret = dvfs_rail_set_voltage(rail, rail->nominal_millivolts);
645         if (ret) {
646                 pr_info("dvfs: failed to set regulator %s to disable "
647                         "voltage %d\n", rail->reg_id,
648                         rail->nominal_millivolts);
649                 return;
650         }
651         rail->disabled = true;
652 }
653
654 /* must be called with dvfs lock held */
655 static void __tegra_dvfs_rail_enable(struct dvfs_rail *rail)
656 {
657         rail->disabled = false;
658         dvfs_rail_update(rail);
659 }
660
661 void tegra_dvfs_rail_enable(struct dvfs_rail *rail)
662 {
663         mutex_lock(&rail_disable_lock);
664
665         if (rail->disabled) {
666                 mutex_lock(&dvfs_lock);
667                 __tegra_dvfs_rail_enable(rail);
668                 mutex_unlock(&dvfs_lock);
669
670                 tegra_dvfs_rail_post_enable(rail);
671         }
672         mutex_unlock(&rail_disable_lock);
673
674 }
675
676 void tegra_dvfs_rail_disable(struct dvfs_rail *rail)
677 {
678         mutex_lock(&rail_disable_lock);
679         if (rail->disabled)
680                 goto out;
681
682         /* rail disable will set it to nominal voltage underneath clock
683            framework - need to re-configure clock rates that are not safe
684            at nominal (yes, unsafe at nominal is ugly, but possible). Rate
685            change must be done outside of dvfs lock. */
686         if (tegra_dvfs_rail_disable_prepare(rail)) {
687                 pr_info("dvfs: failed to prepare regulator %s to disable\n",
688                         rail->reg_id);
689                 goto out;
690         }
691
692         mutex_lock(&dvfs_lock);
693         __tegra_dvfs_rail_disable(rail);
694         mutex_unlock(&dvfs_lock);
695 out:
696         mutex_unlock(&rail_disable_lock);
697 }
698
699 int tegra_dvfs_rail_disable_by_name(const char *reg_id)
700 {
701         struct dvfs_rail *rail = tegra_dvfs_get_rail_by_name(reg_id);
702         if (!rail)
703                 return -EINVAL;
704
705         tegra_dvfs_rail_disable(rail);
706         return 0;
707 }
708
709 struct dvfs_rail *tegra_dvfs_get_rail_by_name(const char *reg_id)
710 {
711         struct dvfs_rail *rail;
712
713         mutex_lock(&dvfs_lock);
714         list_for_each_entry(rail, &dvfs_rail_list, node) {
715                 if (!strcmp(reg_id, rail->reg_id)) {
716                         mutex_unlock(&dvfs_lock);
717                         return rail;
718                 }
719         }
720         mutex_unlock(&dvfs_lock);
721         return NULL;
722 }
723
724 bool tegra_dvfs_rail_updating(struct clk *clk)
725 {
726         return (!clk ? false :
727                 (!clk->dvfs ? false :
728                  (!clk->dvfs->dvfs_rail ? false :
729                   (clk->dvfs->dvfs_rail->updating))));
730 }
731
732 #ifdef CONFIG_OF
733 int __init of_tegra_dvfs_init(const struct of_device_id *matches)
734 {
735         int ret;
736         struct device_node *np;
737
738         for_each_matching_node(np, matches) {
739                 const struct of_device_id *match = of_match_node(matches, np);
740                 of_tegra_dvfs_init_cb_t dvfs_init_cb = match->data;
741                 ret = dvfs_init_cb(np);
742                 if (ret) {
743                         pr_err("dt: Failed to read %s tables from DT\n",
744                                                         match->compatible);
745                         return ret;
746                 }
747         }
748         return 0;
749 }
750 #endif
751 int tegra_dvfs_dfll_mode_set(struct dvfs *d, unsigned long rate)
752 {
753         mutex_lock(&dvfs_lock);
754         if (!d->dvfs_rail->dfll_mode) {
755                 d->dvfs_rail->dfll_mode = true;
756                 __tegra_dvfs_set_rate(d, rate);
757         }
758         mutex_unlock(&dvfs_lock);
759         return 0;
760 }
761
762 int tegra_dvfs_dfll_mode_clear(struct dvfs *d, unsigned long rate)
763 {
764         int ret = 0;
765
766         mutex_lock(&dvfs_lock);
767         if (d->dvfs_rail->dfll_mode) {
768                 d->dvfs_rail->dfll_mode = false;
769                 if (d->dvfs_rail->disabled) {
770                         d->dvfs_rail->disabled = false;
771                         __tegra_dvfs_rail_disable(d->dvfs_rail);
772                 }
773                 ret = __tegra_dvfs_set_rate(d, rate);
774         }
775         mutex_unlock(&dvfs_lock);
776         return ret;
777 }
778
779 /*
780  * Iterate through all the dvfs regulators, finding the regulator exported
781  * by the regulator api for each one.  Must be called in late init, after
782  * all the regulator api's regulators are initialized.
783  */
784 int __init tegra_dvfs_late_init(void)
785 {
786         bool connected = true;
787         struct dvfs_rail *rail;
788
789         mutex_lock(&dvfs_lock);
790
791         list_for_each_entry(rail, &dvfs_rail_list, node)
792                 if (dvfs_rail_connect_to_regulator(rail))
793                         connected = false;
794
795         list_for_each_entry(rail, &dvfs_rail_list, node)
796                 if (connected)
797                         dvfs_rail_update(rail);
798                 else
799                         __tegra_dvfs_rail_disable(rail);
800
801         mutex_unlock(&dvfs_lock);
802
803 #ifdef CONFIG_TEGRA_SILICON_PLATFORM
804         if (!connected)
805                 return -ENODEV;
806 #endif
807         register_pm_notifier(&tegra_dvfs_nb);
808         register_reboot_notifier(&tegra_dvfs_reboot_nb);
809
810         return 0;
811 }
812
813 #ifdef CONFIG_DEBUG_FS
814 static int dvfs_tree_sort_cmp(void *p, struct list_head *a, struct list_head *b)
815 {
816         struct dvfs *da = list_entry(a, struct dvfs, reg_node);
817         struct dvfs *db = list_entry(b, struct dvfs, reg_node);
818         int ret;
819
820         ret = strcmp(da->dvfs_rail->reg_id, db->dvfs_rail->reg_id);
821         if (ret != 0)
822                 return ret;
823
824         if (da->cur_millivolts < db->cur_millivolts)
825                 return 1;
826         if (da->cur_millivolts > db->cur_millivolts)
827                 return -1;
828
829         return strcmp(da->clk_name, db->clk_name);
830 }
831
832 static int dvfs_tree_show(struct seq_file *s, void *data)
833 {
834         struct dvfs *d;
835         struct dvfs_rail *rail;
836         struct dvfs_relationship *rel;
837
838         seq_printf(s, "   clock      rate       mV\n");
839         seq_printf(s, "--------------------------------\n");
840
841         mutex_lock(&dvfs_lock);
842
843         list_for_each_entry(rail, &dvfs_rail_list, node) {
844                 seq_printf(s, "%s %d mV%s:\n", rail->reg_id, rail->millivolts,
845                            rail->dfll_mode ? " dfll mode" :
846                                 rail->disabled ? " disabled" : "");
847                 list_for_each_entry(rel, &rail->relationships_from, from_node) {
848                         seq_printf(s, "   %-10s %-7d mV %-4d mV\n",
849                                 rel->from->reg_id,
850                                 rel->from->millivolts,
851                                 dvfs_solve_relationship(rel));
852                 }
853                 seq_printf(s, "   offset     %-7d mV\n", rail->offs_millivolts);
854
855                 list_sort(NULL, &rail->dvfs, dvfs_tree_sort_cmp);
856
857                 list_for_each_entry(d, &rail->dvfs, reg_node) {
858                         seq_printf(s, "   %-10s %-10lu %-4d mV\n", d->clk_name,
859                                 d->cur_rate, d->cur_millivolts);
860                 }
861         }
862
863         mutex_unlock(&dvfs_lock);
864
865         return 0;
866 }
867
868 static int dvfs_tree_open(struct inode *inode, struct file *file)
869 {
870         return single_open(file, dvfs_tree_show, inode->i_private);
871 }
872
873 static const struct file_operations dvfs_tree_fops = {
874         .open           = dvfs_tree_open,
875         .read           = seq_read,
876         .llseek         = seq_lseek,
877         .release        = single_release,
878 };
879
880 static int rail_stats_show(struct seq_file *s, void *data)
881 {
882         int i;
883         struct dvfs_rail *rail;
884
885         seq_printf(s, "%-12s %-10s (bin: %d.%dmV)\n", "millivolts", "time",
886                    DVFS_RAIL_STATS_BIN / DVFS_RAIL_STATS_SCALE,
887                    ((DVFS_RAIL_STATS_BIN * 100) / DVFS_RAIL_STATS_SCALE) % 100);
888
889         mutex_lock(&dvfs_lock);
890
891         list_for_each_entry(rail, &dvfs_rail_list, node) {
892                 seq_printf(s, "%s\n", rail->reg_id);
893                 dvfs_rail_stats_update(rail, -1, ktime_get());
894
895                 seq_printf(s, "%-12d %-10llu\n", 0,
896                         cputime64_to_clock_t(msecs_to_jiffies(
897                                 ktime_to_ms(rail->stats.time_at_mv[0]))));
898
899                 for (i = 1; i <= DVFS_RAIL_STATS_TOP_BIN; i++) {
900                         ktime_t ktime_zero = ktime_set(0, 0);
901                         if (ktime_equal(rail->stats.time_at_mv[i], ktime_zero))
902                                 continue;
903                         seq_printf(s, "%-12d %-10llu\n",
904                                    rail->min_millivolts + (i - 1) *
905                                    DVFS_RAIL_STATS_BIN / DVFS_RAIL_STATS_SCALE,
906                                 cputime64_to_clock_t(msecs_to_jiffies(
907                                         ktime_to_ms(rail->stats.time_at_mv[i])))
908                         );
909                 }
910         }
911         mutex_unlock(&dvfs_lock);
912         return 0;
913 }
914
915 static int rail_stats_open(struct inode *inode, struct file *file)
916 {
917         return single_open(file, rail_stats_show, inode->i_private);
918 }
919
920 static const struct file_operations rail_stats_fops = {
921         .open           = rail_stats_open,
922         .read           = seq_read,
923         .llseek         = seq_lseek,
924         .release        = single_release,
925 };
926
927 static int cpu_offs_get(void *data, u64 *val)
928 {
929         if (tegra_cpu_rail) {
930                 *val = (u64)tegra_cpu_rail->offs_millivolts;
931                 return 0;
932         }
933         *val = 0;
934         return -ENOENT;
935 }
936 static int cpu_offs_set(void *data, u64 val)
937 {
938         if (tegra_cpu_rail) {
939                 mutex_lock(&dvfs_lock);
940                 tegra_cpu_rail->offs_millivolts = (int)val;
941                 dvfs_rail_update(tegra_cpu_rail);
942                 mutex_unlock(&dvfs_lock);
943                 return 0;
944         }
945         return -ENOENT;
946 }
947 DEFINE_SIMPLE_ATTRIBUTE(cpu_offs_fops, cpu_offs_get, cpu_offs_set, "%lld\n");
948
949 static int core_offs_get(void *data, u64 *val)
950 {
951         if (tegra_core_rail) {
952                 *val = (u64)tegra_core_rail->offs_millivolts;
953                 return 0;
954         }
955         *val = 0;
956         return -ENOENT;
957 }
958 static int core_offs_set(void *data, u64 val)
959 {
960         if (tegra_core_rail) {
961                 mutex_lock(&dvfs_lock);
962                 tegra_core_rail->offs_millivolts = (int)val;
963                 dvfs_rail_update(tegra_core_rail);
964                 mutex_unlock(&dvfs_lock);
965                 return 0;
966         }
967         return -ENOENT;
968 }
969 DEFINE_SIMPLE_ATTRIBUTE(core_offs_fops, core_offs_get, core_offs_set, "%lld\n");
970
971 int __init dvfs_debugfs_init(struct dentry *clk_debugfs_root)
972 {
973         struct dentry *d;
974
975         d = debugfs_create_file("dvfs", S_IRUGO, clk_debugfs_root, NULL,
976                 &dvfs_tree_fops);
977         if (!d)
978                 return -ENOMEM;
979
980         d = debugfs_create_file("rails", S_IRUGO, clk_debugfs_root, NULL,
981                 &rail_stats_fops);
982         if (!d)
983                 return -ENOMEM;
984
985         d = debugfs_create_file("vdd_cpu_offs", S_IRUGO | S_IWUSR,
986                 clk_debugfs_root, NULL, &cpu_offs_fops);
987         if (!d)
988                 return -ENOMEM;
989
990         d = debugfs_create_file("vdd_core_offs", S_IRUGO | S_IWUSR,
991                 clk_debugfs_root, NULL, &core_offs_fops);
992         if (!d)
993                 return -ENOMEM;
994
995         return 0;
996 }
997
998 #endif