ARM: tegra: dvfs: Re-name rail offset variable
[linux-3.10.git] / arch / arm / mach-tegra / dvfs.c
1 /*
2  *
3  * Copyright (C) 2010 Google, Inc.
4  *
5  * Author:
6  *      Colin Cross <ccross@google.com>
7  *
8  * Copyright (C) 2010-2013 NVIDIA CORPORATION. All rights reserved.
9  *
10  * This software is licensed under the terms of the GNU General Public
11  * License version 2, as published by the Free Software Foundation, and
12  * may be copied, distributed, and modified under those terms.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/clk.h>
23 #include <linux/clkdev.h>
24 #include <linux/debugfs.h>
25 #include <linux/init.h>
26 #include <linux/list.h>
27 #include <linux/list_sort.h>
28 #include <linux/module.h>
29 #include <linux/regulator/consumer.h>
30 #include <linux/seq_file.h>
31 #include <linux/slab.h>
32 #include <linux/suspend.h>
33 #include <linux/delay.h>
34 #include <linux/clk/tegra.h>
35 #include <linux/reboot.h>
36 #include <linux/clk/tegra.h>
37 #include <linux/tegra-soc.h>
38
39 #include "board.h"
40 #include "clock.h"
41 #include "dvfs.h"
42
43 #define DVFS_RAIL_STATS_BIN     12500
44
45 struct dvfs_rail *tegra_cpu_rail;
46 struct dvfs_rail *tegra_core_rail;
47 static struct dvfs_rail *tegra_gpu_rail;
48
49 static LIST_HEAD(dvfs_rail_list);
50 static DEFINE_MUTEX(dvfs_lock);
51 static DEFINE_MUTEX(rail_disable_lock);
52
53 static int dvfs_rail_update(struct dvfs_rail *rail);
54
55 static inline int tegra_dvfs_rail_get_disable_level(struct dvfs_rail *rail)
56 {
57         return rail->disable_millivolts ? : rail->nominal_millivolts;
58 }
59
60 static inline int tegra_dvfs_rail_get_suspend_level(struct dvfs_rail *rail)
61 {
62         return rail->suspend_millivolts ? : rail->nominal_millivolts;
63 }
64
65 void tegra_dvfs_add_relationships(struct dvfs_relationship *rels, int n)
66 {
67         int i;
68         struct dvfs_relationship *rel;
69
70         mutex_lock(&dvfs_lock);
71
72         for (i = 0; i < n; i++) {
73                 rel = &rels[i];
74                 list_add_tail(&rel->from_node, &rel->to->relationships_from);
75                 list_add_tail(&rel->to_node, &rel->from->relationships_to);
76         }
77
78         mutex_unlock(&dvfs_lock);
79 }
80
81 /* Make sure there is a matching cooling device for thermal limit profile. */
82 static void dvfs_validate_cdevs(struct dvfs_rail *rail)
83 {
84         if (!rail->therm_mv_caps != !rail->therm_mv_caps_num) {
85                 rail->therm_mv_caps_num = 0;
86                 rail->therm_mv_caps = NULL;
87                 WARN(1, "%s: not matching thermal caps/num\n", rail->reg_id);
88         }
89
90         if (rail->therm_mv_caps && !rail->vmax_cdev)
91                 WARN(1, "%s: missing vmax cooling device\n", rail->reg_id);
92
93         if (!rail->therm_mv_floors != !rail->therm_mv_floors_num) {
94                 rail->therm_mv_floors_num = 0;
95                 rail->therm_mv_floors = NULL;
96                 WARN(1, "%s: not matching thermal floors/num\n", rail->reg_id);
97         }
98
99         if (rail->therm_mv_floors && !rail->vmin_cdev)
100                 WARN(1, "%s: missing vmin cooling device\n", rail->reg_id);
101
102         /* Limit override range to maximum floor */
103         if (rail->therm_mv_floors)
104                 rail->min_override_millivolts = rail->therm_mv_floors[0];
105 }
106
107 int tegra_dvfs_init_rails(struct dvfs_rail *rails[], int n)
108 {
109         int i, mv;
110
111         mutex_lock(&dvfs_lock);
112
113         for (i = 0; i < n; i++) {
114                 INIT_LIST_HEAD(&rails[i]->dvfs);
115                 INIT_LIST_HEAD(&rails[i]->relationships_from);
116                 INIT_LIST_HEAD(&rails[i]->relationships_to);
117
118                 mv = rails[i]->nominal_millivolts;
119                 if (rails[i]->boot_millivolts > mv)
120                         WARN(1, "%s: boot voltage %d above nominal %d\n",
121                              rails[i]->reg_id, rails[i]->boot_millivolts, mv);
122                 if (rails[i]->disable_millivolts > mv)
123                         rails[i]->disable_millivolts = mv;
124                 if (rails[i]->suspend_millivolts > mv)
125                         rails[i]->suspend_millivolts = mv;
126
127                 mv = tegra_dvfs_rail_get_boot_level(rails[i]);
128                 rails[i]->millivolts = mv;
129                 rails[i]->new_millivolts = mv;
130                 if (!rails[i]->step)
131                         rails[i]->step = rails[i]->max_millivolts;
132                 if (!rails[i]->step_up)
133                         rails[i]->step_up = rails[i]->step;
134
135                 list_add_tail(&rails[i]->node, &dvfs_rail_list);
136
137                 dvfs_validate_cdevs(rails[i]);
138
139                 if (!strcmp("vdd_cpu", rails[i]->reg_id))
140                         tegra_cpu_rail = rails[i];
141                 else if (!strcmp("vdd_gpu", rails[i]->reg_id))
142                         tegra_gpu_rail = rails[i];
143                 else if (!strcmp("vdd_core", rails[i]->reg_id))
144                         tegra_core_rail = rails[i];
145         }
146
147         mutex_unlock(&dvfs_lock);
148
149         return 0;
150 };
151
152 static int dvfs_solve_relationship(struct dvfs_relationship *rel)
153 {
154         return rel->solve(rel->from, rel->to);
155 }
156
157 /* rail statistic - called during rail init, or under dfs_lock, or with
158    CPU0 only on-line, and interrupts disabled */
159 static void dvfs_rail_stats_init(struct dvfs_rail *rail, int millivolts)
160 {
161         int dvfs_rail_stats_range;
162
163         if (!rail->stats.bin_uV)
164                 rail->stats.bin_uV = DVFS_RAIL_STATS_BIN;
165
166         dvfs_rail_stats_range =
167                 (DVFS_RAIL_STATS_TOP_BIN - 1) * rail->stats.bin_uV / 1000;
168
169         rail->stats.last_update = ktime_get();
170         if (millivolts >= rail->min_millivolts) {
171                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
172                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
173                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
174         }
175
176         if (rail->max_millivolts >
177             rail->min_millivolts + dvfs_rail_stats_range)
178                 pr_warn("tegra_dvfs: %s: stats above %d mV will be squashed\n",
179                         rail->reg_id,
180                         rail->min_millivolts + dvfs_rail_stats_range);
181 }
182
183 static void dvfs_rail_stats_update(
184         struct dvfs_rail *rail, int millivolts, ktime_t now)
185 {
186         rail->stats.time_at_mv[rail->stats.last_index] = ktime_add(
187                 rail->stats.time_at_mv[rail->stats.last_index], ktime_sub(
188                         now, rail->stats.last_update));
189         rail->stats.last_update = now;
190
191         if (rail->stats.off)
192                 return;
193
194         if (millivolts >= rail->min_millivolts) {
195                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
196                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
197                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
198         } else if (millivolts == 0)
199                         rail->stats.last_index = 0;
200 }
201
202 static void dvfs_rail_stats_pause(struct dvfs_rail *rail,
203                                   ktime_t delta, bool on)
204 {
205         int i = on ? rail->stats.last_index : 0;
206         rail->stats.time_at_mv[i] = ktime_add(rail->stats.time_at_mv[i], delta);
207 }
208
209 void tegra_dvfs_rail_off(struct dvfs_rail *rail, ktime_t now)
210 {
211         if (rail) {
212                 dvfs_rail_stats_update(rail, 0, now);
213                 rail->stats.off = true;
214         }
215 }
216
217 void tegra_dvfs_rail_on(struct dvfs_rail *rail, ktime_t now)
218 {
219         if (rail) {
220                 rail->stats.off = false;
221                 dvfs_rail_stats_update(rail, rail->millivolts, now);
222         }
223 }
224
225 void tegra_dvfs_rail_pause(struct dvfs_rail *rail, ktime_t delta, bool on)
226 {
227         if (rail)
228                 dvfs_rail_stats_pause(rail, delta, on);
229 }
230
231 static int dvfs_rail_set_voltage_reg(struct dvfs_rail *rail, int millivolts)
232 {
233         int ret;
234
235         /*
236          * safely return success for low voltage requests on fixed regulator
237          * (higher requests will go through and fail, as they should)
238          */
239         if (rail->fixed_millivolts && (millivolts <= rail->fixed_millivolts))
240                 return 0;
241
242         rail->updating = true;
243         rail->reg_max_millivolts = rail->reg_max_millivolts ==
244                 rail->max_millivolts ?
245                 rail->max_millivolts + 1 : rail->max_millivolts;
246         ret = regulator_set_voltage(rail->reg,
247                 millivolts * 1000,
248                 rail->reg_max_millivolts * 1000);
249         rail->updating = false;
250
251         return ret;
252 }
253
254 /* Sets the voltage on a dvfs rail to a specific value, and updates any
255  * rails that depend on this rail. */
256 static int dvfs_rail_set_voltage(struct dvfs_rail *rail, int millivolts)
257 {
258         int ret = 0;
259         struct dvfs_relationship *rel;
260         int step, offset;
261         int i;
262         int steps;
263         bool jmp_to_zero;
264
265         if (!rail->reg) {
266                 if (millivolts == rail->millivolts)
267                         return 0;
268                 else
269                         return -EINVAL;
270         }
271
272         if (millivolts > rail->millivolts) {
273                 step = rail->step_up;
274                 offset = step;
275         } else {
276                 step = rail->step;
277                 offset = -step;
278         }
279
280         /*
281          * DFLL adjusts rail voltage automatically, but not exactly to the
282          * expected level - update stats, anyway.
283          */
284         if (rail->dfll_mode) {
285                 rail->millivolts = rail->new_millivolts = millivolts;
286                 dvfs_rail_stats_update(rail, millivolts, ktime_get());
287                 return 0;
288         }
289
290         if (rail->disabled)
291                 return 0;
292
293         rail->resolving_to = true;
294         jmp_to_zero = rail->jmp_to_zero &&
295                         ((millivolts == 0) || (rail->millivolts == 0));
296         steps = jmp_to_zero ? 1 :
297                 DIV_ROUND_UP(abs(millivolts - rail->millivolts), step);
298
299         for (i = 0; i < steps; i++) {
300                 if (!jmp_to_zero &&
301                     (abs(millivolts - rail->millivolts) > step))
302                         rail->new_millivolts = rail->millivolts + offset;
303                 else
304                         rail->new_millivolts = millivolts;
305
306                 /* Before changing the voltage, tell each rail that depends
307                  * on this rail that the voltage will change.
308                  * This rail will be the "from" rail in the relationship,
309                  * the rail that depends on this rail will be the "to" rail.
310                  * from->millivolts will be the old voltage
311                  * from->new_millivolts will be the new voltage */
312                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
313                         ret = dvfs_rail_update(rel->to);
314                         if (ret)
315                                 goto out;
316                 }
317
318                 ret = dvfs_rail_set_voltage_reg(rail, rail->new_millivolts);
319                 if (ret) {
320                         pr_err("Failed to set dvfs regulator %s\n", rail->reg_id);
321                         goto out;
322                 }
323
324                 rail->millivolts = rail->new_millivolts;
325                 dvfs_rail_stats_update(rail, rail->millivolts, ktime_get());
326
327                 /* After changing the voltage, tell each rail that depends
328                  * on this rail that the voltage has changed.
329                  * from->millivolts and from->new_millivolts will be the
330                  * new voltage */
331                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
332                         ret = dvfs_rail_update(rel->to);
333                         if (ret)
334                                 goto out;
335                 }
336         }
337
338         if (unlikely(rail->millivolts != millivolts)) {
339                 pr_err("%s: rail didn't reach target %d in %d steps (%d)\n",
340                         __func__, millivolts, steps, rail->millivolts);
341                 ret = -EINVAL;
342         }
343
344 out:
345         rail->resolving_to = false;
346         return ret;
347 }
348
349 /* Determine the minimum valid voltage for a rail, taking into account
350  * the dvfs clocks and any rails that this rail depends on.  Calls
351  * dvfs_rail_set_voltage with the new voltage, which will call
352  * dvfs_rail_update on any rails that depend on this rail. */
353 static inline int dvfs_rail_apply_limits(struct dvfs_rail *rail, int millivolts)
354 {
355         int min_mv = rail->min_millivolts;
356
357         if (rail->therm_mv_floors) {
358                 int i = rail->therm_floor_idx;
359                 if (i < rail->therm_mv_floors_num)
360                         min_mv = rail->therm_mv_floors[i];
361         }
362
363         if (rail->override_millivolts) {
364                 millivolts = rail->override_millivolts;
365         } else {
366                 /* apply offset and clip up to pll mode fixed mv */
367                 millivolts += rail->dbg_mv_offs;
368                 if (!rail->dfll_mode && rail->fixed_millivolts &&
369                     (millivolts < rail->fixed_millivolts))
370                         millivolts = rail->fixed_millivolts;
371         }
372
373         if (millivolts > rail->max_millivolts)
374                 millivolts = rail->max_millivolts;
375         else if (millivolts < min_mv)
376                 millivolts = min_mv;
377
378         return millivolts;
379 }
380
381 static int dvfs_rail_update(struct dvfs_rail *rail)
382 {
383         int millivolts = 0;
384         struct dvfs *d;
385         struct dvfs_relationship *rel;
386         int ret = 0;
387         int steps;
388
389         /* if dvfs is suspended, return and handle it during resume */
390         if (rail->suspended)
391                 return 0;
392
393         /* if regulators are not connected yet, return and handle it later */
394         if (!rail->reg)
395                 return 0;
396
397         /* if rail update is entered while resolving circular dependencies,
398            abort recursion */
399         if (rail->resolving_to)
400                 return 0;
401
402         /* Find the maximum voltage requested by any clock */
403         list_for_each_entry(d, &rail->dvfs, reg_node)
404                 millivolts = max(d->cur_millivolts, millivolts);
405
406         /* Apply offset and min/max limits if any clock is requesting voltage */
407         if (millivolts)
408                 millivolts = dvfs_rail_apply_limits(rail, millivolts);
409
410         /* retry update if limited by from-relationship to account for
411            circular dependencies */
412         steps = DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
413         for (; steps >= 0; steps--) {
414                 rail->new_millivolts = millivolts;
415
416                 /* Check any rails that this rail depends on */
417                 list_for_each_entry(rel, &rail->relationships_from, from_node)
418                         rail->new_millivolts = dvfs_solve_relationship(rel);
419
420                 if (rail->new_millivolts == rail->millivolts)
421                         break;
422
423                 ret = dvfs_rail_set_voltage(rail, rail->new_millivolts);
424         }
425
426         return ret;
427 }
428
429 static struct regulator *get_fixed_regulator(struct dvfs_rail *rail)
430 {
431         struct regulator *reg;
432         char reg_id[80];
433         struct dvfs *d;
434         int v, i;
435         unsigned long dfll_boost;
436
437         strcpy(reg_id, rail->reg_id);
438         strcat(reg_id, "_fixed");
439         reg = regulator_get(NULL, reg_id);
440         if (IS_ERR(reg))
441                 return reg;
442
443         v = regulator_get_voltage(reg) / 1000;
444         if ((v < rail->min_millivolts) || (v > rail->nominal_millivolts) ||
445             (rail->therm_mv_floors && v < rail->therm_mv_floors[0])) {
446                 pr_err("tegra_dvfs: ivalid fixed %s voltage %d\n",
447                        rail->reg_id, v);
448                 return ERR_PTR(-EINVAL);
449         }
450
451         /*
452          * Only fixed at nominal voltage vdd_core regulator is allowed, same
453          * is true for cpu rail if dfll mode is not supported at all. No thermal
454          * capping can be implemented in this case.
455          */
456         if (!IS_ENABLED(CONFIG_ARCH_TEGRA_HAS_CL_DVFS) ||
457             (rail != tegra_cpu_rail)) {
458                 if (v != rail->nominal_millivolts) {
459                         pr_err("tegra_dvfs: %s fixed below nominal at %d\n",
460                                rail->reg_id, v);
461                         return ERR_PTR(-EINVAL);
462                 }
463                 if (rail->therm_mv_caps) {
464                         pr_err("tegra_dvfs: cannot fix %s with thermal caps\n",
465                                rail->reg_id);
466                         return ERR_PTR(-ENOSYS);
467                 }
468                 return reg;
469         }
470
471         /*
472          * If dfll mode is supported, fixed vdd_cpu regulator may be below
473          * nominal in pll mode - maximum cpu rate in pll mode is limited
474          * respectively. Regulator is required to allow automatic scaling
475          * in dfll mode.
476          *
477          * FIXME: platform data to explicitly identify such "hybrid" regulator?
478          */
479         d = list_first_entry(&rail->dvfs, struct dvfs, reg_node);
480         for (i = 0; i < d->num_freqs; i++) {
481                 if (d->millivolts[i] > v)
482                         break;
483         }
484
485         if (!i) {
486                 pr_err("tegra_dvfs: %s fixed at %d: too low for min rate\n",
487                        rail->reg_id, v);
488                 return ERR_PTR(-EINVAL);
489         }
490
491         dfll_boost = (d->freqs[d->num_freqs - 1] - d->freqs[i - 1]);
492         if (d->dfll_data.max_rate_boost < dfll_boost)
493                 d->dfll_data.max_rate_boost = dfll_boost;
494
495         rail->fixed_millivolts = v;
496         return reg;
497 }
498
499 static int dvfs_rail_connect_to_regulator(struct dvfs_rail *rail)
500 {
501         struct regulator *reg;
502         int v;
503
504         if (!rail->reg) {
505                 reg = regulator_get(NULL, rail->reg_id);
506                 if (IS_ERR(reg)) {
507                         reg = get_fixed_regulator(rail);
508                         if (IS_ERR(reg)) {
509                                 pr_err("tegra_dvfs: failed to connect %s rail\n",
510                                        rail->reg_id);
511                                 return PTR_ERR(reg);
512                         }
513                 }
514                 rail->reg = reg;
515         }
516
517         v = regulator_enable(rail->reg);
518         if (v < 0) {
519                 pr_err("tegra_dvfs: failed on enabling regulator %s\n, err %d",
520                         rail->reg_id, v);
521                 return v;
522         }
523
524         v = regulator_get_voltage(rail->reg);
525         if (v < 0) {
526                 pr_err("tegra_dvfs: failed initial get %s voltage\n",
527                        rail->reg_id);
528                 return v;
529         }
530         rail->millivolts = v / 1000;
531         rail->new_millivolts = rail->millivolts;
532         dvfs_rail_stats_init(rail, rail->millivolts);
533
534         if (rail->boot_millivolts &&
535             (rail->boot_millivolts != rail->millivolts)) {
536                 WARN(1, "%s boot voltage %d does not match expected %d\n",
537                      rail->reg_id, rail->millivolts, rail->boot_millivolts);
538                 rail->boot_millivolts = rail->millivolts;
539         }
540         return 0;
541 }
542
543 static inline unsigned long *dvfs_get_freqs(struct dvfs *d)
544 {
545         return d->alt_freqs ? : &d->freqs[0];
546 }
547
548 static inline const int *dvfs_get_millivolts(struct dvfs *d, unsigned long rate)
549 {
550         if (tegra_dvfs_is_dfll_scale(d, rate))
551                 return d->dfll_millivolts;
552
553         return d->millivolts;
554 }
555
556 static int
557 __tegra_dvfs_set_rate(struct dvfs *d, unsigned long rate)
558 {
559         int i = 0;
560         int ret, mv, detach_mv;
561         unsigned long *freqs = dvfs_get_freqs(d);
562         const int *millivolts = dvfs_get_millivolts(d, rate);
563
564         if (freqs == NULL || millivolts == NULL)
565                 return -ENODEV;
566
567         /* On entry to dfll range limit 1st step to range bottom (full ramp of
568            voltage/rate is completed automatically in dfll mode) */
569         if (tegra_dvfs_is_dfll_range_entry(d, rate))
570                 rate = d->dfll_data.use_dfll_rate_min;
571
572         if (rate > freqs[d->num_freqs - 1]) {
573                 pr_warn("tegra_dvfs: rate %lu too high for dvfs on %s\n", rate,
574                         d->clk_name);
575                 return -EINVAL;
576         }
577
578         if (rate == 0) {
579                 d->cur_millivolts = 0;
580         } else {
581                 while (i < d->num_freqs && rate > freqs[i])
582                         i++;
583
584                 if ((d->max_millivolts) &&
585                     (millivolts[i] > d->max_millivolts)) {
586                         pr_warn("tegra_dvfs: voltage %d too high for dvfs on"
587                                 " %s\n", millivolts[i], d->clk_name);
588                         return -EINVAL;
589                 }
590
591                 mv = millivolts[i];
592                 detach_mv = tegra_dvfs_rail_get_boot_level(d->dvfs_rail);
593                 if (!d->dvfs_rail->reg && (mv > detach_mv)) {
594                         pr_warn("%s: %s: voltage %d above boot limit %d\n",
595                                 __func__, d->clk_name, mv, detach_mv);
596                         return -EINVAL;
597                 }
598
599                 detach_mv = tegra_dvfs_rail_get_disable_level(d->dvfs_rail);
600                 if (d->dvfs_rail->disabled && (mv > detach_mv)) {
601                         pr_warn("%s: %s: voltage %d above disable limit %d\n",
602                                 __func__, d->clk_name, mv, detach_mv);
603                         return -EINVAL;
604                 }
605
606                 detach_mv = tegra_dvfs_rail_get_suspend_level(d->dvfs_rail);
607                 if (d->dvfs_rail->suspended && (mv > detach_mv)) {
608                         pr_warn("%s: %s: voltage %d above disable limit %d\n",
609                                 __func__, d->clk_name, mv, detach_mv);
610                         return -EINVAL;
611                 }
612                 d->cur_millivolts = millivolts[i];
613         }
614
615         d->cur_rate = rate;
616
617         ret = dvfs_rail_update(d->dvfs_rail);
618         if (ret)
619                 pr_err("Failed to set regulator %s for clock %s to %d mV\n",
620                         d->dvfs_rail->reg_id, d->clk_name, d->cur_millivolts);
621
622         return ret;
623 }
624
625 int tegra_dvfs_alt_freqs_set(struct dvfs *d, unsigned long *alt_freqs)
626 {
627         int ret = 0;
628
629         mutex_lock(&dvfs_lock);
630
631         if (d->alt_freqs != alt_freqs) {
632                 d->alt_freqs = alt_freqs;
633                 ret = __tegra_dvfs_set_rate(d, d->cur_rate);
634         }
635
636         mutex_unlock(&dvfs_lock);
637         return ret;
638 }
639
640 static int predict_millivolts(struct clk *c, const int *millivolts,
641                               unsigned long rate)
642 {
643         int i;
644
645         if (!millivolts)
646                 return -ENODEV;
647         /*
648          * Predicted voltage can not be used across the switch to alternative
649          * frequency limits. For now, just fail the call for clock that has
650          * alternative limits initialized.
651          */
652         if (c->dvfs->alt_freqs)
653                 return -ENOSYS;
654
655         for (i = 0; i < c->dvfs->num_freqs; i++) {
656                 if (rate <= c->dvfs->freqs[i])
657                         break;
658         }
659
660         if (i == c->dvfs->num_freqs)
661                 return -EINVAL;
662
663         return millivolts[i];
664 }
665
666 int tegra_dvfs_predict_millivolts(struct clk *c, unsigned long rate)
667 {
668         const int *millivolts;
669
670         if (!rate || !c->dvfs)
671                 return 0;
672
673         millivolts = dvfs_get_millivolts(c->dvfs, rate);
674         return predict_millivolts(c, millivolts, rate);
675 }
676
677 int tegra_dvfs_predict_millivolts_pll(struct clk *c, unsigned long rate)
678 {
679         const int *millivolts;
680
681         if (!rate || !c->dvfs)
682                 return 0;
683
684         millivolts = c->dvfs->millivolts;
685         return predict_millivolts(c, millivolts, rate);
686 }
687
688 int tegra_dvfs_predict_millivolts_dfll(struct clk *c, unsigned long rate)
689 {
690         const int *millivolts;
691
692         if (!rate || !c->dvfs)
693                 return 0;
694
695         millivolts = c->dvfs->dfll_millivolts;
696         return predict_millivolts(c, millivolts, rate);
697 }
698
699 int tegra_dvfs_set_rate(struct clk *c, unsigned long rate)
700 {
701         int ret;
702
703         if (!c->dvfs)
704                 return -EINVAL;
705
706         mutex_lock(&dvfs_lock);
707         ret = __tegra_dvfs_set_rate(c->dvfs, rate);
708         mutex_unlock(&dvfs_lock);
709
710         return ret;
711 }
712 EXPORT_SYMBOL(tegra_dvfs_set_rate);
713
714 int tegra_dvfs_get_freqs(struct clk *c, unsigned long **freqs, int *num_freqs)
715 {
716         if (!c->dvfs)
717                 return -ENOSYS;
718
719         if (c->dvfs->alt_freqs)
720                 return -ENOSYS;
721
722         *num_freqs = c->dvfs->num_freqs;
723         *freqs = c->dvfs->freqs;
724
725         return 0;
726 }
727 EXPORT_SYMBOL(tegra_dvfs_get_freqs);
728
729 #ifdef CONFIG_TEGRA_VDD_CORE_OVERRIDE
730 static DEFINE_MUTEX(rail_override_lock);
731
732 int tegra_dvfs_override_core_voltage(int override_mv)
733 {
734         int ret, floor, ceiling;
735         struct dvfs_rail *rail = tegra_core_rail;
736
737         if (!rail)
738                 return -ENOENT;
739
740         if (rail->fixed_millivolts)
741                 return -ENOSYS;
742
743         floor = rail->min_override_millivolts;
744         ceiling = rail->nominal_millivolts;
745         if (override_mv && ((override_mv < floor) || (override_mv > ceiling))) {
746                 pr_err("%s: override level %d outside the range [%d...%d]\n",
747                        __func__, override_mv, floor, ceiling);
748                 return -EINVAL;
749         }
750
751         mutex_lock(&rail_override_lock);
752
753         if (override_mv == rail->override_millivolts) {
754                 ret = 0;
755                 goto out;
756         }
757
758         if (override_mv) {
759                 ret = tegra_dvfs_core_cap_level_apply(override_mv);
760                 if (ret) {
761                         pr_err("%s: failed to set cap for override level %d\n",
762                                __func__, override_mv);
763                         goto out;
764                 }
765         }
766
767         mutex_lock(&dvfs_lock);
768         if (rail->disabled || rail->suspended) {
769                 pr_err("%s: cannot scale %s rail\n", __func__,
770                        rail->disabled ? "disabled" : "suspended");
771                 ret = -EPERM;
772                 if (!override_mv) {
773                         mutex_unlock(&dvfs_lock);
774                         goto out;
775                 }
776         } else {
777                 rail->override_millivolts = override_mv;
778                 ret = dvfs_rail_update(rail);
779                 if (ret) {
780                         pr_err("%s: failed to set override level %d\n",
781                                __func__, override_mv);
782                         rail->override_millivolts = 0;
783                         dvfs_rail_update(rail);
784                 }
785         }
786         mutex_unlock(&dvfs_lock);
787
788         if (!override_mv || ret)
789                 tegra_dvfs_core_cap_level_apply(0);
790 out:
791         mutex_unlock(&rail_override_lock);
792         return ret;
793 }
794 #else
795 int tegra_dvfs_override_core_voltage(int override_mv)
796 {
797         pr_err("%s: vdd core override is not supported\n", __func__);
798         return -ENOSYS;
799 }
800 #endif
801 EXPORT_SYMBOL(tegra_dvfs_override_core_voltage);
802
803 /* May only be called during clock init, does not take any locks on clock c. */
804 int __init tegra_enable_dvfs_on_clk(struct clk *c, struct dvfs *d)
805 {
806         int i;
807
808         if (c->dvfs) {
809                 pr_err("Error when enabling dvfs on %s for clock %s:\n",
810                         d->dvfs_rail->reg_id, c->name);
811                 pr_err("DVFS already enabled for %s\n",
812                         c->dvfs->dvfs_rail->reg_id);
813                 return -EINVAL;
814         }
815
816         for (i = 0; i < MAX_DVFS_FREQS; i++) {
817                 if (d->millivolts[i] == 0)
818                         break;
819
820                 d->freqs[i] *= d->freqs_mult;
821
822                 /* If final frequencies are 0, pad with previous frequency */
823                 if (d->freqs[i] == 0 && i > 1)
824                         d->freqs[i] = d->freqs[i - 1];
825         }
826         d->num_freqs = i;
827
828         if (d->auto_dvfs) {
829                 c->auto_dvfs = true;
830                 clk_set_cansleep(c);
831         }
832
833         c->dvfs = d;
834
835         /*
836          * Minimum core override level is determined as maximum voltage required
837          * for clocks outside shared buses (shared bus rates can be capped to
838          * safe levels when override limit is set)
839          */
840         if (i && c->ops && !c->ops->shared_bus_update &&
841             !(c->flags & PERIPH_ON_CBUS)) {
842                 int mv = tegra_dvfs_predict_millivolts(c, d->freqs[i-1]);
843                 if (d->dvfs_rail->min_override_millivolts < mv)
844                         d->dvfs_rail->min_override_millivolts = mv;
845         }
846
847         mutex_lock(&dvfs_lock);
848         list_add_tail(&d->reg_node, &d->dvfs_rail->dvfs);
849         mutex_unlock(&dvfs_lock);
850
851         return 0;
852 }
853
854 static bool tegra_dvfs_all_rails_suspended(void)
855 {
856         struct dvfs_rail *rail;
857         bool all_suspended = true;
858
859         list_for_each_entry(rail, &dvfs_rail_list, node)
860                 if (!rail->suspended && !rail->disabled)
861                         all_suspended = false;
862
863         return all_suspended;
864 }
865
866 static bool tegra_dvfs_from_rails_suspended_or_solved(struct dvfs_rail *to)
867 {
868         struct dvfs_relationship *rel;
869         bool all_suspended = true;
870
871         list_for_each_entry(rel, &to->relationships_from, from_node)
872                 if (!rel->from->suspended && !rel->from->disabled &&
873                         !rel->solved_at_nominal)
874                         all_suspended = false;
875
876         return all_suspended;
877 }
878
879 static int tegra_dvfs_suspend_one(void)
880 {
881         struct dvfs_rail *rail;
882         int ret, mv;
883
884         list_for_each_entry(rail, &dvfs_rail_list, node) {
885                 if (!rail->suspended && !rail->disabled &&
886                     tegra_dvfs_from_rails_suspended_or_solved(rail)) {
887                         /* Safe, as pll mode rate is capped to fixed level */
888                         if (!rail->dfll_mode && rail->fixed_millivolts) {
889                                 mv = rail->fixed_millivolts;
890                         } else {
891                                 mv = tegra_dvfs_rail_get_suspend_level(rail);
892                                 mv = dvfs_rail_apply_limits(rail, mv);
893                         }
894
895                         /* apply suspend limit only if it is above current mv */
896                         ret = -EPERM;
897                         if (mv >= rail->millivolts)
898                                 ret = dvfs_rail_set_voltage(rail, mv);
899                         if (ret) {
900                                 pr_err("tegra_dvfs: failed %s suspend at %d\n",
901                                        rail->reg_id, rail->millivolts);
902                                 return ret;
903                         }
904
905                         rail->suspended = true;
906                         return 0;
907                 }
908         }
909
910         return -EINVAL;
911 }
912
913 static void tegra_dvfs_resume(void)
914 {
915         struct dvfs_rail *rail;
916
917         mutex_lock(&dvfs_lock);
918
919         list_for_each_entry(rail, &dvfs_rail_list, node)
920                 rail->suspended = false;
921
922         list_for_each_entry(rail, &dvfs_rail_list, node)
923                 dvfs_rail_update(rail);
924
925         mutex_unlock(&dvfs_lock);
926 }
927
928 static int tegra_dvfs_suspend(void)
929 {
930         int ret = 0;
931
932         mutex_lock(&dvfs_lock);
933
934         while (!tegra_dvfs_all_rails_suspended()) {
935                 ret = tegra_dvfs_suspend_one();
936                 if (ret)
937                         break;
938         }
939
940         mutex_unlock(&dvfs_lock);
941
942         if (ret)
943                 tegra_dvfs_resume();
944
945         return ret;
946 }
947
948 static int tegra_dvfs_pm_suspend(struct notifier_block *nb,
949                                  unsigned long event, void *data)
950 {
951         if (event == PM_SUSPEND_PREPARE) {
952                 if (tegra_dvfs_suspend())
953                         return NOTIFY_STOP;
954                 pr_info("tegra_dvfs: suspended\n");
955         }
956         return NOTIFY_OK;
957 };
958
959 static int tegra_dvfs_pm_resume(struct notifier_block *nb,
960                                 unsigned long event, void *data)
961 {
962         if (event == PM_POST_SUSPEND) {
963                 tegra_dvfs_resume();
964                 pr_info("tegra_dvfs: resumed\n");
965         }
966         return NOTIFY_OK;
967 };
968
969 static struct notifier_block tegra_dvfs_suspend_nb = {
970         .notifier_call = tegra_dvfs_pm_suspend,
971         .priority = -1,
972 };
973
974 static struct notifier_block tegra_dvfs_resume_nb = {
975         .notifier_call = tegra_dvfs_pm_resume,
976         .priority = 1,
977 };
978
979 static int tegra_dvfs_reboot_notify(struct notifier_block *nb,
980                                 unsigned long event, void *data)
981 {
982         switch (event) {
983         case SYS_RESTART:
984         case SYS_HALT:
985         case SYS_POWER_OFF:
986                 tegra_dvfs_suspend();
987                 return NOTIFY_OK;
988         }
989         return NOTIFY_DONE;
990 }
991
992 static struct notifier_block tegra_dvfs_reboot_nb = {
993         .notifier_call = tegra_dvfs_reboot_notify,
994 };
995
996 /* must be called with dvfs lock held */
997 static void __tegra_dvfs_rail_disable(struct dvfs_rail *rail)
998 {
999         int ret = -EPERM;
1000         int mv;
1001
1002         /* don't set voltage in DFLL mode - won't work, but break stats */
1003         if (rail->dfll_mode) {
1004                 rail->disabled = true;
1005                 return;
1006         }
1007
1008         /* Safe, as pll mode rate is capped to fixed level */
1009         if (!rail->dfll_mode && rail->fixed_millivolts) {
1010                 mv = rail->fixed_millivolts;
1011         } else {
1012                 mv = tegra_dvfs_rail_get_disable_level(rail);
1013                 mv = dvfs_rail_apply_limits(rail, mv);
1014         }
1015
1016         /* apply detach mode limit provided it is above current volatge */
1017         if (mv >= rail->millivolts)
1018                 ret = dvfs_rail_set_voltage(rail, mv);
1019         if (ret) {
1020                 pr_err("tegra_dvfs: failed to disable %s at %d\n",
1021                        rail->reg_id, rail->millivolts);
1022                 return;
1023         }
1024         rail->disabled = true;
1025 }
1026
1027 /* must be called with dvfs lock held */
1028 static void __tegra_dvfs_rail_enable(struct dvfs_rail *rail)
1029 {
1030         rail->disabled = false;
1031         dvfs_rail_update(rail);
1032 }
1033
1034 void tegra_dvfs_rail_enable(struct dvfs_rail *rail)
1035 {
1036         if (!rail)
1037                 return;
1038
1039         mutex_lock(&rail_disable_lock);
1040
1041         if (rail->disabled) {
1042                 mutex_lock(&dvfs_lock);
1043                 __tegra_dvfs_rail_enable(rail);
1044                 mutex_unlock(&dvfs_lock);
1045
1046                 tegra_dvfs_rail_post_enable(rail);
1047         }
1048         mutex_unlock(&rail_disable_lock);
1049 }
1050
1051 void tegra_dvfs_rail_disable(struct dvfs_rail *rail)
1052 {
1053         if (!rail)
1054                 return;
1055
1056         mutex_lock(&rail_disable_lock);
1057         if (rail->disabled)
1058                 goto out;
1059
1060         /* rail disable will set it to nominal voltage underneath clock
1061            framework - need to re-configure clock rates that are not safe
1062            at nominal (yes, unsafe at nominal is ugly, but possible). Rate
1063            change must be done outside of dvfs lock. */
1064         if (tegra_dvfs_rail_disable_prepare(rail)) {
1065                 pr_info("dvfs: failed to prepare regulator %s to disable\n",
1066                         rail->reg_id);
1067                 goto out;
1068         }
1069
1070         mutex_lock(&dvfs_lock);
1071         __tegra_dvfs_rail_disable(rail);
1072         mutex_unlock(&dvfs_lock);
1073 out:
1074         mutex_unlock(&rail_disable_lock);
1075 }
1076
1077 int tegra_dvfs_rail_disable_by_name(const char *reg_id)
1078 {
1079         struct dvfs_rail *rail = tegra_dvfs_get_rail_by_name(reg_id);
1080         if (!rail)
1081                 return -EINVAL;
1082
1083         tegra_dvfs_rail_disable(rail);
1084         return 0;
1085 }
1086
1087 struct dvfs_rail *tegra_dvfs_get_rail_by_name(const char *reg_id)
1088 {
1089         struct dvfs_rail *rail;
1090
1091         mutex_lock(&dvfs_lock);
1092         list_for_each_entry(rail, &dvfs_rail_list, node) {
1093                 if (!strcmp(reg_id, rail->reg_id)) {
1094                         mutex_unlock(&dvfs_lock);
1095                         return rail;
1096                 }
1097         }
1098         mutex_unlock(&dvfs_lock);
1099         return NULL;
1100 }
1101
1102 bool tegra_dvfs_rail_updating(struct clk *clk)
1103 {
1104         return (!clk ? false :
1105                 (!clk->dvfs ? false :
1106                  (!clk->dvfs->dvfs_rail ? false :
1107                   (clk->dvfs->dvfs_rail->updating ||
1108                    clk->dvfs->dvfs_rail->dfll_mode_updating))));
1109 }
1110
1111 #ifdef CONFIG_OF
1112 int __init of_tegra_dvfs_init(const struct of_device_id *matches)
1113 {
1114         int ret;
1115         struct device_node *np;
1116
1117         for_each_matching_node(np, matches) {
1118                 const struct of_device_id *match = of_match_node(matches, np);
1119                 of_tegra_dvfs_init_cb_t dvfs_init_cb = match->data;
1120                 ret = dvfs_init_cb(np);
1121                 if (ret) {
1122                         pr_err("dt: Failed to read %s tables from DT\n",
1123                                                         match->compatible);
1124                         return ret;
1125                 }
1126         }
1127         return 0;
1128 }
1129 #endif
1130 int tegra_dvfs_dfll_mode_set(struct dvfs *d, unsigned long rate)
1131 {
1132         mutex_lock(&dvfs_lock);
1133         if (!d->dvfs_rail->dfll_mode) {
1134                 d->dvfs_rail->dfll_mode = true;
1135                 __tegra_dvfs_set_rate(d, rate);
1136         }
1137         mutex_unlock(&dvfs_lock);
1138         return 0;
1139 }
1140
1141 int tegra_dvfs_dfll_mode_clear(struct dvfs *d, unsigned long rate)
1142 {
1143         int ret = 0;
1144
1145         mutex_lock(&dvfs_lock);
1146         if (d->dvfs_rail->dfll_mode) {
1147                 d->dvfs_rail->dfll_mode = false;
1148                 /* avoid false detection of matching target (voltage in dfll
1149                    mode is fluctuating, and recorded level is just estimate) */
1150                 d->dvfs_rail->millivolts--;
1151                 if (d->dvfs_rail->disabled) {
1152                         d->dvfs_rail->disabled = false;
1153                         __tegra_dvfs_rail_disable(d->dvfs_rail);
1154                 }
1155                 ret = __tegra_dvfs_set_rate(d, rate);
1156         }
1157         mutex_unlock(&dvfs_lock);
1158         return ret;
1159 }
1160
1161 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmax_cdev(void)
1162 {
1163         if (tegra_cpu_rail)
1164                 return tegra_cpu_rail->vmax_cdev;
1165         return NULL;
1166 }
1167
1168 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmin_cdev(void)
1169 {
1170         if (tegra_cpu_rail)
1171                 return tegra_cpu_rail->vmin_cdev;
1172         return NULL;
1173 }
1174
1175 struct tegra_cooling_device *tegra_dvfs_get_core_vmin_cdev(void)
1176 {
1177         if (tegra_core_rail)
1178                 return tegra_core_rail->vmin_cdev;
1179         return NULL;
1180 }
1181
1182 struct tegra_cooling_device *tegra_dvfs_get_gpu_vmin_cdev(void)
1183 {
1184         if (tegra_gpu_rail)
1185                 return tegra_gpu_rail->vmin_cdev;
1186         return NULL;
1187 }
1188
1189 #ifdef CONFIG_THERMAL
1190 /* Cooling device limits minimum rail voltage at cold temperature in pll mode */
1191 static int tegra_dvfs_rail_get_vmin_cdev_max_state(
1192         struct thermal_cooling_device *cdev, unsigned long *max_state)
1193 {
1194         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1195         *max_state = rail->vmin_cdev->trip_temperatures_num;
1196         return 0;
1197 }
1198
1199 static int tegra_dvfs_rail_get_vmin_cdev_cur_state(
1200         struct thermal_cooling_device *cdev, unsigned long *cur_state)
1201 {
1202         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1203         *cur_state = rail->therm_floor_idx;
1204         return 0;
1205 }
1206
1207 static int tegra_dvfs_rail_set_vmin_cdev_state(
1208         struct thermal_cooling_device *cdev, unsigned long cur_state)
1209 {
1210         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1211
1212         mutex_lock(&dvfs_lock);
1213         if (rail->therm_floor_idx != cur_state) {
1214                 rail->therm_floor_idx = cur_state;
1215                 dvfs_rail_update(rail);
1216         }
1217         mutex_unlock(&dvfs_lock);
1218         return 0;
1219 }
1220
1221 static struct thermal_cooling_device_ops tegra_dvfs_rail_cooling_ops = {
1222         .get_max_state = tegra_dvfs_rail_get_vmin_cdev_max_state,
1223         .get_cur_state = tegra_dvfs_rail_get_vmin_cdev_cur_state,
1224         .set_cur_state = tegra_dvfs_rail_set_vmin_cdev_state,
1225 };
1226
1227 static void tegra_dvfs_rail_register_vmin_cdev(struct dvfs_rail *rail)
1228 {
1229         if (!rail->vmin_cdev)
1230                 return;
1231
1232         /* just report error - initialized for cold temperature, anyway */
1233         if (IS_ERR_OR_NULL(thermal_cooling_device_register(
1234                 rail->vmin_cdev->cdev_type, (void *)rail,
1235                 &tegra_dvfs_rail_cooling_ops)))
1236                 pr_err("tegra cooling device %s failed to register\n",
1237                        rail->vmin_cdev->cdev_type);
1238 }
1239
1240 #else
1241 #define tegra_dvfs_rail_register_vmin_cdev(rail)
1242 #endif
1243
1244 /*
1245  * Validate rail thermal profile, and get its size. Valid profile:
1246  * - voltage limits are descending with temperature increasing
1247  * - the lowest limit is above rail minimum voltage in pll and
1248  *   in dfll mode (if applicable)
1249  * - the highest limit is below rail nominal voltage
1250  */
1251 static int __init get_thermal_profile_size(
1252         int *trips_table, int *limits_table,
1253         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1254 {
1255         int i, min_mv;
1256
1257         for (i = 0; i < MAX_THERMAL_LIMITS - 1; i++) {
1258                 if (!limits_table[i+1])
1259                         break;
1260
1261                 if ((trips_table[i] >= trips_table[i+1]) ||
1262                     (limits_table[i] < limits_table[i+1])) {
1263                         pr_warn("%s: not ordered profile\n", rail->reg_id);
1264                         return -EINVAL;
1265                 }
1266         }
1267
1268         min_mv = max(rail->min_millivolts, d ? d->min_millivolts : 0);
1269         if (limits_table[i] < min_mv) {
1270                 pr_warn("%s: thermal profile below Vmin\n", rail->reg_id);
1271                 return -EINVAL;
1272         }
1273
1274         if (limits_table[0] > rail->nominal_millivolts) {
1275                 pr_warn("%s: thermal profile above Vmax\n", rail->reg_id);
1276                 return -EINVAL;
1277         }
1278         return i + 1;
1279 }
1280
1281 void __init tegra_dvfs_rail_init_vmax_thermal_profile(
1282         int *therm_trips_table, int *therm_caps_table,
1283         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1284 {
1285         int i = get_thermal_profile_size(therm_trips_table,
1286                                          therm_caps_table, rail, d);
1287         if (i <= 0) {
1288                 rail->vmax_cdev = NULL;
1289                 WARN(1, "%s: invalid Vmax thermal profile\n", rail->reg_id);
1290                 return;
1291         }
1292
1293         /* Install validated thermal caps */
1294         rail->therm_mv_caps = therm_caps_table;
1295         rail->therm_mv_caps_num = i;
1296
1297         /* Setup trip-points if applicable */
1298         if (rail->vmax_cdev) {
1299                 rail->vmax_cdev->trip_temperatures_num = i;
1300                 rail->vmax_cdev->trip_temperatures = therm_trips_table;
1301         }
1302 }
1303
1304 void __init tegra_dvfs_rail_init_vmin_thermal_profile(
1305         int *therm_trips_table, int *therm_floors_table,
1306         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1307 {
1308         int i = get_thermal_profile_size(therm_trips_table,
1309                                          therm_floors_table, rail, d);
1310         if (i <= 0) {
1311                 rail->vmin_cdev = NULL;
1312                 WARN(1, "%s: invalid Vmin thermal profile\n", rail->reg_id);
1313                 return;
1314         }
1315
1316         /* Install validated thermal floors */
1317         rail->therm_mv_floors = therm_floors_table;
1318         rail->therm_mv_floors_num = i;
1319
1320         /* Setup trip-points if applicable */
1321         if (rail->vmin_cdev) {
1322                 rail->vmin_cdev->trip_temperatures_num = i;
1323                 rail->vmin_cdev->trip_temperatures = therm_trips_table;
1324         }
1325 }
1326
1327 /* Directly set cold temperature limit in dfll mode */
1328 int tegra_dvfs_rail_dfll_mode_set_cold(struct dvfs_rail *rail)
1329 {
1330         int ret = 0;
1331
1332         /* No thermal floors - nothing to do */
1333         if (!rail || !rail->therm_mv_floors)
1334                 return ret;
1335
1336         /*
1337          * Since cooling thresholds are the same in pll and dfll modes, pll mode
1338          * thermal index can be used to decide if cold limit should be set in
1339          * dfll mode.
1340          */
1341         mutex_lock(&dvfs_lock);
1342         if (rail->dfll_mode &&
1343             (rail->therm_floor_idx < rail->therm_mv_floors_num)) {
1344                         int mv = rail->therm_mv_floors[rail->therm_floor_idx];
1345                         ret = dvfs_rail_set_voltage_reg(rail, mv);
1346         }
1347         mutex_unlock(&dvfs_lock);
1348
1349         return ret;
1350 }
1351
1352 /*
1353  * Iterate through all the dvfs regulators, finding the regulator exported
1354  * by the regulator api for each one.  Must be called in late init, after
1355  * all the regulator api's regulators are initialized.
1356  */
1357 int __init tegra_dvfs_late_init(void)
1358 {
1359         bool connected = true;
1360         struct dvfs_rail *rail;
1361
1362         mutex_lock(&dvfs_lock);
1363
1364         list_for_each_entry(rail, &dvfs_rail_list, node)
1365                 if (dvfs_rail_connect_to_regulator(rail))
1366                         connected = false;
1367
1368         list_for_each_entry(rail, &dvfs_rail_list, node)
1369                 if (connected)
1370                         dvfs_rail_update(rail);
1371                 else
1372                         __tegra_dvfs_rail_disable(rail);
1373
1374         mutex_unlock(&dvfs_lock);
1375
1376         if (!connected && tegra_platform_is_silicon()) {
1377                 pr_warn("tegra_dvfs: DVFS regulators connection failed\n"
1378                         "            !!!! voltage scaling is disabled !!!!\n");
1379                 return -ENODEV;
1380         }
1381
1382         register_pm_notifier(&tegra_dvfs_suspend_nb);
1383         register_pm_notifier(&tegra_dvfs_resume_nb);
1384         register_reboot_notifier(&tegra_dvfs_reboot_nb);
1385
1386         list_for_each_entry(rail, &dvfs_rail_list, node)
1387                 tegra_dvfs_rail_register_vmin_cdev(rail);
1388
1389         return 0;
1390 }
1391
1392 static int rail_stats_save_to_buf(char *buf, int len)
1393 {
1394         int i;
1395         struct dvfs_rail *rail;
1396         char *str = buf;
1397         char *end = buf + len;
1398
1399         str += scnprintf(str, end - str, "%-12s %-10s\n", "millivolts", "time");
1400
1401         mutex_lock(&dvfs_lock);
1402
1403         list_for_each_entry(rail, &dvfs_rail_list, node) {
1404                 str += scnprintf(str, end - str, "%s (bin: %d.%dmV)\n",
1405                            rail->reg_id,
1406                            rail->stats.bin_uV / 1000,
1407                            (rail->stats.bin_uV / 10) % 100);
1408
1409                 dvfs_rail_stats_update(rail, -1, ktime_get());
1410
1411                 str += scnprintf(str, end - str, "%-12d %-10llu\n", 0,
1412                         cputime64_to_clock_t(msecs_to_jiffies(
1413                                 ktime_to_ms(rail->stats.time_at_mv[0]))));
1414
1415                 for (i = 1; i <= DVFS_RAIL_STATS_TOP_BIN; i++) {
1416                         ktime_t ktime_zero = ktime_set(0, 0);
1417                         if (ktime_equal(rail->stats.time_at_mv[i], ktime_zero))
1418                                 continue;
1419                         str += scnprintf(str, end - str, "%-12d %-10llu\n",
1420                                 rail->min_millivolts +
1421                                 (i - 1) * rail->stats.bin_uV / 1000,
1422                                 cputime64_to_clock_t(msecs_to_jiffies(
1423                                         ktime_to_ms(rail->stats.time_at_mv[i])))
1424                         );
1425                 }
1426         }
1427         mutex_unlock(&dvfs_lock);
1428         return str - buf;
1429 }
1430
1431 #ifdef CONFIG_DEBUG_FS
1432 static int dvfs_tree_sort_cmp(void *p, struct list_head *a, struct list_head *b)
1433 {
1434         struct dvfs *da = list_entry(a, struct dvfs, reg_node);
1435         struct dvfs *db = list_entry(b, struct dvfs, reg_node);
1436         int ret;
1437
1438         ret = strcmp(da->dvfs_rail->reg_id, db->dvfs_rail->reg_id);
1439         if (ret != 0)
1440                 return ret;
1441
1442         if (da->cur_millivolts < db->cur_millivolts)
1443                 return 1;
1444         if (da->cur_millivolts > db->cur_millivolts)
1445                 return -1;
1446
1447         return strcmp(da->clk_name, db->clk_name);
1448 }
1449
1450 static int dvfs_tree_show(struct seq_file *s, void *data)
1451 {
1452         struct dvfs *d;
1453         struct dvfs_rail *rail;
1454         struct dvfs_relationship *rel;
1455
1456         seq_printf(s, "   clock      rate       mV\n");
1457         seq_printf(s, "--------------------------------\n");
1458
1459         mutex_lock(&dvfs_lock);
1460
1461         list_for_each_entry(rail, &dvfs_rail_list, node) {
1462                 int thermal_mv_floor = 0;
1463
1464                 seq_printf(s, "%s %d mV%s:\n", rail->reg_id, rail->millivolts,
1465                            rail->dfll_mode ? " dfll mode" :
1466                                 rail->disabled ? " disabled" : "");
1467                 list_for_each_entry(rel, &rail->relationships_from, from_node) {
1468                         seq_printf(s, "   %-10s %-7d mV %-4d mV\n",
1469                                 rel->from->reg_id, rel->from->millivolts,
1470                                 dvfs_solve_relationship(rel));
1471                 }
1472                 seq_printf(s, "   offset     %-7d mV\n", rail->dbg_mv_offs);
1473
1474                 if (rail->therm_mv_floors) {
1475                         int i = rail->therm_floor_idx;
1476                         if (i < rail->therm_mv_floors_num)
1477                                 thermal_mv_floor = rail->therm_mv_floors[i];
1478                 }
1479                 seq_printf(s, "   thermal    %-7d mV\n", thermal_mv_floor);
1480
1481                 if (rail == tegra_core_rail) {
1482                         seq_printf(s, "   override   %-7d mV [%-4d...%-4d]\n",
1483                                    rail->override_millivolts,
1484                                    rail->min_override_millivolts,
1485                                    rail->nominal_millivolts);
1486                 }
1487
1488                 list_sort(NULL, &rail->dvfs, dvfs_tree_sort_cmp);
1489
1490                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1491                         seq_printf(s, "   %-10s %-10lu %-4d mV\n", d->clk_name,
1492                                 d->cur_rate, d->cur_millivolts);
1493                 }
1494         }
1495
1496         mutex_unlock(&dvfs_lock);
1497
1498         return 0;
1499 }
1500
1501 static int dvfs_tree_open(struct inode *inode, struct file *file)
1502 {
1503         return single_open(file, dvfs_tree_show, inode->i_private);
1504 }
1505
1506 static const struct file_operations dvfs_tree_fops = {
1507         .open           = dvfs_tree_open,
1508         .read           = seq_read,
1509         .llseek         = seq_lseek,
1510         .release        = single_release,
1511 };
1512
1513 static int rail_stats_show(struct seq_file *s, void *data)
1514 {
1515         char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
1516         int size = 0;
1517
1518         if (!buf)
1519                 return -ENOMEM;
1520
1521         size = rail_stats_save_to_buf(buf, PAGE_SIZE);
1522         seq_write(s, buf, size);
1523         kfree(buf);
1524         return 0;
1525 }
1526
1527 static int rail_stats_open(struct inode *inode, struct file *file)
1528 {
1529         return single_open(file, rail_stats_show, inode->i_private);
1530 }
1531
1532 static const struct file_operations rail_stats_fops = {
1533         .open           = rail_stats_open,
1534         .read           = seq_read,
1535         .llseek         = seq_lseek,
1536         .release        = single_release,
1537 };
1538
1539 static int gpu_dvfs_show(struct seq_file *s, void *data)
1540 {
1541         int idx;
1542         int *millivolts;
1543         unsigned long *freqs;
1544
1545         if (read_gpu_dvfs_table(&millivolts, &freqs)) {
1546                 seq_printf(s, "Only supported for T124 or higher\n");
1547                 return 0;
1548         }
1549
1550         seq_printf(s, "millivolts \t \t frequency\n");
1551         seq_printf(s, "=====================================\n");
1552
1553         for (idx = 0; millivolts[idx]; idx++)
1554                 seq_printf(s, "%d mV \t \t %lu Hz\n", millivolts[idx],
1555                                 freqs[idx]);
1556
1557         return 0;
1558 }
1559
1560 static int gpu_dvfs_open(struct inode *inode, struct file *file)
1561 {
1562         return single_open(file, gpu_dvfs_show, NULL);
1563 }
1564
1565 static const struct file_operations gpu_dvfs_fops = {
1566         .open           = gpu_dvfs_open,
1567         .read           = seq_read,
1568         .llseek         = seq_lseek,
1569         .release        = single_release,
1570 };
1571
1572 static int rail_offs_set(struct dvfs_rail *rail, int offs)
1573 {
1574         if (rail) {
1575                 mutex_lock(&dvfs_lock);
1576                 rail->dbg_mv_offs = offs;
1577                 dvfs_rail_update(rail);
1578                 mutex_unlock(&dvfs_lock);
1579                 return 0;
1580         }
1581         return -ENOENT;
1582 }
1583
1584 static int cpu_offs_get(void *data, u64 *val)
1585 {
1586         if (tegra_cpu_rail) {
1587                 *val = (u64)tegra_cpu_rail->dbg_mv_offs;
1588                 return 0;
1589         }
1590         *val = 0;
1591         return -ENOENT;
1592 }
1593 static int cpu_offs_set(void *data, u64 val)
1594 {
1595         return rail_offs_set(tegra_cpu_rail, (int)val);
1596 }
1597 DEFINE_SIMPLE_ATTRIBUTE(cpu_offs_fops, cpu_offs_get, cpu_offs_set, "%lld\n");
1598
1599 static int gpu_offs_get(void *data, u64 *val)
1600 {
1601         if (tegra_gpu_rail) {
1602                 *val = (u64)tegra_gpu_rail->dbg_mv_offs;
1603                 return 0;
1604         }
1605         *val = 0;
1606         return -ENOENT;
1607 }
1608 static int gpu_offs_set(void *data, u64 val)
1609 {
1610         return rail_offs_set(tegra_gpu_rail, (int)val);
1611 }
1612 DEFINE_SIMPLE_ATTRIBUTE(gpu_offs_fops, gpu_offs_get, gpu_offs_set, "%lld\n");
1613
1614 static int core_offs_get(void *data, u64 *val)
1615 {
1616         if (tegra_core_rail) {
1617                 *val = (u64)tegra_core_rail->dbg_mv_offs;
1618                 return 0;
1619         }
1620         *val = 0;
1621         return -ENOENT;
1622 }
1623 static int core_offs_set(void *data, u64 val)
1624 {
1625         return rail_offs_set(tegra_core_rail, (int)val);
1626 }
1627 DEFINE_SIMPLE_ATTRIBUTE(core_offs_fops, core_offs_get, core_offs_set, "%lld\n");
1628
1629 static int core_override_get(void *data, u64 *val)
1630 {
1631         if (tegra_core_rail) {
1632                 *val = (u64)tegra_core_rail->override_millivolts;
1633                 return 0;
1634         }
1635         *val = 0;
1636         return -ENOENT;
1637 }
1638 static int core_override_set(void *data, u64 val)
1639 {
1640         return tegra_dvfs_override_core_voltage((int)val);
1641 }
1642 DEFINE_SIMPLE_ATTRIBUTE(core_override_fops,
1643                         core_override_get, core_override_set, "%llu\n");
1644
1645 static int dvfs_table_show(struct seq_file *s, void *data)
1646 {
1647         int i;
1648         struct dvfs *d;
1649         struct dvfs_rail *rail;
1650
1651         seq_printf(s, "DVFS tables: units mV/MHz\n\n");
1652
1653         mutex_lock(&dvfs_lock);
1654
1655         list_for_each_entry(rail, &dvfs_rail_list, node) {
1656                 bool mv_done = false;
1657                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1658                         if (!mv_done) {
1659                                 mv_done = true;
1660                                 seq_printf(s, "%-16s", rail->reg_id);
1661                                 for (i = 0; i < d->num_freqs; i++) {
1662                                         int mv = d->millivolts[i];
1663                                         seq_printf(s, "%7d", mv);
1664                                 }
1665                                 seq_printf(s, "\n");
1666                                 if (d->dfll_millivolts) {
1667                                         seq_printf(s, "%-8s (dfll) ",
1668                                                    rail->reg_id);
1669                                         for (i = 0; i < d->num_freqs; i++) {
1670                                                 int mv = d->dfll_millivolts[i];
1671                                                 seq_printf(s, "%7d", mv);
1672                                         }
1673                                         seq_printf(s, "\n");
1674                                 }
1675                         }
1676
1677                         seq_printf(s, "%-16s", d->clk_name);
1678                         for (i = 0; i < d->num_freqs; i++) {
1679                                 unsigned int f = d->freqs[i]/100000;
1680                                 seq_printf(s, " %4u.%u", f/10, f%10);
1681                         }
1682                         seq_printf(s, "\n");
1683                 }
1684                 seq_printf(s, "\n");
1685         }
1686
1687         mutex_unlock(&dvfs_lock);
1688
1689         return 0;
1690 }
1691
1692 static int dvfs_table_open(struct inode *inode, struct file *file)
1693 {
1694         return single_open(file, dvfs_table_show, inode->i_private);
1695 }
1696
1697 static const struct file_operations dvfs_table_fops = {
1698         .open           = dvfs_table_open,
1699         .read           = seq_read,
1700         .llseek         = seq_lseek,
1701         .release        = single_release,
1702 };
1703
1704 int __init dvfs_debugfs_init(struct dentry *clk_debugfs_root)
1705 {
1706         struct dentry *d;
1707
1708         d = debugfs_create_file("dvfs", S_IRUGO, clk_debugfs_root, NULL,
1709                 &dvfs_tree_fops);
1710         if (!d)
1711                 return -ENOMEM;
1712
1713         d = debugfs_create_file("rails", S_IRUGO, clk_debugfs_root, NULL,
1714                 &rail_stats_fops);
1715         if (!d)
1716                 return -ENOMEM;
1717
1718         d = debugfs_create_file("vdd_cpu_offs", S_IRUGO | S_IWUSR,
1719                 clk_debugfs_root, NULL, &cpu_offs_fops);
1720         if (!d)
1721                 return -ENOMEM;
1722
1723         d = debugfs_create_file("vdd_gpu_offs", S_IRUGO | S_IWUSR,
1724                 clk_debugfs_root, NULL, &gpu_offs_fops);
1725         if (!d)
1726                 return -ENOMEM;
1727
1728         d = debugfs_create_file("vdd_core_offs", S_IRUGO | S_IWUSR,
1729                 clk_debugfs_root, NULL, &core_offs_fops);
1730         if (!d)
1731                 return -ENOMEM;
1732
1733         d = debugfs_create_file("vdd_core_override", S_IRUGO | S_IWUSR,
1734                 clk_debugfs_root, NULL, &core_override_fops);
1735         if (!d)
1736                 return -ENOMEM;
1737
1738         d = debugfs_create_file("gpu_dvfs", S_IRUGO | S_IWUSR,
1739                 clk_debugfs_root, NULL, &gpu_dvfs_fops);
1740         if (!d)
1741                 return -ENOMEM;
1742
1743         d = debugfs_create_file("dvfs_table", S_IRUGO, clk_debugfs_root, NULL,
1744                 &dvfs_table_fops);
1745         if (!d)
1746                 return -ENOMEM;
1747
1748         return 0;
1749 }
1750
1751 #endif
1752
1753 #ifdef CONFIG_PM
1754 static ssize_t tegra_rail_stats_show(struct kobject *kobj,
1755                                         struct kobj_attribute *attr,
1756                                         char *buf)
1757 {
1758         return rail_stats_save_to_buf(buf, PAGE_SIZE);
1759 }
1760
1761 static struct kobj_attribute rail_stats_attr =
1762                 __ATTR_RO(tegra_rail_stats);
1763
1764 static int __init tegra_dvfs_sysfs_stats_init(void)
1765 {
1766         int error;
1767         error = sysfs_create_file(power_kobj, &rail_stats_attr.attr);
1768         return 0;
1769 }
1770 late_initcall(tegra_dvfs_sysfs_stats_init);
1771 #endif