ARM: tegra: dvfs: Account for GPU rail suspend time
[linux-3.10.git] / arch / arm / mach-tegra / dvfs.c
1 /*
2  *
3  * Copyright (C) 2010 Google, Inc.
4  *
5  * Author:
6  *      Colin Cross <ccross@google.com>
7  *
8  * Copyright (C) 2010-2013 NVIDIA CORPORATION. All rights reserved.
9  *
10  * This software is licensed under the terms of the GNU General Public
11  * License version 2, as published by the Free Software Foundation, and
12  * may be copied, distributed, and modified under those terms.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/clk.h>
23 #include <linux/clkdev.h>
24 #include <linux/debugfs.h>
25 #include <linux/init.h>
26 #include <linux/list.h>
27 #include <linux/list_sort.h>
28 #include <linux/module.h>
29 #include <linux/regulator/consumer.h>
30 #include <linux/seq_file.h>
31 #include <linux/slab.h>
32 #include <linux/suspend.h>
33 #include <linux/delay.h>
34 #include <linux/clk/tegra.h>
35 #include <linux/reboot.h>
36 #include <linux/clk/tegra.h>
37 #include <linux/tegra-soc.h>
38
39 #include "board.h"
40 #include "clock.h"
41 #include "dvfs.h"
42
43 #define DVFS_RAIL_STATS_BIN     12500
44
45 struct dvfs_rail *tegra_cpu_rail;
46 struct dvfs_rail *tegra_core_rail;
47 struct dvfs_rail *tegra_gpu_rail;
48
49 static LIST_HEAD(dvfs_rail_list);
50 static DEFINE_MUTEX(dvfs_lock);
51 static DEFINE_MUTEX(rail_disable_lock);
52
53 static int dvfs_rail_update(struct dvfs_rail *rail);
54
55 static inline int tegra_dvfs_rail_get_disable_level(struct dvfs_rail *rail)
56 {
57         return rail->disable_millivolts ? : rail->nominal_millivolts;
58 }
59
60 static inline int tegra_dvfs_rail_get_suspend_level(struct dvfs_rail *rail)
61 {
62         return rail->suspend_millivolts ? : rail->nominal_millivolts;
63 }
64
65 void tegra_dvfs_add_relationships(struct dvfs_relationship *rels, int n)
66 {
67         int i;
68         struct dvfs_relationship *rel;
69
70         mutex_lock(&dvfs_lock);
71
72         for (i = 0; i < n; i++) {
73                 rel = &rels[i];
74                 list_add_tail(&rel->from_node, &rel->to->relationships_from);
75                 list_add_tail(&rel->to_node, &rel->from->relationships_to);
76         }
77
78         mutex_unlock(&dvfs_lock);
79 }
80
81 /* Make sure there is a matching cooling device for thermal limit profile. */
82 static void dvfs_validate_cdevs(struct dvfs_rail *rail)
83 {
84         if (!rail->therm_mv_caps != !rail->therm_mv_caps_num) {
85                 rail->therm_mv_caps_num = 0;
86                 rail->therm_mv_caps = NULL;
87                 WARN(1, "%s: not matching thermal caps/num\n", rail->reg_id);
88         }
89
90         if (rail->therm_mv_caps && !rail->vmax_cdev)
91                 WARN(1, "%s: missing vmax cooling device\n", rail->reg_id);
92
93         if (!rail->therm_mv_floors != !rail->therm_mv_floors_num) {
94                 rail->therm_mv_floors_num = 0;
95                 rail->therm_mv_floors = NULL;
96                 WARN(1, "%s: not matching thermal floors/num\n", rail->reg_id);
97         }
98
99         if (rail->therm_mv_floors && !rail->vmin_cdev)
100                 WARN(1, "%s: missing vmin cooling device\n", rail->reg_id);
101
102         /* Limit override range to maximum floor */
103         if (rail->therm_mv_floors)
104                 rail->min_override_millivolts = rail->therm_mv_floors[0];
105 }
106
107 int tegra_dvfs_init_rails(struct dvfs_rail *rails[], int n)
108 {
109         int i, mv;
110
111         mutex_lock(&dvfs_lock);
112
113         for (i = 0; i < n; i++) {
114                 INIT_LIST_HEAD(&rails[i]->dvfs);
115                 INIT_LIST_HEAD(&rails[i]->relationships_from);
116                 INIT_LIST_HEAD(&rails[i]->relationships_to);
117
118                 mv = rails[i]->nominal_millivolts;
119                 if (rails[i]->boot_millivolts > mv)
120                         WARN(1, "%s: boot voltage %d above nominal %d\n",
121                              rails[i]->reg_id, rails[i]->boot_millivolts, mv);
122                 if (rails[i]->disable_millivolts > mv)
123                         rails[i]->disable_millivolts = mv;
124                 if (rails[i]->suspend_millivolts > mv)
125                         rails[i]->suspend_millivolts = mv;
126
127                 mv = tegra_dvfs_rail_get_boot_level(rails[i]);
128                 rails[i]->millivolts = mv;
129                 rails[i]->new_millivolts = mv;
130                 if (!rails[i]->step)
131                         rails[i]->step = rails[i]->max_millivolts;
132                 if (!rails[i]->step_up)
133                         rails[i]->step_up = rails[i]->step;
134
135                 list_add_tail(&rails[i]->node, &dvfs_rail_list);
136
137                 dvfs_validate_cdevs(rails[i]);
138
139                 if (!strcmp("vdd_cpu", rails[i]->reg_id))
140                         tegra_cpu_rail = rails[i];
141                 else if (!strcmp("vdd_gpu", rails[i]->reg_id))
142                         tegra_gpu_rail = rails[i];
143                 else if (!strcmp("vdd_core", rails[i]->reg_id))
144                         tegra_core_rail = rails[i];
145         }
146
147         mutex_unlock(&dvfs_lock);
148
149         return 0;
150 };
151
152 static int dvfs_solve_relationship(struct dvfs_relationship *rel)
153 {
154         return rel->solve(rel->from, rel->to);
155 }
156
157 /* rail statistic - called during rail init, or under dfs_lock, or with
158    CPU0 only on-line, and interrupts disabled */
159 static void dvfs_rail_stats_init(struct dvfs_rail *rail, int millivolts)
160 {
161         int dvfs_rail_stats_range;
162
163         if (!rail->stats.bin_uV)
164                 rail->stats.bin_uV = DVFS_RAIL_STATS_BIN;
165
166         dvfs_rail_stats_range =
167                 (DVFS_RAIL_STATS_TOP_BIN - 1) * rail->stats.bin_uV / 1000;
168
169         rail->stats.last_update = ktime_get();
170         if (millivolts >= rail->min_millivolts) {
171                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
172                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
173                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
174         }
175
176         if (rail->max_millivolts >
177             rail->min_millivolts + dvfs_rail_stats_range)
178                 pr_warn("tegra_dvfs: %s: stats above %d mV will be squashed\n",
179                         rail->reg_id,
180                         rail->min_millivolts + dvfs_rail_stats_range);
181 }
182
183 static void dvfs_rail_stats_update(
184         struct dvfs_rail *rail, int millivolts, ktime_t now)
185 {
186         rail->stats.time_at_mv[rail->stats.last_index] = ktime_add(
187                 rail->stats.time_at_mv[rail->stats.last_index], ktime_sub(
188                         now, rail->stats.last_update));
189         rail->stats.last_update = now;
190
191         if (rail->stats.off)
192                 return;
193
194         if (millivolts >= rail->min_millivolts) {
195                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
196                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
197                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
198         } else if (millivolts == 0)
199                         rail->stats.last_index = 0;
200 }
201
202 static void dvfs_rail_stats_pause(struct dvfs_rail *rail,
203                                   ktime_t delta, bool on)
204 {
205         int i = on ? rail->stats.last_index : 0;
206         rail->stats.time_at_mv[i] = ktime_add(rail->stats.time_at_mv[i], delta);
207 }
208
209 void tegra_dvfs_rail_off(struct dvfs_rail *rail, ktime_t now)
210 {
211         if (rail) {
212                 dvfs_rail_stats_update(rail, 0, now);
213                 rail->stats.off = true;
214         }
215 }
216
217 void tegra_dvfs_rail_on(struct dvfs_rail *rail, ktime_t now)
218 {
219         if (rail) {
220                 rail->stats.off = false;
221                 dvfs_rail_stats_update(rail, rail->millivolts, now);
222         }
223 }
224
225 void tegra_dvfs_rail_pause(struct dvfs_rail *rail, ktime_t delta, bool on)
226 {
227         if (rail)
228                 dvfs_rail_stats_pause(rail, delta, on);
229 }
230
231 static int dvfs_rail_set_voltage_reg(struct dvfs_rail *rail, int millivolts)
232 {
233         int ret;
234
235         /*
236          * safely return success for low voltage requests on fixed regulator
237          * (higher requests will go through and fail, as they should)
238          */
239         if (rail->fixed_millivolts && (millivolts <= rail->fixed_millivolts))
240                 return 0;
241
242         rail->updating = true;
243         rail->reg_max_millivolts = rail->reg_max_millivolts ==
244                 rail->max_millivolts ?
245                 rail->max_millivolts + 1 : rail->max_millivolts;
246         ret = regulator_set_voltage(rail->reg,
247                 millivolts * 1000,
248                 rail->reg_max_millivolts * 1000);
249         rail->updating = false;
250
251         return ret;
252 }
253
254 /* Sets the voltage on a dvfs rail to a specific value, and updates any
255  * rails that depend on this rail. */
256 static int dvfs_rail_set_voltage(struct dvfs_rail *rail, int millivolts)
257 {
258         int ret = 0;
259         struct dvfs_relationship *rel;
260         int step, offset;
261         int i;
262         int steps;
263         bool jmp_to_zero;
264
265         if (!rail->reg) {
266                 if (millivolts == rail->millivolts)
267                         return 0;
268                 else
269                         return -EINVAL;
270         }
271
272         if (millivolts > rail->millivolts) {
273                 step = rail->step_up;
274                 offset = step;
275         } else {
276                 step = rail->step;
277                 offset = -step;
278         }
279
280         /*
281          * DFLL adjusts rail voltage automatically, but not exactly to the
282          * expected level - update stats, anyway.
283          */
284         if (rail->dfll_mode) {
285                 rail->millivolts = rail->new_millivolts = millivolts;
286                 dvfs_rail_stats_update(rail, millivolts, ktime_get());
287                 return 0;
288         }
289
290         if (rail->disabled)
291                 return 0;
292
293         rail->resolving_to = true;
294         jmp_to_zero = rail->jmp_to_zero &&
295                         ((millivolts == 0) || (rail->millivolts == 0));
296         steps = jmp_to_zero ? 1 :
297                 DIV_ROUND_UP(abs(millivolts - rail->millivolts), step);
298
299         for (i = 0; i < steps; i++) {
300                 if (!jmp_to_zero &&
301                     (abs(millivolts - rail->millivolts) > step))
302                         rail->new_millivolts = rail->millivolts + offset;
303                 else
304                         rail->new_millivolts = millivolts;
305
306                 /* Before changing the voltage, tell each rail that depends
307                  * on this rail that the voltage will change.
308                  * This rail will be the "from" rail in the relationship,
309                  * the rail that depends on this rail will be the "to" rail.
310                  * from->millivolts will be the old voltage
311                  * from->new_millivolts will be the new voltage */
312                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
313                         ret = dvfs_rail_update(rel->to);
314                         if (ret)
315                                 goto out;
316                 }
317
318                 ret = dvfs_rail_set_voltage_reg(rail, rail->new_millivolts);
319                 if (ret) {
320                         pr_err("Failed to set dvfs regulator %s\n", rail->reg_id);
321                         goto out;
322                 }
323
324                 rail->millivolts = rail->new_millivolts;
325                 dvfs_rail_stats_update(rail, rail->millivolts, ktime_get());
326
327                 /* After changing the voltage, tell each rail that depends
328                  * on this rail that the voltage has changed.
329                  * from->millivolts and from->new_millivolts will be the
330                  * new voltage */
331                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
332                         ret = dvfs_rail_update(rel->to);
333                         if (ret)
334                                 goto out;
335                 }
336         }
337
338         if (unlikely(rail->millivolts != millivolts)) {
339                 pr_err("%s: rail didn't reach target %d in %d steps (%d)\n",
340                         __func__, millivolts, steps, rail->millivolts);
341                 ret = -EINVAL;
342         }
343
344 out:
345         rail->resolving_to = false;
346         return ret;
347 }
348
349 /* Determine the minimum valid voltage for a rail, taking into account
350  * the dvfs clocks and any rails that this rail depends on.  Calls
351  * dvfs_rail_set_voltage with the new voltage, which will call
352  * dvfs_rail_update on any rails that depend on this rail. */
353 static inline int dvfs_rail_apply_limits(struct dvfs_rail *rail, int millivolts)
354 {
355         int min_mv = rail->min_millivolts;
356
357         if (rail->therm_mv_floors) {
358                 int i = rail->therm_floor_idx;
359                 if (i < rail->therm_mv_floors_num)
360                         min_mv = rail->therm_mv_floors[i];
361         }
362
363         if (rail->override_millivolts) {
364                 millivolts = rail->override_millivolts;
365         } else {
366                 /* apply offset and clip up to pll mode fixed mv */
367                 millivolts += rail->dbg_mv_offs;
368                 if (!rail->dfll_mode && rail->fixed_millivolts &&
369                     (millivolts < rail->fixed_millivolts))
370                         millivolts = rail->fixed_millivolts;
371         }
372
373         if (millivolts < min_mv)
374                 millivolts = min_mv;
375
376         return millivolts;
377 }
378
379 static int dvfs_rail_update(struct dvfs_rail *rail)
380 {
381         int millivolts = 0;
382         struct dvfs *d;
383         struct dvfs_relationship *rel;
384         int ret = 0;
385         int steps;
386
387         /* if dvfs is suspended, return and handle it during resume */
388         if (rail->suspended)
389                 return 0;
390
391         /* if regulators are not connected yet, return and handle it later */
392         if (!rail->reg)
393                 return 0;
394
395         /* if rail update is entered while resolving circular dependencies,
396            abort recursion */
397         if (rail->resolving_to)
398                 return 0;
399
400         /* Find the maximum voltage requested by any clock */
401         list_for_each_entry(d, &rail->dvfs, reg_node)
402                 millivolts = max(d->cur_millivolts, millivolts);
403
404         /* Apply offset and min/max limits if any clock is requesting voltage */
405         if (millivolts)
406                 millivolts = dvfs_rail_apply_limits(rail, millivolts);
407
408         /* retry update if limited by from-relationship to account for
409            circular dependencies */
410         steps = DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
411         for (; steps >= 0; steps--) {
412                 rail->new_millivolts = millivolts;
413
414                 /* Check any rails that this rail depends on */
415                 list_for_each_entry(rel, &rail->relationships_from, from_node)
416                         rail->new_millivolts = dvfs_solve_relationship(rel);
417
418                 if (rail->new_millivolts == rail->millivolts)
419                         break;
420
421                 ret = dvfs_rail_set_voltage(rail, rail->new_millivolts);
422         }
423
424         return ret;
425 }
426
427 static struct regulator *get_fixed_regulator(struct dvfs_rail *rail)
428 {
429         struct regulator *reg;
430         char reg_id[80];
431         struct dvfs *d;
432         int v, i;
433         unsigned long dfll_boost;
434
435         strcpy(reg_id, rail->reg_id);
436         strcat(reg_id, "_fixed");
437         reg = regulator_get(NULL, reg_id);
438         if (IS_ERR(reg))
439                 return reg;
440
441         v = regulator_get_voltage(reg) / 1000;
442         if ((v < rail->min_millivolts) || (v > rail->nominal_millivolts) ||
443             (rail->therm_mv_floors && v < rail->therm_mv_floors[0])) {
444                 pr_err("tegra_dvfs: ivalid fixed %s voltage %d\n",
445                        rail->reg_id, v);
446                 return ERR_PTR(-EINVAL);
447         }
448
449         /*
450          * Only fixed at nominal voltage vdd_core regulator is allowed, same
451          * is true for cpu rail if dfll mode is not supported at all. No thermal
452          * capping can be implemented in this case.
453          */
454         if (!IS_ENABLED(CONFIG_ARCH_TEGRA_HAS_CL_DVFS) ||
455             (rail != tegra_cpu_rail)) {
456                 if (v != rail->nominal_millivolts) {
457                         pr_err("tegra_dvfs: %s fixed below nominal at %d\n",
458                                rail->reg_id, v);
459                         return ERR_PTR(-EINVAL);
460                 }
461                 if (rail->therm_mv_caps) {
462                         pr_err("tegra_dvfs: cannot fix %s with thermal caps\n",
463                                rail->reg_id);
464                         return ERR_PTR(-ENOSYS);
465                 }
466                 return reg;
467         }
468
469         /*
470          * If dfll mode is supported, fixed vdd_cpu regulator may be below
471          * nominal in pll mode - maximum cpu rate in pll mode is limited
472          * respectively. Regulator is required to allow automatic scaling
473          * in dfll mode.
474          *
475          * FIXME: platform data to explicitly identify such "hybrid" regulator?
476          */
477         d = list_first_entry(&rail->dvfs, struct dvfs, reg_node);
478         for (i = 0; i < d->num_freqs; i++) {
479                 if (d->millivolts[i] > v)
480                         break;
481         }
482
483         if (!i) {
484                 pr_err("tegra_dvfs: %s fixed at %d: too low for min rate\n",
485                        rail->reg_id, v);
486                 return ERR_PTR(-EINVAL);
487         }
488
489         dfll_boost = (d->freqs[d->num_freqs - 1] - d->freqs[i - 1]);
490         if (d->dfll_data.max_rate_boost < dfll_boost)
491                 d->dfll_data.max_rate_boost = dfll_boost;
492
493         rail->fixed_millivolts = v;
494         return reg;
495 }
496
497 static int dvfs_rail_connect_to_regulator(struct dvfs_rail *rail)
498 {
499         struct regulator *reg;
500         int v;
501
502         if (!rail->reg) {
503                 reg = regulator_get(NULL, rail->reg_id);
504                 if (IS_ERR(reg)) {
505                         reg = get_fixed_regulator(rail);
506                         if (IS_ERR(reg)) {
507                                 pr_err("tegra_dvfs: failed to connect %s rail\n",
508                                        rail->reg_id);
509                                 return PTR_ERR(reg);
510                         }
511                 }
512                 rail->reg = reg;
513         }
514
515         v = regulator_enable(rail->reg);
516         if (v < 0) {
517                 pr_err("tegra_dvfs: failed on enabling regulator %s\n, err %d",
518                         rail->reg_id, v);
519                 return v;
520         }
521
522         v = regulator_get_voltage(rail->reg);
523         if (v < 0) {
524                 pr_err("tegra_dvfs: failed initial get %s voltage\n",
525                        rail->reg_id);
526                 return v;
527         }
528         rail->millivolts = v / 1000;
529         rail->new_millivolts = rail->millivolts;
530         dvfs_rail_stats_init(rail, rail->millivolts);
531
532         if (rail->boot_millivolts &&
533             (rail->boot_millivolts != rail->millivolts)) {
534                 WARN(1, "%s boot voltage %d does not match expected %d\n",
535                      rail->reg_id, rail->millivolts, rail->boot_millivolts);
536                 rail->boot_millivolts = rail->millivolts;
537         }
538         return 0;
539 }
540
541 static inline unsigned long *dvfs_get_freqs(struct dvfs *d)
542 {
543         return d->alt_freqs ? : &d->freqs[0];
544 }
545
546 static inline const int *dvfs_get_millivolts(struct dvfs *d, unsigned long rate)
547 {
548         if (tegra_dvfs_is_dfll_scale(d, rate))
549                 return d->dfll_millivolts;
550
551         return d->millivolts;
552 }
553
554 static int
555 __tegra_dvfs_set_rate(struct dvfs *d, unsigned long rate)
556 {
557         int i = 0;
558         int ret, mv, detach_mv;
559         unsigned long *freqs = dvfs_get_freqs(d);
560         const int *millivolts = dvfs_get_millivolts(d, rate);
561
562         if (freqs == NULL || millivolts == NULL)
563                 return -ENODEV;
564
565         /* On entry to dfll range limit 1st step to range bottom (full ramp of
566            voltage/rate is completed automatically in dfll mode) */
567         if (tegra_dvfs_is_dfll_range_entry(d, rate))
568                 rate = d->dfll_data.use_dfll_rate_min;
569
570         if (rate > freqs[d->num_freqs - 1]) {
571                 pr_warn("tegra_dvfs: rate %lu too high for dvfs on %s\n", rate,
572                         d->clk_name);
573                 return -EINVAL;
574         }
575
576         if (rate == 0) {
577                 d->cur_millivolts = 0;
578         } else {
579                 while (i < d->num_freqs && rate > freqs[i])
580                         i++;
581
582                 if ((d->max_millivolts) &&
583                     (millivolts[i] > d->max_millivolts)) {
584                         pr_warn("tegra_dvfs: voltage %d too high for dvfs on"
585                                 " %s\n", millivolts[i], d->clk_name);
586                         return -EINVAL;
587                 }
588
589                 mv = millivolts[i];
590                 detach_mv = tegra_dvfs_rail_get_boot_level(d->dvfs_rail);
591                 if (!d->dvfs_rail->reg && (mv > detach_mv)) {
592                         pr_warn("%s: %s: voltage %d above boot limit %d\n",
593                                 __func__, d->clk_name, mv, detach_mv);
594                         return -EINVAL;
595                 }
596
597                 detach_mv = tegra_dvfs_rail_get_disable_level(d->dvfs_rail);
598                 if (d->dvfs_rail->disabled && (mv > detach_mv)) {
599                         pr_warn("%s: %s: voltage %d above disable limit %d\n",
600                                 __func__, d->clk_name, mv, detach_mv);
601                         return -EINVAL;
602                 }
603
604                 detach_mv = tegra_dvfs_rail_get_suspend_level(d->dvfs_rail);
605                 if (d->dvfs_rail->suspended && (mv > detach_mv)) {
606                         pr_warn("%s: %s: voltage %d above disable limit %d\n",
607                                 __func__, d->clk_name, mv, detach_mv);
608                         return -EINVAL;
609                 }
610                 d->cur_millivolts = millivolts[i];
611         }
612
613         d->cur_rate = rate;
614
615         ret = dvfs_rail_update(d->dvfs_rail);
616         if (ret)
617                 pr_err("Failed to set regulator %s for clock %s to %d mV\n",
618                         d->dvfs_rail->reg_id, d->clk_name, d->cur_millivolts);
619
620         return ret;
621 }
622
623 int tegra_dvfs_alt_freqs_set(struct dvfs *d, unsigned long *alt_freqs)
624 {
625         int ret = 0;
626
627         mutex_lock(&dvfs_lock);
628
629         if (d->alt_freqs != alt_freqs) {
630                 d->alt_freqs = alt_freqs;
631                 ret = __tegra_dvfs_set_rate(d, d->cur_rate);
632         }
633
634         mutex_unlock(&dvfs_lock);
635         return ret;
636 }
637
638 static int predict_millivolts(struct clk *c, const int *millivolts,
639                               unsigned long rate)
640 {
641         int i;
642
643         if (!millivolts)
644                 return -ENODEV;
645         /*
646          * Predicted voltage can not be used across the switch to alternative
647          * frequency limits. For now, just fail the call for clock that has
648          * alternative limits initialized.
649          */
650         if (c->dvfs->alt_freqs)
651                 return -ENOSYS;
652
653         for (i = 0; i < c->dvfs->num_freqs; i++) {
654                 if (rate <= c->dvfs->freqs[i])
655                         break;
656         }
657
658         if (i == c->dvfs->num_freqs)
659                 return -EINVAL;
660
661         return millivolts[i];
662 }
663
664 int tegra_dvfs_predict_millivolts(struct clk *c, unsigned long rate)
665 {
666         const int *millivolts;
667
668         if (!rate || !c->dvfs)
669                 return 0;
670
671         millivolts = dvfs_get_millivolts(c->dvfs, rate);
672         return predict_millivolts(c, millivolts, rate);
673 }
674
675 int tegra_dvfs_predict_millivolts_pll(struct clk *c, unsigned long rate)
676 {
677         const int *millivolts;
678
679         if (!rate || !c->dvfs)
680                 return 0;
681
682         millivolts = c->dvfs->millivolts;
683         return predict_millivolts(c, millivolts, rate);
684 }
685
686 int tegra_dvfs_predict_millivolts_dfll(struct clk *c, unsigned long rate)
687 {
688         const int *millivolts;
689
690         if (!rate || !c->dvfs)
691                 return 0;
692
693         millivolts = c->dvfs->dfll_millivolts;
694         return predict_millivolts(c, millivolts, rate);
695 }
696
697 int tegra_dvfs_set_rate(struct clk *c, unsigned long rate)
698 {
699         int ret;
700
701         if (!c->dvfs)
702                 return -EINVAL;
703
704         mutex_lock(&dvfs_lock);
705         ret = __tegra_dvfs_set_rate(c->dvfs, rate);
706         mutex_unlock(&dvfs_lock);
707
708         return ret;
709 }
710 EXPORT_SYMBOL(tegra_dvfs_set_rate);
711
712 int tegra_dvfs_get_freqs(struct clk *c, unsigned long **freqs, int *num_freqs)
713 {
714         if (!c->dvfs)
715                 return -ENOSYS;
716
717         if (c->dvfs->alt_freqs)
718                 return -ENOSYS;
719
720         *num_freqs = c->dvfs->num_freqs;
721         *freqs = c->dvfs->freqs;
722
723         return 0;
724 }
725 EXPORT_SYMBOL(tegra_dvfs_get_freqs);
726
727 #ifdef CONFIG_TEGRA_VDD_CORE_OVERRIDE
728 static DEFINE_MUTEX(rail_override_lock);
729
730 int tegra_dvfs_override_core_voltage(int override_mv)
731 {
732         int ret, floor, ceiling;
733         struct dvfs_rail *rail = tegra_core_rail;
734
735         if (!rail)
736                 return -ENOENT;
737
738         if (rail->fixed_millivolts)
739                 return -ENOSYS;
740
741         floor = rail->min_override_millivolts;
742         ceiling = rail->nominal_millivolts;
743         if (override_mv && ((override_mv < floor) || (override_mv > ceiling))) {
744                 pr_err("%s: override level %d outside the range [%d...%d]\n",
745                        __func__, override_mv, floor, ceiling);
746                 return -EINVAL;
747         }
748
749         mutex_lock(&rail_override_lock);
750
751         if (override_mv == rail->override_millivolts) {
752                 ret = 0;
753                 goto out;
754         }
755
756         if (override_mv) {
757                 ret = tegra_dvfs_core_cap_level_apply(override_mv);
758                 if (ret) {
759                         pr_err("%s: failed to set cap for override level %d\n",
760                                __func__, override_mv);
761                         goto out;
762                 }
763         }
764
765         mutex_lock(&dvfs_lock);
766         if (rail->disabled || rail->suspended) {
767                 pr_err("%s: cannot scale %s rail\n", __func__,
768                        rail->disabled ? "disabled" : "suspended");
769                 ret = -EPERM;
770                 if (!override_mv) {
771                         mutex_unlock(&dvfs_lock);
772                         goto out;
773                 }
774         } else {
775                 rail->override_millivolts = override_mv;
776                 ret = dvfs_rail_update(rail);
777                 if (ret) {
778                         pr_err("%s: failed to set override level %d\n",
779                                __func__, override_mv);
780                         rail->override_millivolts = 0;
781                         dvfs_rail_update(rail);
782                 }
783         }
784         mutex_unlock(&dvfs_lock);
785
786         if (!override_mv || ret)
787                 tegra_dvfs_core_cap_level_apply(0);
788 out:
789         mutex_unlock(&rail_override_lock);
790         return ret;
791 }
792 #else
793 int tegra_dvfs_override_core_voltage(int override_mv)
794 {
795         pr_err("%s: vdd core override is not supported\n", __func__);
796         return -ENOSYS;
797 }
798 #endif
799 EXPORT_SYMBOL(tegra_dvfs_override_core_voltage);
800
801 /* May only be called during clock init, does not take any locks on clock c. */
802 int __init tegra_enable_dvfs_on_clk(struct clk *c, struct dvfs *d)
803 {
804         int i;
805
806         if (c->dvfs) {
807                 pr_err("Error when enabling dvfs on %s for clock %s:\n",
808                         d->dvfs_rail->reg_id, c->name);
809                 pr_err("DVFS already enabled for %s\n",
810                         c->dvfs->dvfs_rail->reg_id);
811                 return -EINVAL;
812         }
813
814         for (i = 0; i < MAX_DVFS_FREQS; i++) {
815                 if (d->millivolts[i] == 0)
816                         break;
817
818                 d->freqs[i] *= d->freqs_mult;
819
820                 /* If final frequencies are 0, pad with previous frequency */
821                 if (d->freqs[i] == 0 && i > 1)
822                         d->freqs[i] = d->freqs[i - 1];
823         }
824         d->num_freqs = i;
825
826         if (d->auto_dvfs) {
827                 c->auto_dvfs = true;
828                 clk_set_cansleep(c);
829         }
830
831         c->dvfs = d;
832
833         /*
834          * Minimum core override level is determined as maximum voltage required
835          * for clocks outside shared buses (shared bus rates can be capped to
836          * safe levels when override limit is set)
837          */
838         if (i && c->ops && !c->ops->shared_bus_update &&
839             !(c->flags & PERIPH_ON_CBUS)) {
840                 int mv = tegra_dvfs_predict_millivolts(c, d->freqs[i-1]);
841                 if (d->dvfs_rail->min_override_millivolts < mv)
842                         d->dvfs_rail->min_override_millivolts = mv;
843         }
844
845         mutex_lock(&dvfs_lock);
846         list_add_tail(&d->reg_node, &d->dvfs_rail->dvfs);
847         mutex_unlock(&dvfs_lock);
848
849         return 0;
850 }
851
852 static bool tegra_dvfs_all_rails_suspended(void)
853 {
854         struct dvfs_rail *rail;
855         bool all_suspended = true;
856
857         list_for_each_entry(rail, &dvfs_rail_list, node)
858                 if (!rail->suspended && !rail->disabled)
859                         all_suspended = false;
860
861         return all_suspended;
862 }
863
864 static bool tegra_dvfs_from_rails_suspended_or_solved(struct dvfs_rail *to)
865 {
866         struct dvfs_relationship *rel;
867         bool all_suspended = true;
868
869         list_for_each_entry(rel, &to->relationships_from, from_node)
870                 if (!rel->from->suspended && !rel->from->disabled &&
871                         !rel->solved_at_nominal)
872                         all_suspended = false;
873
874         return all_suspended;
875 }
876
877 static int tegra_dvfs_suspend_one(void)
878 {
879         struct dvfs_rail *rail;
880         int ret, mv;
881
882         list_for_each_entry(rail, &dvfs_rail_list, node) {
883                 if (!rail->suspended && !rail->disabled &&
884                     tegra_dvfs_from_rails_suspended_or_solved(rail)) {
885                         /* Safe, as pll mode rate is capped to fixed level */
886                         if (!rail->dfll_mode && rail->fixed_millivolts) {
887                                 mv = rail->fixed_millivolts;
888                         } else {
889                                 mv = tegra_dvfs_rail_get_suspend_level(rail);
890                                 mv = dvfs_rail_apply_limits(rail, mv);
891                         }
892
893                         /* apply suspend limit only if it is above current mv */
894                         ret = -EPERM;
895                         if (mv >= rail->millivolts)
896                                 ret = dvfs_rail_set_voltage(rail, mv);
897                         if (ret) {
898                                 pr_err("tegra_dvfs: failed %s suspend at %d\n",
899                                        rail->reg_id, rail->millivolts);
900                                 return ret;
901                         }
902
903                         rail->suspended = true;
904                         return 0;
905                 }
906         }
907
908         return -EINVAL;
909 }
910
911 static void tegra_dvfs_resume(void)
912 {
913         struct dvfs_rail *rail;
914
915         mutex_lock(&dvfs_lock);
916
917         list_for_each_entry(rail, &dvfs_rail_list, node)
918                 rail->suspended = false;
919
920         list_for_each_entry(rail, &dvfs_rail_list, node)
921                 dvfs_rail_update(rail);
922
923         mutex_unlock(&dvfs_lock);
924 }
925
926 static int tegra_dvfs_suspend(void)
927 {
928         int ret = 0;
929
930         mutex_lock(&dvfs_lock);
931
932         while (!tegra_dvfs_all_rails_suspended()) {
933                 ret = tegra_dvfs_suspend_one();
934                 if (ret)
935                         break;
936         }
937
938         mutex_unlock(&dvfs_lock);
939
940         if (ret)
941                 tegra_dvfs_resume();
942
943         return ret;
944 }
945
946 static int tegra_dvfs_pm_suspend(struct notifier_block *nb,
947                                  unsigned long event, void *data)
948 {
949         if (event == PM_SUSPEND_PREPARE) {
950                 if (tegra_dvfs_suspend())
951                         return NOTIFY_STOP;
952                 pr_info("tegra_dvfs: suspended\n");
953         }
954         return NOTIFY_OK;
955 };
956
957 static int tegra_dvfs_pm_resume(struct notifier_block *nb,
958                                 unsigned long event, void *data)
959 {
960         if (event == PM_POST_SUSPEND) {
961                 tegra_dvfs_resume();
962                 pr_info("tegra_dvfs: resumed\n");
963         }
964         return NOTIFY_OK;
965 };
966
967 static struct notifier_block tegra_dvfs_suspend_nb = {
968         .notifier_call = tegra_dvfs_pm_suspend,
969         .priority = -1,
970 };
971
972 static struct notifier_block tegra_dvfs_resume_nb = {
973         .notifier_call = tegra_dvfs_pm_resume,
974         .priority = 1,
975 };
976
977 static int tegra_dvfs_reboot_notify(struct notifier_block *nb,
978                                 unsigned long event, void *data)
979 {
980         switch (event) {
981         case SYS_RESTART:
982         case SYS_HALT:
983         case SYS_POWER_OFF:
984                 tegra_dvfs_suspend();
985                 return NOTIFY_OK;
986         }
987         return NOTIFY_DONE;
988 }
989
990 static struct notifier_block tegra_dvfs_reboot_nb = {
991         .notifier_call = tegra_dvfs_reboot_notify,
992 };
993
994 /* must be called with dvfs lock held */
995 static void __tegra_dvfs_rail_disable(struct dvfs_rail *rail)
996 {
997         int ret = -EPERM;
998         int mv;
999
1000         /* don't set voltage in DFLL mode - won't work, but break stats */
1001         if (rail->dfll_mode) {
1002                 rail->disabled = true;
1003                 return;
1004         }
1005
1006         /* Safe, as pll mode rate is capped to fixed level */
1007         if (!rail->dfll_mode && rail->fixed_millivolts) {
1008                 mv = rail->fixed_millivolts;
1009         } else {
1010                 mv = tegra_dvfs_rail_get_disable_level(rail);
1011                 mv = dvfs_rail_apply_limits(rail, mv);
1012         }
1013
1014         /* apply detach mode limit provided it is above current volatge */
1015         if (mv >= rail->millivolts)
1016                 ret = dvfs_rail_set_voltage(rail, mv);
1017         if (ret) {
1018                 pr_err("tegra_dvfs: failed to disable %s at %d\n",
1019                        rail->reg_id, rail->millivolts);
1020                 return;
1021         }
1022         rail->disabled = true;
1023 }
1024
1025 /* must be called with dvfs lock held */
1026 static void __tegra_dvfs_rail_enable(struct dvfs_rail *rail)
1027 {
1028         rail->disabled = false;
1029         dvfs_rail_update(rail);
1030 }
1031
1032 void tegra_dvfs_rail_enable(struct dvfs_rail *rail)
1033 {
1034         if (!rail)
1035                 return;
1036
1037         mutex_lock(&rail_disable_lock);
1038
1039         if (rail->disabled) {
1040                 mutex_lock(&dvfs_lock);
1041                 __tegra_dvfs_rail_enable(rail);
1042                 mutex_unlock(&dvfs_lock);
1043
1044                 tegra_dvfs_rail_post_enable(rail);
1045         }
1046         mutex_unlock(&rail_disable_lock);
1047 }
1048
1049 void tegra_dvfs_rail_disable(struct dvfs_rail *rail)
1050 {
1051         if (!rail)
1052                 return;
1053
1054         mutex_lock(&rail_disable_lock);
1055         if (rail->disabled)
1056                 goto out;
1057
1058         /* rail disable will set it to nominal voltage underneath clock
1059            framework - need to re-configure clock rates that are not safe
1060            at nominal (yes, unsafe at nominal is ugly, but possible). Rate
1061            change must be done outside of dvfs lock. */
1062         if (tegra_dvfs_rail_disable_prepare(rail)) {
1063                 pr_info("dvfs: failed to prepare regulator %s to disable\n",
1064                         rail->reg_id);
1065                 goto out;
1066         }
1067
1068         mutex_lock(&dvfs_lock);
1069         __tegra_dvfs_rail_disable(rail);
1070         mutex_unlock(&dvfs_lock);
1071 out:
1072         mutex_unlock(&rail_disable_lock);
1073 }
1074
1075 int tegra_dvfs_rail_disable_by_name(const char *reg_id)
1076 {
1077         struct dvfs_rail *rail = tegra_dvfs_get_rail_by_name(reg_id);
1078         if (!rail)
1079                 return -EINVAL;
1080
1081         tegra_dvfs_rail_disable(rail);
1082         return 0;
1083 }
1084
1085 struct dvfs_rail *tegra_dvfs_get_rail_by_name(const char *reg_id)
1086 {
1087         struct dvfs_rail *rail;
1088
1089         mutex_lock(&dvfs_lock);
1090         list_for_each_entry(rail, &dvfs_rail_list, node) {
1091                 if (!strcmp(reg_id, rail->reg_id)) {
1092                         mutex_unlock(&dvfs_lock);
1093                         return rail;
1094                 }
1095         }
1096         mutex_unlock(&dvfs_lock);
1097         return NULL;
1098 }
1099
1100 int tegra_dvfs_rail_power_up(struct dvfs_rail *rail)
1101 {
1102         int ret = -ENOENT;
1103
1104         if (!rail || !rail->in_band_pm)
1105                 return -ENOSYS;
1106
1107         mutex_lock(&dvfs_lock);
1108         if (rail->reg) {
1109                 ret = regulator_enable(rail->reg);
1110                 if (!ret && !timekeeping_suspended)
1111                         tegra_dvfs_rail_on(rail, ktime_get());
1112         }
1113         mutex_unlock(&dvfs_lock);
1114         return ret;
1115 }
1116
1117 int tegra_dvfs_rail_power_down(struct dvfs_rail *rail)
1118 {
1119         int ret = -ENOENT;
1120
1121         if (!rail || !rail->in_band_pm)
1122                 return -ENOSYS;
1123
1124         mutex_lock(&dvfs_lock);
1125         if (rail->reg) {
1126                 ret = regulator_disable(rail->reg);
1127                 if (!ret && !timekeeping_suspended)
1128                         tegra_dvfs_rail_off(rail, ktime_get());
1129         }
1130         mutex_unlock(&dvfs_lock);
1131         return ret;
1132 }
1133
1134 bool tegra_dvfs_is_rail_up(struct dvfs_rail *rail)
1135 {
1136         bool ret = false;
1137
1138         if (!rail)
1139                 return false;
1140
1141         if (!rail->in_band_pm)
1142                 return true;
1143
1144         mutex_lock(&dvfs_lock);
1145         if (rail->reg)
1146                 ret = regulator_is_enabled(rail->reg) > 0;
1147         mutex_unlock(&dvfs_lock);
1148         return ret;
1149 }
1150
1151 bool tegra_dvfs_rail_updating(struct clk *clk)
1152 {
1153         return (!clk ? false :
1154                 (!clk->dvfs ? false :
1155                  (!clk->dvfs->dvfs_rail ? false :
1156                   (clk->dvfs->dvfs_rail->updating ||
1157                    clk->dvfs->dvfs_rail->dfll_mode_updating))));
1158 }
1159
1160 #ifdef CONFIG_OF
1161 int __init of_tegra_dvfs_init(const struct of_device_id *matches)
1162 {
1163         int ret;
1164         struct device_node *np;
1165
1166         for_each_matching_node(np, matches) {
1167                 const struct of_device_id *match = of_match_node(matches, np);
1168                 of_tegra_dvfs_init_cb_t dvfs_init_cb = match->data;
1169                 ret = dvfs_init_cb(np);
1170                 if (ret) {
1171                         pr_err("dt: Failed to read %s tables from DT\n",
1172                                                         match->compatible);
1173                         return ret;
1174                 }
1175         }
1176         return 0;
1177 }
1178 #endif
1179 int tegra_dvfs_dfll_mode_set(struct dvfs *d, unsigned long rate)
1180 {
1181         mutex_lock(&dvfs_lock);
1182         if (!d->dvfs_rail->dfll_mode) {
1183                 d->dvfs_rail->dfll_mode = true;
1184                 __tegra_dvfs_set_rate(d, rate);
1185         }
1186         mutex_unlock(&dvfs_lock);
1187         return 0;
1188 }
1189
1190 int tegra_dvfs_dfll_mode_clear(struct dvfs *d, unsigned long rate)
1191 {
1192         int ret = 0;
1193
1194         mutex_lock(&dvfs_lock);
1195         if (d->dvfs_rail->dfll_mode) {
1196                 d->dvfs_rail->dfll_mode = false;
1197                 /* avoid false detection of matching target (voltage in dfll
1198                    mode is fluctuating, and recorded level is just estimate) */
1199                 d->dvfs_rail->millivolts--;
1200                 if (d->dvfs_rail->disabled) {
1201                         d->dvfs_rail->disabled = false;
1202                         __tegra_dvfs_rail_disable(d->dvfs_rail);
1203                 }
1204                 ret = __tegra_dvfs_set_rate(d, rate);
1205         }
1206         mutex_unlock(&dvfs_lock);
1207         return ret;
1208 }
1209
1210 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmax_cdev(void)
1211 {
1212         if (tegra_cpu_rail)
1213                 return tegra_cpu_rail->vmax_cdev;
1214         return NULL;
1215 }
1216
1217 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmin_cdev(void)
1218 {
1219         if (tegra_cpu_rail)
1220                 return tegra_cpu_rail->vmin_cdev;
1221         return NULL;
1222 }
1223
1224 struct tegra_cooling_device *tegra_dvfs_get_core_vmin_cdev(void)
1225 {
1226         if (tegra_core_rail)
1227                 return tegra_core_rail->vmin_cdev;
1228         return NULL;
1229 }
1230
1231 struct tegra_cooling_device *tegra_dvfs_get_gpu_vmin_cdev(void)
1232 {
1233         if (tegra_gpu_rail)
1234                 return tegra_gpu_rail->vmin_cdev;
1235         return NULL;
1236 }
1237
1238 #ifdef CONFIG_THERMAL
1239 /* Cooling device limits minimum rail voltage at cold temperature in pll mode */
1240 static int tegra_dvfs_rail_get_vmin_cdev_max_state(
1241         struct thermal_cooling_device *cdev, unsigned long *max_state)
1242 {
1243         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1244         *max_state = rail->vmin_cdev->trip_temperatures_num;
1245         return 0;
1246 }
1247
1248 static int tegra_dvfs_rail_get_vmin_cdev_cur_state(
1249         struct thermal_cooling_device *cdev, unsigned long *cur_state)
1250 {
1251         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1252         *cur_state = rail->therm_floor_idx;
1253         return 0;
1254 }
1255
1256 static int tegra_dvfs_rail_set_vmin_cdev_state(
1257         struct thermal_cooling_device *cdev, unsigned long cur_state)
1258 {
1259         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1260
1261         mutex_lock(&dvfs_lock);
1262         if (rail->therm_floor_idx != cur_state) {
1263                 rail->therm_floor_idx = cur_state;
1264                 dvfs_rail_update(rail);
1265         }
1266         mutex_unlock(&dvfs_lock);
1267         return 0;
1268 }
1269
1270 static struct thermal_cooling_device_ops tegra_dvfs_rail_cooling_ops = {
1271         .get_max_state = tegra_dvfs_rail_get_vmin_cdev_max_state,
1272         .get_cur_state = tegra_dvfs_rail_get_vmin_cdev_cur_state,
1273         .set_cur_state = tegra_dvfs_rail_set_vmin_cdev_state,
1274 };
1275
1276 static void tegra_dvfs_rail_register_vmin_cdev(struct dvfs_rail *rail)
1277 {
1278         if (!rail->vmin_cdev)
1279                 return;
1280
1281         /* just report error - initialized for cold temperature, anyway */
1282         if (IS_ERR_OR_NULL(thermal_cooling_device_register(
1283                 rail->vmin_cdev->cdev_type, (void *)rail,
1284                 &tegra_dvfs_rail_cooling_ops)))
1285                 pr_err("tegra cooling device %s failed to register\n",
1286                        rail->vmin_cdev->cdev_type);
1287 }
1288
1289 #else
1290 #define tegra_dvfs_rail_register_vmin_cdev(rail)
1291 #endif
1292
1293 /*
1294  * Validate rail thermal profile, and get its size. Valid profile:
1295  * - voltage limits are descending with temperature increasing
1296  * - the lowest limit is above rail minimum voltage in pll and
1297  *   in dfll mode (if applicable)
1298  * - the highest limit is below rail nominal voltage
1299  */
1300 static int __init get_thermal_profile_size(
1301         int *trips_table, int *limits_table,
1302         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1303 {
1304         int i, min_mv;
1305
1306         for (i = 0; i < MAX_THERMAL_LIMITS - 1; i++) {
1307                 if (!limits_table[i+1])
1308                         break;
1309
1310                 if ((trips_table[i] >= trips_table[i+1]) ||
1311                     (limits_table[i] < limits_table[i+1])) {
1312                         pr_warn("%s: not ordered profile\n", rail->reg_id);
1313                         return -EINVAL;
1314                 }
1315         }
1316
1317         min_mv = max(rail->min_millivolts, d ? d->min_millivolts : 0);
1318         if (limits_table[i] < min_mv) {
1319                 pr_warn("%s: thermal profile below Vmin\n", rail->reg_id);
1320                 return -EINVAL;
1321         }
1322
1323         if (limits_table[0] > rail->nominal_millivolts) {
1324                 pr_warn("%s: thermal profile above Vmax\n", rail->reg_id);
1325                 return -EINVAL;
1326         }
1327         return i + 1;
1328 }
1329
1330 void __init tegra_dvfs_rail_init_vmax_thermal_profile(
1331         int *therm_trips_table, int *therm_caps_table,
1332         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1333 {
1334         int i = get_thermal_profile_size(therm_trips_table,
1335                                          therm_caps_table, rail, d);
1336         if (i <= 0) {
1337                 rail->vmax_cdev = NULL;
1338                 WARN(1, "%s: invalid Vmax thermal profile\n", rail->reg_id);
1339                 return;
1340         }
1341
1342         /* Install validated thermal caps */
1343         rail->therm_mv_caps = therm_caps_table;
1344         rail->therm_mv_caps_num = i;
1345
1346         /* Setup trip-points if applicable */
1347         if (rail->vmax_cdev) {
1348                 rail->vmax_cdev->trip_temperatures_num = i;
1349                 rail->vmax_cdev->trip_temperatures = therm_trips_table;
1350         }
1351 }
1352
1353 void __init tegra_dvfs_rail_init_vmin_thermal_profile(
1354         int *therm_trips_table, int *therm_floors_table,
1355         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1356 {
1357         int i = get_thermal_profile_size(therm_trips_table,
1358                                          therm_floors_table, rail, d);
1359         if (i <= 0) {
1360                 rail->vmin_cdev = NULL;
1361                 WARN(1, "%s: invalid Vmin thermal profile\n", rail->reg_id);
1362                 return;
1363         }
1364
1365         /* Install validated thermal floors */
1366         rail->therm_mv_floors = therm_floors_table;
1367         rail->therm_mv_floors_num = i;
1368
1369         /* Setup trip-points if applicable */
1370         if (rail->vmin_cdev) {
1371                 rail->vmin_cdev->trip_temperatures_num = i;
1372                 rail->vmin_cdev->trip_temperatures = therm_trips_table;
1373         }
1374 }
1375
1376 /* Directly set cold temperature limit in dfll mode */
1377 int tegra_dvfs_rail_dfll_mode_set_cold(struct dvfs_rail *rail)
1378 {
1379         int ret = 0;
1380
1381         /* No thermal floors - nothing to do */
1382         if (!rail || !rail->therm_mv_floors)
1383                 return ret;
1384
1385         /*
1386          * Since cooling thresholds are the same in pll and dfll modes, pll mode
1387          * thermal index can be used to decide if cold limit should be set in
1388          * dfll mode.
1389          */
1390         mutex_lock(&dvfs_lock);
1391         if (rail->dfll_mode &&
1392             (rail->therm_floor_idx < rail->therm_mv_floors_num)) {
1393                         int mv = rail->therm_mv_floors[rail->therm_floor_idx];
1394                         ret = dvfs_rail_set_voltage_reg(rail, mv);
1395         }
1396         mutex_unlock(&dvfs_lock);
1397
1398         return ret;
1399 }
1400
1401 /*
1402  * Iterate through all the dvfs regulators, finding the regulator exported
1403  * by the regulator api for each one.  Must be called in late init, after
1404  * all the regulator api's regulators are initialized.
1405  */
1406 int __init tegra_dvfs_late_init(void)
1407 {
1408         bool connected = true;
1409         struct dvfs_rail *rail;
1410
1411         mutex_lock(&dvfs_lock);
1412
1413         list_for_each_entry(rail, &dvfs_rail_list, node)
1414                 if (dvfs_rail_connect_to_regulator(rail))
1415                         connected = false;
1416
1417         list_for_each_entry(rail, &dvfs_rail_list, node)
1418                 if (connected)
1419                         dvfs_rail_update(rail);
1420                 else
1421                         __tegra_dvfs_rail_disable(rail);
1422
1423         mutex_unlock(&dvfs_lock);
1424
1425         if (!connected && tegra_platform_is_silicon()) {
1426                 pr_warn("tegra_dvfs: DVFS regulators connection failed\n"
1427                         "            !!!! voltage scaling is disabled !!!!\n");
1428                 return -ENODEV;
1429         }
1430
1431         register_pm_notifier(&tegra_dvfs_suspend_nb);
1432         register_pm_notifier(&tegra_dvfs_resume_nb);
1433         register_reboot_notifier(&tegra_dvfs_reboot_nb);
1434
1435         list_for_each_entry(rail, &dvfs_rail_list, node)
1436                 tegra_dvfs_rail_register_vmin_cdev(rail);
1437
1438         return 0;
1439 }
1440
1441 static int rail_stats_save_to_buf(char *buf, int len)
1442 {
1443         int i;
1444         struct dvfs_rail *rail;
1445         char *str = buf;
1446         char *end = buf + len;
1447
1448         str += scnprintf(str, end - str, "%-12s %-10s\n", "millivolts", "time");
1449
1450         mutex_lock(&dvfs_lock);
1451
1452         list_for_each_entry(rail, &dvfs_rail_list, node) {
1453                 str += scnprintf(str, end - str, "%s (bin: %d.%dmV)\n",
1454                            rail->reg_id,
1455                            rail->stats.bin_uV / 1000,
1456                            (rail->stats.bin_uV / 10) % 100);
1457
1458                 dvfs_rail_stats_update(rail, -1, ktime_get());
1459
1460                 str += scnprintf(str, end - str, "%-12d %-10llu\n", 0,
1461                         cputime64_to_clock_t(msecs_to_jiffies(
1462                                 ktime_to_ms(rail->stats.time_at_mv[0]))));
1463
1464                 for (i = 1; i <= DVFS_RAIL_STATS_TOP_BIN; i++) {
1465                         ktime_t ktime_zero = ktime_set(0, 0);
1466                         if (ktime_equal(rail->stats.time_at_mv[i], ktime_zero))
1467                                 continue;
1468                         str += scnprintf(str, end - str, "%-12d %-10llu\n",
1469                                 rail->min_millivolts +
1470                                 (i - 1) * rail->stats.bin_uV / 1000,
1471                                 cputime64_to_clock_t(msecs_to_jiffies(
1472                                         ktime_to_ms(rail->stats.time_at_mv[i])))
1473                         );
1474                 }
1475         }
1476         mutex_unlock(&dvfs_lock);
1477         return str - buf;
1478 }
1479
1480 #ifdef CONFIG_DEBUG_FS
1481 static int dvfs_tree_sort_cmp(void *p, struct list_head *a, struct list_head *b)
1482 {
1483         struct dvfs *da = list_entry(a, struct dvfs, reg_node);
1484         struct dvfs *db = list_entry(b, struct dvfs, reg_node);
1485         int ret;
1486
1487         ret = strcmp(da->dvfs_rail->reg_id, db->dvfs_rail->reg_id);
1488         if (ret != 0)
1489                 return ret;
1490
1491         if (da->cur_millivolts < db->cur_millivolts)
1492                 return 1;
1493         if (da->cur_millivolts > db->cur_millivolts)
1494                 return -1;
1495
1496         return strcmp(da->clk_name, db->clk_name);
1497 }
1498
1499 static int dvfs_tree_show(struct seq_file *s, void *data)
1500 {
1501         struct dvfs *d;
1502         struct dvfs_rail *rail;
1503         struct dvfs_relationship *rel;
1504
1505         seq_printf(s, "   clock      rate       mV\n");
1506         seq_printf(s, "--------------------------------\n");
1507
1508         mutex_lock(&dvfs_lock);
1509
1510         list_for_each_entry(rail, &dvfs_rail_list, node) {
1511                 int thermal_mv_floor = 0;
1512
1513                 seq_printf(s, "%s %d mV%s:\n", rail->reg_id, rail->millivolts,
1514                            rail->dfll_mode ? " dfll mode" :
1515                                 rail->disabled ? " disabled" : "");
1516                 list_for_each_entry(rel, &rail->relationships_from, from_node) {
1517                         seq_printf(s, "   %-10s %-7d mV %-4d mV\n",
1518                                 rel->from->reg_id, rel->from->millivolts,
1519                                 dvfs_solve_relationship(rel));
1520                 }
1521                 seq_printf(s, "   offset     %-7d mV\n", rail->dbg_mv_offs);
1522
1523                 if (rail->therm_mv_floors) {
1524                         int i = rail->therm_floor_idx;
1525                         if (i < rail->therm_mv_floors_num)
1526                                 thermal_mv_floor = rail->therm_mv_floors[i];
1527                 }
1528                 seq_printf(s, "   thermal    %-7d mV\n", thermal_mv_floor);
1529
1530                 if (rail == tegra_core_rail) {
1531                         seq_printf(s, "   override   %-7d mV [%-4d...%-4d]\n",
1532                                    rail->override_millivolts,
1533                                    rail->min_override_millivolts,
1534                                    rail->nominal_millivolts);
1535                 }
1536
1537                 list_sort(NULL, &rail->dvfs, dvfs_tree_sort_cmp);
1538
1539                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1540                         seq_printf(s, "   %-10s %-10lu %-4d mV\n", d->clk_name,
1541                                 d->cur_rate, d->cur_millivolts);
1542                 }
1543         }
1544
1545         mutex_unlock(&dvfs_lock);
1546
1547         return 0;
1548 }
1549
1550 static int dvfs_tree_open(struct inode *inode, struct file *file)
1551 {
1552         return single_open(file, dvfs_tree_show, inode->i_private);
1553 }
1554
1555 static const struct file_operations dvfs_tree_fops = {
1556         .open           = dvfs_tree_open,
1557         .read           = seq_read,
1558         .llseek         = seq_lseek,
1559         .release        = single_release,
1560 };
1561
1562 static int rail_stats_show(struct seq_file *s, void *data)
1563 {
1564         char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
1565         int size = 0;
1566
1567         if (!buf)
1568                 return -ENOMEM;
1569
1570         size = rail_stats_save_to_buf(buf, PAGE_SIZE);
1571         seq_write(s, buf, size);
1572         kfree(buf);
1573         return 0;
1574 }
1575
1576 static int rail_stats_open(struct inode *inode, struct file *file)
1577 {
1578         return single_open(file, rail_stats_show, inode->i_private);
1579 }
1580
1581 static const struct file_operations rail_stats_fops = {
1582         .open           = rail_stats_open,
1583         .read           = seq_read,
1584         .llseek         = seq_lseek,
1585         .release        = single_release,
1586 };
1587
1588 static int gpu_dvfs_show(struct seq_file *s, void *data)
1589 {
1590         int idx;
1591         int *millivolts;
1592         unsigned long *freqs;
1593
1594         if (read_gpu_dvfs_table(&millivolts, &freqs)) {
1595                 seq_printf(s, "Only supported for T124 or higher\n");
1596                 return 0;
1597         }
1598
1599         seq_printf(s, "millivolts \t \t frequency\n");
1600         seq_printf(s, "=====================================\n");
1601
1602         for (idx = 0; millivolts[idx]; idx++)
1603                 seq_printf(s, "%d mV \t \t %lu Hz\n", millivolts[idx],
1604                                 freqs[idx]);
1605
1606         return 0;
1607 }
1608
1609 static int gpu_dvfs_open(struct inode *inode, struct file *file)
1610 {
1611         return single_open(file, gpu_dvfs_show, NULL);
1612 }
1613
1614 static const struct file_operations gpu_dvfs_fops = {
1615         .open           = gpu_dvfs_open,
1616         .read           = seq_read,
1617         .llseek         = seq_lseek,
1618         .release        = single_release,
1619 };
1620
1621 static int rail_offs_set(struct dvfs_rail *rail, int offs)
1622 {
1623         if (rail) {
1624                 mutex_lock(&dvfs_lock);
1625                 rail->dbg_mv_offs = offs;
1626                 dvfs_rail_update(rail);
1627                 mutex_unlock(&dvfs_lock);
1628                 return 0;
1629         }
1630         return -ENOENT;
1631 }
1632
1633 static int cpu_offs_get(void *data, u64 *val)
1634 {
1635         if (tegra_cpu_rail) {
1636                 *val = (u64)tegra_cpu_rail->dbg_mv_offs;
1637                 return 0;
1638         }
1639         *val = 0;
1640         return -ENOENT;
1641 }
1642 static int cpu_offs_set(void *data, u64 val)
1643 {
1644         return rail_offs_set(tegra_cpu_rail, (int)val);
1645 }
1646 DEFINE_SIMPLE_ATTRIBUTE(cpu_offs_fops, cpu_offs_get, cpu_offs_set, "%lld\n");
1647
1648 static int gpu_offs_get(void *data, u64 *val)
1649 {
1650         if (tegra_gpu_rail) {
1651                 *val = (u64)tegra_gpu_rail->dbg_mv_offs;
1652                 return 0;
1653         }
1654         *val = 0;
1655         return -ENOENT;
1656 }
1657 static int gpu_offs_set(void *data, u64 val)
1658 {
1659         return rail_offs_set(tegra_gpu_rail, (int)val);
1660 }
1661 DEFINE_SIMPLE_ATTRIBUTE(gpu_offs_fops, gpu_offs_get, gpu_offs_set, "%lld\n");
1662
1663 static int core_offs_get(void *data, u64 *val)
1664 {
1665         if (tegra_core_rail) {
1666                 *val = (u64)tegra_core_rail->dbg_mv_offs;
1667                 return 0;
1668         }
1669         *val = 0;
1670         return -ENOENT;
1671 }
1672 static int core_offs_set(void *data, u64 val)
1673 {
1674         return rail_offs_set(tegra_core_rail, (int)val);
1675 }
1676 DEFINE_SIMPLE_ATTRIBUTE(core_offs_fops, core_offs_get, core_offs_set, "%lld\n");
1677
1678 static int core_override_get(void *data, u64 *val)
1679 {
1680         if (tegra_core_rail) {
1681                 *val = (u64)tegra_core_rail->override_millivolts;
1682                 return 0;
1683         }
1684         *val = 0;
1685         return -ENOENT;
1686 }
1687 static int core_override_set(void *data, u64 val)
1688 {
1689         return tegra_dvfs_override_core_voltage((int)val);
1690 }
1691 DEFINE_SIMPLE_ATTRIBUTE(core_override_fops,
1692                         core_override_get, core_override_set, "%llu\n");
1693
1694 static int dvfs_table_show(struct seq_file *s, void *data)
1695 {
1696         int i;
1697         struct dvfs *d;
1698         struct dvfs_rail *rail;
1699
1700         seq_printf(s, "DVFS tables: units mV/MHz\n\n");
1701
1702         mutex_lock(&dvfs_lock);
1703
1704         list_for_each_entry(rail, &dvfs_rail_list, node) {
1705                 bool mv_done = false;
1706                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1707                         if (!mv_done) {
1708                                 mv_done = true;
1709                                 seq_printf(s, "%-16s", rail->reg_id);
1710                                 for (i = 0; i < d->num_freqs; i++) {
1711                                         int mv = d->millivolts[i];
1712                                         seq_printf(s, "%7d", mv);
1713                                 }
1714                                 seq_printf(s, "\n");
1715                                 if (d->dfll_millivolts) {
1716                                         seq_printf(s, "%-8s (dfll) ",
1717                                                    rail->reg_id);
1718                                         for (i = 0; i < d->num_freqs; i++) {
1719                                                 int mv = d->dfll_millivolts[i];
1720                                                 seq_printf(s, "%7d", mv);
1721                                         }
1722                                         seq_printf(s, "\n");
1723                                 }
1724                         }
1725
1726                         seq_printf(s, "%-16s", d->clk_name);
1727                         for (i = 0; i < d->num_freqs; i++) {
1728                                 unsigned int f = d->freqs[i]/100000;
1729                                 seq_printf(s, " %4u.%u", f/10, f%10);
1730                         }
1731                         seq_printf(s, "\n");
1732                 }
1733                 seq_printf(s, "\n");
1734         }
1735
1736         mutex_unlock(&dvfs_lock);
1737
1738         return 0;
1739 }
1740
1741 static int dvfs_table_open(struct inode *inode, struct file *file)
1742 {
1743         return single_open(file, dvfs_table_show, inode->i_private);
1744 }
1745
1746 static const struct file_operations dvfs_table_fops = {
1747         .open           = dvfs_table_open,
1748         .read           = seq_read,
1749         .llseek         = seq_lseek,
1750         .release        = single_release,
1751 };
1752
1753 int __init dvfs_debugfs_init(struct dentry *clk_debugfs_root)
1754 {
1755         struct dentry *d;
1756
1757         d = debugfs_create_file("dvfs", S_IRUGO, clk_debugfs_root, NULL,
1758                 &dvfs_tree_fops);
1759         if (!d)
1760                 return -ENOMEM;
1761
1762         d = debugfs_create_file("rails", S_IRUGO, clk_debugfs_root, NULL,
1763                 &rail_stats_fops);
1764         if (!d)
1765                 return -ENOMEM;
1766
1767         d = debugfs_create_file("vdd_cpu_offs", S_IRUGO | S_IWUSR,
1768                 clk_debugfs_root, NULL, &cpu_offs_fops);
1769         if (!d)
1770                 return -ENOMEM;
1771
1772         d = debugfs_create_file("vdd_gpu_offs", S_IRUGO | S_IWUSR,
1773                 clk_debugfs_root, NULL, &gpu_offs_fops);
1774         if (!d)
1775                 return -ENOMEM;
1776
1777         d = debugfs_create_file("vdd_core_offs", S_IRUGO | S_IWUSR,
1778                 clk_debugfs_root, NULL, &core_offs_fops);
1779         if (!d)
1780                 return -ENOMEM;
1781
1782         d = debugfs_create_file("vdd_core_override", S_IRUGO | S_IWUSR,
1783                 clk_debugfs_root, NULL, &core_override_fops);
1784         if (!d)
1785                 return -ENOMEM;
1786
1787         d = debugfs_create_file("gpu_dvfs", S_IRUGO | S_IWUSR,
1788                 clk_debugfs_root, NULL, &gpu_dvfs_fops);
1789         if (!d)
1790                 return -ENOMEM;
1791
1792         d = debugfs_create_file("dvfs_table", S_IRUGO, clk_debugfs_root, NULL,
1793                 &dvfs_table_fops);
1794         if (!d)
1795                 return -ENOMEM;
1796
1797         return 0;
1798 }
1799
1800 #endif
1801
1802 #ifdef CONFIG_PM
1803 static ssize_t tegra_rail_stats_show(struct kobject *kobj,
1804                                         struct kobj_attribute *attr,
1805                                         char *buf)
1806 {
1807         return rail_stats_save_to_buf(buf, PAGE_SIZE);
1808 }
1809
1810 static struct kobj_attribute rail_stats_attr =
1811                 __ATTR_RO(tegra_rail_stats);
1812
1813 static int __init tegra_dvfs_sysfs_stats_init(void)
1814 {
1815         int error;
1816         error = sysfs_create_file(power_kobj, &rail_stats_attr.attr);
1817         return 0;
1818 }
1819 late_initcall(tegra_dvfs_sysfs_stats_init);
1820 #endif