ARM: tegra: dvfs: Defer 1st rail update
[linux-3.10.git] / arch / arm / mach-tegra / dvfs.c
1 /*
2  *
3  * Copyright (C) 2010 Google, Inc.
4  *
5  * Author:
6  *      Colin Cross <ccross@google.com>
7  *
8  * Copyright (C) 2010-2013 NVIDIA CORPORATION. All rights reserved.
9  *
10  * This software is licensed under the terms of the GNU General Public
11  * License version 2, as published by the Free Software Foundation, and
12  * may be copied, distributed, and modified under those terms.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/clk.h>
23 #include <linux/clkdev.h>
24 #include <linux/debugfs.h>
25 #include <linux/init.h>
26 #include <linux/list.h>
27 #include <linux/list_sort.h>
28 #include <linux/module.h>
29 #include <linux/regulator/consumer.h>
30 #include <linux/seq_file.h>
31 #include <linux/slab.h>
32 #include <linux/suspend.h>
33 #include <linux/delay.h>
34 #include <linux/clk/tegra.h>
35 #include <linux/reboot.h>
36 #include <linux/clk/tegra.h>
37 #include <linux/tegra-soc.h>
38
39 #include "board.h"
40 #include "clock.h"
41 #include "dvfs.h"
42
43 #define DVFS_RAIL_STATS_BIN     12500
44
45 struct dvfs_rail *tegra_cpu_rail;
46 struct dvfs_rail *tegra_core_rail;
47 struct dvfs_rail *tegra_gpu_rail;
48
49 static LIST_HEAD(dvfs_rail_list);
50 static DEFINE_MUTEX(dvfs_lock);
51 static DEFINE_MUTEX(rail_disable_lock);
52
53 static int dvfs_rail_update(struct dvfs_rail *rail);
54
55 static inline int tegra_dvfs_rail_get_disable_level(struct dvfs_rail *rail)
56 {
57         return rail->disable_millivolts ? : rail->nominal_millivolts;
58 }
59
60 static inline int tegra_dvfs_rail_get_suspend_level(struct dvfs_rail *rail)
61 {
62         return rail->suspend_millivolts ? : rail->nominal_millivolts;
63 }
64
65 void tegra_dvfs_add_relationships(struct dvfs_relationship *rels, int n)
66 {
67         int i;
68         struct dvfs_relationship *rel;
69
70         mutex_lock(&dvfs_lock);
71
72         for (i = 0; i < n; i++) {
73                 rel = &rels[i];
74                 list_add_tail(&rel->from_node, &rel->to->relationships_from);
75                 list_add_tail(&rel->to_node, &rel->from->relationships_to);
76         }
77
78         mutex_unlock(&dvfs_lock);
79 }
80
81 /* Make sure there is a matching cooling device for thermal limit profile. */
82 static void dvfs_validate_cdevs(struct dvfs_rail *rail)
83 {
84         if (!rail->therm_mv_caps != !rail->therm_mv_caps_num) {
85                 rail->therm_mv_caps_num = 0;
86                 rail->therm_mv_caps = NULL;
87                 WARN(1, "%s: not matching thermal caps/num\n", rail->reg_id);
88         }
89
90         if (rail->therm_mv_caps && !rail->vmax_cdev)
91                 WARN(1, "%s: missing vmax cooling device\n", rail->reg_id);
92
93         if (!rail->therm_mv_floors != !rail->therm_mv_floors_num) {
94                 rail->therm_mv_floors_num = 0;
95                 rail->therm_mv_floors = NULL;
96                 WARN(1, "%s: not matching thermal floors/num\n", rail->reg_id);
97         }
98
99         if (rail->therm_mv_floors && !rail->vmin_cdev)
100                 WARN(1, "%s: missing vmin cooling device\n", rail->reg_id);
101
102         /* Limit override range to maximum floor */
103         if (rail->therm_mv_floors)
104                 rail->min_override_millivolts = rail->therm_mv_floors[0];
105 }
106
107 int tegra_dvfs_init_rails(struct dvfs_rail *rails[], int n)
108 {
109         int i, mv;
110
111         mutex_lock(&dvfs_lock);
112
113         for (i = 0; i < n; i++) {
114                 INIT_LIST_HEAD(&rails[i]->dvfs);
115                 INIT_LIST_HEAD(&rails[i]->relationships_from);
116                 INIT_LIST_HEAD(&rails[i]->relationships_to);
117
118                 mv = rails[i]->nominal_millivolts;
119                 if (rails[i]->boot_millivolts > mv)
120                         WARN(1, "%s: boot voltage %d above nominal %d\n",
121                              rails[i]->reg_id, rails[i]->boot_millivolts, mv);
122                 if (rails[i]->disable_millivolts > mv)
123                         rails[i]->disable_millivolts = mv;
124                 if (rails[i]->suspend_millivolts > mv)
125                         rails[i]->suspend_millivolts = mv;
126
127                 mv = tegra_dvfs_rail_get_boot_level(rails[i]);
128                 rails[i]->millivolts = mv;
129                 rails[i]->new_millivolts = mv;
130                 if (!rails[i]->step)
131                         rails[i]->step = rails[i]->max_millivolts;
132                 if (!rails[i]->step_up)
133                         rails[i]->step_up = rails[i]->step;
134
135                 list_add_tail(&rails[i]->node, &dvfs_rail_list);
136
137                 dvfs_validate_cdevs(rails[i]);
138
139                 if (!strcmp("vdd_cpu", rails[i]->reg_id))
140                         tegra_cpu_rail = rails[i];
141                 else if (!strcmp("vdd_gpu", rails[i]->reg_id))
142                         tegra_gpu_rail = rails[i];
143                 else if (!strcmp("vdd_core", rails[i]->reg_id))
144                         tegra_core_rail = rails[i];
145         }
146
147         mutex_unlock(&dvfs_lock);
148
149         return 0;
150 };
151
152 static int dvfs_solve_relationship(struct dvfs_relationship *rel)
153 {
154         return rel->solve(rel->from, rel->to);
155 }
156
157 /* rail statistic - called during rail init, or under dfs_lock, or with
158    CPU0 only on-line, and interrupts disabled */
159 static void dvfs_rail_stats_init(struct dvfs_rail *rail, int millivolts)
160 {
161         int dvfs_rail_stats_range;
162
163         if (!rail->stats.bin_uV)
164                 rail->stats.bin_uV = DVFS_RAIL_STATS_BIN;
165
166         dvfs_rail_stats_range =
167                 (DVFS_RAIL_STATS_TOP_BIN - 1) * rail->stats.bin_uV / 1000;
168
169         rail->stats.last_update = ktime_get();
170         if (millivolts >= rail->min_millivolts) {
171                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
172                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
173                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
174         }
175
176         if (rail->max_millivolts >
177             rail->min_millivolts + dvfs_rail_stats_range)
178                 pr_warn("tegra_dvfs: %s: stats above %d mV will be squashed\n",
179                         rail->reg_id,
180                         rail->min_millivolts + dvfs_rail_stats_range);
181 }
182
183 static void dvfs_rail_stats_update(
184         struct dvfs_rail *rail, int millivolts, ktime_t now)
185 {
186         rail->stats.time_at_mv[rail->stats.last_index] = ktime_add(
187                 rail->stats.time_at_mv[rail->stats.last_index], ktime_sub(
188                         now, rail->stats.last_update));
189         rail->stats.last_update = now;
190
191         if (rail->stats.off)
192                 return;
193
194         if (millivolts >= rail->min_millivolts) {
195                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
196                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
197                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
198         } else if (millivolts == 0)
199                         rail->stats.last_index = 0;
200 }
201
202 static void dvfs_rail_stats_pause(struct dvfs_rail *rail,
203                                   ktime_t delta, bool on)
204 {
205         int i = on ? rail->stats.last_index : 0;
206         rail->stats.time_at_mv[i] = ktime_add(rail->stats.time_at_mv[i], delta);
207 }
208
209 void tegra_dvfs_rail_off(struct dvfs_rail *rail, ktime_t now)
210 {
211         if (rail) {
212                 dvfs_rail_stats_update(rail, 0, now);
213                 rail->stats.off = true;
214         }
215 }
216
217 void tegra_dvfs_rail_on(struct dvfs_rail *rail, ktime_t now)
218 {
219         if (rail) {
220                 rail->stats.off = false;
221                 dvfs_rail_stats_update(rail, rail->millivolts, now);
222         }
223 }
224
225 void tegra_dvfs_rail_pause(struct dvfs_rail *rail, ktime_t delta, bool on)
226 {
227         if (rail)
228                 dvfs_rail_stats_pause(rail, delta, on);
229 }
230
231 static int dvfs_rail_set_voltage_reg(struct dvfs_rail *rail, int millivolts)
232 {
233         int ret;
234
235         /*
236          * safely return success for low voltage requests on fixed regulator
237          * (higher requests will go through and fail, as they should)
238          */
239         if (rail->fixed_millivolts && (millivolts <= rail->fixed_millivolts))
240                 return 0;
241
242         rail->updating = true;
243         rail->reg_max_millivolts = rail->reg_max_millivolts ==
244                 rail->max_millivolts ?
245                 rail->max_millivolts + 1 : rail->max_millivolts;
246         ret = regulator_set_voltage(rail->reg,
247                 millivolts * 1000,
248                 rail->reg_max_millivolts * 1000);
249         rail->updating = false;
250
251         return ret;
252 }
253
254 /* Sets the voltage on a dvfs rail to a specific value, and updates any
255  * rails that depend on this rail. */
256 static int dvfs_rail_set_voltage(struct dvfs_rail *rail, int millivolts)
257 {
258         int ret = 0;
259         struct dvfs_relationship *rel;
260         int step, offset;
261         int i;
262         int steps;
263         bool jmp_to_zero;
264
265         if (!rail->reg) {
266                 if (millivolts == rail->millivolts)
267                         return 0;
268                 else
269                         return -EINVAL;
270         }
271
272         if (millivolts > rail->millivolts) {
273                 step = rail->step_up;
274                 offset = step;
275         } else {
276                 step = rail->step;
277                 offset = -step;
278         }
279
280         /*
281          * DFLL adjusts rail voltage automatically, but not exactly to the
282          * expected level - update stats, anyway.
283          */
284         if (rail->dfll_mode) {
285                 rail->millivolts = rail->new_millivolts = millivolts;
286                 dvfs_rail_stats_update(rail, millivolts, ktime_get());
287                 return 0;
288         }
289
290         if (rail->disabled)
291                 return 0;
292
293         rail->resolving_to = true;
294         jmp_to_zero = rail->jmp_to_zero &&
295                         ((millivolts == 0) || (rail->millivolts == 0));
296         steps = jmp_to_zero ? 1 :
297                 DIV_ROUND_UP(abs(millivolts - rail->millivolts), step);
298
299         for (i = 0; i < steps; i++) {
300                 if (!jmp_to_zero &&
301                     (abs(millivolts - rail->millivolts) > step))
302                         rail->new_millivolts = rail->millivolts + offset;
303                 else
304                         rail->new_millivolts = millivolts;
305
306                 /* Before changing the voltage, tell each rail that depends
307                  * on this rail that the voltage will change.
308                  * This rail will be the "from" rail in the relationship,
309                  * the rail that depends on this rail will be the "to" rail.
310                  * from->millivolts will be the old voltage
311                  * from->new_millivolts will be the new voltage */
312                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
313                         ret = dvfs_rail_update(rel->to);
314                         if (ret)
315                                 goto out;
316                 }
317
318                 ret = dvfs_rail_set_voltage_reg(rail, rail->new_millivolts);
319                 if (ret) {
320                         pr_err("Failed to set dvfs regulator %s\n", rail->reg_id);
321                         goto out;
322                 }
323
324                 rail->millivolts = rail->new_millivolts;
325                 dvfs_rail_stats_update(rail, rail->millivolts, ktime_get());
326
327                 /* After changing the voltage, tell each rail that depends
328                  * on this rail that the voltage has changed.
329                  * from->millivolts and from->new_millivolts will be the
330                  * new voltage */
331                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
332                         ret = dvfs_rail_update(rel->to);
333                         if (ret)
334                                 goto out;
335                 }
336         }
337
338         if (unlikely(rail->millivolts != millivolts)) {
339                 pr_err("%s: rail didn't reach target %d in %d steps (%d)\n",
340                         __func__, millivolts, steps, rail->millivolts);
341                 ret = -EINVAL;
342         }
343
344 out:
345         rail->resolving_to = false;
346         return ret;
347 }
348
349 /* Determine the minimum valid voltage for a rail, taking into account
350  * the dvfs clocks and any rails that this rail depends on.  Calls
351  * dvfs_rail_set_voltage with the new voltage, which will call
352  * dvfs_rail_update on any rails that depend on this rail. */
353 static inline int dvfs_rail_apply_limits(struct dvfs_rail *rail, int millivolts)
354 {
355         int min_mv = rail->min_millivolts;
356
357         if (rail->therm_mv_floors) {
358                 int i = rail->therm_floor_idx;
359                 if (i < rail->therm_mv_floors_num)
360                         min_mv = rail->therm_mv_floors[i];
361         }
362
363         if (rail->override_millivolts) {
364                 millivolts = rail->override_millivolts;
365         } else {
366                 /* apply offset and clip up to pll mode fixed mv */
367                 millivolts += rail->dbg_mv_offs;
368                 if (!rail->dfll_mode && rail->fixed_millivolts &&
369                     (millivolts < rail->fixed_millivolts))
370                         millivolts = rail->fixed_millivolts;
371         }
372
373         if (millivolts < min_mv)
374                 millivolts = min_mv;
375
376         return millivolts;
377 }
378
379 static int dvfs_rail_update(struct dvfs_rail *rail)
380 {
381         int millivolts = 0;
382         struct dvfs *d;
383         struct dvfs_relationship *rel;
384         int ret = 0;
385         int steps;
386
387         /* if dvfs is suspended, return and handle it during resume */
388         if (rail->suspended)
389                 return 0;
390
391         /* if regulators are not connected yet, return and handle it later */
392         if (!rail->reg)
393                 return 0;
394
395         /* if no clock has requested voltage since boot, defer update */
396         if (!rail->rate_set)
397                 return 0;
398
399         /* if rail update is entered while resolving circular dependencies,
400            abort recursion */
401         if (rail->resolving_to)
402                 return 0;
403
404         /* Find the maximum voltage requested by any clock */
405         list_for_each_entry(d, &rail->dvfs, reg_node)
406                 millivolts = max(d->cur_millivolts, millivolts);
407
408         /* Apply offset and min/max limits if any clock is requesting voltage */
409         if (millivolts)
410                 millivolts = dvfs_rail_apply_limits(rail, millivolts);
411
412         /* retry update if limited by from-relationship to account for
413            circular dependencies */
414         steps = DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
415         for (; steps >= 0; steps--) {
416                 rail->new_millivolts = millivolts;
417
418                 /* Check any rails that this rail depends on */
419                 list_for_each_entry(rel, &rail->relationships_from, from_node)
420                         rail->new_millivolts = dvfs_solve_relationship(rel);
421
422                 if (rail->new_millivolts == rail->millivolts)
423                         break;
424
425                 ret = dvfs_rail_set_voltage(rail, rail->new_millivolts);
426         }
427
428         return ret;
429 }
430
431 static struct regulator *get_fixed_regulator(struct dvfs_rail *rail)
432 {
433         struct regulator *reg;
434         char reg_id[80];
435         struct dvfs *d;
436         int v, i;
437         unsigned long dfll_boost;
438
439         strcpy(reg_id, rail->reg_id);
440         strcat(reg_id, "_fixed");
441         reg = regulator_get(NULL, reg_id);
442         if (IS_ERR(reg))
443                 return reg;
444
445         v = regulator_get_voltage(reg) / 1000;
446         if ((v < rail->min_millivolts) || (v > rail->nominal_millivolts) ||
447             (rail->therm_mv_floors && v < rail->therm_mv_floors[0])) {
448                 pr_err("tegra_dvfs: ivalid fixed %s voltage %d\n",
449                        rail->reg_id, v);
450                 return ERR_PTR(-EINVAL);
451         }
452
453         /*
454          * Only fixed at nominal voltage vdd_core regulator is allowed, same
455          * is true for cpu rail if dfll mode is not supported at all. No thermal
456          * capping can be implemented in this case.
457          */
458         if (!IS_ENABLED(CONFIG_ARCH_TEGRA_HAS_CL_DVFS) ||
459             (rail != tegra_cpu_rail)) {
460                 if (v != rail->nominal_millivolts) {
461                         pr_err("tegra_dvfs: %s fixed below nominal at %d\n",
462                                rail->reg_id, v);
463                         return ERR_PTR(-EINVAL);
464                 }
465                 if (rail->therm_mv_caps) {
466                         pr_err("tegra_dvfs: cannot fix %s with thermal caps\n",
467                                rail->reg_id);
468                         return ERR_PTR(-ENOSYS);
469                 }
470                 return reg;
471         }
472
473         /*
474          * If dfll mode is supported, fixed vdd_cpu regulator may be below
475          * nominal in pll mode - maximum cpu rate in pll mode is limited
476          * respectively. Regulator is required to allow automatic scaling
477          * in dfll mode.
478          *
479          * FIXME: platform data to explicitly identify such "hybrid" regulator?
480          */
481         d = list_first_entry(&rail->dvfs, struct dvfs, reg_node);
482         for (i = 0; i < d->num_freqs; i++) {
483                 if (d->millivolts[i] > v)
484                         break;
485         }
486
487         if (!i) {
488                 pr_err("tegra_dvfs: %s fixed at %d: too low for min rate\n",
489                        rail->reg_id, v);
490                 return ERR_PTR(-EINVAL);
491         }
492
493         dfll_boost = (d->freqs[d->num_freqs - 1] - d->freqs[i - 1]);
494         if (d->dfll_data.max_rate_boost < dfll_boost)
495                 d->dfll_data.max_rate_boost = dfll_boost;
496
497         rail->fixed_millivolts = v;
498         return reg;
499 }
500
501 static int dvfs_rail_connect_to_regulator(struct dvfs_rail *rail)
502 {
503         struct regulator *reg;
504         int v;
505
506         if (!rail->reg) {
507                 reg = regulator_get(NULL, rail->reg_id);
508                 if (IS_ERR(reg)) {
509                         reg = get_fixed_regulator(rail);
510                         if (IS_ERR(reg)) {
511                                 pr_err("tegra_dvfs: failed to connect %s rail\n",
512                                        rail->reg_id);
513                                 return PTR_ERR(reg);
514                         }
515                 }
516                 rail->reg = reg;
517         }
518
519         v = regulator_enable(rail->reg);
520         if (v < 0) {
521                 pr_err("tegra_dvfs: failed on enabling regulator %s\n, err %d",
522                         rail->reg_id, v);
523                 return v;
524         }
525
526         v = regulator_get_voltage(rail->reg);
527         if (v < 0) {
528                 pr_err("tegra_dvfs: failed initial get %s voltage\n",
529                        rail->reg_id);
530                 return v;
531         }
532         rail->millivolts = v / 1000;
533         rail->new_millivolts = rail->millivolts;
534         dvfs_rail_stats_init(rail, rail->millivolts);
535
536         if (rail->boot_millivolts &&
537             (rail->boot_millivolts != rail->millivolts)) {
538                 WARN(1, "%s boot voltage %d does not match expected %d\n",
539                      rail->reg_id, rail->millivolts, rail->boot_millivolts);
540                 rail->boot_millivolts = rail->millivolts;
541         }
542         return 0;
543 }
544
545 static inline unsigned long *dvfs_get_freqs(struct dvfs *d)
546 {
547         return d->alt_freqs ? : &d->freqs[0];
548 }
549
550 static inline const int *dvfs_get_millivolts(struct dvfs *d, unsigned long rate)
551 {
552         if (tegra_dvfs_is_dfll_scale(d, rate))
553                 return d->dfll_millivolts;
554
555         return d->millivolts;
556 }
557
558 static int
559 __tegra_dvfs_set_rate(struct dvfs *d, unsigned long rate)
560 {
561         int i = 0;
562         int ret, mv, detach_mv;
563         unsigned long *freqs = dvfs_get_freqs(d);
564         const int *millivolts = dvfs_get_millivolts(d, rate);
565
566         if (freqs == NULL || millivolts == NULL)
567                 return -ENODEV;
568
569         /* On entry to dfll range limit 1st step to range bottom (full ramp of
570            voltage/rate is completed automatically in dfll mode) */
571         if (tegra_dvfs_is_dfll_range_entry(d, rate))
572                 rate = d->dfll_data.use_dfll_rate_min;
573
574         if (rate > freqs[d->num_freqs - 1]) {
575                 pr_warn("tegra_dvfs: rate %lu too high for dvfs on %s\n", rate,
576                         d->clk_name);
577                 return -EINVAL;
578         }
579
580         if (rate == 0) {
581                 d->cur_millivolts = 0;
582         } else {
583                 while (i < d->num_freqs && rate > freqs[i])
584                         i++;
585
586                 if ((d->max_millivolts) &&
587                     (millivolts[i] > d->max_millivolts)) {
588                         pr_warn("tegra_dvfs: voltage %d too high for dvfs on"
589                                 " %s\n", millivolts[i], d->clk_name);
590                         return -EINVAL;
591                 }
592
593                 mv = millivolts[i];
594                 detach_mv = tegra_dvfs_rail_get_boot_level(d->dvfs_rail);
595                 if (!d->dvfs_rail->reg && (mv > detach_mv)) {
596                         pr_warn("%s: %s: voltage %d above boot limit %d\n",
597                                 __func__, d->clk_name, mv, detach_mv);
598                         return -EINVAL;
599                 }
600
601                 detach_mv = tegra_dvfs_rail_get_disable_level(d->dvfs_rail);
602                 if (d->dvfs_rail->disabled && (mv > detach_mv)) {
603                         pr_warn("%s: %s: voltage %d above disable limit %d\n",
604                                 __func__, d->clk_name, mv, detach_mv);
605                         return -EINVAL;
606                 }
607
608                 detach_mv = tegra_dvfs_rail_get_suspend_level(d->dvfs_rail);
609                 if (d->dvfs_rail->suspended && (mv > detach_mv)) {
610                         pr_warn("%s: %s: voltage %d above disable limit %d\n",
611                                 __func__, d->clk_name, mv, detach_mv);
612                         return -EINVAL;
613                 }
614                 d->cur_millivolts = millivolts[i];
615         }
616
617         d->cur_rate = rate;
618
619         d->dvfs_rail->rate_set = true;
620         ret = dvfs_rail_update(d->dvfs_rail);
621         if (ret)
622                 pr_err("Failed to set regulator %s for clock %s to %d mV\n",
623                         d->dvfs_rail->reg_id, d->clk_name, d->cur_millivolts);
624
625         return ret;
626 }
627
628 int tegra_dvfs_alt_freqs_set(struct dvfs *d, unsigned long *alt_freqs)
629 {
630         int ret = 0;
631
632         mutex_lock(&dvfs_lock);
633
634         if (d->alt_freqs != alt_freqs) {
635                 d->alt_freqs = alt_freqs;
636                 ret = __tegra_dvfs_set_rate(d, d->cur_rate);
637         }
638
639         mutex_unlock(&dvfs_lock);
640         return ret;
641 }
642
643 static int predict_millivolts(struct clk *c, const int *millivolts,
644                               unsigned long rate)
645 {
646         int i;
647
648         if (!millivolts)
649                 return -ENODEV;
650         /*
651          * Predicted voltage can not be used across the switch to alternative
652          * frequency limits. For now, just fail the call for clock that has
653          * alternative limits initialized.
654          */
655         if (c->dvfs->alt_freqs)
656                 return -ENOSYS;
657
658         for (i = 0; i < c->dvfs->num_freqs; i++) {
659                 if (rate <= c->dvfs->freqs[i])
660                         break;
661         }
662
663         if (i == c->dvfs->num_freqs)
664                 return -EINVAL;
665
666         return millivolts[i];
667 }
668
669 int tegra_dvfs_predict_millivolts(struct clk *c, unsigned long rate)
670 {
671         const int *millivolts;
672
673         if (!rate || !c->dvfs)
674                 return 0;
675
676         millivolts = dvfs_get_millivolts(c->dvfs, rate);
677         return predict_millivolts(c, millivolts, rate);
678 }
679
680 int tegra_dvfs_predict_millivolts_pll(struct clk *c, unsigned long rate)
681 {
682         const int *millivolts;
683
684         if (!rate || !c->dvfs)
685                 return 0;
686
687         millivolts = c->dvfs->millivolts;
688         return predict_millivolts(c, millivolts, rate);
689 }
690
691 int tegra_dvfs_predict_millivolts_dfll(struct clk *c, unsigned long rate)
692 {
693         const int *millivolts;
694
695         if (!rate || !c->dvfs)
696                 return 0;
697
698         millivolts = c->dvfs->dfll_millivolts;
699         return predict_millivolts(c, millivolts, rate);
700 }
701
702 int tegra_dvfs_set_rate(struct clk *c, unsigned long rate)
703 {
704         int ret;
705
706         if (!c->dvfs)
707                 return -EINVAL;
708
709         mutex_lock(&dvfs_lock);
710         ret = __tegra_dvfs_set_rate(c->dvfs, rate);
711         mutex_unlock(&dvfs_lock);
712
713         return ret;
714 }
715 EXPORT_SYMBOL(tegra_dvfs_set_rate);
716
717 int tegra_dvfs_get_freqs(struct clk *c, unsigned long **freqs, int *num_freqs)
718 {
719         if (!c->dvfs)
720                 return -ENOSYS;
721
722         if (c->dvfs->alt_freqs)
723                 return -ENOSYS;
724
725         *num_freqs = c->dvfs->num_freqs;
726         *freqs = c->dvfs->freqs;
727
728         return 0;
729 }
730 EXPORT_SYMBOL(tegra_dvfs_get_freqs);
731
732 #ifdef CONFIG_TEGRA_VDD_CORE_OVERRIDE
733 static DEFINE_MUTEX(rail_override_lock);
734
735 int tegra_dvfs_override_core_voltage(int override_mv)
736 {
737         int ret, floor, ceiling;
738         struct dvfs_rail *rail = tegra_core_rail;
739
740         if (!rail)
741                 return -ENOENT;
742
743         if (rail->fixed_millivolts)
744                 return -ENOSYS;
745
746         floor = rail->min_override_millivolts;
747         ceiling = rail->nominal_millivolts;
748         if (override_mv && ((override_mv < floor) || (override_mv > ceiling))) {
749                 pr_err("%s: override level %d outside the range [%d...%d]\n",
750                        __func__, override_mv, floor, ceiling);
751                 return -EINVAL;
752         }
753
754         mutex_lock(&rail_override_lock);
755
756         if (override_mv == rail->override_millivolts) {
757                 ret = 0;
758                 goto out;
759         }
760
761         if (override_mv) {
762                 ret = tegra_dvfs_core_cap_level_apply(override_mv);
763                 if (ret) {
764                         pr_err("%s: failed to set cap for override level %d\n",
765                                __func__, override_mv);
766                         goto out;
767                 }
768         }
769
770         mutex_lock(&dvfs_lock);
771         if (rail->disabled || rail->suspended) {
772                 pr_err("%s: cannot scale %s rail\n", __func__,
773                        rail->disabled ? "disabled" : "suspended");
774                 ret = -EPERM;
775                 if (!override_mv) {
776                         mutex_unlock(&dvfs_lock);
777                         goto out;
778                 }
779         } else {
780                 rail->override_millivolts = override_mv;
781                 ret = dvfs_rail_update(rail);
782                 if (ret) {
783                         pr_err("%s: failed to set override level %d\n",
784                                __func__, override_mv);
785                         rail->override_millivolts = 0;
786                         dvfs_rail_update(rail);
787                 }
788         }
789         mutex_unlock(&dvfs_lock);
790
791         if (!override_mv || ret)
792                 tegra_dvfs_core_cap_level_apply(0);
793 out:
794         mutex_unlock(&rail_override_lock);
795         return ret;
796 }
797 #else
798 int tegra_dvfs_override_core_voltage(int override_mv)
799 {
800         pr_err("%s: vdd core override is not supported\n", __func__);
801         return -ENOSYS;
802 }
803 #endif
804 EXPORT_SYMBOL(tegra_dvfs_override_core_voltage);
805
806 /* May only be called during clock init, does not take any locks on clock c. */
807 int __init tegra_enable_dvfs_on_clk(struct clk *c, struct dvfs *d)
808 {
809         int i;
810
811         if (c->dvfs) {
812                 pr_err("Error when enabling dvfs on %s for clock %s:\n",
813                         d->dvfs_rail->reg_id, c->name);
814                 pr_err("DVFS already enabled for %s\n",
815                         c->dvfs->dvfs_rail->reg_id);
816                 return -EINVAL;
817         }
818
819         for (i = 0; i < MAX_DVFS_FREQS; i++) {
820                 if (d->millivolts[i] == 0)
821                         break;
822
823                 d->freqs[i] *= d->freqs_mult;
824
825                 /* If final frequencies are 0, pad with previous frequency */
826                 if (d->freqs[i] == 0 && i > 1)
827                         d->freqs[i] = d->freqs[i - 1];
828         }
829         d->num_freqs = i;
830
831         if (d->auto_dvfs) {
832                 c->auto_dvfs = true;
833                 clk_set_cansleep(c);
834         }
835
836         c->dvfs = d;
837
838         /*
839          * Minimum core override level is determined as maximum voltage required
840          * for clocks outside shared buses (shared bus rates can be capped to
841          * safe levels when override limit is set)
842          */
843         if (i && c->ops && !c->ops->shared_bus_update &&
844             !(c->flags & PERIPH_ON_CBUS)) {
845                 int mv = tegra_dvfs_predict_millivolts(c, d->freqs[i-1]);
846                 if (d->dvfs_rail->min_override_millivolts < mv)
847                         d->dvfs_rail->min_override_millivolts = mv;
848         }
849
850         mutex_lock(&dvfs_lock);
851         list_add_tail(&d->reg_node, &d->dvfs_rail->dvfs);
852         mutex_unlock(&dvfs_lock);
853
854         return 0;
855 }
856
857 static bool tegra_dvfs_all_rails_suspended(void)
858 {
859         struct dvfs_rail *rail;
860         bool all_suspended = true;
861
862         list_for_each_entry(rail, &dvfs_rail_list, node)
863                 if (!rail->suspended && !rail->disabled)
864                         all_suspended = false;
865
866         return all_suspended;
867 }
868
869 static bool tegra_dvfs_from_rails_suspended_or_solved(struct dvfs_rail *to)
870 {
871         struct dvfs_relationship *rel;
872         bool all_suspended = true;
873
874         list_for_each_entry(rel, &to->relationships_from, from_node)
875                 if (!rel->from->suspended && !rel->from->disabled &&
876                         !rel->solved_at_nominal)
877                         all_suspended = false;
878
879         return all_suspended;
880 }
881
882 static int tegra_dvfs_suspend_one(void)
883 {
884         struct dvfs_rail *rail;
885         int ret, mv;
886
887         list_for_each_entry(rail, &dvfs_rail_list, node) {
888                 if (!rail->suspended && !rail->disabled &&
889                     tegra_dvfs_from_rails_suspended_or_solved(rail)) {
890                         /* Safe, as pll mode rate is capped to fixed level */
891                         if (!rail->dfll_mode && rail->fixed_millivolts) {
892                                 mv = rail->fixed_millivolts;
893                         } else {
894                                 mv = tegra_dvfs_rail_get_suspend_level(rail);
895                                 mv = dvfs_rail_apply_limits(rail, mv);
896                         }
897
898                         /* apply suspend limit only if it is above current mv */
899                         ret = -EPERM;
900                         if (mv >= rail->millivolts)
901                                 ret = dvfs_rail_set_voltage(rail, mv);
902                         if (ret) {
903                                 pr_err("tegra_dvfs: failed %s suspend at %d\n",
904                                        rail->reg_id, rail->millivolts);
905                                 return ret;
906                         }
907
908                         rail->suspended = true;
909                         return 0;
910                 }
911         }
912
913         return -EINVAL;
914 }
915
916 static void tegra_dvfs_resume(void)
917 {
918         struct dvfs_rail *rail;
919
920         mutex_lock(&dvfs_lock);
921
922         list_for_each_entry(rail, &dvfs_rail_list, node)
923                 rail->suspended = false;
924
925         list_for_each_entry(rail, &dvfs_rail_list, node)
926                 dvfs_rail_update(rail);
927
928         mutex_unlock(&dvfs_lock);
929 }
930
931 static int tegra_dvfs_suspend(void)
932 {
933         int ret = 0;
934
935         mutex_lock(&dvfs_lock);
936
937         while (!tegra_dvfs_all_rails_suspended()) {
938                 ret = tegra_dvfs_suspend_one();
939                 if (ret)
940                         break;
941         }
942
943         mutex_unlock(&dvfs_lock);
944
945         if (ret)
946                 tegra_dvfs_resume();
947
948         return ret;
949 }
950
951 static int tegra_dvfs_pm_suspend(struct notifier_block *nb,
952                                  unsigned long event, void *data)
953 {
954         if (event == PM_SUSPEND_PREPARE) {
955                 if (tegra_dvfs_suspend())
956                         return NOTIFY_STOP;
957                 pr_info("tegra_dvfs: suspended\n");
958         }
959         return NOTIFY_OK;
960 };
961
962 static int tegra_dvfs_pm_resume(struct notifier_block *nb,
963                                 unsigned long event, void *data)
964 {
965         if (event == PM_POST_SUSPEND) {
966                 tegra_dvfs_resume();
967                 pr_info("tegra_dvfs: resumed\n");
968         }
969         return NOTIFY_OK;
970 };
971
972 static struct notifier_block tegra_dvfs_suspend_nb = {
973         .notifier_call = tegra_dvfs_pm_suspend,
974         .priority = -1,
975 };
976
977 static struct notifier_block tegra_dvfs_resume_nb = {
978         .notifier_call = tegra_dvfs_pm_resume,
979         .priority = 1,
980 };
981
982 static int tegra_dvfs_reboot_notify(struct notifier_block *nb,
983                                 unsigned long event, void *data)
984 {
985         switch (event) {
986         case SYS_RESTART:
987         case SYS_HALT:
988         case SYS_POWER_OFF:
989                 tegra_dvfs_suspend();
990                 return NOTIFY_OK;
991         }
992         return NOTIFY_DONE;
993 }
994
995 static struct notifier_block tegra_dvfs_reboot_nb = {
996         .notifier_call = tegra_dvfs_reboot_notify,
997 };
998
999 /* must be called with dvfs lock held */
1000 static void __tegra_dvfs_rail_disable(struct dvfs_rail *rail)
1001 {
1002         int ret = -EPERM;
1003         int mv;
1004
1005         /* don't set voltage in DFLL mode - won't work, but break stats */
1006         if (rail->dfll_mode) {
1007                 rail->disabled = true;
1008                 return;
1009         }
1010
1011         /* Safe, as pll mode rate is capped to fixed level */
1012         if (!rail->dfll_mode && rail->fixed_millivolts) {
1013                 mv = rail->fixed_millivolts;
1014         } else {
1015                 mv = tegra_dvfs_rail_get_disable_level(rail);
1016                 mv = dvfs_rail_apply_limits(rail, mv);
1017         }
1018
1019         /* apply detach mode limit provided it is above current volatge */
1020         if (mv >= rail->millivolts)
1021                 ret = dvfs_rail_set_voltage(rail, mv);
1022         if (ret) {
1023                 pr_err("tegra_dvfs: failed to disable %s at %d\n",
1024                        rail->reg_id, rail->millivolts);
1025                 return;
1026         }
1027         rail->disabled = true;
1028 }
1029
1030 /* must be called with dvfs lock held */
1031 static void __tegra_dvfs_rail_enable(struct dvfs_rail *rail)
1032 {
1033         rail->disabled = false;
1034         dvfs_rail_update(rail);
1035 }
1036
1037 void tegra_dvfs_rail_enable(struct dvfs_rail *rail)
1038 {
1039         if (!rail)
1040                 return;
1041
1042         mutex_lock(&rail_disable_lock);
1043
1044         if (rail->disabled) {
1045                 mutex_lock(&dvfs_lock);
1046                 __tegra_dvfs_rail_enable(rail);
1047                 mutex_unlock(&dvfs_lock);
1048
1049                 tegra_dvfs_rail_post_enable(rail);
1050         }
1051         mutex_unlock(&rail_disable_lock);
1052 }
1053
1054 void tegra_dvfs_rail_disable(struct dvfs_rail *rail)
1055 {
1056         if (!rail)
1057                 return;
1058
1059         mutex_lock(&rail_disable_lock);
1060         if (rail->disabled)
1061                 goto out;
1062
1063         /* rail disable will set it to nominal voltage underneath clock
1064            framework - need to re-configure clock rates that are not safe
1065            at nominal (yes, unsafe at nominal is ugly, but possible). Rate
1066            change must be done outside of dvfs lock. */
1067         if (tegra_dvfs_rail_disable_prepare(rail)) {
1068                 pr_info("dvfs: failed to prepare regulator %s to disable\n",
1069                         rail->reg_id);
1070                 goto out;
1071         }
1072
1073         mutex_lock(&dvfs_lock);
1074         __tegra_dvfs_rail_disable(rail);
1075         mutex_unlock(&dvfs_lock);
1076 out:
1077         mutex_unlock(&rail_disable_lock);
1078 }
1079
1080 int tegra_dvfs_rail_disable_by_name(const char *reg_id)
1081 {
1082         struct dvfs_rail *rail = tegra_dvfs_get_rail_by_name(reg_id);
1083         if (!rail)
1084                 return -EINVAL;
1085
1086         tegra_dvfs_rail_disable(rail);
1087         return 0;
1088 }
1089
1090 struct dvfs_rail *tegra_dvfs_get_rail_by_name(const char *reg_id)
1091 {
1092         struct dvfs_rail *rail;
1093
1094         mutex_lock(&dvfs_lock);
1095         list_for_each_entry(rail, &dvfs_rail_list, node) {
1096                 if (!strcmp(reg_id, rail->reg_id)) {
1097                         mutex_unlock(&dvfs_lock);
1098                         return rail;
1099                 }
1100         }
1101         mutex_unlock(&dvfs_lock);
1102         return NULL;
1103 }
1104
1105 int tegra_dvfs_rail_power_up(struct dvfs_rail *rail)
1106 {
1107         int ret = -ENOENT;
1108
1109         if (!rail || !rail->in_band_pm)
1110                 return -ENOSYS;
1111
1112         mutex_lock(&dvfs_lock);
1113         if (rail->reg) {
1114                 ret = regulator_enable(rail->reg);
1115                 if (!ret && !timekeeping_suspended)
1116                         tegra_dvfs_rail_on(rail, ktime_get());
1117         }
1118         mutex_unlock(&dvfs_lock);
1119         return ret;
1120 }
1121
1122 int tegra_dvfs_rail_power_down(struct dvfs_rail *rail)
1123 {
1124         int ret = -ENOENT;
1125
1126         if (!rail || !rail->in_band_pm)
1127                 return -ENOSYS;
1128
1129         mutex_lock(&dvfs_lock);
1130         if (rail->reg) {
1131                 ret = regulator_disable(rail->reg);
1132                 if (!ret && !timekeeping_suspended)
1133                         tegra_dvfs_rail_off(rail, ktime_get());
1134         }
1135         mutex_unlock(&dvfs_lock);
1136         return ret;
1137 }
1138
1139 bool tegra_dvfs_is_rail_up(struct dvfs_rail *rail)
1140 {
1141         bool ret = false;
1142
1143         if (!rail)
1144                 return false;
1145
1146         if (!rail->in_band_pm)
1147                 return true;
1148
1149         mutex_lock(&dvfs_lock);
1150         if (rail->reg)
1151                 ret = regulator_is_enabled(rail->reg) > 0;
1152         mutex_unlock(&dvfs_lock);
1153         return ret;
1154 }
1155
1156 bool tegra_dvfs_rail_updating(struct clk *clk)
1157 {
1158         return (!clk ? false :
1159                 (!clk->dvfs ? false :
1160                  (!clk->dvfs->dvfs_rail ? false :
1161                   (clk->dvfs->dvfs_rail->updating ||
1162                    clk->dvfs->dvfs_rail->dfll_mode_updating))));
1163 }
1164
1165 #ifdef CONFIG_OF
1166 int __init of_tegra_dvfs_init(const struct of_device_id *matches)
1167 {
1168         int ret;
1169         struct device_node *np;
1170
1171         for_each_matching_node(np, matches) {
1172                 const struct of_device_id *match = of_match_node(matches, np);
1173                 of_tegra_dvfs_init_cb_t dvfs_init_cb = match->data;
1174                 ret = dvfs_init_cb(np);
1175                 if (ret) {
1176                         pr_err("dt: Failed to read %s tables from DT\n",
1177                                                         match->compatible);
1178                         return ret;
1179                 }
1180         }
1181         return 0;
1182 }
1183 #endif
1184 int tegra_dvfs_dfll_mode_set(struct dvfs *d, unsigned long rate)
1185 {
1186         mutex_lock(&dvfs_lock);
1187         if (!d->dvfs_rail->dfll_mode) {
1188                 d->dvfs_rail->dfll_mode = true;
1189                 __tegra_dvfs_set_rate(d, rate);
1190         }
1191         mutex_unlock(&dvfs_lock);
1192         return 0;
1193 }
1194
1195 int tegra_dvfs_dfll_mode_clear(struct dvfs *d, unsigned long rate)
1196 {
1197         int ret = 0;
1198
1199         mutex_lock(&dvfs_lock);
1200         if (d->dvfs_rail->dfll_mode) {
1201                 d->dvfs_rail->dfll_mode = false;
1202                 /* avoid false detection of matching target (voltage in dfll
1203                    mode is fluctuating, and recorded level is just estimate) */
1204                 d->dvfs_rail->millivolts--;
1205                 if (d->dvfs_rail->disabled) {
1206                         d->dvfs_rail->disabled = false;
1207                         __tegra_dvfs_rail_disable(d->dvfs_rail);
1208                 }
1209                 ret = __tegra_dvfs_set_rate(d, rate);
1210         }
1211         mutex_unlock(&dvfs_lock);
1212         return ret;
1213 }
1214
1215 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmax_cdev(void)
1216 {
1217         if (tegra_cpu_rail)
1218                 return tegra_cpu_rail->vmax_cdev;
1219         return NULL;
1220 }
1221
1222 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmin_cdev(void)
1223 {
1224         if (tegra_cpu_rail)
1225                 return tegra_cpu_rail->vmin_cdev;
1226         return NULL;
1227 }
1228
1229 struct tegra_cooling_device *tegra_dvfs_get_core_vmin_cdev(void)
1230 {
1231         if (tegra_core_rail)
1232                 return tegra_core_rail->vmin_cdev;
1233         return NULL;
1234 }
1235
1236 struct tegra_cooling_device *tegra_dvfs_get_gpu_vmin_cdev(void)
1237 {
1238         if (tegra_gpu_rail)
1239                 return tegra_gpu_rail->vmin_cdev;
1240         return NULL;
1241 }
1242
1243 #ifdef CONFIG_THERMAL
1244 /* Cooling device limits minimum rail voltage at cold temperature in pll mode */
1245 static int tegra_dvfs_rail_get_vmin_cdev_max_state(
1246         struct thermal_cooling_device *cdev, unsigned long *max_state)
1247 {
1248         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1249         *max_state = rail->vmin_cdev->trip_temperatures_num;
1250         return 0;
1251 }
1252
1253 static int tegra_dvfs_rail_get_vmin_cdev_cur_state(
1254         struct thermal_cooling_device *cdev, unsigned long *cur_state)
1255 {
1256         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1257         *cur_state = rail->therm_floor_idx;
1258         return 0;
1259 }
1260
1261 static int tegra_dvfs_rail_set_vmin_cdev_state(
1262         struct thermal_cooling_device *cdev, unsigned long cur_state)
1263 {
1264         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1265
1266         mutex_lock(&dvfs_lock);
1267         if (rail->therm_floor_idx != cur_state) {
1268                 rail->therm_floor_idx = cur_state;
1269                 dvfs_rail_update(rail);
1270         }
1271         mutex_unlock(&dvfs_lock);
1272         return 0;
1273 }
1274
1275 static struct thermal_cooling_device_ops tegra_dvfs_rail_cooling_ops = {
1276         .get_max_state = tegra_dvfs_rail_get_vmin_cdev_max_state,
1277         .get_cur_state = tegra_dvfs_rail_get_vmin_cdev_cur_state,
1278         .set_cur_state = tegra_dvfs_rail_set_vmin_cdev_state,
1279 };
1280
1281 static void tegra_dvfs_rail_register_vmin_cdev(struct dvfs_rail *rail)
1282 {
1283         if (!rail->vmin_cdev)
1284                 return;
1285
1286         /* just report error - initialized for cold temperature, anyway */
1287         if (IS_ERR_OR_NULL(thermal_cooling_device_register(
1288                 rail->vmin_cdev->cdev_type, (void *)rail,
1289                 &tegra_dvfs_rail_cooling_ops)))
1290                 pr_err("tegra cooling device %s failed to register\n",
1291                        rail->vmin_cdev->cdev_type);
1292 }
1293
1294 #else
1295 #define tegra_dvfs_rail_register_vmin_cdev(rail)
1296 #endif
1297
1298 /*
1299  * Validate rail thermal profile, and get its size. Valid profile:
1300  * - voltage limits are descending with temperature increasing
1301  * - the lowest limit is above rail minimum voltage in pll and
1302  *   in dfll mode (if applicable)
1303  * - the highest limit is below rail nominal voltage
1304  */
1305 static int __init get_thermal_profile_size(
1306         int *trips_table, int *limits_table,
1307         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1308 {
1309         int i, min_mv;
1310
1311         for (i = 0; i < MAX_THERMAL_LIMITS - 1; i++) {
1312                 if (!limits_table[i+1])
1313                         break;
1314
1315                 if ((trips_table[i] >= trips_table[i+1]) ||
1316                     (limits_table[i] < limits_table[i+1])) {
1317                         pr_warn("%s: not ordered profile\n", rail->reg_id);
1318                         return -EINVAL;
1319                 }
1320         }
1321
1322         min_mv = max(rail->min_millivolts, d ? d->min_millivolts : 0);
1323         if (limits_table[i] < min_mv) {
1324                 pr_warn("%s: thermal profile below Vmin\n", rail->reg_id);
1325                 return -EINVAL;
1326         }
1327
1328         if (limits_table[0] > rail->nominal_millivolts) {
1329                 pr_warn("%s: thermal profile above Vmax\n", rail->reg_id);
1330                 return -EINVAL;
1331         }
1332         return i + 1;
1333 }
1334
1335 void __init tegra_dvfs_rail_init_vmax_thermal_profile(
1336         int *therm_trips_table, int *therm_caps_table,
1337         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1338 {
1339         int i = get_thermal_profile_size(therm_trips_table,
1340                                          therm_caps_table, rail, d);
1341         if (i <= 0) {
1342                 rail->vmax_cdev = NULL;
1343                 WARN(1, "%s: invalid Vmax thermal profile\n", rail->reg_id);
1344                 return;
1345         }
1346
1347         /* Install validated thermal caps */
1348         rail->therm_mv_caps = therm_caps_table;
1349         rail->therm_mv_caps_num = i;
1350
1351         /* Setup trip-points if applicable */
1352         if (rail->vmax_cdev) {
1353                 rail->vmax_cdev->trip_temperatures_num = i;
1354                 rail->vmax_cdev->trip_temperatures = therm_trips_table;
1355         }
1356 }
1357
1358 void __init tegra_dvfs_rail_init_vmin_thermal_profile(
1359         int *therm_trips_table, int *therm_floors_table,
1360         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1361 {
1362         int i = get_thermal_profile_size(therm_trips_table,
1363                                          therm_floors_table, rail, d);
1364         if (i <= 0) {
1365                 rail->vmin_cdev = NULL;
1366                 WARN(1, "%s: invalid Vmin thermal profile\n", rail->reg_id);
1367                 return;
1368         }
1369
1370         /* Install validated thermal floors */
1371         rail->therm_mv_floors = therm_floors_table;
1372         rail->therm_mv_floors_num = i;
1373
1374         /* Setup trip-points if applicable */
1375         if (rail->vmin_cdev) {
1376                 rail->vmin_cdev->trip_temperatures_num = i;
1377                 rail->vmin_cdev->trip_temperatures = therm_trips_table;
1378         }
1379 }
1380
1381 /* Directly set cold temperature limit in dfll mode */
1382 int tegra_dvfs_rail_dfll_mode_set_cold(struct dvfs_rail *rail)
1383 {
1384         int ret = 0;
1385
1386         /* No thermal floors - nothing to do */
1387         if (!rail || !rail->therm_mv_floors)
1388                 return ret;
1389
1390         /*
1391          * Since cooling thresholds are the same in pll and dfll modes, pll mode
1392          * thermal index can be used to decide if cold limit should be set in
1393          * dfll mode.
1394          */
1395         mutex_lock(&dvfs_lock);
1396         if (rail->dfll_mode &&
1397             (rail->therm_floor_idx < rail->therm_mv_floors_num)) {
1398                         int mv = rail->therm_mv_floors[rail->therm_floor_idx];
1399                         ret = dvfs_rail_set_voltage_reg(rail, mv);
1400         }
1401         mutex_unlock(&dvfs_lock);
1402
1403         return ret;
1404 }
1405
1406 /*
1407  * Iterate through all the dvfs regulators, finding the regulator exported
1408  * by the regulator api for each one.  Must be called in late init, after
1409  * all the regulator api's regulators are initialized.
1410  */
1411 int __init tegra_dvfs_late_init(void)
1412 {
1413         bool connected = true;
1414         struct dvfs_rail *rail;
1415
1416         mutex_lock(&dvfs_lock);
1417
1418         list_for_each_entry(rail, &dvfs_rail_list, node)
1419                 if (dvfs_rail_connect_to_regulator(rail))
1420                         connected = false;
1421
1422         list_for_each_entry(rail, &dvfs_rail_list, node)
1423                 if (connected)
1424                         dvfs_rail_update(rail);
1425                 else
1426                         __tegra_dvfs_rail_disable(rail);
1427
1428         mutex_unlock(&dvfs_lock);
1429
1430         if (!connected && tegra_platform_is_silicon()) {
1431                 pr_warn("tegra_dvfs: DVFS regulators connection failed\n"
1432                         "            !!!! voltage scaling is disabled !!!!\n");
1433                 return -ENODEV;
1434         }
1435
1436         register_pm_notifier(&tegra_dvfs_suspend_nb);
1437         register_pm_notifier(&tegra_dvfs_resume_nb);
1438         register_reboot_notifier(&tegra_dvfs_reboot_nb);
1439
1440         list_for_each_entry(rail, &dvfs_rail_list, node)
1441                 tegra_dvfs_rail_register_vmin_cdev(rail);
1442
1443         return 0;
1444 }
1445
1446 static int rail_stats_save_to_buf(char *buf, int len)
1447 {
1448         int i;
1449         struct dvfs_rail *rail;
1450         char *str = buf;
1451         char *end = buf + len;
1452
1453         str += scnprintf(str, end - str, "%-12s %-10s\n", "millivolts", "time");
1454
1455         mutex_lock(&dvfs_lock);
1456
1457         list_for_each_entry(rail, &dvfs_rail_list, node) {
1458                 str += scnprintf(str, end - str, "%s (bin: %d.%dmV)\n",
1459                            rail->reg_id,
1460                            rail->stats.bin_uV / 1000,
1461                            (rail->stats.bin_uV / 10) % 100);
1462
1463                 dvfs_rail_stats_update(rail, -1, ktime_get());
1464
1465                 str += scnprintf(str, end - str, "%-12d %-10llu\n", 0,
1466                         cputime64_to_clock_t(msecs_to_jiffies(
1467                                 ktime_to_ms(rail->stats.time_at_mv[0]))));
1468
1469                 for (i = 1; i <= DVFS_RAIL_STATS_TOP_BIN; i++) {
1470                         ktime_t ktime_zero = ktime_set(0, 0);
1471                         if (ktime_equal(rail->stats.time_at_mv[i], ktime_zero))
1472                                 continue;
1473                         str += scnprintf(str, end - str, "%-12d %-10llu\n",
1474                                 rail->min_millivolts +
1475                                 (i - 1) * rail->stats.bin_uV / 1000,
1476                                 cputime64_to_clock_t(msecs_to_jiffies(
1477                                         ktime_to_ms(rail->stats.time_at_mv[i])))
1478                         );
1479                 }
1480         }
1481         mutex_unlock(&dvfs_lock);
1482         return str - buf;
1483 }
1484
1485 #ifdef CONFIG_DEBUG_FS
1486 static int dvfs_tree_sort_cmp(void *p, struct list_head *a, struct list_head *b)
1487 {
1488         struct dvfs *da = list_entry(a, struct dvfs, reg_node);
1489         struct dvfs *db = list_entry(b, struct dvfs, reg_node);
1490         int ret;
1491
1492         ret = strcmp(da->dvfs_rail->reg_id, db->dvfs_rail->reg_id);
1493         if (ret != 0)
1494                 return ret;
1495
1496         if (da->cur_millivolts < db->cur_millivolts)
1497                 return 1;
1498         if (da->cur_millivolts > db->cur_millivolts)
1499                 return -1;
1500
1501         return strcmp(da->clk_name, db->clk_name);
1502 }
1503
1504 static int dvfs_tree_show(struct seq_file *s, void *data)
1505 {
1506         struct dvfs *d;
1507         struct dvfs_rail *rail;
1508         struct dvfs_relationship *rel;
1509
1510         seq_printf(s, "   clock      rate       mV\n");
1511         seq_printf(s, "--------------------------------\n");
1512
1513         mutex_lock(&dvfs_lock);
1514
1515         list_for_each_entry(rail, &dvfs_rail_list, node) {
1516                 int thermal_mv_floor = 0;
1517
1518                 seq_printf(s, "%s %d mV%s:\n", rail->reg_id, rail->millivolts,
1519                            rail->dfll_mode ? " dfll mode" :
1520                                 rail->disabled ? " disabled" : "");
1521                 list_for_each_entry(rel, &rail->relationships_from, from_node) {
1522                         seq_printf(s, "   %-10s %-7d mV %-4d mV\n",
1523                                 rel->from->reg_id, rel->from->millivolts,
1524                                 dvfs_solve_relationship(rel));
1525                 }
1526                 seq_printf(s, "   offset     %-7d mV\n", rail->dbg_mv_offs);
1527
1528                 if (rail->therm_mv_floors) {
1529                         int i = rail->therm_floor_idx;
1530                         if (i < rail->therm_mv_floors_num)
1531                                 thermal_mv_floor = rail->therm_mv_floors[i];
1532                 }
1533                 seq_printf(s, "   thermal    %-7d mV\n", thermal_mv_floor);
1534
1535                 if (rail == tegra_core_rail) {
1536                         seq_printf(s, "   override   %-7d mV [%-4d...%-4d]\n",
1537                                    rail->override_millivolts,
1538                                    rail->min_override_millivolts,
1539                                    rail->nominal_millivolts);
1540                 }
1541
1542                 list_sort(NULL, &rail->dvfs, dvfs_tree_sort_cmp);
1543
1544                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1545                         seq_printf(s, "   %-10s %-10lu %-4d mV\n", d->clk_name,
1546                                 d->cur_rate, d->cur_millivolts);
1547                 }
1548         }
1549
1550         mutex_unlock(&dvfs_lock);
1551
1552         return 0;
1553 }
1554
1555 static int dvfs_tree_open(struct inode *inode, struct file *file)
1556 {
1557         return single_open(file, dvfs_tree_show, inode->i_private);
1558 }
1559
1560 static const struct file_operations dvfs_tree_fops = {
1561         .open           = dvfs_tree_open,
1562         .read           = seq_read,
1563         .llseek         = seq_lseek,
1564         .release        = single_release,
1565 };
1566
1567 static int rail_stats_show(struct seq_file *s, void *data)
1568 {
1569         char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
1570         int size = 0;
1571
1572         if (!buf)
1573                 return -ENOMEM;
1574
1575         size = rail_stats_save_to_buf(buf, PAGE_SIZE);
1576         seq_write(s, buf, size);
1577         kfree(buf);
1578         return 0;
1579 }
1580
1581 static int rail_stats_open(struct inode *inode, struct file *file)
1582 {
1583         return single_open(file, rail_stats_show, inode->i_private);
1584 }
1585
1586 static const struct file_operations rail_stats_fops = {
1587         .open           = rail_stats_open,
1588         .read           = seq_read,
1589         .llseek         = seq_lseek,
1590         .release        = single_release,
1591 };
1592
1593 static int gpu_dvfs_show(struct seq_file *s, void *data)
1594 {
1595         int idx;
1596         int *millivolts;
1597         unsigned long *freqs;
1598
1599         if (read_gpu_dvfs_table(&millivolts, &freqs)) {
1600                 seq_printf(s, "Only supported for T124 or higher\n");
1601                 return 0;
1602         }
1603
1604         seq_printf(s, "millivolts \t \t frequency\n");
1605         seq_printf(s, "=====================================\n");
1606
1607         for (idx = 0; millivolts[idx]; idx++)
1608                 seq_printf(s, "%d mV \t \t %lu Hz\n", millivolts[idx],
1609                                 freqs[idx]);
1610
1611         return 0;
1612 }
1613
1614 static int gpu_dvfs_open(struct inode *inode, struct file *file)
1615 {
1616         return single_open(file, gpu_dvfs_show, NULL);
1617 }
1618
1619 static const struct file_operations gpu_dvfs_fops = {
1620         .open           = gpu_dvfs_open,
1621         .read           = seq_read,
1622         .llseek         = seq_lseek,
1623         .release        = single_release,
1624 };
1625
1626 static int rail_offs_set(struct dvfs_rail *rail, int offs)
1627 {
1628         if (rail) {
1629                 mutex_lock(&dvfs_lock);
1630                 rail->dbg_mv_offs = offs;
1631                 dvfs_rail_update(rail);
1632                 mutex_unlock(&dvfs_lock);
1633                 return 0;
1634         }
1635         return -ENOENT;
1636 }
1637
1638 static int cpu_offs_get(void *data, u64 *val)
1639 {
1640         if (tegra_cpu_rail) {
1641                 *val = (u64)tegra_cpu_rail->dbg_mv_offs;
1642                 return 0;
1643         }
1644         *val = 0;
1645         return -ENOENT;
1646 }
1647 static int cpu_offs_set(void *data, u64 val)
1648 {
1649         return rail_offs_set(tegra_cpu_rail, (int)val);
1650 }
1651 DEFINE_SIMPLE_ATTRIBUTE(cpu_offs_fops, cpu_offs_get, cpu_offs_set, "%lld\n");
1652
1653 static int gpu_offs_get(void *data, u64 *val)
1654 {
1655         if (tegra_gpu_rail) {
1656                 *val = (u64)tegra_gpu_rail->dbg_mv_offs;
1657                 return 0;
1658         }
1659         *val = 0;
1660         return -ENOENT;
1661 }
1662 static int gpu_offs_set(void *data, u64 val)
1663 {
1664         return rail_offs_set(tegra_gpu_rail, (int)val);
1665 }
1666 DEFINE_SIMPLE_ATTRIBUTE(gpu_offs_fops, gpu_offs_get, gpu_offs_set, "%lld\n");
1667
1668 static int core_offs_get(void *data, u64 *val)
1669 {
1670         if (tegra_core_rail) {
1671                 *val = (u64)tegra_core_rail->dbg_mv_offs;
1672                 return 0;
1673         }
1674         *val = 0;
1675         return -ENOENT;
1676 }
1677 static int core_offs_set(void *data, u64 val)
1678 {
1679         return rail_offs_set(tegra_core_rail, (int)val);
1680 }
1681 DEFINE_SIMPLE_ATTRIBUTE(core_offs_fops, core_offs_get, core_offs_set, "%lld\n");
1682
1683 static int core_override_get(void *data, u64 *val)
1684 {
1685         if (tegra_core_rail) {
1686                 *val = (u64)tegra_core_rail->override_millivolts;
1687                 return 0;
1688         }
1689         *val = 0;
1690         return -ENOENT;
1691 }
1692 static int core_override_set(void *data, u64 val)
1693 {
1694         return tegra_dvfs_override_core_voltage((int)val);
1695 }
1696 DEFINE_SIMPLE_ATTRIBUTE(core_override_fops,
1697                         core_override_get, core_override_set, "%llu\n");
1698
1699 static int dvfs_table_show(struct seq_file *s, void *data)
1700 {
1701         int i;
1702         struct dvfs *d;
1703         struct dvfs_rail *rail;
1704
1705         seq_printf(s, "DVFS tables: units mV/MHz\n\n");
1706
1707         mutex_lock(&dvfs_lock);
1708
1709         list_for_each_entry(rail, &dvfs_rail_list, node) {
1710                 bool mv_done = false;
1711                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1712                         if (!mv_done) {
1713                                 mv_done = true;
1714                                 seq_printf(s, "%-16s", rail->reg_id);
1715                                 for (i = 0; i < d->num_freqs; i++) {
1716                                         int mv = d->millivolts[i];
1717                                         seq_printf(s, "%7d", mv);
1718                                 }
1719                                 seq_printf(s, "\n");
1720                                 if (d->dfll_millivolts) {
1721                                         seq_printf(s, "%-8s (dfll) ",
1722                                                    rail->reg_id);
1723                                         for (i = 0; i < d->num_freqs; i++) {
1724                                                 int mv = d->dfll_millivolts[i];
1725                                                 seq_printf(s, "%7d", mv);
1726                                         }
1727                                         seq_printf(s, "\n");
1728                                 }
1729                         }
1730
1731                         seq_printf(s, "%-16s", d->clk_name);
1732                         for (i = 0; i < d->num_freqs; i++) {
1733                                 unsigned int f = d->freqs[i]/100000;
1734                                 seq_printf(s, " %4u.%u", f/10, f%10);
1735                         }
1736                         seq_printf(s, "\n");
1737                 }
1738                 seq_printf(s, "\n");
1739         }
1740
1741         mutex_unlock(&dvfs_lock);
1742
1743         return 0;
1744 }
1745
1746 static int dvfs_table_open(struct inode *inode, struct file *file)
1747 {
1748         return single_open(file, dvfs_table_show, inode->i_private);
1749 }
1750
1751 static const struct file_operations dvfs_table_fops = {
1752         .open           = dvfs_table_open,
1753         .read           = seq_read,
1754         .llseek         = seq_lseek,
1755         .release        = single_release,
1756 };
1757
1758 int __init dvfs_debugfs_init(struct dentry *clk_debugfs_root)
1759 {
1760         struct dentry *d;
1761
1762         d = debugfs_create_file("dvfs", S_IRUGO, clk_debugfs_root, NULL,
1763                 &dvfs_tree_fops);
1764         if (!d)
1765                 return -ENOMEM;
1766
1767         d = debugfs_create_file("rails", S_IRUGO, clk_debugfs_root, NULL,
1768                 &rail_stats_fops);
1769         if (!d)
1770                 return -ENOMEM;
1771
1772         d = debugfs_create_file("vdd_cpu_offs", S_IRUGO | S_IWUSR,
1773                 clk_debugfs_root, NULL, &cpu_offs_fops);
1774         if (!d)
1775                 return -ENOMEM;
1776
1777         d = debugfs_create_file("vdd_gpu_offs", S_IRUGO | S_IWUSR,
1778                 clk_debugfs_root, NULL, &gpu_offs_fops);
1779         if (!d)
1780                 return -ENOMEM;
1781
1782         d = debugfs_create_file("vdd_core_offs", S_IRUGO | S_IWUSR,
1783                 clk_debugfs_root, NULL, &core_offs_fops);
1784         if (!d)
1785                 return -ENOMEM;
1786
1787         d = debugfs_create_file("vdd_core_override", S_IRUGO | S_IWUSR,
1788                 clk_debugfs_root, NULL, &core_override_fops);
1789         if (!d)
1790                 return -ENOMEM;
1791
1792         d = debugfs_create_file("gpu_dvfs", S_IRUGO | S_IWUSR,
1793                 clk_debugfs_root, NULL, &gpu_dvfs_fops);
1794         if (!d)
1795                 return -ENOMEM;
1796
1797         d = debugfs_create_file("dvfs_table", S_IRUGO, clk_debugfs_root, NULL,
1798                 &dvfs_table_fops);
1799         if (!d)
1800                 return -ENOMEM;
1801
1802         return 0;
1803 }
1804
1805 #endif
1806
1807 #ifdef CONFIG_PM
1808 static ssize_t tegra_rail_stats_show(struct kobject *kobj,
1809                                         struct kobj_attribute *attr,
1810                                         char *buf)
1811 {
1812         return rail_stats_save_to_buf(buf, PAGE_SIZE);
1813 }
1814
1815 static struct kobj_attribute rail_stats_attr =
1816                 __ATTR_RO(tegra_rail_stats);
1817
1818 static int __init tegra_dvfs_sysfs_stats_init(void)
1819 {
1820         int error;
1821         error = sysfs_create_file(power_kobj, &rail_stats_attr.attr);
1822         return 0;
1823 }
1824 late_initcall(tegra_dvfs_sysfs_stats_init);
1825 #endif