ARM: tegra: dvfs: Add thermal dvfs cooling device ops
[linux-3.10.git] / arch / arm / mach-tegra / dvfs.c
1 /*
2  *
3  * Copyright (C) 2010 Google, Inc.
4  *
5  * Author:
6  *      Colin Cross <ccross@google.com>
7  *
8  * Copyright (C) 2010-2013 NVIDIA CORPORATION. All rights reserved.
9  *
10  * This software is licensed under the terms of the GNU General Public
11  * License version 2, as published by the Free Software Foundation, and
12  * may be copied, distributed, and modified under those terms.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/clk.h>
23 #include <linux/clkdev.h>
24 #include <linux/debugfs.h>
25 #include <linux/init.h>
26 #include <linux/list.h>
27 #include <linux/list_sort.h>
28 #include <linux/module.h>
29 #include <linux/regulator/consumer.h>
30 #include <linux/seq_file.h>
31 #include <linux/slab.h>
32 #include <linux/suspend.h>
33 #include <linux/delay.h>
34 #include <linux/clk/tegra.h>
35 #include <linux/reboot.h>
36 #include <linux/clk/tegra.h>
37 #include <linux/tegra-soc.h>
38
39 #include "board.h"
40 #include "clock.h"
41 #include "dvfs.h"
42
43 #define DVFS_RAIL_STATS_BIN     12500
44
45 struct dvfs_rail *tegra_cpu_rail;
46 struct dvfs_rail *tegra_core_rail;
47 struct dvfs_rail *tegra_gpu_rail;
48
49 static LIST_HEAD(dvfs_rail_list);
50 static DEFINE_MUTEX(dvfs_lock);
51 static DEFINE_MUTEX(rail_disable_lock);
52
53 static int dvfs_rail_update(struct dvfs_rail *rail);
54
55 static inline int tegra_dvfs_rail_get_disable_level(struct dvfs_rail *rail)
56 {
57         return rail->disable_millivolts ? : rail->nominal_millivolts;
58 }
59
60 static inline int tegra_dvfs_rail_get_suspend_level(struct dvfs_rail *rail)
61 {
62         return rail->suspend_millivolts ? : rail->nominal_millivolts;
63 }
64
65 void tegra_dvfs_add_relationships(struct dvfs_relationship *rels, int n)
66 {
67         int i;
68         struct dvfs_relationship *rel;
69
70         mutex_lock(&dvfs_lock);
71
72         for (i = 0; i < n; i++) {
73                 rel = &rels[i];
74                 list_add_tail(&rel->from_node, &rel->to->relationships_from);
75                 list_add_tail(&rel->to_node, &rel->from->relationships_to);
76         }
77
78         mutex_unlock(&dvfs_lock);
79 }
80
81 /* Make sure there is a matching cooling device for thermal limit profile. */
82 static void dvfs_validate_cdevs(struct dvfs_rail *rail)
83 {
84         if (!rail->therm_mv_caps != !rail->therm_mv_caps_num) {
85                 rail->therm_mv_caps_num = 0;
86                 rail->therm_mv_caps = NULL;
87                 WARN(1, "%s: not matching thermal caps/num\n", rail->reg_id);
88         }
89
90         if (rail->therm_mv_caps && !rail->vmax_cdev)
91                 WARN(1, "%s: missing vmax cooling device\n", rail->reg_id);
92
93         if (!rail->therm_mv_floors != !rail->therm_mv_floors_num) {
94                 rail->therm_mv_floors_num = 0;
95                 rail->therm_mv_floors = NULL;
96                 WARN(1, "%s: not matching thermal floors/num\n", rail->reg_id);
97         }
98
99         if (rail->therm_mv_floors && !rail->vmin_cdev)
100                 WARN(1, "%s: missing vmin cooling device\n", rail->reg_id);
101
102         /* Limit override range to maximum floor */
103         if (rail->therm_mv_floors)
104                 rail->min_override_millivolts = rail->therm_mv_floors[0];
105 }
106
107 int tegra_dvfs_init_rails(struct dvfs_rail *rails[], int n)
108 {
109         int i, mv;
110
111         mutex_lock(&dvfs_lock);
112
113         for (i = 0; i < n; i++) {
114                 INIT_LIST_HEAD(&rails[i]->dvfs);
115                 INIT_LIST_HEAD(&rails[i]->relationships_from);
116                 INIT_LIST_HEAD(&rails[i]->relationships_to);
117
118                 mv = rails[i]->nominal_millivolts;
119                 if (rails[i]->boot_millivolts > mv)
120                         WARN(1, "%s: boot voltage %d above nominal %d\n",
121                              rails[i]->reg_id, rails[i]->boot_millivolts, mv);
122                 if (rails[i]->disable_millivolts > mv)
123                         rails[i]->disable_millivolts = mv;
124                 if (rails[i]->suspend_millivolts > mv)
125                         rails[i]->suspend_millivolts = mv;
126
127                 mv = tegra_dvfs_rail_get_boot_level(rails[i]);
128                 rails[i]->millivolts = mv;
129                 rails[i]->new_millivolts = mv;
130                 if (!rails[i]->step)
131                         rails[i]->step = rails[i]->max_millivolts;
132                 if (!rails[i]->step_up)
133                         rails[i]->step_up = rails[i]->step;
134
135                 list_add_tail(&rails[i]->node, &dvfs_rail_list);
136
137                 dvfs_validate_cdevs(rails[i]);
138
139                 if (!strcmp("vdd_cpu", rails[i]->reg_id))
140                         tegra_cpu_rail = rails[i];
141                 else if (!strcmp("vdd_gpu", rails[i]->reg_id))
142                         tegra_gpu_rail = rails[i];
143                 else if (!strcmp("vdd_core", rails[i]->reg_id))
144                         tegra_core_rail = rails[i];
145         }
146
147         mutex_unlock(&dvfs_lock);
148
149         return 0;
150 };
151
152 static int dvfs_solve_relationship(struct dvfs_relationship *rel)
153 {
154         return rel->solve(rel->from, rel->to);
155 }
156
157 /* rail statistic - called during rail init, or under dfs_lock, or with
158    CPU0 only on-line, and interrupts disabled */
159 static void dvfs_rail_stats_init(struct dvfs_rail *rail, int millivolts)
160 {
161         int dvfs_rail_stats_range;
162
163         if (!rail->stats.bin_uV)
164                 rail->stats.bin_uV = DVFS_RAIL_STATS_BIN;
165
166         dvfs_rail_stats_range =
167                 (DVFS_RAIL_STATS_TOP_BIN - 1) * rail->stats.bin_uV / 1000;
168
169         rail->stats.last_update = ktime_get();
170         if (millivolts >= rail->min_millivolts) {
171                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
172                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
173                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
174         }
175
176         if (rail->max_millivolts >
177             rail->min_millivolts + dvfs_rail_stats_range)
178                 pr_warn("tegra_dvfs: %s: stats above %d mV will be squashed\n",
179                         rail->reg_id,
180                         rail->min_millivolts + dvfs_rail_stats_range);
181 }
182
183 static void dvfs_rail_stats_update(
184         struct dvfs_rail *rail, int millivolts, ktime_t now)
185 {
186         rail->stats.time_at_mv[rail->stats.last_index] = ktime_add(
187                 rail->stats.time_at_mv[rail->stats.last_index], ktime_sub(
188                         now, rail->stats.last_update));
189         rail->stats.last_update = now;
190
191         if (rail->stats.off)
192                 return;
193
194         if (millivolts >= rail->min_millivolts) {
195                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
196                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
197                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
198         } else if (millivolts == 0)
199                         rail->stats.last_index = 0;
200 }
201
202 static void dvfs_rail_stats_pause(struct dvfs_rail *rail,
203                                   ktime_t delta, bool on)
204 {
205         int i = on ? rail->stats.last_index : 0;
206         rail->stats.time_at_mv[i] = ktime_add(rail->stats.time_at_mv[i], delta);
207 }
208
209 void tegra_dvfs_rail_off(struct dvfs_rail *rail, ktime_t now)
210 {
211         if (rail) {
212                 dvfs_rail_stats_update(rail, 0, now);
213                 rail->stats.off = true;
214         }
215 }
216
217 void tegra_dvfs_rail_on(struct dvfs_rail *rail, ktime_t now)
218 {
219         if (rail) {
220                 rail->stats.off = false;
221                 dvfs_rail_stats_update(rail, rail->millivolts, now);
222         }
223 }
224
225 void tegra_dvfs_rail_pause(struct dvfs_rail *rail, ktime_t delta, bool on)
226 {
227         if (rail)
228                 dvfs_rail_stats_pause(rail, delta, on);
229 }
230
231 static int dvfs_rail_set_voltage_reg(struct dvfs_rail *rail, int millivolts)
232 {
233         int ret;
234
235         /*
236          * safely return success for low voltage requests on fixed regulator
237          * (higher requests will go through and fail, as they should)
238          */
239         if (rail->fixed_millivolts && (millivolts <= rail->fixed_millivolts))
240                 return 0;
241
242         rail->updating = true;
243         rail->reg_max_millivolts = rail->reg_max_millivolts ==
244                 rail->max_millivolts ?
245                 rail->max_millivolts + 1 : rail->max_millivolts;
246         ret = regulator_set_voltage(rail->reg,
247                 millivolts * 1000,
248                 rail->reg_max_millivolts * 1000);
249         rail->updating = false;
250
251         return ret;
252 }
253
254 /* Sets the voltage on a dvfs rail to a specific value, and updates any
255  * rails that depend on this rail. */
256 static int dvfs_rail_set_voltage(struct dvfs_rail *rail, int millivolts)
257 {
258         int ret = 0;
259         struct dvfs_relationship *rel;
260         int step, offset;
261         int i;
262         int steps;
263         bool jmp_to_zero;
264
265         if (!rail->reg) {
266                 if (millivolts == rail->millivolts)
267                         return 0;
268                 else
269                         return -EINVAL;
270         }
271
272         if (millivolts > rail->millivolts) {
273                 step = rail->step_up;
274                 offset = step;
275         } else {
276                 step = rail->step;
277                 offset = -step;
278         }
279
280         /*
281          * DFLL adjusts rail voltage automatically, but not exactly to the
282          * expected level - update stats, anyway.
283          */
284         if (rail->dfll_mode) {
285                 rail->millivolts = rail->new_millivolts = millivolts;
286                 dvfs_rail_stats_update(rail, millivolts, ktime_get());
287                 return 0;
288         }
289
290         if (rail->disabled)
291                 return 0;
292
293         rail->resolving_to = true;
294         jmp_to_zero = rail->jmp_to_zero &&
295                         ((millivolts == 0) || (rail->millivolts == 0));
296         steps = jmp_to_zero ? 1 :
297                 DIV_ROUND_UP(abs(millivolts - rail->millivolts), step);
298
299         for (i = 0; i < steps; i++) {
300                 if (!jmp_to_zero &&
301                     (abs(millivolts - rail->millivolts) > step))
302                         rail->new_millivolts = rail->millivolts + offset;
303                 else
304                         rail->new_millivolts = millivolts;
305
306                 /* Before changing the voltage, tell each rail that depends
307                  * on this rail that the voltage will change.
308                  * This rail will be the "from" rail in the relationship,
309                  * the rail that depends on this rail will be the "to" rail.
310                  * from->millivolts will be the old voltage
311                  * from->new_millivolts will be the new voltage */
312                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
313                         ret = dvfs_rail_update(rel->to);
314                         if (ret)
315                                 goto out;
316                 }
317
318                 ret = dvfs_rail_set_voltage_reg(rail, rail->new_millivolts);
319                 if (ret) {
320                         pr_err("Failed to set dvfs regulator %s\n", rail->reg_id);
321                         goto out;
322                 }
323
324                 rail->millivolts = rail->new_millivolts;
325                 dvfs_rail_stats_update(rail, rail->millivolts, ktime_get());
326
327                 /* After changing the voltage, tell each rail that depends
328                  * on this rail that the voltage has changed.
329                  * from->millivolts and from->new_millivolts will be the
330                  * new voltage */
331                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
332                         ret = dvfs_rail_update(rel->to);
333                         if (ret)
334                                 goto out;
335                 }
336         }
337
338         if (unlikely(rail->millivolts != millivolts)) {
339                 pr_err("%s: rail didn't reach target %d in %d steps (%d)\n",
340                         __func__, millivolts, steps, rail->millivolts);
341                 ret = -EINVAL;
342         }
343
344 out:
345         rail->resolving_to = false;
346         return ret;
347 }
348
349 /* Determine the minimum valid voltage for a rail, taking into account
350  * the dvfs clocks and any rails that this rail depends on.  Calls
351  * dvfs_rail_set_voltage with the new voltage, which will call
352  * dvfs_rail_update on any rails that depend on this rail. */
353 static inline int dvfs_rail_apply_limits(struct dvfs_rail *rail, int millivolts)
354 {
355         int min_mv = rail->min_millivolts;
356
357         if (rail->therm_mv_floors) {
358                 int i = rail->therm_floor_idx;
359                 if (i < rail->therm_mv_floors_num)
360                         min_mv = rail->therm_mv_floors[i];
361         }
362
363         if (rail->override_millivolts) {
364                 millivolts = rail->override_millivolts;
365         } else {
366                 /* apply offset and clip up to pll mode fixed mv */
367                 millivolts += rail->dbg_mv_offs;
368                 if (!rail->dfll_mode && rail->fixed_millivolts &&
369                     (millivolts < rail->fixed_millivolts))
370                         millivolts = rail->fixed_millivolts;
371         }
372
373         if (millivolts < min_mv)
374                 millivolts = min_mv;
375
376         return millivolts;
377 }
378
379 static int dvfs_rail_update(struct dvfs_rail *rail)
380 {
381         int millivolts = 0;
382         struct dvfs *d;
383         struct dvfs_relationship *rel;
384         int ret = 0;
385         int steps;
386
387         /* if dvfs is suspended, return and handle it during resume */
388         if (rail->suspended)
389                 return 0;
390
391         /* if regulators are not connected yet, return and handle it later */
392         if (!rail->reg)
393                 return 0;
394
395         /* if no clock has requested voltage since boot, defer update */
396         if (!rail->rate_set)
397                 return 0;
398
399         /* if rail update is entered while resolving circular dependencies,
400            abort recursion */
401         if (rail->resolving_to)
402                 return 0;
403
404         /* Find the maximum voltage requested by any clock */
405         list_for_each_entry(d, &rail->dvfs, reg_node)
406                 millivolts = max(d->cur_millivolts, millivolts);
407
408         /* Apply offset and min/max limits if any clock is requesting voltage */
409         if (millivolts)
410                 millivolts = dvfs_rail_apply_limits(rail, millivolts);
411         /* Keep current voltage if regulator is to be disabled via explicitly */
412         else if (rail->in_band_pm)
413                 return 0;
414         /* Keep current voltage if regulator must not be disabled at run time */
415         else if (!rail->jmp_to_zero) {
416                 WARN(1, "%s cannot be turned off by dvfs\n");
417                 return 0;
418         }
419         /* else: fall thru if regulator is turned off by side band signaling */
420
421         /* retry update if limited by from-relationship to account for
422            circular dependencies */
423         steps = DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
424         for (; steps >= 0; steps--) {
425                 rail->new_millivolts = millivolts;
426
427                 /* Check any rails that this rail depends on */
428                 list_for_each_entry(rel, &rail->relationships_from, from_node)
429                         rail->new_millivolts = dvfs_solve_relationship(rel);
430
431                 if (rail->new_millivolts == rail->millivolts)
432                         break;
433
434                 ret = dvfs_rail_set_voltage(rail, rail->new_millivolts);
435         }
436
437         return ret;
438 }
439
440 static struct regulator *get_fixed_regulator(struct dvfs_rail *rail)
441 {
442         struct regulator *reg;
443         char reg_id[80];
444         struct dvfs *d;
445         int v, i;
446         unsigned long dfll_boost;
447
448         strcpy(reg_id, rail->reg_id);
449         strcat(reg_id, "_fixed");
450         reg = regulator_get(NULL, reg_id);
451         if (IS_ERR(reg))
452                 return reg;
453
454         v = regulator_get_voltage(reg) / 1000;
455         if ((v < rail->min_millivolts) || (v > rail->nominal_millivolts) ||
456             (rail->therm_mv_floors && v < rail->therm_mv_floors[0])) {
457                 pr_err("tegra_dvfs: ivalid fixed %s voltage %d\n",
458                        rail->reg_id, v);
459                 return ERR_PTR(-EINVAL);
460         }
461
462         /*
463          * Only fixed at nominal voltage vdd_core regulator is allowed, same
464          * is true for cpu rail if dfll mode is not supported at all. No thermal
465          * capping can be implemented in this case.
466          */
467         if (!IS_ENABLED(CONFIG_ARCH_TEGRA_HAS_CL_DVFS) ||
468             (rail != tegra_cpu_rail)) {
469                 if (v != rail->nominal_millivolts) {
470                         pr_err("tegra_dvfs: %s fixed below nominal at %d\n",
471                                rail->reg_id, v);
472                         return ERR_PTR(-EINVAL);
473                 }
474                 if (rail->therm_mv_caps) {
475                         pr_err("tegra_dvfs: cannot fix %s with thermal caps\n",
476                                rail->reg_id);
477                         return ERR_PTR(-ENOSYS);
478                 }
479                 return reg;
480         }
481
482         /*
483          * If dfll mode is supported, fixed vdd_cpu regulator may be below
484          * nominal in pll mode - maximum cpu rate in pll mode is limited
485          * respectively. Regulator is required to allow automatic scaling
486          * in dfll mode.
487          *
488          * FIXME: platform data to explicitly identify such "hybrid" regulator?
489          */
490         d = list_first_entry(&rail->dvfs, struct dvfs, reg_node);
491         for (i = 0; i < d->num_freqs; i++) {
492                 if (d->millivolts[i] > v)
493                         break;
494         }
495
496         if (!i) {
497                 pr_err("tegra_dvfs: %s fixed at %d: too low for min rate\n",
498                        rail->reg_id, v);
499                 return ERR_PTR(-EINVAL);
500         }
501
502         dfll_boost = (d->freqs[d->num_freqs - 1] - d->freqs[i - 1]);
503         if (d->dfll_data.max_rate_boost < dfll_boost)
504                 d->dfll_data.max_rate_boost = dfll_boost;
505
506         rail->fixed_millivolts = v;
507         return reg;
508 }
509
510 static int dvfs_rail_connect_to_regulator(struct dvfs_rail *rail)
511 {
512         struct regulator *reg;
513         int v;
514
515         if (!rail->reg) {
516                 reg = regulator_get(NULL, rail->reg_id);
517                 if (IS_ERR(reg)) {
518                         reg = get_fixed_regulator(rail);
519                         if (IS_ERR(reg)) {
520                                 pr_err("tegra_dvfs: failed to connect %s rail\n",
521                                        rail->reg_id);
522                                 return PTR_ERR(reg);
523                         }
524                 }
525                 rail->reg = reg;
526         }
527
528         v = regulator_enable(rail->reg);
529         if (v < 0) {
530                 pr_err("tegra_dvfs: failed on enabling regulator %s\n, err %d",
531                         rail->reg_id, v);
532                 return v;
533         }
534
535         v = regulator_get_voltage(rail->reg);
536         if (v < 0) {
537                 pr_err("tegra_dvfs: failed initial get %s voltage\n",
538                        rail->reg_id);
539                 return v;
540         }
541         rail->millivolts = v / 1000;
542         rail->new_millivolts = rail->millivolts;
543         dvfs_rail_stats_init(rail, rail->millivolts);
544
545         if (rail->boot_millivolts &&
546             (rail->boot_millivolts != rail->millivolts)) {
547                 WARN(1, "%s boot voltage %d does not match expected %d\n",
548                      rail->reg_id, rail->millivolts, rail->boot_millivolts);
549                 rail->boot_millivolts = rail->millivolts;
550         }
551         return 0;
552 }
553
554 static inline unsigned long *dvfs_get_freqs(struct dvfs *d)
555 {
556         return d->alt_freqs ? : &d->freqs[0];
557 }
558
559 static inline const int *dvfs_get_millivolts(struct dvfs *d, unsigned long rate)
560 {
561         if (tegra_dvfs_is_dfll_scale(d, rate))
562                 return d->dfll_millivolts;
563
564         return d->millivolts;
565 }
566
567 static int
568 __tegra_dvfs_set_rate(struct dvfs *d, unsigned long rate)
569 {
570         int i = 0;
571         int ret, mv, detach_mv;
572         unsigned long *freqs = dvfs_get_freqs(d);
573         const int *millivolts = dvfs_get_millivolts(d, rate);
574
575         if (freqs == NULL || millivolts == NULL)
576                 return -ENODEV;
577
578         /* On entry to dfll range limit 1st step to range bottom (full ramp of
579            voltage/rate is completed automatically in dfll mode) */
580         if (tegra_dvfs_is_dfll_range_entry(d, rate))
581                 rate = d->dfll_data.use_dfll_rate_min;
582
583         if (rate > freqs[d->num_freqs - 1]) {
584                 pr_warn("tegra_dvfs: rate %lu too high for dvfs on %s\n", rate,
585                         d->clk_name);
586                 return -EINVAL;
587         }
588
589         if (rate == 0) {
590                 d->cur_millivolts = 0;
591         } else {
592                 while (i < d->num_freqs && rate > freqs[i])
593                         i++;
594
595                 if ((d->max_millivolts) &&
596                     (millivolts[i] > d->max_millivolts)) {
597                         pr_warn("tegra_dvfs: voltage %d too high for dvfs on"
598                                 " %s\n", millivolts[i], d->clk_name);
599                         return -EINVAL;
600                 }
601
602                 mv = millivolts[i];
603                 detach_mv = tegra_dvfs_rail_get_boot_level(d->dvfs_rail);
604                 if (!d->dvfs_rail->reg && (mv > detach_mv)) {
605                         pr_warn("%s: %s: voltage %d above boot limit %d\n",
606                                 __func__, d->clk_name, mv, detach_mv);
607                         return -EINVAL;
608                 }
609
610                 detach_mv = tegra_dvfs_rail_get_disable_level(d->dvfs_rail);
611                 if (d->dvfs_rail->disabled && (mv > detach_mv)) {
612                         pr_warn("%s: %s: voltage %d above disable limit %d\n",
613                                 __func__, d->clk_name, mv, detach_mv);
614                         return -EINVAL;
615                 }
616
617                 detach_mv = tegra_dvfs_rail_get_suspend_level(d->dvfs_rail);
618                 if (d->dvfs_rail->suspended && (mv > detach_mv)) {
619                         pr_warn("%s: %s: voltage %d above disable limit %d\n",
620                                 __func__, d->clk_name, mv, detach_mv);
621                         return -EINVAL;
622                 }
623                 d->cur_millivolts = millivolts[i];
624         }
625
626         d->cur_rate = rate;
627
628         d->dvfs_rail->rate_set = true;
629         ret = dvfs_rail_update(d->dvfs_rail);
630         if (ret)
631                 pr_err("Failed to set regulator %s for clock %s to %d mV\n",
632                         d->dvfs_rail->reg_id, d->clk_name, d->cur_millivolts);
633
634         return ret;
635 }
636
637 int tegra_dvfs_alt_freqs_set(struct dvfs *d, unsigned long *alt_freqs)
638 {
639         int ret = 0;
640
641         mutex_lock(&dvfs_lock);
642
643         if (d->alt_freqs != alt_freqs) {
644                 d->alt_freqs = alt_freqs;
645                 ret = __tegra_dvfs_set_rate(d, d->cur_rate);
646         }
647
648         mutex_unlock(&dvfs_lock);
649         return ret;
650 }
651
652 static int predict_millivolts(struct clk *c, const int *millivolts,
653                               unsigned long rate)
654 {
655         int i;
656
657         if (!millivolts)
658                 return -ENODEV;
659         /*
660          * Predicted voltage can not be used across the switch to alternative
661          * frequency limits. For now, just fail the call for clock that has
662          * alternative limits initialized.
663          */
664         if (c->dvfs->alt_freqs)
665                 return -ENOSYS;
666
667         for (i = 0; i < c->dvfs->num_freqs; i++) {
668                 if (rate <= c->dvfs->freqs[i])
669                         break;
670         }
671
672         if (i == c->dvfs->num_freqs)
673                 return -EINVAL;
674
675         return millivolts[i];
676 }
677
678 int tegra_dvfs_predict_millivolts(struct clk *c, unsigned long rate)
679 {
680         const int *millivolts;
681
682         if (!rate || !c->dvfs)
683                 return 0;
684
685         millivolts = dvfs_get_millivolts(c->dvfs, rate);
686         return predict_millivolts(c, millivolts, rate);
687 }
688
689 int tegra_dvfs_predict_millivolts_pll(struct clk *c, unsigned long rate)
690 {
691         const int *millivolts;
692
693         if (!rate || !c->dvfs)
694                 return 0;
695
696         millivolts = c->dvfs->millivolts;
697         return predict_millivolts(c, millivolts, rate);
698 }
699
700 int tegra_dvfs_predict_millivolts_dfll(struct clk *c, unsigned long rate)
701 {
702         const int *millivolts;
703
704         if (!rate || !c->dvfs)
705                 return 0;
706
707         millivolts = c->dvfs->dfll_millivolts;
708         return predict_millivolts(c, millivolts, rate);
709 }
710
711 int tegra_dvfs_set_rate(struct clk *c, unsigned long rate)
712 {
713         int ret;
714
715         if (!c->dvfs)
716                 return -EINVAL;
717
718         mutex_lock(&dvfs_lock);
719         ret = __tegra_dvfs_set_rate(c->dvfs, rate);
720         mutex_unlock(&dvfs_lock);
721
722         return ret;
723 }
724 EXPORT_SYMBOL(tegra_dvfs_set_rate);
725
726 int tegra_dvfs_get_freqs(struct clk *c, unsigned long **freqs, int *num_freqs)
727 {
728         if (!c->dvfs)
729                 return -ENOSYS;
730
731         if (c->dvfs->alt_freqs)
732                 return -ENOSYS;
733
734         *num_freqs = c->dvfs->num_freqs;
735         *freqs = c->dvfs->freqs;
736
737         return 0;
738 }
739 EXPORT_SYMBOL(tegra_dvfs_get_freqs);
740
741 #ifdef CONFIG_TEGRA_VDD_CORE_OVERRIDE
742 static DEFINE_MUTEX(rail_override_lock);
743
744 static int dvfs_override_core_voltage(int override_mv)
745 {
746         int ret, floor, ceiling;
747         struct dvfs_rail *rail = tegra_core_rail;
748
749         if (!rail)
750                 return -ENOENT;
751
752         if (rail->fixed_millivolts)
753                 return -ENOSYS;
754
755         floor = rail->min_override_millivolts;
756         ceiling = rail->nominal_millivolts;
757         if (override_mv && ((override_mv < floor) || (override_mv > ceiling))) {
758                 pr_err("%s: override level %d outside the range [%d...%d]\n",
759                        __func__, override_mv, floor, ceiling);
760                 return -EINVAL;
761         }
762
763         mutex_lock(&rail_override_lock);
764
765         if (override_mv == rail->override_millivolts) {
766                 ret = 0;
767                 goto out;
768         }
769
770         if (override_mv) {
771                 ret = tegra_dvfs_core_cap_level_apply(override_mv);
772                 if (ret) {
773                         pr_err("%s: failed to set cap for override level %d\n",
774                                __func__, override_mv);
775                         goto out;
776                 }
777         }
778
779         mutex_lock(&dvfs_lock);
780         if (rail->disabled || rail->suspended) {
781                 pr_err("%s: cannot scale %s rail\n", __func__,
782                        rail->disabled ? "disabled" : "suspended");
783                 ret = -EPERM;
784                 if (!override_mv) {
785                         mutex_unlock(&dvfs_lock);
786                         goto out;
787                 }
788         } else {
789                 rail->override_millivolts = override_mv;
790                 ret = dvfs_rail_update(rail);
791                 if (ret) {
792                         pr_err("%s: failed to set override level %d\n",
793                                __func__, override_mv);
794                         rail->override_millivolts = 0;
795                         dvfs_rail_update(rail);
796                 }
797         }
798         mutex_unlock(&dvfs_lock);
799
800         if (!override_mv || ret)
801                 tegra_dvfs_core_cap_level_apply(0);
802 out:
803         mutex_unlock(&rail_override_lock);
804         return ret;
805 }
806 #else
807 static int dvfs_override_core_voltage(int override_mv)
808 {
809         pr_err("%s: vdd core override is not supported\n", __func__);
810         return -ENOSYS;
811 }
812 #endif
813
814 int tegra_dvfs_override_core_voltage(struct clk *c, int override_mv)
815 {
816         if (!c->dvfs || !c->dvfs->can_override) {
817                 pr_err("%s: %s cannot override vdd core\n", __func__, c->name);
818                 return -EPERM;
819         }
820         return dvfs_override_core_voltage(override_mv);
821 }
822 EXPORT_SYMBOL(tegra_dvfs_override_core_voltage);
823
824 /* May only be called during clock init, does not take any locks on clock c. */
825 int __init tegra_enable_dvfs_on_clk(struct clk *c, struct dvfs *d)
826 {
827         int i;
828
829         if (c->dvfs) {
830                 pr_err("Error when enabling dvfs on %s for clock %s:\n",
831                         d->dvfs_rail->reg_id, c->name);
832                 pr_err("DVFS already enabled for %s\n",
833                         c->dvfs->dvfs_rail->reg_id);
834                 return -EINVAL;
835         }
836
837         for (i = 0; i < MAX_DVFS_FREQS; i++) {
838                 if (d->millivolts[i] == 0)
839                         break;
840
841                 d->freqs[i] *= d->freqs_mult;
842
843                 /* If final frequencies are 0, pad with previous frequency */
844                 if (d->freqs[i] == 0 && i > 1)
845                         d->freqs[i] = d->freqs[i - 1];
846         }
847         d->num_freqs = i;
848
849         if (d->auto_dvfs) {
850                 c->auto_dvfs = true;
851                 clk_set_cansleep(c);
852         }
853
854         c->dvfs = d;
855
856         /*
857          * Minimum core override level is determined as maximum voltage required
858          * for clocks outside shared buses (shared bus rates can be capped to
859          * safe levels when override limit is set)
860          */
861         if (i && c->ops && !c->ops->shared_bus_update &&
862             !(c->flags & PERIPH_ON_CBUS) && !d->can_override) {
863                 int mv = tegra_dvfs_predict_millivolts(c, d->freqs[i-1]);
864                 if (d->dvfs_rail->min_override_millivolts < mv)
865                         d->dvfs_rail->min_override_millivolts = mv;
866         }
867
868         mutex_lock(&dvfs_lock);
869         list_add_tail(&d->reg_node, &d->dvfs_rail->dvfs);
870         mutex_unlock(&dvfs_lock);
871
872         return 0;
873 }
874
875 static bool tegra_dvfs_all_rails_suspended(void)
876 {
877         struct dvfs_rail *rail;
878         bool all_suspended = true;
879
880         list_for_each_entry(rail, &dvfs_rail_list, node)
881                 if (!rail->suspended && !rail->disabled)
882                         all_suspended = false;
883
884         return all_suspended;
885 }
886
887 static bool tegra_dvfs_from_rails_suspended_or_solved(struct dvfs_rail *to)
888 {
889         struct dvfs_relationship *rel;
890         bool all_suspended = true;
891
892         list_for_each_entry(rel, &to->relationships_from, from_node)
893                 if (!rel->from->suspended && !rel->from->disabled &&
894                         !rel->solved_at_nominal)
895                         all_suspended = false;
896
897         return all_suspended;
898 }
899
900 static int tegra_dvfs_suspend_one(void)
901 {
902         struct dvfs_rail *rail;
903         int ret, mv;
904
905         list_for_each_entry(rail, &dvfs_rail_list, node) {
906                 if (!rail->suspended && !rail->disabled &&
907                     tegra_dvfs_from_rails_suspended_or_solved(rail)) {
908                         /* Safe, as pll mode rate is capped to fixed level */
909                         if (!rail->dfll_mode && rail->fixed_millivolts) {
910                                 mv = rail->fixed_millivolts;
911                         } else {
912                                 mv = tegra_dvfs_rail_get_suspend_level(rail);
913                                 mv = dvfs_rail_apply_limits(rail, mv);
914                         }
915
916                         /* apply suspend limit only if it is above current mv */
917                         ret = -EPERM;
918                         if (mv >= rail->millivolts)
919                                 ret = dvfs_rail_set_voltage(rail, mv);
920                         if (ret) {
921                                 pr_err("tegra_dvfs: failed %s suspend at %d\n",
922                                        rail->reg_id, rail->millivolts);
923                                 return ret;
924                         }
925
926                         rail->suspended = true;
927                         return 0;
928                 }
929         }
930
931         return -EINVAL;
932 }
933
934 static void tegra_dvfs_resume(void)
935 {
936         struct dvfs_rail *rail;
937
938         mutex_lock(&dvfs_lock);
939
940         list_for_each_entry(rail, &dvfs_rail_list, node)
941                 rail->suspended = false;
942
943         list_for_each_entry(rail, &dvfs_rail_list, node)
944                 dvfs_rail_update(rail);
945
946         mutex_unlock(&dvfs_lock);
947 }
948
949 static int tegra_dvfs_suspend(void)
950 {
951         int ret = 0;
952
953         mutex_lock(&dvfs_lock);
954
955         while (!tegra_dvfs_all_rails_suspended()) {
956                 ret = tegra_dvfs_suspend_one();
957                 if (ret)
958                         break;
959         }
960
961         mutex_unlock(&dvfs_lock);
962
963         if (ret)
964                 tegra_dvfs_resume();
965
966         return ret;
967 }
968
969 static int tegra_dvfs_pm_suspend(struct notifier_block *nb,
970                                  unsigned long event, void *data)
971 {
972         if (event == PM_SUSPEND_PREPARE) {
973                 if (tegra_dvfs_suspend())
974                         return NOTIFY_STOP;
975                 pr_info("tegra_dvfs: suspended\n");
976         }
977         return NOTIFY_OK;
978 };
979
980 static int tegra_dvfs_pm_resume(struct notifier_block *nb,
981                                 unsigned long event, void *data)
982 {
983         if (event == PM_POST_SUSPEND) {
984                 tegra_dvfs_resume();
985                 pr_info("tegra_dvfs: resumed\n");
986         }
987         return NOTIFY_OK;
988 };
989
990 static struct notifier_block tegra_dvfs_suspend_nb = {
991         .notifier_call = tegra_dvfs_pm_suspend,
992         .priority = -1,
993 };
994
995 static struct notifier_block tegra_dvfs_resume_nb = {
996         .notifier_call = tegra_dvfs_pm_resume,
997         .priority = 1,
998 };
999
1000 static int tegra_dvfs_reboot_notify(struct notifier_block *nb,
1001                                 unsigned long event, void *data)
1002 {
1003         switch (event) {
1004         case SYS_RESTART:
1005         case SYS_HALT:
1006         case SYS_POWER_OFF:
1007                 tegra_dvfs_suspend();
1008                 return NOTIFY_OK;
1009         }
1010         return NOTIFY_DONE;
1011 }
1012
1013 static struct notifier_block tegra_dvfs_reboot_nb = {
1014         .notifier_call = tegra_dvfs_reboot_notify,
1015 };
1016
1017 /* must be called with dvfs lock held */
1018 static void __tegra_dvfs_rail_disable(struct dvfs_rail *rail)
1019 {
1020         int ret = -EPERM;
1021         int mv;
1022
1023         /* don't set voltage in DFLL mode - won't work, but break stats */
1024         if (rail->dfll_mode) {
1025                 rail->disabled = true;
1026                 return;
1027         }
1028
1029         /* Safe, as pll mode rate is capped to fixed level */
1030         if (!rail->dfll_mode && rail->fixed_millivolts) {
1031                 mv = rail->fixed_millivolts;
1032         } else {
1033                 mv = tegra_dvfs_rail_get_disable_level(rail);
1034                 mv = dvfs_rail_apply_limits(rail, mv);
1035         }
1036
1037         /* apply detach mode limit provided it is above current volatge */
1038         if (mv >= rail->millivolts)
1039                 ret = dvfs_rail_set_voltage(rail, mv);
1040         if (ret) {
1041                 pr_err("tegra_dvfs: failed to disable %s at %d\n",
1042                        rail->reg_id, rail->millivolts);
1043                 return;
1044         }
1045         rail->disabled = true;
1046 }
1047
1048 /* must be called with dvfs lock held */
1049 static void __tegra_dvfs_rail_enable(struct dvfs_rail *rail)
1050 {
1051         rail->disabled = false;
1052         dvfs_rail_update(rail);
1053 }
1054
1055 void tegra_dvfs_rail_enable(struct dvfs_rail *rail)
1056 {
1057         if (!rail)
1058                 return;
1059
1060         mutex_lock(&rail_disable_lock);
1061
1062         if (rail->disabled) {
1063                 mutex_lock(&dvfs_lock);
1064                 __tegra_dvfs_rail_enable(rail);
1065                 mutex_unlock(&dvfs_lock);
1066
1067                 tegra_dvfs_rail_post_enable(rail);
1068         }
1069         mutex_unlock(&rail_disable_lock);
1070 }
1071
1072 void tegra_dvfs_rail_disable(struct dvfs_rail *rail)
1073 {
1074         if (!rail)
1075                 return;
1076
1077         mutex_lock(&rail_disable_lock);
1078         if (rail->disabled)
1079                 goto out;
1080
1081         /* rail disable will set it to nominal voltage underneath clock
1082            framework - need to re-configure clock rates that are not safe
1083            at nominal (yes, unsafe at nominal is ugly, but possible). Rate
1084            change must be done outside of dvfs lock. */
1085         if (tegra_dvfs_rail_disable_prepare(rail)) {
1086                 pr_info("dvfs: failed to prepare regulator %s to disable\n",
1087                         rail->reg_id);
1088                 goto out;
1089         }
1090
1091         mutex_lock(&dvfs_lock);
1092         __tegra_dvfs_rail_disable(rail);
1093         mutex_unlock(&dvfs_lock);
1094 out:
1095         mutex_unlock(&rail_disable_lock);
1096 }
1097
1098 int tegra_dvfs_rail_disable_by_name(const char *reg_id)
1099 {
1100         struct dvfs_rail *rail = tegra_dvfs_get_rail_by_name(reg_id);
1101         if (!rail)
1102                 return -EINVAL;
1103
1104         tegra_dvfs_rail_disable(rail);
1105         return 0;
1106 }
1107
1108 struct dvfs_rail *tegra_dvfs_get_rail_by_name(const char *reg_id)
1109 {
1110         struct dvfs_rail *rail;
1111
1112         mutex_lock(&dvfs_lock);
1113         list_for_each_entry(rail, &dvfs_rail_list, node) {
1114                 if (!strcmp(reg_id, rail->reg_id)) {
1115                         mutex_unlock(&dvfs_lock);
1116                         return rail;
1117                 }
1118         }
1119         mutex_unlock(&dvfs_lock);
1120         return NULL;
1121 }
1122
1123 int tegra_dvfs_rail_power_up(struct dvfs_rail *rail)
1124 {
1125         int ret = -ENOENT;
1126
1127         if (!rail || !rail->in_band_pm)
1128                 return -ENOSYS;
1129
1130         mutex_lock(&dvfs_lock);
1131         if (rail->reg) {
1132                 ret = regulator_enable(rail->reg);
1133                 if (!ret && !timekeeping_suspended)
1134                         tegra_dvfs_rail_on(rail, ktime_get());
1135         }
1136         mutex_unlock(&dvfs_lock);
1137         return ret;
1138 }
1139
1140 int tegra_dvfs_rail_power_down(struct dvfs_rail *rail)
1141 {
1142         int ret = -ENOENT;
1143
1144         if (!rail || !rail->in_band_pm)
1145                 return -ENOSYS;
1146
1147         mutex_lock(&dvfs_lock);
1148         if (rail->reg) {
1149                 ret = regulator_disable(rail->reg);
1150                 if (!ret && !timekeeping_suspended)
1151                         tegra_dvfs_rail_off(rail, ktime_get());
1152         }
1153         mutex_unlock(&dvfs_lock);
1154         return ret;
1155 }
1156
1157 bool tegra_dvfs_is_rail_up(struct dvfs_rail *rail)
1158 {
1159         bool ret = false;
1160
1161         if (!rail)
1162                 return false;
1163
1164         if (!rail->in_band_pm)
1165                 return true;
1166
1167         mutex_lock(&dvfs_lock);
1168         if (rail->reg)
1169                 ret = regulator_is_enabled(rail->reg) > 0;
1170         mutex_unlock(&dvfs_lock);
1171         return ret;
1172 }
1173
1174 bool tegra_dvfs_rail_updating(struct clk *clk)
1175 {
1176         return (!clk ? false :
1177                 (!clk->dvfs ? false :
1178                  (!clk->dvfs->dvfs_rail ? false :
1179                   (clk->dvfs->dvfs_rail->updating ||
1180                    clk->dvfs->dvfs_rail->dfll_mode_updating))));
1181 }
1182
1183 #ifdef CONFIG_OF
1184 int __init of_tegra_dvfs_init(const struct of_device_id *matches)
1185 {
1186         int ret;
1187         struct device_node *np;
1188
1189         for_each_matching_node(np, matches) {
1190                 const struct of_device_id *match = of_match_node(matches, np);
1191                 of_tegra_dvfs_init_cb_t dvfs_init_cb = match->data;
1192                 ret = dvfs_init_cb(np);
1193                 if (ret) {
1194                         pr_err("dt: Failed to read %s tables from DT\n",
1195                                                         match->compatible);
1196                         return ret;
1197                 }
1198         }
1199         return 0;
1200 }
1201 #endif
1202 int tegra_dvfs_dfll_mode_set(struct dvfs *d, unsigned long rate)
1203 {
1204         mutex_lock(&dvfs_lock);
1205         if (!d->dvfs_rail->dfll_mode) {
1206                 d->dvfs_rail->dfll_mode = true;
1207                 __tegra_dvfs_set_rate(d, rate);
1208         }
1209         mutex_unlock(&dvfs_lock);
1210         return 0;
1211 }
1212
1213 int tegra_dvfs_dfll_mode_clear(struct dvfs *d, unsigned long rate)
1214 {
1215         int ret = 0;
1216
1217         mutex_lock(&dvfs_lock);
1218         if (d->dvfs_rail->dfll_mode) {
1219                 d->dvfs_rail->dfll_mode = false;
1220                 /* avoid false detection of matching target (voltage in dfll
1221                    mode is fluctuating, and recorded level is just estimate) */
1222                 d->dvfs_rail->millivolts--;
1223                 if (d->dvfs_rail->disabled) {
1224                         d->dvfs_rail->disabled = false;
1225                         __tegra_dvfs_rail_disable(d->dvfs_rail);
1226                 }
1227                 ret = __tegra_dvfs_set_rate(d, rate);
1228         }
1229         mutex_unlock(&dvfs_lock);
1230         return ret;
1231 }
1232
1233 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmax_cdev(void)
1234 {
1235         if (tegra_cpu_rail)
1236                 return tegra_cpu_rail->vmax_cdev;
1237         return NULL;
1238 }
1239
1240 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmin_cdev(void)
1241 {
1242         if (tegra_cpu_rail)
1243                 return tegra_cpu_rail->vmin_cdev;
1244         return NULL;
1245 }
1246
1247 struct tegra_cooling_device *tegra_dvfs_get_core_vmin_cdev(void)
1248 {
1249         if (tegra_core_rail)
1250                 return tegra_core_rail->vmin_cdev;
1251         return NULL;
1252 }
1253
1254 struct tegra_cooling_device *tegra_dvfs_get_gpu_vmin_cdev(void)
1255 {
1256         if (tegra_gpu_rail)
1257                 return tegra_gpu_rail->vmin_cdev;
1258         return NULL;
1259 }
1260
1261 #ifdef CONFIG_THERMAL
1262 /* Cooling device limits minimum rail voltage at cold temperature in pll mode */
1263 static int tegra_dvfs_rail_get_vmin_cdev_max_state(
1264         struct thermal_cooling_device *cdev, unsigned long *max_state)
1265 {
1266         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1267         *max_state = rail->vmin_cdev->trip_temperatures_num;
1268         return 0;
1269 }
1270
1271 static int tegra_dvfs_rail_get_vmin_cdev_cur_state(
1272         struct thermal_cooling_device *cdev, unsigned long *cur_state)
1273 {
1274         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1275         *cur_state = rail->therm_floor_idx;
1276         return 0;
1277 }
1278
1279 static int tegra_dvfs_rail_set_vmin_cdev_state(
1280         struct thermal_cooling_device *cdev, unsigned long cur_state)
1281 {
1282         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1283
1284         mutex_lock(&dvfs_lock);
1285         if (rail->therm_floor_idx != cur_state) {
1286                 rail->therm_floor_idx = cur_state;
1287                 dvfs_rail_update(rail);
1288         }
1289         mutex_unlock(&dvfs_lock);
1290         return 0;
1291 }
1292
1293 static struct thermal_cooling_device_ops tegra_dvfs_vmin_cooling_ops = {
1294         .get_max_state = tegra_dvfs_rail_get_vmin_cdev_max_state,
1295         .get_cur_state = tegra_dvfs_rail_get_vmin_cdev_cur_state,
1296         .set_cur_state = tegra_dvfs_rail_set_vmin_cdev_state,
1297 };
1298
1299 static void tegra_dvfs_rail_register_vmin_cdev(struct dvfs_rail *rail)
1300 {
1301         if (!rail->vmin_cdev)
1302                 return;
1303
1304         /* just report error - initialized for cold temperature, anyway */
1305         if (IS_ERR_OR_NULL(thermal_cooling_device_register(
1306                 rail->vmin_cdev->cdev_type, (void *)rail,
1307                 &tegra_dvfs_vmin_cooling_ops)))
1308                 pr_err("tegra cooling device %s failed to register\n",
1309                        rail->vmin_cdev->cdev_type);
1310 }
1311
1312 /* Cooling device to scale voltage with temperature in pll mode */
1313 static int tegra_dvfs_rail_get_vts_cdev_max_state(
1314         struct thermal_cooling_device *cdev, unsigned long *max_state)
1315 {
1316         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1317         *max_state = rail->vts_cdev->trip_temperatures_num;
1318         return 0;
1319 }
1320
1321 static int tegra_dvfs_rail_get_vts_cdev_cur_state(
1322         struct thermal_cooling_device *cdev, unsigned long *cur_state)
1323 {
1324         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1325         *cur_state = rail->therm_scale_idx;
1326         return 0;
1327 }
1328
1329 static int tegra_dvfs_rail_set_vts_cdev_state(
1330         struct thermal_cooling_device *cdev, unsigned long cur_state)
1331 {
1332         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
1333         struct dvfs *d;
1334
1335         mutex_lock(&dvfs_lock);
1336         if (rail->therm_scale_idx != cur_state) {
1337                 rail->therm_scale_idx = cur_state;
1338                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1339                         if (d->therm_dvfs)
1340                                 __tegra_dvfs_set_rate(d, d->cur_rate);
1341                 }
1342         }
1343         mutex_unlock(&dvfs_lock);
1344         return 0;
1345 }
1346
1347 static struct thermal_cooling_device_ops tegra_dvfs_vts_cooling_ops = {
1348         .get_max_state = tegra_dvfs_rail_get_vts_cdev_max_state,
1349         .get_cur_state = tegra_dvfs_rail_get_vts_cdev_cur_state,
1350         .set_cur_state = tegra_dvfs_rail_set_vts_cdev_state,
1351 };
1352
1353 static void tegra_dvfs_rail_register_vts_cdev(struct dvfs_rail *rail)
1354 {
1355         if (!rail->vts_cdev)
1356                 return;
1357
1358         /* just report error - initialized for cold temperature, anyway */
1359         if (IS_ERR_OR_NULL(thermal_cooling_device_register(
1360                 rail->vts_cdev->cdev_type, (void *)rail,
1361                 &tegra_dvfs_vts_cooling_ops)))
1362                 pr_err("tegra cooling device %s failed to register\n",
1363                        rail->vts_cdev->cdev_type);
1364 }
1365
1366 #else
1367 #define tegra_dvfs_rail_register_vmin_cdev(rail)
1368 #define tegra_dvfs_rail_register_vts_cdev(rail)
1369 #endif
1370
1371 /*
1372  * Validate rail thermal profile, and get its size. Valid profile:
1373  * - voltage limits are descending with temperature increasing
1374  * - the lowest limit is above rail minimum voltage in pll and
1375  *   in dfll mode (if applicable)
1376  * - the highest limit is below rail nominal voltage
1377  */
1378 static int __init get_thermal_profile_size(
1379         int *trips_table, int *limits_table,
1380         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1381 {
1382         int i, min_mv;
1383
1384         for (i = 0; i < MAX_THERMAL_LIMITS - 1; i++) {
1385                 if (!limits_table[i+1])
1386                         break;
1387
1388                 if ((trips_table[i] >= trips_table[i+1]) ||
1389                     (limits_table[i] < limits_table[i+1])) {
1390                         pr_warn("%s: not ordered profile\n", rail->reg_id);
1391                         return -EINVAL;
1392                 }
1393         }
1394
1395         min_mv = max(rail->min_millivolts, d ? d->min_millivolts : 0);
1396         if (limits_table[i] < min_mv) {
1397                 pr_warn("%s: thermal profile below Vmin\n", rail->reg_id);
1398                 return -EINVAL;
1399         }
1400
1401         if (limits_table[0] > rail->nominal_millivolts) {
1402                 pr_warn("%s: thermal profile above Vmax\n", rail->reg_id);
1403                 return -EINVAL;
1404         }
1405         return i + 1;
1406 }
1407
1408 void __init tegra_dvfs_rail_init_vmax_thermal_profile(
1409         int *therm_trips_table, int *therm_caps_table,
1410         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1411 {
1412         int i = get_thermal_profile_size(therm_trips_table,
1413                                          therm_caps_table, rail, d);
1414         if (i <= 0) {
1415                 rail->vmax_cdev = NULL;
1416                 WARN(1, "%s: invalid Vmax thermal profile\n", rail->reg_id);
1417                 return;
1418         }
1419
1420         /* Install validated thermal caps */
1421         rail->therm_mv_caps = therm_caps_table;
1422         rail->therm_mv_caps_num = i;
1423
1424         /* Setup trip-points if applicable */
1425         if (rail->vmax_cdev) {
1426                 rail->vmax_cdev->trip_temperatures_num = i;
1427                 rail->vmax_cdev->trip_temperatures = therm_trips_table;
1428         }
1429 }
1430
1431 void __init tegra_dvfs_rail_init_vmin_thermal_profile(
1432         int *therm_trips_table, int *therm_floors_table,
1433         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
1434 {
1435         int i = get_thermal_profile_size(therm_trips_table,
1436                                          therm_floors_table, rail, d);
1437         if (i <= 0) {
1438                 rail->vmin_cdev = NULL;
1439                 WARN(1, "%s: invalid Vmin thermal profile\n", rail->reg_id);
1440                 return;
1441         }
1442
1443         /* Install validated thermal floors */
1444         rail->therm_mv_floors = therm_floors_table;
1445         rail->therm_mv_floors_num = i;
1446
1447         /* Setup trip-points if applicable */
1448         if (rail->vmin_cdev) {
1449                 rail->vmin_cdev->trip_temperatures_num = i;
1450                 rail->vmin_cdev->trip_temperatures = therm_trips_table;
1451         }
1452 }
1453
1454 /* Directly set cold temperature limit in dfll mode */
1455 int tegra_dvfs_rail_dfll_mode_set_cold(struct dvfs_rail *rail)
1456 {
1457         int ret = 0;
1458
1459         /* No thermal floors - nothing to do */
1460         if (!rail || !rail->therm_mv_floors)
1461                 return ret;
1462
1463         /*
1464          * Since cooling thresholds are the same in pll and dfll modes, pll mode
1465          * thermal index can be used to decide if cold limit should be set in
1466          * dfll mode.
1467          */
1468         mutex_lock(&dvfs_lock);
1469         if (rail->dfll_mode &&
1470             (rail->therm_floor_idx < rail->therm_mv_floors_num)) {
1471                         int mv = rail->therm_mv_floors[rail->therm_floor_idx];
1472                         ret = dvfs_rail_set_voltage_reg(rail, mv);
1473         }
1474         mutex_unlock(&dvfs_lock);
1475
1476         return ret;
1477 }
1478
1479 /*
1480  * Iterate through all the dvfs regulators, finding the regulator exported
1481  * by the regulator api for each one.  Must be called in late init, after
1482  * all the regulator api's regulators are initialized.
1483  */
1484 int __init tegra_dvfs_late_init(void)
1485 {
1486         bool connected = true;
1487         struct dvfs_rail *rail;
1488
1489         mutex_lock(&dvfs_lock);
1490
1491         list_for_each_entry(rail, &dvfs_rail_list, node)
1492                 if (dvfs_rail_connect_to_regulator(rail))
1493                         connected = false;
1494
1495         list_for_each_entry(rail, &dvfs_rail_list, node)
1496                 if (connected)
1497                         dvfs_rail_update(rail);
1498                 else
1499                         __tegra_dvfs_rail_disable(rail);
1500
1501         mutex_unlock(&dvfs_lock);
1502
1503         if (!connected && tegra_platform_is_silicon()) {
1504                 pr_warn("tegra_dvfs: DVFS regulators connection failed\n"
1505                         "            !!!! voltage scaling is disabled !!!!\n");
1506                 return -ENODEV;
1507         }
1508
1509         register_pm_notifier(&tegra_dvfs_suspend_nb);
1510         register_pm_notifier(&tegra_dvfs_resume_nb);
1511         register_reboot_notifier(&tegra_dvfs_reboot_nb);
1512
1513         list_for_each_entry(rail, &dvfs_rail_list, node) {
1514                         tegra_dvfs_rail_register_vmin_cdev(rail);
1515                         tegra_dvfs_rail_register_vts_cdev(rail);
1516         }
1517
1518         return 0;
1519 }
1520
1521 static int rail_stats_save_to_buf(char *buf, int len)
1522 {
1523         int i;
1524         struct dvfs_rail *rail;
1525         char *str = buf;
1526         char *end = buf + len;
1527
1528         str += scnprintf(str, end - str, "%-12s %-10s\n", "millivolts", "time");
1529
1530         mutex_lock(&dvfs_lock);
1531
1532         list_for_each_entry(rail, &dvfs_rail_list, node) {
1533                 str += scnprintf(str, end - str, "%s (bin: %d.%dmV)\n",
1534                            rail->reg_id,
1535                            rail->stats.bin_uV / 1000,
1536                            (rail->stats.bin_uV / 10) % 100);
1537
1538                 dvfs_rail_stats_update(rail, -1, ktime_get());
1539
1540                 str += scnprintf(str, end - str, "%-12d %-10llu\n", 0,
1541                         cputime64_to_clock_t(msecs_to_jiffies(
1542                                 ktime_to_ms(rail->stats.time_at_mv[0]))));
1543
1544                 for (i = 1; i <= DVFS_RAIL_STATS_TOP_BIN; i++) {
1545                         ktime_t ktime_zero = ktime_set(0, 0);
1546                         if (ktime_equal(rail->stats.time_at_mv[i], ktime_zero))
1547                                 continue;
1548                         str += scnprintf(str, end - str, "%-12d %-10llu\n",
1549                                 rail->min_millivolts +
1550                                 (i - 1) * rail->stats.bin_uV / 1000,
1551                                 cputime64_to_clock_t(msecs_to_jiffies(
1552                                         ktime_to_ms(rail->stats.time_at_mv[i])))
1553                         );
1554                 }
1555         }
1556         mutex_unlock(&dvfs_lock);
1557         return str - buf;
1558 }
1559
1560 #ifdef CONFIG_DEBUG_FS
1561 static int dvfs_tree_sort_cmp(void *p, struct list_head *a, struct list_head *b)
1562 {
1563         struct dvfs *da = list_entry(a, struct dvfs, reg_node);
1564         struct dvfs *db = list_entry(b, struct dvfs, reg_node);
1565         int ret;
1566
1567         ret = strcmp(da->dvfs_rail->reg_id, db->dvfs_rail->reg_id);
1568         if (ret != 0)
1569                 return ret;
1570
1571         if (da->cur_millivolts < db->cur_millivolts)
1572                 return 1;
1573         if (da->cur_millivolts > db->cur_millivolts)
1574                 return -1;
1575
1576         return strcmp(da->clk_name, db->clk_name);
1577 }
1578
1579 static int dvfs_tree_show(struct seq_file *s, void *data)
1580 {
1581         struct dvfs *d;
1582         struct dvfs_rail *rail;
1583         struct dvfs_relationship *rel;
1584
1585         seq_printf(s, "   clock      rate       mV\n");
1586         seq_printf(s, "--------------------------------\n");
1587
1588         mutex_lock(&dvfs_lock);
1589
1590         list_for_each_entry(rail, &dvfs_rail_list, node) {
1591                 int thermal_mv_floor = 0;
1592
1593                 seq_printf(s, "%s %d mV%s:\n", rail->reg_id,
1594                            rail->stats.off ? 0 : rail->millivolts,
1595                            rail->dfll_mode ? " dfll mode" :
1596                                 rail->disabled ? " disabled" : "");
1597                 list_for_each_entry(rel, &rail->relationships_from, from_node) {
1598                         seq_printf(s, "   %-10s %-7d mV %-4d mV\n",
1599                                 rel->from->reg_id, rel->from->millivolts,
1600                                 dvfs_solve_relationship(rel));
1601                 }
1602                 seq_printf(s, "   offset     %-7d mV\n", rail->dbg_mv_offs);
1603
1604                 if (rail->therm_mv_floors) {
1605                         int i = rail->therm_floor_idx;
1606                         if (i < rail->therm_mv_floors_num)
1607                                 thermal_mv_floor = rail->therm_mv_floors[i];
1608                 }
1609                 seq_printf(s, "   thermal    %-7d mV\n", thermal_mv_floor);
1610
1611                 if (rail == tegra_core_rail) {
1612                         seq_printf(s, "   override   %-7d mV [%-4d...%-4d]\n",
1613                                    rail->override_millivolts,
1614                                    rail->min_override_millivolts,
1615                                    rail->nominal_millivolts);
1616                 }
1617
1618                 list_sort(NULL, &rail->dvfs, dvfs_tree_sort_cmp);
1619
1620                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1621                         seq_printf(s, "   %-10s %-10lu %-4d mV\n", d->clk_name,
1622                                 d->cur_rate, d->cur_millivolts);
1623                 }
1624         }
1625
1626         mutex_unlock(&dvfs_lock);
1627
1628         return 0;
1629 }
1630
1631 static int dvfs_tree_open(struct inode *inode, struct file *file)
1632 {
1633         return single_open(file, dvfs_tree_show, inode->i_private);
1634 }
1635
1636 static const struct file_operations dvfs_tree_fops = {
1637         .open           = dvfs_tree_open,
1638         .read           = seq_read,
1639         .llseek         = seq_lseek,
1640         .release        = single_release,
1641 };
1642
1643 static int rail_stats_show(struct seq_file *s, void *data)
1644 {
1645         char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
1646         int size = 0;
1647
1648         if (!buf)
1649                 return -ENOMEM;
1650
1651         size = rail_stats_save_to_buf(buf, PAGE_SIZE);
1652         seq_write(s, buf, size);
1653         kfree(buf);
1654         return 0;
1655 }
1656
1657 static int rail_stats_open(struct inode *inode, struct file *file)
1658 {
1659         return single_open(file, rail_stats_show, inode->i_private);
1660 }
1661
1662 static const struct file_operations rail_stats_fops = {
1663         .open           = rail_stats_open,
1664         .read           = seq_read,
1665         .llseek         = seq_lseek,
1666         .release        = single_release,
1667 };
1668
1669 static int gpu_dvfs_show(struct seq_file *s, void *data)
1670 {
1671         int idx;
1672         int *millivolts;
1673         unsigned long *freqs;
1674
1675         if (read_gpu_dvfs_table(&millivolts, &freqs)) {
1676                 seq_printf(s, "Only supported for T124 or higher\n");
1677                 return 0;
1678         }
1679
1680         seq_printf(s, "millivolts \t \t frequency\n");
1681         seq_printf(s, "=====================================\n");
1682
1683         for (idx = 0; millivolts[idx]; idx++)
1684                 seq_printf(s, "%d mV \t \t %lu Hz\n", millivolts[idx],
1685                                 freqs[idx]);
1686
1687         return 0;
1688 }
1689
1690 static int gpu_dvfs_open(struct inode *inode, struct file *file)
1691 {
1692         return single_open(file, gpu_dvfs_show, NULL);
1693 }
1694
1695 static const struct file_operations gpu_dvfs_fops = {
1696         .open           = gpu_dvfs_open,
1697         .read           = seq_read,
1698         .llseek         = seq_lseek,
1699         .release        = single_release,
1700 };
1701
1702 static int rail_offs_set(struct dvfs_rail *rail, int offs)
1703 {
1704         if (rail) {
1705                 mutex_lock(&dvfs_lock);
1706                 rail->dbg_mv_offs = offs;
1707                 dvfs_rail_update(rail);
1708                 mutex_unlock(&dvfs_lock);
1709                 return 0;
1710         }
1711         return -ENOENT;
1712 }
1713
1714 static int cpu_offs_get(void *data, u64 *val)
1715 {
1716         if (tegra_cpu_rail) {
1717                 *val = (u64)tegra_cpu_rail->dbg_mv_offs;
1718                 return 0;
1719         }
1720         *val = 0;
1721         return -ENOENT;
1722 }
1723 static int cpu_offs_set(void *data, u64 val)
1724 {
1725         return rail_offs_set(tegra_cpu_rail, (int)val);
1726 }
1727 DEFINE_SIMPLE_ATTRIBUTE(cpu_offs_fops, cpu_offs_get, cpu_offs_set, "%lld\n");
1728
1729 static int gpu_offs_get(void *data, u64 *val)
1730 {
1731         if (tegra_gpu_rail) {
1732                 *val = (u64)tegra_gpu_rail->dbg_mv_offs;
1733                 return 0;
1734         }
1735         *val = 0;
1736         return -ENOENT;
1737 }
1738 static int gpu_offs_set(void *data, u64 val)
1739 {
1740         return rail_offs_set(tegra_gpu_rail, (int)val);
1741 }
1742 DEFINE_SIMPLE_ATTRIBUTE(gpu_offs_fops, gpu_offs_get, gpu_offs_set, "%lld\n");
1743
1744 static int core_offs_get(void *data, u64 *val)
1745 {
1746         if (tegra_core_rail) {
1747                 *val = (u64)tegra_core_rail->dbg_mv_offs;
1748                 return 0;
1749         }
1750         *val = 0;
1751         return -ENOENT;
1752 }
1753 static int core_offs_set(void *data, u64 val)
1754 {
1755         return rail_offs_set(tegra_core_rail, (int)val);
1756 }
1757 DEFINE_SIMPLE_ATTRIBUTE(core_offs_fops, core_offs_get, core_offs_set, "%lld\n");
1758
1759 static int core_override_get(void *data, u64 *val)
1760 {
1761         if (tegra_core_rail) {
1762                 *val = (u64)tegra_core_rail->override_millivolts;
1763                 return 0;
1764         }
1765         *val = 0;
1766         return -ENOENT;
1767 }
1768 static int core_override_set(void *data, u64 val)
1769 {
1770         return dvfs_override_core_voltage((int)val);
1771 }
1772 DEFINE_SIMPLE_ATTRIBUTE(core_override_fops,
1773                         core_override_get, core_override_set, "%llu\n");
1774
1775 static int dvfs_table_show(struct seq_file *s, void *data)
1776 {
1777         int i;
1778         struct dvfs *d;
1779         struct dvfs_rail *rail;
1780
1781         seq_printf(s, "DVFS tables: units mV/MHz\n\n");
1782
1783         mutex_lock(&dvfs_lock);
1784
1785         list_for_each_entry(rail, &dvfs_rail_list, node) {
1786                 bool mv_done = false;
1787                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1788                         if (!mv_done) {
1789                                 mv_done = true;
1790                                 seq_printf(s, "%-16s", rail->reg_id);
1791                                 for (i = 0; i < d->num_freqs; i++) {
1792                                         int mv = d->millivolts[i];
1793                                         seq_printf(s, "%7d", mv);
1794                                 }
1795                                 seq_printf(s, "\n");
1796                                 if (d->dfll_millivolts) {
1797                                         seq_printf(s, "%-8s (dfll) ",
1798                                                    rail->reg_id);
1799                                         for (i = 0; i < d->num_freqs; i++) {
1800                                                 int mv = d->dfll_millivolts[i];
1801                                                 seq_printf(s, "%7d", mv);
1802                                         }
1803                                         seq_printf(s, "\n");
1804                                 }
1805                         }
1806
1807                         seq_printf(s, "%-16s", d->clk_name);
1808                         for (i = 0; i < d->num_freqs; i++) {
1809                                 unsigned int f = d->freqs[i]/100000;
1810                                 seq_printf(s, " %4u.%u", f/10, f%10);
1811                         }
1812                         seq_printf(s, "\n");
1813                 }
1814                 seq_printf(s, "\n");
1815         }
1816
1817         mutex_unlock(&dvfs_lock);
1818
1819         return 0;
1820 }
1821
1822 static int dvfs_table_open(struct inode *inode, struct file *file)
1823 {
1824         return single_open(file, dvfs_table_show, inode->i_private);
1825 }
1826
1827 static const struct file_operations dvfs_table_fops = {
1828         .open           = dvfs_table_open,
1829         .read           = seq_read,
1830         .llseek         = seq_lseek,
1831         .release        = single_release,
1832 };
1833
1834 int __init dvfs_debugfs_init(struct dentry *clk_debugfs_root)
1835 {
1836         struct dentry *d;
1837
1838         d = debugfs_create_file("dvfs", S_IRUGO, clk_debugfs_root, NULL,
1839                 &dvfs_tree_fops);
1840         if (!d)
1841                 return -ENOMEM;
1842
1843         d = debugfs_create_file("rails", S_IRUGO, clk_debugfs_root, NULL,
1844                 &rail_stats_fops);
1845         if (!d)
1846                 return -ENOMEM;
1847
1848         d = debugfs_create_file("vdd_cpu_offs", S_IRUGO | S_IWUSR,
1849                 clk_debugfs_root, NULL, &cpu_offs_fops);
1850         if (!d)
1851                 return -ENOMEM;
1852
1853         d = debugfs_create_file("vdd_gpu_offs", S_IRUGO | S_IWUSR,
1854                 clk_debugfs_root, NULL, &gpu_offs_fops);
1855         if (!d)
1856                 return -ENOMEM;
1857
1858         d = debugfs_create_file("vdd_core_offs", S_IRUGO | S_IWUSR,
1859                 clk_debugfs_root, NULL, &core_offs_fops);
1860         if (!d)
1861                 return -ENOMEM;
1862
1863         d = debugfs_create_file("vdd_core_override", S_IRUGO | S_IWUSR,
1864                 clk_debugfs_root, NULL, &core_override_fops);
1865         if (!d)
1866                 return -ENOMEM;
1867
1868         d = debugfs_create_file("gpu_dvfs", S_IRUGO | S_IWUSR,
1869                 clk_debugfs_root, NULL, &gpu_dvfs_fops);
1870         if (!d)
1871                 return -ENOMEM;
1872
1873         d = debugfs_create_file("dvfs_table", S_IRUGO, clk_debugfs_root, NULL,
1874                 &dvfs_table_fops);
1875         if (!d)
1876                 return -ENOMEM;
1877
1878         return 0;
1879 }
1880
1881 #endif
1882
1883 #ifdef CONFIG_PM
1884 static ssize_t tegra_rail_stats_show(struct kobject *kobj,
1885                                         struct kobj_attribute *attr,
1886                                         char *buf)
1887 {
1888         return rail_stats_save_to_buf(buf, PAGE_SIZE);
1889 }
1890
1891 static struct kobj_attribute rail_stats_attr =
1892                 __ATTR_RO(tegra_rail_stats);
1893
1894 static int __init tegra_dvfs_sysfs_stats_init(void)
1895 {
1896         int error;
1897         error = sysfs_create_file(power_kobj, &rail_stats_attr.attr);
1898         return 0;
1899 }
1900 late_initcall(tegra_dvfs_sysfs_stats_init);
1901 #endif