dvfs: tegra21: Rename thermal safe maximum frequency
[linux-3.10.git] / drivers / platform / tegra / dvfs.c
1 /*
2  *
3  * Copyright (C) 2010 Google, Inc.
4  *
5  * Author:
6  *      Colin Cross <ccross@google.com>
7  *
8  * Copyright (C) 2010-2014 NVIDIA CORPORATION. All rights reserved.
9  *
10  * This software is licensed under the terms of the GNU General Public
11  * License version 2, as published by the Free Software Foundation, and
12  * may be copied, distributed, and modified under those terms.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/clk.h>
23 #include <linux/clkdev.h>
24 #include <linux/debugfs.h>
25 #include <linux/init.h>
26 #include <linux/list.h>
27 #include <linux/list_sort.h>
28 #include <linux/module.h>
29 #include <linux/regulator/consumer.h>
30 #include <linux/seq_file.h>
31 #include <linux/slab.h>
32 #include <linux/suspend.h>
33 #include <linux/delay.h>
34 #include <linux/clk/tegra.h>
35 #include <linux/reboot.h>
36 #include <linux/clk/tegra.h>
37 #include <linux/tegra-soc.h>
38 #include <linux/of_platform.h>
39
40 #include "board.h"
41 #include <linux/platform/tegra/clock.h>
42 #include <linux/platform/tegra/dvfs.h>
43 #include <linux/platform/tegra/tegra_cl_dvfs.h>
44
45 #define DVFS_RAIL_STATS_BIN     12500
46
47 struct dvfs_rail *tegra_cpu_rail;
48 struct dvfs_rail *tegra_core_rail;
49 struct dvfs_rail *tegra_gpu_rail;
50
51 static LIST_HEAD(dvfs_rail_list);
52 static DEFINE_MUTEX(dvfs_lock);
53 static DEFINE_MUTEX(rail_disable_lock);
54
55 static int dvfs_rail_update(struct dvfs_rail *rail);
56
57 static int tegra_dvfs_rail_get_disable_level(struct dvfs_rail *rail)
58 {
59         return rail->disable_millivolts ? : rail->nominal_millivolts;
60 }
61
62 static int tegra_dvfs_rail_get_suspend_level(struct dvfs_rail *rail)
63 {
64         return rail->suspend_millivolts ? : rail->nominal_millivolts;
65 }
66
67 int tegra_dvfs_get_core_override_floor(void)
68 {
69         return tegra_dvfs_rail_get_override_floor(tegra_core_rail);
70 }
71 EXPORT_SYMBOL(tegra_dvfs_get_core_override_floor);
72
73 int tegra_dvfs_get_core_nominal_millivolts(void)
74 {
75         return tegra_dvfs_rail_get_nominal_millivolts(tegra_core_rail);
76 }
77 EXPORT_SYMBOL(tegra_dvfs_get_core_nominal_millivolts);
78
79 int tegra_dvfs_get_core_boot_level(void)
80 {
81         return tegra_dvfs_rail_get_boot_level(tegra_core_rail);
82 }
83 EXPORT_SYMBOL(tegra_dvfs_get_core_boot_level);
84
85 unsigned long tegra_dvfs_get_fmax_at_vmin_safe_t(struct clk *c)
86 {
87         if (!c->dvfs)
88                 return 0;
89
90         return c->dvfs->fmax_at_vmin_safe_t;
91 }
92 EXPORT_SYMBOL(tegra_dvfs_get_fmax_at_vmin_safe_t);
93
94 void tegra_dvfs_add_relationships(struct dvfs_relationship *rels, int n)
95 {
96         int i;
97         struct dvfs_relationship *rel;
98
99         mutex_lock(&dvfs_lock);
100
101         for (i = 0; i < n; i++) {
102                 rel = &rels[i];
103                 list_add_tail(&rel->from_node, &rel->to->relationships_from);
104                 list_add_tail(&rel->to_node, &rel->from->relationships_to);
105
106                 /* Overriding dependent rail below nominal may not be safe */
107                 rel->to->min_override_millivolts = rel->to->nominal_millivolts;
108         }
109
110         mutex_unlock(&dvfs_lock);
111 }
112
113 /* Make sure there is a matching cooling device for thermal limit profile. */
114 static void dvfs_validate_cdevs(struct dvfs_rail *rail)
115 {
116         if (!rail->therm_mv_caps != !rail->therm_mv_caps_num) {
117                 rail->therm_mv_caps_num = 0;
118                 rail->therm_mv_caps = NULL;
119                 WARN(1, "%s: not matching thermal caps/num\n", rail->reg_id);
120         }
121
122         if (rail->therm_mv_caps && !rail->vmax_cdev)
123                 WARN(1, "%s: missing vmax cooling device\n", rail->reg_id);
124
125         if (!rail->therm_mv_floors != !rail->therm_mv_floors_num) {
126                 rail->therm_mv_floors_num = 0;
127                 rail->therm_mv_floors = NULL;
128                 WARN(1, "%s: not matching thermal floors/num\n", rail->reg_id);
129         }
130
131         if (rail->therm_mv_floors && !rail->vmin_cdev)
132                 WARN(1, "%s: missing vmin cooling device\n", rail->reg_id);
133
134         /* Limit override range to maximum floor */
135         if (rail->therm_mv_floors)
136                 rail->min_override_millivolts = rail->therm_mv_floors[0];
137
138         /* Only GPU thermal dvfs is supported */
139         if (rail->vts_cdev && (rail != tegra_gpu_rail)) {
140                 rail->vts_cdev = NULL;
141                 WARN(1, "%s: thermal dvfs is not supported\n", rail->reg_id);
142         }
143
144         /* Thermal clock switch is only supported for CPU */
145         if (rail->clk_switch_cdev && (rail != tegra_cpu_rail)) {
146                 rail->clk_switch_cdev = NULL;
147                  WARN(1, "%s: thermal clock switch is not supported\n",
148                                 rail->reg_id);
149         }
150
151         if (!rail->simon_vmin_offsets != !rail->simon_vmin_offs_num) {
152                 rail->simon_vmin_offs_num = 0;
153                 rail->simon_vmin_offsets = NULL;
154                 WARN(1, "%s: not matching simon offsets/num\n", rail->reg_id);
155         }
156 }
157
158 int tegra_dvfs_init_rails(struct dvfs_rail *rails[], int n)
159 {
160         int i, mv;
161
162         mutex_lock(&dvfs_lock);
163
164         for (i = 0; i < n; i++) {
165                 INIT_LIST_HEAD(&rails[i]->dvfs);
166                 INIT_LIST_HEAD(&rails[i]->relationships_from);
167                 INIT_LIST_HEAD(&rails[i]->relationships_to);
168
169                 mv = rails[i]->nominal_millivolts;
170                 if (rails[i]->boot_millivolts > mv)
171                         WARN(1, "%s: boot voltage %d above nominal %d\n",
172                              rails[i]->reg_id, rails[i]->boot_millivolts, mv);
173                 if (rails[i]->disable_millivolts > mv)
174                         rails[i]->disable_millivolts = mv;
175                 if (rails[i]->suspend_millivolts > mv)
176                         rails[i]->suspend_millivolts = mv;
177
178                 mv = tegra_dvfs_rail_get_boot_level(rails[i]);
179                 rails[i]->millivolts = mv;
180                 rails[i]->new_millivolts = mv;
181                 if (!rails[i]->step)
182                         rails[i]->step = rails[i]->max_millivolts;
183                 if (!rails[i]->step_up)
184                         rails[i]->step_up = rails[i]->step;
185
186                 list_add_tail(&rails[i]->node, &dvfs_rail_list);
187
188                 if (!strcmp("vdd_cpu", rails[i]->reg_id))
189                         tegra_cpu_rail = rails[i];
190                 else if (!strcmp("vdd_gpu", rails[i]->reg_id))
191                         tegra_gpu_rail = rails[i];
192                 else if (!strcmp("vdd_core", rails[i]->reg_id))
193                         tegra_core_rail = rails[i];
194
195                 dvfs_validate_cdevs(rails[i]);
196         }
197
198         mutex_unlock(&dvfs_lock);
199
200         return 0;
201 };
202
203 static int dvfs_solve_relationship(struct dvfs_relationship *rel)
204 {
205         return rel->solve(rel->from, rel->to);
206 }
207
208 /* rail statistic - called during rail init, or under dfs_lock, or with
209    CPU0 only on-line, and interrupts disabled */
210 static void dvfs_rail_stats_init(struct dvfs_rail *rail, int millivolts)
211 {
212         int dvfs_rail_stats_range;
213
214         if (!rail->stats.bin_uV)
215                 rail->stats.bin_uV = DVFS_RAIL_STATS_BIN;
216
217         dvfs_rail_stats_range =
218                 (DVFS_RAIL_STATS_TOP_BIN - 1) * rail->stats.bin_uV / 1000;
219
220         rail->stats.last_update = ktime_get();
221         if (millivolts >= rail->min_millivolts) {
222                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
223                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
224                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
225         }
226
227         if (rail->max_millivolts >
228             rail->min_millivolts + dvfs_rail_stats_range)
229                 pr_warn("tegra_dvfs: %s: stats above %d mV will be squashed\n",
230                         rail->reg_id,
231                         rail->min_millivolts + dvfs_rail_stats_range);
232 }
233
234 static void dvfs_rail_stats_update(
235         struct dvfs_rail *rail, int millivolts, ktime_t now)
236 {
237         rail->stats.time_at_mv[rail->stats.last_index] = ktime_add(
238                 rail->stats.time_at_mv[rail->stats.last_index], ktime_sub(
239                         now, rail->stats.last_update));
240         rail->stats.last_update = now;
241
242         if (rail->stats.off)
243                 return;
244
245         if (millivolts >= rail->min_millivolts) {
246                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
247                              rail->stats.bin_uV) / (2 * rail->stats.bin_uV);
248                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
249         } else if (millivolts == 0)
250                         rail->stats.last_index = 0;
251 }
252
253 static void dvfs_rail_stats_pause(struct dvfs_rail *rail,
254                                   ktime_t delta, bool on)
255 {
256         int i = on ? rail->stats.last_index : 0;
257         rail->stats.time_at_mv[i] = ktime_add(rail->stats.time_at_mv[i], delta);
258 }
259
260 void tegra_dvfs_rail_off(struct dvfs_rail *rail, ktime_t now)
261 {
262         if (rail) {
263                 dvfs_rail_stats_update(rail, 0, now);
264                 rail->stats.off = true;
265         }
266 }
267
268 void tegra_dvfs_rail_on(struct dvfs_rail *rail, ktime_t now)
269 {
270         if (rail) {
271                 rail->stats.off = false;
272                 dvfs_rail_stats_update(rail, rail->millivolts, now);
273         }
274 }
275
276 void tegra_dvfs_rail_pause(struct dvfs_rail *rail, ktime_t delta, bool on)
277 {
278         if (rail)
279                 dvfs_rail_stats_pause(rail, delta, on);
280 }
281
282 static int dvfs_rail_set_voltage_reg(struct dvfs_rail *rail, int millivolts)
283 {
284         int ret;
285
286         /*
287          * safely return success for low voltage requests on fixed regulator
288          * (higher requests will go through and fail, as they should)
289          */
290         if (rail->fixed_millivolts && (millivolts <= rail->fixed_millivolts))
291                 return 0;
292
293         rail->updating = true;
294         rail->reg_max_millivolts = rail->reg_max_millivolts ==
295                 rail->max_millivolts ?
296                 rail->max_millivolts + 1 : rail->max_millivolts;
297         ret = regulator_set_voltage(rail->reg,
298                 millivolts * 1000,
299                 rail->reg_max_millivolts * 1000);
300         rail->updating = false;
301
302         pr_debug("%s: request_mV [%d, %d] from %s regulator (%d)\n", __func__,
303                  millivolts, rail->reg_max_millivolts, rail->reg_id, ret);
304
305         return ret;
306 }
307
308 /* Sets the voltage on a dvfs rail to a specific value, and updates any
309  * rails that depend on this rail. */
310 static int dvfs_rail_set_voltage(struct dvfs_rail *rail, int millivolts)
311 {
312         int ret = 0;
313         struct dvfs_relationship *rel;
314         int step, offset;
315         int i;
316         int steps;
317         bool jmp_to_zero;
318
319         if (!rail->reg) {
320                 if (millivolts == rail->millivolts)
321                         return 0;
322                 else
323                         return -EINVAL;
324         }
325
326         if (millivolts > rail->millivolts) {
327                 step = rail->step_up;
328                 offset = step;
329         } else {
330                 step = rail->step;
331                 offset = -step;
332         }
333
334         /* Voltage change is always happening in DFLL mode */
335         if (rail->disabled && !rail->dfll_mode)
336                 return 0;
337
338         rail->resolving_to = true;
339         jmp_to_zero = rail->jmp_to_zero &&
340                         ((millivolts == 0) || (rail->millivolts == 0));
341
342         if (jmp_to_zero || rail->dfll_mode)
343                 steps = 1;
344         else
345                 steps = DIV_ROUND_UP(abs(millivolts - rail->millivolts), step);
346
347         for (i = 0; i < steps; i++) {
348                 if (steps > (i + 1))
349                         rail->new_millivolts = rail->millivolts + offset;
350                 else
351                         rail->new_millivolts = millivolts;
352
353                 /* Before changing the voltage, tell each rail that depends
354                  * on this rail that the voltage will change.
355                  * This rail will be the "from" rail in the relationship,
356                  * the rail that depends on this rail will be the "to" rail.
357                  * from->millivolts will be the old voltage
358                  * from->new_millivolts will be the new voltage */
359                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
360                         ret = dvfs_rail_update(rel->to);
361                         if (ret)
362                                 goto out;
363                 }
364
365                 /*
366                  * DFLL adjusts voltage automatically - don't touch regulator,
367                  * but update stats, anyway.
368                  */
369                 if (!rail->dfll_mode) {
370                         ret = dvfs_rail_set_voltage_reg(rail,
371                                                         rail->new_millivolts);
372                         if (ret) {
373                                 pr_err("Failed to set dvfs regulator %s\n",
374                                        rail->reg_id);
375                                 goto out;
376                         }
377                 }
378
379                 rail->millivolts = rail->new_millivolts;
380                 dvfs_rail_stats_update(rail, rail->millivolts, ktime_get());
381
382                 /* After changing the voltage, tell each rail that depends
383                  * on this rail that the voltage has changed.
384                  * from->millivolts and from->new_millivolts will be the
385                  * new voltage */
386                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
387                         ret = dvfs_rail_update(rel->to);
388                         if (ret)
389                                 goto out;
390                 }
391         }
392
393         if (unlikely(rail->millivolts != millivolts)) {
394                 pr_err("%s: rail didn't reach target %d in %d steps (%d)\n",
395                         __func__, millivolts, steps, rail->millivolts);
396                 ret = -EINVAL;
397         }
398
399 out:
400         rail->resolving_to = false;
401         return ret;
402 }
403
404 /* Determine the minimum valid voltage for a rail, taking into account
405  * the dvfs clocks and any rails that this rail depends on.  Calls
406  * dvfs_rail_set_voltage with the new voltage, which will call
407  * dvfs_rail_update on any rails that depend on this rail. */
408 static int dvfs_rail_apply_limits(struct dvfs_rail *rail, int millivolts)
409 {
410         int min_mv = rail->min_millivolts;
411         min_mv = max(min_mv, tegra_dvfs_rail_get_thermal_floor(rail));
412
413         if (rail->override_millivolts) {
414                 millivolts = rail->override_millivolts;
415         } else if (!rail->dfll_mode && rail->fixed_millivolts) {
416                 /* clip up to pll mode fixed mv */
417                 if (millivolts < rail->fixed_millivolts)
418                         millivolts = rail->fixed_millivolts;
419         } else if (rail->dbg_mv_offs) {
420                 /* apply offset and ignore minimum limit */
421                 millivolts += rail->dbg_mv_offs;
422                 return millivolts;
423         }
424
425         if (millivolts < min_mv)
426                 millivolts = min_mv;
427
428         return millivolts;
429 }
430
431 static int dvfs_rail_update(struct dvfs_rail *rail)
432 {
433         int millivolts = 0;
434         struct dvfs *d;
435         struct dvfs_relationship *rel;
436         int ret = 0;
437         int steps;
438
439         /* if dvfs is suspended, return and handle it during resume */
440         if (rail->suspended)
441                 return 0;
442
443         /* if regulators are not connected yet, return and handle it later */
444         if (!rail->reg)
445                 return 0;
446
447         /* if no clock has requested voltage since boot, defer update */
448         if (!rail->rate_set)
449                 return 0;
450
451         /* if rail update is entered while resolving circular dependencies,
452            abort recursion */
453         if (rail->resolving_to)
454                 return 0;
455
456         /* Find the maximum voltage requested by any clock */
457         list_for_each_entry(d, &rail->dvfs, reg_node)
458                 millivolts = max(d->cur_millivolts, millivolts);
459
460         /* Apply offset and min/max limits if any clock is requesting voltage */
461         if (millivolts)
462                 millivolts = dvfs_rail_apply_limits(rail, millivolts);
463         /* Keep current voltage if regulator is to be disabled via explicitly */
464         else if (rail->in_band_pm)
465                 return 0;
466         /* Keep current voltage if regulator must not be disabled at run time */
467         else if (!rail->jmp_to_zero) {
468                 WARN(!rail->disabled,
469                         "%s cannot be turned off by dvfs\n", rail->reg_id);
470                 return 0;
471         }
472         /* else: fall thru if regulator is turned off by side band signaling */
473
474         /* retry update if limited by from-relationship to account for
475            circular dependencies */
476         steps = DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
477         for (; steps >= 0; steps--) {
478                 rail->new_millivolts = millivolts;
479
480                 /* Check any rails that this rail depends on */
481                 list_for_each_entry(rel, &rail->relationships_from, from_node)
482                         rail->new_millivolts = dvfs_solve_relationship(rel);
483
484                 if (rail->new_millivolts == rail->millivolts)
485                         break;
486
487                 ret = dvfs_rail_set_voltage(rail, rail->new_millivolts);
488         }
489
490         return ret;
491 }
492
493 static struct regulator *get_fixed_regulator(struct dvfs_rail *rail,
494                                              struct device *dev)
495 {
496         struct regulator *reg;
497         char reg_id[80];
498         struct dvfs *d;
499         int v, i;
500         unsigned long dfll_boost;
501
502         strcpy(reg_id, rail->reg_id);
503         if (!dev)
504                 strcat(reg_id, "_fixed");
505         reg = regulator_get(dev, reg_id);
506         if (IS_ERR(reg))
507                 return reg;
508
509         v = regulator_get_voltage(reg) / 1000;
510         if ((v < rail->min_millivolts) || (v > rail->nominal_millivolts) ||
511             (rail->therm_mv_floors && v < rail->therm_mv_floors[0])) {
512                 pr_err("tegra_dvfs: ivalid fixed %s voltage %d\n",
513                        rail->reg_id, v);
514                 return ERR_PTR(-EINVAL);
515         }
516
517         /*
518          * Only fixed at nominal voltage vdd_core regulator is allowed, same
519          * is true for cpu rail if dfll mode is not supported at all. No thermal
520          * capping can be implemented in this case.
521          */
522         if (!IS_ENABLED(CONFIG_ARCH_TEGRA_HAS_CL_DVFS) ||
523             (rail != tegra_cpu_rail)) {
524                 if (v != rail->nominal_millivolts) {
525                         pr_err("tegra_dvfs: %s fixed below nominal at %d\n",
526                                rail->reg_id, v);
527                         return ERR_PTR(-EINVAL);
528                 }
529                 if (rail->therm_mv_caps) {
530                         pr_err("tegra_dvfs: cannot fix %s with thermal caps\n",
531                                rail->reg_id);
532                         return ERR_PTR(-ENOSYS);
533                 }
534                 return reg;
535         }
536
537         /*
538          * If dfll mode is supported, fixed vdd_cpu regulator may be below
539          * nominal in pll mode - maximum cpu rate in pll mode is limited
540          * respectively. Regulator is required to allow automatic scaling
541          * in dfll mode.
542          *
543          * FIXME: platform data to explicitly identify such "hybrid" regulator?
544          */
545         d = list_first_entry(&rail->dvfs, struct dvfs, reg_node);
546         for (i = 0; i < d->num_freqs; i++) {
547                 if (d->millivolts[i] > v)
548                         break;
549         }
550
551         if (!i) {
552                 pr_err("tegra_dvfs: %s fixed at %d: too low for min rate\n",
553                        rail->reg_id, v);
554                 return ERR_PTR(-EINVAL);
555         }
556
557         dfll_boost = (d->freqs[d->num_freqs - 1] - d->freqs[i - 1]);
558         if (d->dfll_data.max_rate_boost < dfll_boost)
559                 d->dfll_data.max_rate_boost = dfll_boost;
560
561         rail->fixed_millivolts = v;
562         return reg;
563 }
564
565 static int connect_to_regulator(struct dvfs_rail *rail, struct device *dev,
566                                 bool fixed)
567 {
568         struct regulator *reg;
569         int v;
570
571         if (!rail->reg) {
572                 reg = fixed ? get_fixed_regulator(rail, dev) :
573                         regulator_get(dev, rail->reg_id);
574
575                 if (IS_ERR(reg))
576                         return PTR_ERR(reg);
577                 rail->reg = reg;
578         }
579
580         v = regulator_enable(rail->reg);
581         if (v < 0) {
582                 pr_err("tegra_dvfs: failed on enabling regulator %s\n, err %d",
583                         rail->reg_id, v);
584                 return v;
585         }
586
587         v = regulator_get_voltage(rail->reg);
588         if (v < 0) {
589                 pr_err("tegra_dvfs: failed initial get %s voltage\n",
590                        rail->reg_id);
591                 return v;
592         }
593
594         return v;
595 }
596
597 static int dvfs_rail_connect_to_regulator(struct dvfs_rail *rail)
598 {
599         int v;
600         struct device *dev = NULL;
601         bool fixed = rail->dt_reg_fixed; /* false, if rail is not in DT */
602
603 #ifdef CONFIG_OF
604         /* Find dvfs rail device, if the respective node is present in DT */
605         if (rail->dt_node) {
606                 struct platform_device *pdev =
607                         of_find_device_by_node(rail->dt_node);
608                 if (pdev)
609                         dev = &pdev->dev;
610         }
611 #endif
612         v = connect_to_regulator(rail, dev, fixed);
613         if ((v < 0) && !dev) {
614                 /*
615                  * If connecting just by supply name, re-try fixed voltage
616                  * regulator with "_fixed" suffix.
617                  */
618                 rail->reg = NULL;
619                 fixed = true;
620                 v = connect_to_regulator(rail, dev, fixed);
621         }
622
623         if (v < 0) {
624                 pr_err("tegra_dvfs: failed to connect %s rail\n", rail->reg_id);
625                 return v;
626         }
627
628         rail->millivolts = v / 1000;
629         rail->new_millivolts = rail->millivolts;
630         dvfs_rail_stats_init(rail, rail->millivolts);
631
632         if (rail->boot_millivolts &&
633             (rail->boot_millivolts != rail->millivolts)) {
634                 WARN(1, "%s boot voltage %d does not match expected %d\n",
635                      rail->reg_id, rail->millivolts, rail->boot_millivolts);
636                 rail->boot_millivolts = rail->millivolts;
637         }
638
639         pr_info("tegra_dvfs: %s connected to regulator %s\n",
640                 dev ? dev_name(dev) : rail->reg_id,
641                 fixed ? "in fixed pll mode" : "");
642         return 0;
643 }
644
645 /* Get dvfs rail thermal floor helpers in pll and dfll modes */
646 static int dvfs_rail_get_thermal_floor_pll(struct dvfs_rail *rail)
647 {
648         if (rail->therm_mv_floors &&
649             (rail->therm_floor_idx < rail->therm_mv_floors_num))
650                 return rail->therm_mv_floors[rail->therm_floor_idx];
651         return 0;
652 }
653
654 static int dvfs_rail_get_thermal_floor_dfll(struct dvfs_rail *rail)
655 {
656         if (rail->therm_mv_dfll_floors &&
657             (rail->therm_floor_idx < rail->therm_mv_floors_num))
658                 return rail->therm_mv_dfll_floors[rail->therm_floor_idx];
659         return 0;
660 }
661
662 /* Get dvfs clock V/F curve helpers in pll and dfll modes */
663 static unsigned long *dvfs_get_freqs(struct dvfs *d)
664 {
665         return d->alt_freqs && d->use_alt_freqs ? d->alt_freqs : &d->freqs[0];
666 }
667
668 static const int *dvfs_get_millivolts_dfll(struct dvfs *d)
669 {
670         return d->dfll_millivolts;
671 }
672
673 static const int *dvfs_get_millivolts_pll(struct dvfs *d)
674 {
675         if (d->therm_dvfs) {
676                 int therm_idx = d->dvfs_rail->therm_scale_idx;
677                 return d->millivolts + therm_idx * MAX_DVFS_FREQS;
678         }
679         return d->millivolts;
680 }
681
682 static const int *dvfs_get_millivolts(struct dvfs *d, unsigned long rate)
683 {
684         if (tegra_dvfs_is_dfll_scale(d, rate))
685                 return dvfs_get_millivolts_dfll(d);
686
687         return dvfs_get_millivolts_pll(d);
688 }
689
690 static int
691 __tegra_dvfs_set_rate(struct dvfs *d, unsigned long rate)
692 {
693         int i = 0;
694         int ret, mv, detach_mv;
695         unsigned long *freqs = dvfs_get_freqs(d);
696         const int *millivolts = dvfs_get_millivolts(d, rate);
697
698         if (freqs == NULL || millivolts == NULL)
699                 return -ENODEV;
700
701         /* On entry to dfll range limit 1st step to range bottom (full ramp of
702            voltage/rate is completed automatically in dfll mode) */
703         if (tegra_dvfs_is_dfll_range_entry(d, rate))
704                 rate = d->dfll_data.use_dfll_rate_min;
705
706         if (rate > freqs[d->num_freqs - 1]) {
707                 pr_warn("tegra_dvfs: rate %lu too high for dvfs on %s\n", rate,
708                         d->clk_name);
709                 return -EINVAL;
710         }
711
712         if (rate == 0) {
713                 d->cur_millivolts = 0;
714                 /*
715                  * For single clock GPU rail keep DVFS rate unchanged when clock
716                  * is disabled. Rail is turned off explicitly, in any case, but
717                  * with non-zero rate voltage level at regulator is updated when
718                  * temperature is changes while rail is off.
719                  */
720                 if (d->dvfs_rail == tegra_gpu_rail)
721                         rate = d->cur_rate;
722         }
723
724         if (rate != 0) {
725                 while (i < d->num_freqs && rate > freqs[i])
726                         i++;
727
728                 mv = millivolts[i];
729
730                 if ((d->max_millivolts) && (mv > d->max_millivolts)) {
731                         pr_warn("tegra_dvfs: voltage %d too high for dvfs on %s\n",
732                                 mv, d->clk_name);
733                         return -EINVAL;
734                 }
735
736                 detach_mv = tegra_dvfs_rail_get_boot_level(d->dvfs_rail);
737                 if (!d->dvfs_rail->reg && (mv > detach_mv)) {
738                         pr_warn("%s: %s: voltage %d above boot limit %d\n",
739                                 __func__, d->clk_name, mv, detach_mv);
740                         return -EINVAL;
741                 }
742
743                 detach_mv = tegra_dvfs_rail_get_disable_level(d->dvfs_rail);
744                 if (d->dvfs_rail->disabled && (mv > detach_mv)) {
745                         pr_warn("%s: %s: voltage %d above disable limit %d\n",
746                                 __func__, d->clk_name, mv, detach_mv);
747                         return -EINVAL;
748                 }
749
750                 detach_mv = tegra_dvfs_rail_get_suspend_level(d->dvfs_rail);
751                 if (d->dvfs_rail->suspended && (mv > detach_mv)) {
752                         pr_warn("%s: %s: voltage %d above disable limit %d\n",
753                                 __func__, d->clk_name, mv, detach_mv);
754                         return -EINVAL;
755                 }
756
757                 detach_mv = d->dvfs_rail->override_millivolts;
758                 if (detach_mv && (mv > detach_mv)) {
759                         pr_warn("%s: %s: voltage %d above override level %d\n",
760                                 __func__, d->clk_name, mv, detach_mv);
761                         return -EINVAL;
762                 }
763                 d->cur_millivolts = mv;
764         }
765
766         d->cur_rate = rate;
767
768         d->dvfs_rail->rate_set = true;
769         ret = dvfs_rail_update(d->dvfs_rail);
770         if (ret)
771                 pr_err("Failed to set regulator %s for clock %s to %d mV\n",
772                         d->dvfs_rail->reg_id, d->clk_name, d->cur_millivolts);
773
774         return ret;
775 }
776
777 /*
778  * Some clocks may have alternative frequency ladder that provides lower minimum
779  * voltage at the same rate (or complimentary: higher maximum rate at the same
780  * voltage). Interfaces below allows dvfs clients to install such ladder, and
781  * switch between primary and alternative frequencies in flight.
782  */
783 static int alt_freqs_validate(struct dvfs *d, unsigned long *alt_freqs)
784 {
785         int i;
786
787         if (alt_freqs) {
788                 for (i = 0; i < d->num_freqs; i++) {
789                         if (d->freqs[i] > alt_freqs[i]) {
790                                 pr_err("%s: Invalid alt freqs for %s\n",
791                                        __func__, d->clk_name);
792                                 return -EINVAL;
793                         }
794                 }
795         }
796         return 0;
797 }
798
799 int tegra_dvfs_alt_freqs_install(struct dvfs *d, unsigned long *alt_freqs)
800 {
801         int ret = 0;
802
803         mutex_lock(&dvfs_lock);
804
805         ret = alt_freqs_validate(d, alt_freqs);
806         if (!ret)
807                 d->alt_freqs = alt_freqs;
808
809         mutex_unlock(&dvfs_lock);
810         return ret;
811 }
812
813 int tegra_dvfs_use_alt_freqs_on_clk(struct clk *c, bool use_alt_freq)
814 {
815         int ret = -ENOENT;
816         struct dvfs *d = c->dvfs;
817
818         mutex_lock(&dvfs_lock);
819
820         if (d && d->alt_freqs) {
821                 ret = 0;
822                 if (d->use_alt_freqs != use_alt_freq) {
823                         d->use_alt_freqs = use_alt_freq;
824                         ret = __tegra_dvfs_set_rate(d, d->cur_rate);
825                 }
826         }
827
828         mutex_unlock(&dvfs_lock);
829         return ret;
830 }
831
832 int tegra_dvfs_alt_freqs_set(struct dvfs *d, unsigned long *alt_freqs)
833 {
834         int ret = 0;
835
836         mutex_lock(&dvfs_lock);
837
838         if (d->alt_freqs != alt_freqs) {
839                 ret = alt_freqs_validate(d, alt_freqs);
840                 if (!ret) {
841                         d->use_alt_freqs = !!alt_freqs;
842                         d->alt_freqs = alt_freqs;
843                         ret = __tegra_dvfs_set_rate(d, d->cur_rate);
844                 }
845         }
846
847         mutex_unlock(&dvfs_lock);
848         return ret;
849 }
850
851 /*
852  * Some clocks may need run-time voltage ladder replacement. Allow it only if
853  * peak voltages across all possible ladders are specified, and new voltages
854  * do not violate peaks.
855  */
856 static int new_voltages_validate(struct dvfs *d, const int *new_millivolts,
857                                  int freqs_num, int ranges_num)
858 {
859         const int *millivolts;
860         int freq_idx, therm_idx;
861
862         for (therm_idx = 0; therm_idx < ranges_num; therm_idx++) {
863                 millivolts = new_millivolts + therm_idx * MAX_DVFS_FREQS;
864                 for (freq_idx = 0; freq_idx < freqs_num; freq_idx++) {
865                         if (millivolts[freq_idx] >
866                             d->peak_millivolts[freq_idx]) {
867                                 pr_err("%s: Invalid new voltages for %s\n",
868                                        __func__, d->clk_name);
869                                 return -EINVAL;
870                         }
871                 }
872         }
873         return 0;
874 }
875
876 int tegra_dvfs_replace_voltage_table(struct dvfs *d, const int *new_millivolts)
877 {
878         int ret = 0;
879         int ranges_num = 1;
880
881         mutex_lock(&dvfs_lock);
882
883         if (!d->peak_millivolts) {
884                 ret = -EINVAL;
885                 goto out;
886         }
887
888         if (d->therm_dvfs && d->dvfs_rail->vts_cdev)
889                 ranges_num += d->dvfs_rail->vts_cdev->trip_temperatures_num;
890
891         if (new_voltages_validate(d, new_millivolts,
892                                   d->num_freqs, ranges_num)) {
893                 ret = -EINVAL;
894                 goto out;
895         }
896
897         d->millivolts = new_millivolts;
898         if (__tegra_dvfs_set_rate(d, d->cur_rate))
899                 ret = -EAGAIN;
900 out:
901         mutex_unlock(&dvfs_lock);
902         return ret;
903 }
904
905 /*
906  *  Using non alt frequencies always results in peak voltage
907  * (enforced by alt_freqs_validate())
908  */
909 static int predict_non_alt_millivolts(struct clk *c, const int *millivolts,
910                                       unsigned long rate)
911 {
912         int i;
913
914         if (!millivolts)
915                 return -ENODEV;
916
917         for (i = 0; i < c->dvfs->num_freqs; i++) {
918                 if (rate <= c->dvfs->freqs[i])
919                         break;
920         }
921
922         if (i == c->dvfs->num_freqs)
923                 i--;
924
925         return max(millivolts[i], c->dvfs->dvfs_rail->min_millivolts);
926 }
927
928 static int predict_millivolts(struct clk *c, const int *millivolts,
929                               unsigned long rate)
930 {
931         /*
932          * Predicted voltage can not be used across the switch to alternative
933          * frequency limits. For now, just fail the call for clock that has
934          * alternative limits initialized.
935          */
936         if (c->dvfs->alt_freqs)
937                 return -ENOSYS;
938
939         return predict_non_alt_millivolts(c, millivolts, rate);
940 }
941
942 /*
943  * Predict minimum voltage required to run specified clock at target rate.
944  * Evaluate individual clock domain V/F relation; does not apply thermal
945  * floor. Should be used for core domains to evaluate per-clock voltage
946  * requirements.
947  */
948 int tegra_dvfs_predict_millivolts(struct clk *c, unsigned long rate)
949 {
950         int mv;
951         const int *millivolts;
952
953         if (!rate || !c->dvfs)
954                 return 0;
955
956         if ((c->dvfs->dvfs_rail == tegra_gpu_rail) ||
957             (c->dvfs->dvfs_rail == tegra_cpu_rail)) {
958                 pr_warn("%s does not apply thermal floors - use %s\n",
959                          __func__, "tegra_dvfs_predict_millivolts_t");
960         }
961
962         mutex_lock(&dvfs_lock);
963         millivolts = tegra_dvfs_is_dfll_range(c->dvfs, rate) ?
964                 dvfs_get_millivolts_dfll(c->dvfs) :
965                 dvfs_get_millivolts_pll(c->dvfs);
966         mv = predict_millivolts(c, millivolts, rate);
967         mutex_unlock(&dvfs_lock);
968
969         return mv;
970 }
971 EXPORT_SYMBOL(tegra_dvfs_predict_millivolts);
972
973 /*
974  * Predict minimum voltage required to run specified clock at target rate
975  * including effect of thermal floor at current temperature.
976  */
977 int tegra_dvfs_predict_millivolts_t(struct clk *c, unsigned long rate)
978 {
979         int mv;
980         const int *millivolts;
981         struct dvfs_rail *rail;
982
983         if (!rate || !c->dvfs)
984                 return 0;
985
986         rail = c->dvfs->dvfs_rail;
987
988         mutex_lock(&dvfs_lock);
989         if (tegra_dvfs_is_dfll_range(c->dvfs, rate)) {
990                 millivolts = dvfs_get_millivolts_dfll(c->dvfs);
991                 mv = predict_millivolts(c, millivolts, rate);
992                 if (mv >= 0)
993                         mv = max(mv, dvfs_rail_get_thermal_floor_dfll(rail));
994         } else {
995                 millivolts = dvfs_get_millivolts_pll(c->dvfs);
996                 mv = predict_millivolts(c, millivolts, rate);
997                 if (mv >= 0)
998                         mv = max(mv, dvfs_rail_get_thermal_floor_pll(rail));
999         }
1000         mutex_unlock(&dvfs_lock);
1001
1002         return mv;
1003 }
1004 EXPORT_SYMBOL(tegra_dvfs_predict_millivolts_t);
1005
1006 /*
1007  * Predict minimum voltage required to run specified clock at target rate
1008  * including effect of thermal floors across all temperature ranges.
1009  */
1010 static int dvfs_predict_peak_millivolts(struct clk *c, unsigned long rate)
1011 {
1012         int mv;
1013         const int *millivolts;
1014         bool dfll_range;
1015
1016         if (!rate || !c->dvfs)
1017                 return 0;
1018
1019         dfll_range = tegra_dvfs_is_dfll_range(c->dvfs, rate);
1020         millivolts = dfll_range ? dvfs_get_millivolts_dfll(c->dvfs) :
1021                 c->dvfs->peak_millivolts ? : dvfs_get_millivolts_pll(c->dvfs);
1022
1023         mv = predict_non_alt_millivolts(c, millivolts, rate);
1024         if (mv < 0)
1025                 return mv;
1026
1027         if (!dfll_range && c->dvfs->dvfs_rail->therm_mv_floors)
1028                 mv = max(mv, c->dvfs->dvfs_rail->therm_mv_floors[0]);
1029         if (dfll_range && c->dvfs->dvfs_rail->therm_mv_dfll_floors)
1030                 mv = max(mv, c->dvfs->dvfs_rail->therm_mv_dfll_floors[0]);
1031         return mv;
1032 }
1033
1034 int tegra_dvfs_predict_peak_millivolts(struct clk *c, unsigned long rate)
1035 {
1036         int mv;
1037
1038         mutex_lock(&dvfs_lock);
1039         mv = dvfs_predict_peak_millivolts(c, rate);
1040         mutex_unlock(&dvfs_lock);
1041
1042         return mv;
1043 }
1044
1045 /* Set DVFS rate and update voltage accordingly */
1046 int tegra_dvfs_set_rate(struct clk *c, unsigned long rate)
1047 {
1048         int ret;
1049         bool suspended;
1050
1051         if (!c->dvfs)
1052                 return -EINVAL;
1053
1054         suspended = timekeeping_suspended && c->dvfs->dvfs_rail->suspended;
1055         if (suspended) {
1056                 if (mutex_is_locked(&dvfs_lock))
1057                         WARN(1, "%s: Entered suspend with DVFS mutex locked\n",
1058                              __func__);
1059         } else {
1060                 mutex_lock(&dvfs_lock);
1061         }
1062
1063         ret = __tegra_dvfs_set_rate(c->dvfs, rate);
1064
1065         if (!suspended)
1066                 mutex_unlock(&dvfs_lock);
1067
1068         return ret;
1069 }
1070 EXPORT_SYMBOL(tegra_dvfs_set_rate);
1071
1072 int tegra_dvfs_get_freqs(struct clk *c, unsigned long **freqs, int *num_freqs)
1073 {
1074         if (!c->dvfs)
1075                 return -ENOSYS;
1076
1077         if (c->dvfs->alt_freqs)
1078                 return -ENOSYS;
1079
1080         *num_freqs = c->dvfs->num_freqs;
1081         *freqs = c->dvfs->freqs;
1082
1083         return 0;
1084 }
1085 EXPORT_SYMBOL(tegra_dvfs_get_freqs);
1086
1087 static int dvfs_rail_get_override_floor(struct dvfs_rail *rail)
1088 {
1089         return rail->override_unresolved ? rail->nominal_millivolts :
1090                 rail->min_override_millivolts;
1091 }
1092
1093 #ifdef CONFIG_TEGRA_VDD_CORE_OVERRIDE
1094 static DEFINE_MUTEX(rail_override_lock);
1095
1096 static int dvfs_override_core_voltage(int override_mv)
1097 {
1098         int ret, floor, ceiling;
1099         struct dvfs_rail *rail = tegra_core_rail;
1100
1101         if (!rail)
1102                 return -ENOENT;
1103
1104         if (rail->fixed_millivolts)
1105                 return -ENOSYS;
1106
1107         mutex_lock(&rail_override_lock);
1108
1109         floor = dvfs_rail_get_override_floor(rail);
1110         ceiling = rail->nominal_millivolts;
1111         if (override_mv && ((override_mv < floor) || (override_mv > ceiling))) {
1112                 pr_err("%s: override level %d outside the range [%d...%d]\n",
1113                        __func__, override_mv, floor, ceiling);
1114                 mutex_unlock(&rail_override_lock);
1115                 return -EINVAL;
1116         }
1117
1118         if (override_mv == rail->override_millivolts) {
1119                 ret = 0;
1120                 goto out;
1121         }
1122
1123         if (override_mv) {
1124                 ret = tegra_dvfs_override_core_cap_apply(override_mv);
1125                 if (ret) {
1126                         pr_err("%s: failed to set cap for override level %d\n",
1127                                __func__, override_mv);
1128                         goto out;
1129                 }
1130         }
1131
1132         mutex_lock(&dvfs_lock);
1133         if (rail->disabled || rail->suspended) {
1134                 pr_err("%s: cannot scale %s rail\n", __func__,
1135                        rail->disabled ? "disabled" : "suspended");
1136                 ret = -EPERM;
1137                 if (!override_mv) {
1138                         mutex_unlock(&dvfs_lock);
1139                         goto out;
1140                 }
1141         } else {
1142                 rail->override_millivolts = override_mv;
1143                 ret = dvfs_rail_update(rail);
1144                 if (ret) {
1145                         pr_err("%s: failed to set override level %d\n",
1146                                __func__, override_mv);
1147                         rail->override_millivolts = 0;
1148                         dvfs_rail_update(rail);
1149                 }
1150         }
1151         mutex_unlock(&dvfs_lock);
1152
1153         if (!override_mv || ret)
1154                 tegra_dvfs_override_core_cap_apply(0);
1155 out:
1156         mutex_unlock(&rail_override_lock);
1157         return ret;
1158 }
1159
1160 int tegra_dvfs_resolve_override(struct clk *c, unsigned long max_rate)
1161 {
1162         int mv;
1163         struct dvfs *d = c->dvfs;
1164         struct dvfs_rail *rail;
1165
1166         if (!d)
1167                 return 0;
1168         rail = d->dvfs_rail;
1169
1170         mutex_lock(&rail_override_lock);
1171         mutex_lock(&dvfs_lock);
1172
1173         if (d->defer_override && rail->override_unresolved) {
1174                 d->defer_override = false;
1175
1176                 mv = dvfs_predict_peak_millivolts(c, max_rate);
1177                 if (rail->min_override_millivolts < mv)
1178                         rail->min_override_millivolts = mv;
1179
1180                 rail->override_unresolved--;
1181                 if (!rail->override_unresolved && rail->resolve_override)
1182                         rail->resolve_override(rail->min_override_millivolts);
1183         }
1184         mutex_unlock(&dvfs_lock);
1185         mutex_unlock(&rail_override_lock);
1186         return 0;
1187 }
1188
1189 int tegra_dvfs_rail_get_override_floor(struct dvfs_rail *rail)
1190 {
1191         if (rail) {
1192                 int mv;
1193                 mutex_lock(&rail_override_lock);
1194                 mv = dvfs_rail_get_override_floor(rail);
1195                 mutex_unlock(&rail_override_lock);
1196                 return mv;
1197         }
1198         return -ENOENT;
1199 }
1200
1201 static int dvfs_set_fmax_at_vmin(struct clk *c, unsigned long f_max, int v_min)
1202 {
1203         int i, ret = 0;
1204         struct dvfs *d = c->dvfs;
1205         unsigned long f_min = 1000;     /* 1kHz min rate in DVFS tables */
1206
1207         mutex_lock(&rail_override_lock);
1208         mutex_lock(&dvfs_lock);
1209
1210         if (v_min > d->dvfs_rail->override_millivolts) {
1211                 pr_err("%s: new %s vmin %dmV is above override voltage %dmV\n",
1212                        __func__, c->name, v_min,
1213                        d->dvfs_rail->override_millivolts);
1214                 ret = -EPERM;
1215                 goto out;
1216         }
1217
1218         if (v_min >= d->max_millivolts) {
1219                 pr_err("%s: new %s vmin %dmV is at/above max voltage %dmV\n",
1220                        __func__, c->name, v_min, d->max_millivolts);
1221                 ret = -EINVAL;
1222                 goto out;
1223         }
1224
1225         /*
1226          * dvfs table update:
1227          * - for voltages below new v_min the respective frequencies are shifted
1228          * below new f_max to the levels already present in the table; if the
1229          * 1st table entry has frequency above new fmax, all entries below v_min
1230          * are filled in with 1kHz (min rate used in DVFS tables).
1231          * - for voltages above new v_min, the respective frequencies are
1232          * increased to at least new f_max
1233          * - if new v_min is already in the table set the respective frequency
1234          * to new f_max
1235          */
1236         for (i = 0; i < d->num_freqs; i++) {
1237                 int mv = d->millivolts[i];
1238                 unsigned long f = d->freqs[i];
1239
1240                 if (mv < v_min) {
1241                         if (d->freqs[i] >= f_max)
1242                                 d->freqs[i] = i ? d->freqs[i-1] : f_min;
1243                 } else if (mv > v_min) {
1244                         d->freqs[i] = max(f, f_max);
1245                 } else {
1246                         d->freqs[i] = f_max;
1247                 }
1248                 ret = __tegra_dvfs_set_rate(d, d->cur_rate);
1249         }
1250 out:
1251         mutex_unlock(&dvfs_lock);
1252         mutex_unlock(&rail_override_lock);
1253
1254         return ret;
1255 }
1256 #else
1257 static int dvfs_override_core_voltage(int override_mv)
1258 {
1259         pr_err("%s: vdd core override is not supported\n", __func__);
1260         return -ENOSYS;
1261 }
1262
1263 static int dvfs_set_fmax_at_vmin(struct clk *c, unsigned long f_max, int v_min)
1264 {
1265         pr_err("%s: vdd core override is not supported\n", __func__);
1266         return -ENOSYS;
1267 }
1268 #endif
1269
1270 int tegra_dvfs_override_core_voltage(struct clk *c, int override_mv)
1271 {
1272         if (!c->dvfs || !c->dvfs->can_override) {
1273                 pr_err("%s: %s cannot override vdd core\n", __func__, c->name);
1274                 return -EPERM;
1275         }
1276         return dvfs_override_core_voltage(override_mv);
1277 }
1278 EXPORT_SYMBOL(tegra_dvfs_override_core_voltage);
1279
1280 int tegra_dvfs_set_fmax_at_vmin(struct clk *c, unsigned long f_max, int v_min)
1281 {
1282         if (!c->dvfs || !c->dvfs->can_override) {
1283                 pr_err("%s: %s cannot set fmax_at_vmin)\n", __func__, c->name);
1284                 return -EPERM;
1285         }
1286         return dvfs_set_fmax_at_vmin(c, f_max, v_min);
1287 }
1288 EXPORT_SYMBOL(tegra_dvfs_set_fmax_at_vmin);
1289
1290 /* May only be called during clock init, does not take any locks on clock c. */
1291 static int __init enable_dvfs_on_clk(struct clk *c, struct dvfs *d)
1292 {
1293         int i;
1294
1295         if (c->dvfs) {
1296                 pr_err("Error when enabling dvfs on %s for clock %s:\n",
1297                         d->dvfs_rail->reg_id, c->name);
1298                 pr_err("DVFS already enabled for %s\n",
1299                         c->dvfs->dvfs_rail->reg_id);
1300                 return -EINVAL;
1301         }
1302
1303         for (i = 0; i < MAX_DVFS_FREQS; i++) {
1304                 if (d->millivolts[i] == 0)
1305                         break;
1306
1307                 d->freqs[i] *= d->freqs_mult;
1308
1309                 /* If final frequencies are 0, pad with previous frequency */
1310                 if (d->freqs[i] == 0 && i > 1)
1311                         d->freqs[i] = d->freqs[i - 1];
1312         }
1313         d->num_freqs = i;
1314
1315         if (d->auto_dvfs) {
1316                 c->auto_dvfs = true;
1317                 clk_set_cansleep(c);
1318         }
1319
1320         c->dvfs = d;
1321
1322         /*
1323          * Minimum core override level is determined as maximum voltage required
1324          * for clocks outside shared buses (shared bus rates can be capped to
1325          * safe levels when override limit is set)
1326          */
1327         if (i && c->ops && !c->ops->shared_bus_update &&
1328             !(c->flags & PERIPH_ON_CBUS) && !d->can_override) {
1329                 int mv = dvfs_predict_peak_millivolts(c, d->freqs[i-1]);
1330                 struct dvfs_rail *rail = d->dvfs_rail;
1331                 if (d->defer_override)
1332                         rail->override_unresolved++;
1333                 else if (rail->min_override_millivolts < mv)
1334                         rail->min_override_millivolts =
1335                                 min(mv, rail->nominal_millivolts);
1336         }
1337
1338         mutex_lock(&dvfs_lock);
1339         list_add_tail(&d->reg_node, &d->dvfs_rail->dvfs);
1340         mutex_unlock(&dvfs_lock);
1341
1342         /* Init common DVFS lock pointer */
1343         d->lock = &dvfs_lock;
1344
1345         return 0;
1346 }
1347
1348 static bool __init can_update_max_rate(struct clk *c, struct dvfs *d)
1349 {
1350         /* Don't update manual dvfs, non-shared clocks */
1351         if (!d->auto_dvfs && !c->ops->shared_bus_update)
1352                 return false;
1353
1354         /*
1355          * Don't update EMC shared bus, since EMC dvfs is board dependent: max
1356          * rate and EMC scaling frequencies are determined by tegra BCT (flashed
1357          * together with the image) and board specific EMC DFS table; we will
1358          * check the scaling ladder against nominal core voltage when the table
1359          * is loaded (and if on particular board the table is not loaded, EMC
1360          * scaling is disabled).
1361          */
1362         if (c->ops->shared_bus_update && (c->flags & PERIPH_EMC_ENB))
1363                 return false;
1364
1365         /*
1366          * Don't update shared cbus, and don't propagate common cbus dvfs
1367          * limit down to shared users, but set maximum rate for each user
1368          * equal to the respective client limit.
1369          */
1370         if (c->ops->shared_bus_update && (c->flags & PERIPH_ON_CBUS)) {
1371                 struct clk *user;
1372                 unsigned long rate;
1373
1374                 list_for_each_entry(
1375                         user, &c->shared_bus_list, u.shared_bus_user.node) {
1376                         if (user->u.shared_bus_user.client) {
1377                                 rate = user->u.shared_bus_user.client->max_rate;
1378                                 user->max_rate = rate;
1379                                 user->u.shared_bus_user.rate = rate;
1380                         }
1381                 }
1382                 return false;
1383         }
1384
1385         /* Other, than EMC and cbus, auto-dvfs clocks can be updated */
1386         return true;
1387 }
1388
1389 void __init tegra_init_dvfs_one(struct dvfs *d, int max_freq_index)
1390 {
1391         int ret;
1392         struct clk *c = tegra_get_clock_by_name(d->clk_name);
1393
1394         if (!c) {
1395                 pr_debug("tegra_dvfs: no clock found for %s\n", d->clk_name);
1396                 return;
1397         }
1398
1399         /* Update max rate for auto-dvfs clocks, with shared bus exceptions */
1400         if (can_update_max_rate(c, d)) {
1401                 BUG_ON(!d->freqs[max_freq_index]);
1402                 tegra_init_max_rate(
1403                         c, d->freqs[max_freq_index] * d->freqs_mult);
1404         }
1405         d->max_millivolts = d->dvfs_rail->nominal_millivolts;
1406
1407         ret = enable_dvfs_on_clk(c, d);
1408         if (ret)
1409                 pr_err("tegra_dvfs: failed to enable dvfs on %s\n", c->name);
1410
1411         /* Limit DVFS mapping to maximum frequency */
1412         d->num_freqs = min(d->num_freqs, max_freq_index + 1);
1413 }
1414
1415 static bool tegra_dvfs_all_rails_suspended(void)
1416 {
1417         struct dvfs_rail *rail;
1418         bool all_suspended = true;
1419
1420         list_for_each_entry(rail, &dvfs_rail_list, node)
1421                 if (!rail->suspended && !rail->disabled)
1422                         all_suspended = false;
1423
1424         return all_suspended;
1425 }
1426
1427 static bool is_solved_at_suspend(struct dvfs_rail *to,
1428                                  struct dvfs_relationship *rel)
1429 {
1430         if (rel->solved_at_suspend)
1431                 return true;
1432
1433         if (rel->solved_at_nominal) {
1434                 int mv = tegra_dvfs_rail_get_suspend_level(to);
1435                 if (mv == to->nominal_millivolts)
1436                         return true;
1437         }
1438         return false;
1439 }
1440
1441 static bool tegra_dvfs_from_rails_suspended_or_solved(struct dvfs_rail *to)
1442 {
1443         struct dvfs_relationship *rel;
1444         bool all_suspended = true;
1445
1446         list_for_each_entry(rel, &to->relationships_from, from_node)
1447                 if (!rel->from->suspended && !rel->from->disabled &&
1448                         !is_solved_at_suspend(to, rel))
1449                         all_suspended = false;
1450
1451         return all_suspended;
1452 }
1453
1454 static int tegra_dvfs_suspend_one(void)
1455 {
1456         struct dvfs_rail *rail;
1457         int ret, mv;
1458
1459         list_for_each_entry(rail, &dvfs_rail_list, node) {
1460                 if (!rail->suspended && !rail->disabled &&
1461                     tegra_dvfs_from_rails_suspended_or_solved(rail)) {
1462                         if (rail->dfll_mode) {
1463                                 /* s/w doesn't change voltage in dfll mode */
1464                                 mv = rail->millivolts;
1465                         } else if (rail->fixed_millivolts) {
1466                                 /* Safe: pll mode rate capped to fixed level */
1467                                 mv = rail->fixed_millivolts;
1468                         } else {
1469                                 mv = tegra_dvfs_rail_get_suspend_level(rail);
1470                                 mv = dvfs_rail_apply_limits(rail, mv);
1471                         }
1472
1473                         /* apply suspend limit only if it is above current mv */
1474                         ret = -EPERM;
1475                         if (mv >= rail->millivolts)
1476                                 ret = dvfs_rail_set_voltage(rail, mv);
1477                         if (ret) {
1478                                 pr_err("tegra_dvfs: failed %s suspend at %d\n",
1479                                        rail->reg_id, rail->millivolts);
1480                                 return ret;
1481                         }
1482
1483                         rail->suspended = true;
1484                         return 0;
1485                 }
1486         }
1487
1488         return -EINVAL;
1489 }
1490
1491 static void tegra_dvfs_resume(void)
1492 {
1493         struct dvfs_rail *rail;
1494
1495         mutex_lock(&dvfs_lock);
1496
1497         list_for_each_entry(rail, &dvfs_rail_list, node)
1498                 rail->suspended = false;
1499
1500         list_for_each_entry(rail, &dvfs_rail_list, node)
1501                 dvfs_rail_update(rail);
1502
1503         mutex_unlock(&dvfs_lock);
1504 }
1505
1506 static int tegra_dvfs_suspend(void)
1507 {
1508         int ret = 0;
1509
1510         mutex_lock(&dvfs_lock);
1511
1512         while (!tegra_dvfs_all_rails_suspended()) {
1513                 ret = tegra_dvfs_suspend_one();
1514                 if (ret)
1515                         break;
1516         }
1517
1518         mutex_unlock(&dvfs_lock);
1519
1520         if (ret)
1521                 tegra_dvfs_resume();
1522
1523         return ret;
1524 }
1525
1526 static int tegra_dvfs_pm_suspend(struct notifier_block *nb,
1527                                  unsigned long event, void *data)
1528 {
1529         if (event == PM_SUSPEND_PREPARE) {
1530                 if (tegra_dvfs_suspend())
1531                         return NOTIFY_STOP;
1532                 pr_info("tegra_dvfs: suspended\n");
1533         }
1534         return NOTIFY_OK;
1535 };
1536
1537 static int tegra_dvfs_pm_resume(struct notifier_block *nb,
1538                                 unsigned long event, void *data)
1539 {
1540         if (event == PM_POST_SUSPEND) {
1541                 tegra_dvfs_resume();
1542                 pr_info("tegra_dvfs: resumed\n");
1543         }
1544         return NOTIFY_OK;
1545 };
1546
1547 static struct notifier_block tegra_dvfs_suspend_nb = {
1548         .notifier_call = tegra_dvfs_pm_suspend,
1549         .priority = -1,
1550 };
1551
1552 static struct notifier_block tegra_dvfs_resume_nb = {
1553         .notifier_call = tegra_dvfs_pm_resume,
1554         .priority = 1,
1555 };
1556
1557 static int tegra_dvfs_reboot_notify(struct notifier_block *nb,
1558                                 unsigned long event, void *data)
1559 {
1560         switch (event) {
1561         case SYS_RESTART:
1562         case SYS_HALT:
1563         case SYS_POWER_OFF:
1564                 tegra_dvfs_suspend();
1565                 return NOTIFY_OK;
1566         }
1567         return NOTIFY_DONE;
1568 }
1569
1570 static struct notifier_block tegra_dvfs_reboot_nb = {
1571         .notifier_call = tegra_dvfs_reboot_notify,
1572 };
1573
1574 /* must be called with dvfs lock held */
1575 static void dvfs_rail_disable(struct dvfs_rail *rail, bool force)
1576 {
1577         int ret = -EPERM;
1578         int mv;
1579
1580         /* don't set voltage in DFLL mode - won't work, but break stats */
1581         if (rail->dfll_mode) {
1582                 rail->disabled = true;
1583                 return;
1584         }
1585
1586         /* Safe, as pll mode rate is capped to fixed level */
1587         if (!rail->dfll_mode && rail->fixed_millivolts) {
1588                 mv = rail->fixed_millivolts;
1589         } else {
1590                 mv = tegra_dvfs_rail_get_disable_level(rail);
1591                 mv = dvfs_rail_apply_limits(rail, mv);
1592         }
1593
1594         /* apply detach mode limit if it is enforced or above current volatge */
1595         if (force || (mv >= rail->millivolts))
1596                 ret = dvfs_rail_set_voltage(rail, mv);
1597         if (ret) {
1598                 pr_err("tegra_dvfs: failed to disable %s at %d\n",
1599                        rail->reg_id, rail->millivolts);
1600                 return;
1601         }
1602         rail->disabled = true;
1603 }
1604
1605 static void __tegra_dvfs_rail_disable(struct dvfs_rail *rail)
1606 {
1607         dvfs_rail_disable(rail, false);
1608 }
1609
1610 static void __tegra_dvfs_rail_force_disable(struct dvfs_rail *rail)
1611 {
1612         dvfs_rail_disable(rail, true);
1613 }
1614
1615 /* must be called with dvfs lock held */
1616 static void __tegra_dvfs_rail_enable(struct dvfs_rail *rail)
1617 {
1618         rail->disabled = false;
1619         dvfs_rail_update(rail);
1620 }
1621
1622 void tegra_dvfs_rail_enable(struct dvfs_rail *rail)
1623 {
1624         if (!rail)
1625                 return;
1626
1627         mutex_lock(&rail_disable_lock);
1628
1629         if (rail->disabled) {
1630                 mutex_lock(&dvfs_lock);
1631                 __tegra_dvfs_rail_enable(rail);
1632                 mutex_unlock(&dvfs_lock);
1633
1634                 tegra_dvfs_rail_post_enable(rail);
1635         }
1636         mutex_unlock(&rail_disable_lock);
1637 }
1638
1639 void tegra_dvfs_rail_disable(struct dvfs_rail *rail)
1640 {
1641         if (!rail)
1642                 return;
1643
1644         mutex_lock(&rail_disable_lock);
1645         if (rail->disabled)
1646                 goto out;
1647
1648         /* rail disable will set it to nominal voltage underneath clock
1649            framework - need to re-configure clock rates that are not safe
1650            at nominal (yes, unsafe at nominal is ugly, but possible). Rate
1651            change must be done outside of dvfs lock. */
1652         if (tegra_dvfs_rail_disable_prepare(rail)) {
1653                 pr_info("dvfs: failed to prepare regulator %s to disable\n",
1654                         rail->reg_id);
1655                 goto out;
1656         }
1657
1658         mutex_lock(&dvfs_lock);
1659         __tegra_dvfs_rail_disable(rail);
1660         mutex_unlock(&dvfs_lock);
1661 out:
1662         mutex_unlock(&rail_disable_lock);
1663 }
1664
1665 int tegra_dvfs_rail_disable_by_name(const char *reg_id)
1666 {
1667         struct dvfs_rail *rail = tegra_dvfs_get_rail_by_name(reg_id);
1668         if (!rail)
1669                 return -EINVAL;
1670
1671         tegra_dvfs_rail_disable(rail);
1672         return 0;
1673 }
1674
1675 struct dvfs_rail *tegra_dvfs_get_rail_by_name(const char *reg_id)
1676 {
1677         struct dvfs_rail *rail;
1678
1679         mutex_lock(&dvfs_lock);
1680         list_for_each_entry(rail, &dvfs_rail_list, node) {
1681                 if (!strcmp(reg_id, rail->reg_id)) {
1682                         mutex_unlock(&dvfs_lock);
1683                         return rail;
1684                 }
1685         }
1686         mutex_unlock(&dvfs_lock);
1687         return NULL;
1688 }
1689
1690 int tegra_dvfs_rail_power_up(struct dvfs_rail *rail)
1691 {
1692         int ret = -ENOENT;
1693
1694         if (!rail || !rail->in_band_pm)
1695                 return -ENOSYS;
1696
1697         mutex_lock(&dvfs_lock);
1698         if (rail->reg) {
1699                 ret = regulator_enable(rail->reg);
1700                 if (!ret && !timekeeping_suspended)
1701                         tegra_dvfs_rail_on(rail, ktime_get());
1702         }
1703         mutex_unlock(&dvfs_lock);
1704         return ret;
1705 }
1706
1707 int tegra_dvfs_rail_power_down(struct dvfs_rail *rail)
1708 {
1709         int ret = -ENOENT;
1710
1711         if (!rail || !rail->in_band_pm)
1712                 return -ENOSYS;
1713
1714         mutex_lock(&dvfs_lock);
1715         if (rail->reg) {
1716                 ret = regulator_disable(rail->reg);
1717                 if (!ret && !timekeeping_suspended)
1718                         tegra_dvfs_rail_off(rail, ktime_get());
1719         }
1720         mutex_unlock(&dvfs_lock);
1721         return ret;
1722 }
1723
1724 bool tegra_dvfs_is_rail_up(struct dvfs_rail *rail)
1725 {
1726         bool ret = false;
1727
1728         if (!rail)
1729                 return false;
1730
1731         if (!rail->in_band_pm)
1732                 return true;
1733
1734         mutex_lock(&dvfs_lock);
1735         if (rail->reg)
1736                 ret = regulator_is_enabled(rail->reg) > 0;
1737         mutex_unlock(&dvfs_lock);
1738         return ret;
1739 }
1740
1741 int tegra_dvfs_rail_set_mode(struct dvfs_rail *rail, unsigned int mode)
1742 {
1743         int ret = -ENOENT;
1744         unsigned int cur_mode;
1745
1746         if (!rail || !rail->reg)
1747                 return ret;
1748
1749         if (regulator_can_set_mode(rail->reg)) {
1750                 pr_debug("%s: updating %s mode to %u\n", __func__,
1751                          rail->reg_id, mode);
1752                 ret = regulator_set_mode(rail->reg, mode);
1753                 if (ret)
1754                         pr_err("%s: failed to set dvfs regulator %s mode %u\n",
1755                                 __func__, rail->reg_id, mode);
1756                 return ret;
1757         }
1758
1759         /*
1760          * Set mode is not supported - check request against current mode
1761          * (if the latter is unknown, assume NORMAL).
1762          */
1763         cur_mode = regulator_get_mode(rail->reg);
1764         if (IS_ERR_VALUE(cur_mode))
1765                 cur_mode = REGULATOR_MODE_NORMAL;
1766
1767         if (WARN_ONCE(cur_mode != mode,
1768                   "%s: dvfs regulator %s cannot change mode from %u\n",
1769                   __func__, rail->reg_id, cur_mode))
1770                 return -EINVAL;
1771
1772         return 0;
1773 }
1774
1775 int tegra_dvfs_rail_register_notifier(struct dvfs_rail *rail,
1776                                       struct notifier_block *nb)
1777 {
1778         if (!rail || !rail->reg)
1779                 return -ENOENT;
1780
1781         return regulator_register_notifier(rail->reg, nb);
1782 }
1783
1784 int tegra_dvfs_rail_unregister_notifier(struct dvfs_rail *rail,
1785                                         struct notifier_block *nb)
1786 {
1787         if (!rail || !rail->reg)
1788                 return -ENOENT;
1789
1790         return regulator_unregister_notifier(rail->reg, nb);
1791 }
1792
1793 bool tegra_dvfs_rail_updating(struct clk *clk)
1794 {
1795         return (!clk ? false :
1796                 (!clk->dvfs ? false :
1797                  (!clk->dvfs->dvfs_rail ? false :
1798                   (clk->dvfs->dvfs_rail->updating ||
1799                    clk->dvfs->dvfs_rail->dfll_mode_updating))));
1800 }
1801
1802 #ifdef CONFIG_OF
1803 int __init of_tegra_dvfs_init(const struct of_device_id *matches)
1804 {
1805         int ret;
1806         struct device_node *np;
1807
1808         for_each_matching_node(np, matches) {
1809                 const struct of_device_id *match = of_match_node(matches, np);
1810                 of_tegra_dvfs_init_cb_t dvfs_init_cb = match->data;
1811                 ret = dvfs_init_cb(np);
1812                 if (ret) {
1813                         pr_err("dt: Failed to read %s data from DT\n",
1814                                match->compatible);
1815                         return ret;
1816                 }
1817         }
1818         return 0;
1819 }
1820
1821 static int __init of_rail_align(struct device_node *reg_dn,
1822                                 struct dvfs_rail *rail)
1823 {
1824         u32 vmin, vmax, n;
1825         int step, ret = 0;
1826
1827         ret |= of_property_read_u32(reg_dn, "regulator-n-voltages", &n);
1828         ret |= of_property_read_u32(reg_dn, "regulator-min-microvolt", &vmin);
1829         ret |= of_property_read_u32(reg_dn, "regulator-max-microvolt", &vmax);
1830         if (ret || (n <= 1)) {
1831                 ret = -EINVAL;
1832                 goto _out;
1833         }
1834
1835         step = (vmax - vmin) / (n - 1);
1836         if (step <= 0) {
1837                 ret = -EINVAL;
1838                 goto _out;
1839         }
1840
1841         rail->alignment.offset_uv = vmin;
1842         rail->alignment.step_uv = step;
1843
1844 _out:
1845         of_node_put(reg_dn);
1846         return ret;
1847 }
1848
1849 int __init of_tegra_dvfs_rail_node_parse(struct device_node *rail_dn,
1850                                          struct dvfs_rail *rail)
1851 {
1852         u32 val;
1853         struct device_node *reg_dn;
1854         char prop_name[80];
1855
1856         snprintf(prop_name, 80, "%s-supply", rail->reg_id);
1857         reg_dn = of_parse_phandle(rail_dn, prop_name, 0);
1858         if (!reg_dn)
1859                 return -ENODEV;
1860
1861         /*
1862          * This is called in early DVFS init before device tree population.
1863          * Hence, if rail node supply is matching rail regulator id, save rail
1864          * node to be used when DVFS is connected to regulators in late init,
1865          * and align rail to regulator DT data.
1866          *
1867          * If CPU rail supply is specified as fixed regulator on platforms that
1868          * have DFLL clock source with CL-DVFS h/w control, a separate DFLL mode
1869          * regulator node must be present and used for rail alignment.
1870          */
1871         rail->dt_node = rail_dn;
1872         rail->dt_reg_fixed = of_device_is_compatible(reg_dn, "regulator-fixed");
1873         rail->dt_reg_pwm = of_device_is_compatible(reg_dn, "regulator-pwm");
1874         if (!of_property_read_u32(reg_dn, "regulator-min-microvolt", &val)) {
1875                 if (rail->min_millivolts < val / 1000)
1876                         rail->min_millivolts = val / 1000;
1877         }
1878
1879 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1880         if (rail->dt_reg_fixed && !strcmp("vdd_cpu", rail->reg_id)) {
1881                 of_node_put(reg_dn);
1882                 snprintf(prop_name, 80, "%s_dfll-supply", rail->reg_id);
1883                 reg_dn = of_parse_phandle(rail_dn, prop_name, 0);
1884                 if (!reg_dn) {
1885                         pr_err("tegra_dvfs: missed %s DFLL mode supply\n",
1886                                rail->reg_id);
1887                         return -ENODEV;
1888                 }
1889         }
1890 #endif
1891         of_rail_align(reg_dn, rail);
1892         return 0;
1893
1894 }
1895 #endif
1896
1897 int tegra_dvfs_dfll_mode_set(struct dvfs *d, unsigned long rate)
1898 {
1899         mutex_lock(&dvfs_lock);
1900         if (!d->dvfs_rail->dfll_mode) {
1901                 d->dvfs_rail->dfll_mode = true;
1902                 __tegra_dvfs_set_rate(d, rate);
1903
1904                 /*
1905                  * Report error, but continue: DFLL is functional, anyway, and
1906                  * no error with proper regulator driver update
1907                  */
1908                 if (regulator_set_vsel_volatile(d->dvfs_rail->reg, true))
1909                         WARN_ONCE(1, "%s: failed to set vsel volatile\n",
1910                                   __func__);
1911         }
1912         mutex_unlock(&dvfs_lock);
1913         return 0;
1914 }
1915
1916 int tegra_dvfs_dfll_mode_clear(struct dvfs *d, unsigned long rate)
1917 {
1918         int ret = 0;
1919
1920         mutex_lock(&dvfs_lock);
1921         if (d->dvfs_rail->dfll_mode) {
1922                 d->dvfs_rail->dfll_mode = false;
1923                 regulator_set_vsel_volatile(d->dvfs_rail->reg, false);
1924
1925                 /*
1926                  * avoid false detection of matching target (voltage in
1927                  * dfll mode is fluctuating, and recorded level is just
1928                  * estimate)
1929                  */
1930                 d->dvfs_rail->millivolts--;
1931
1932                 /* Update voltage using pll dvfs table per caller request */
1933                 if (rate) {
1934                         if (d->dvfs_rail->disabled) {
1935                                 d->dvfs_rail->disabled = false;
1936                                 __tegra_dvfs_rail_disable(d->dvfs_rail);
1937                         }
1938                         ret = __tegra_dvfs_set_rate(d, rate);
1939                 }
1940         }
1941         mutex_unlock(&dvfs_lock);
1942         return ret;
1943 }
1944
1945 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmax_cdev(void)
1946 {
1947         if (tegra_cpu_rail)
1948                 return tegra_cpu_rail->vmax_cdev;
1949         return NULL;
1950 }
1951
1952 struct tegra_cooling_device *tegra_dvfs_get_cpu_vmin_cdev(void)
1953 {
1954         if (tegra_cpu_rail)
1955                 return tegra_cpu_rail->vmin_cdev;
1956         return NULL;
1957 }
1958
1959 struct tegra_cooling_device *tegra_dvfs_get_core_vmax_cdev(void)
1960 {
1961         if (tegra_core_rail)
1962                 return tegra_core_rail->vmax_cdev;
1963         return NULL;
1964 }
1965
1966 struct tegra_cooling_device *tegra_dvfs_get_core_vmin_cdev(void)
1967 {
1968         if (tegra_core_rail)
1969                 return tegra_core_rail->vmin_cdev;
1970         return NULL;
1971 }
1972
1973 struct tegra_cooling_device *tegra_dvfs_get_gpu_vmin_cdev(void)
1974 {
1975         if (tegra_gpu_rail)
1976                 return tegra_gpu_rail->vmin_cdev;
1977         return NULL;
1978 }
1979
1980 struct tegra_cooling_device *tegra_dvfs_get_gpu_vts_cdev(void)
1981 {
1982         if (tegra_gpu_rail)
1983                 return tegra_gpu_rail->vts_cdev;
1984         return NULL;
1985 }
1986
1987 struct tegra_cooling_device *tegra_dvfs_get_cpu_clk_switch_cdev(void)
1988 {
1989         if (tegra_cpu_rail)
1990                 return tegra_cpu_rail->clk_switch_cdev;
1991         return NULL;
1992 }
1993
1994 static void make_safe_thermal_dvfs(struct dvfs_rail *rail)
1995 {
1996         struct dvfs *d;
1997
1998         mutex_lock(&dvfs_lock);
1999         list_for_each_entry(d, &rail->dvfs, reg_node) {
2000                 if (d->therm_dvfs) {
2001                         BUG_ON(!d->peak_millivolts);
2002                         d->millivolts = d->peak_millivolts;
2003                         d->therm_dvfs = false;
2004                 }
2005         }
2006         mutex_unlock(&dvfs_lock);
2007 }
2008
2009 #ifdef CONFIG_THERMAL
2010 /* Cooling device limits minimum rail voltage at cold temperature in pll mode */
2011 static int tegra_dvfs_rail_get_vmin_cdev_max_state(
2012         struct thermal_cooling_device *cdev, unsigned long *max_state)
2013 {
2014         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
2015         *max_state = rail->vmin_cdev->trip_temperatures_num;
2016         return 0;
2017 }
2018
2019 static int tegra_dvfs_rail_get_vmin_cdev_cur_state(
2020         struct thermal_cooling_device *cdev, unsigned long *cur_state)
2021 {
2022         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
2023         *cur_state = rail->therm_floor_idx;
2024         return 0;
2025 }
2026
2027 static int tegra_dvfs_rail_set_vmin_cdev_state(
2028         struct thermal_cooling_device *cdev, unsigned long cur_state)
2029 {
2030         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
2031
2032         mutex_lock(&dvfs_lock);
2033         if (rail->therm_floor_idx != cur_state) {
2034                 rail->therm_floor_idx = cur_state;
2035                 dvfs_rail_update(rail);
2036         }
2037         mutex_unlock(&dvfs_lock);
2038         return 0;
2039 }
2040
2041 static struct thermal_cooling_device_ops tegra_dvfs_vmin_cooling_ops = {
2042         .get_max_state = tegra_dvfs_rail_get_vmin_cdev_max_state,
2043         .get_cur_state = tegra_dvfs_rail_get_vmin_cdev_cur_state,
2044         .set_cur_state = tegra_dvfs_rail_set_vmin_cdev_state,
2045 };
2046
2047 static void tegra_dvfs_rail_register_vmin_cdev(struct dvfs_rail *rail)
2048 {
2049         if (!rail->vmin_cdev)
2050                 return;
2051
2052         /* just report error - initialized for cold temperature, anyway */
2053         if (IS_ERR_OR_NULL(thermal_cooling_device_register(
2054                 rail->vmin_cdev->cdev_type, (void *)rail,
2055                 &tegra_dvfs_vmin_cooling_ops)))
2056                 pr_err("tegra cooling device %s failed to register\n",
2057                        rail->vmin_cdev->cdev_type);
2058 }
2059
2060 /*
2061  * Cooling device limits frequencies of the clocks in pll mode based on rail
2062  * vmax thermal profile. Supported for core rail only, and applied only to
2063  * shared buses selected by platform specific code.
2064  */
2065 static int tegra_dvfs_rail_get_vmax_cdev_max_state(
2066         struct thermal_cooling_device *cdev, unsigned long *max_state)
2067 {
2068         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
2069         *max_state = rail->vmax_cdev->trip_temperatures_num;
2070         return 0;
2071 }
2072
2073 static int tegra_dvfs_rail_get_vmax_cdev_cur_state(
2074         struct thermal_cooling_device *cdev, unsigned long *cur_state)
2075 {
2076         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
2077         *cur_state = rail->therm_cap_idx;
2078         return 0;
2079 }
2080
2081 static int tegra_dvfs_rail_set_vmax_cdev_state(
2082         struct thermal_cooling_device *cdev, unsigned long cur_state)
2083 {
2084         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
2085         int cur_cap = cur_state ? rail->therm_mv_caps[cur_state - 1] : 0;
2086
2087         return tegra_dvfs_therm_vmax_core_cap_apply(&rail->therm_cap_idx,
2088                                                     cur_state, cur_cap);
2089 }
2090
2091 static struct thermal_cooling_device_ops tegra_dvfs_vmax_cooling_ops = {
2092         .get_max_state = tegra_dvfs_rail_get_vmax_cdev_max_state,
2093         .get_cur_state = tegra_dvfs_rail_get_vmax_cdev_cur_state,
2094         .set_cur_state = tegra_dvfs_rail_set_vmax_cdev_state,
2095 };
2096
2097 void tegra_dvfs_rail_register_vmax_cdev(struct dvfs_rail *rail)
2098 {
2099         struct thermal_cooling_device *dev;
2100
2101         if (!rail || !rail->vmax_cdev || (rail != tegra_core_rail))
2102                 return;
2103
2104         dev = thermal_cooling_device_register(rail->vmax_cdev->cdev_type,
2105                 (void *)rail, &tegra_dvfs_vmax_cooling_ops);
2106
2107         if (IS_ERR_OR_NULL(dev) || list_empty(&dev->thermal_instances)) {
2108                 /* report error & set the most agressive caps */
2109                 int cur_state = rail->vmax_cdev->trip_temperatures_num;
2110                 int cur_cap = rail->therm_mv_caps[cur_state - 1];
2111                 tegra_dvfs_therm_vmax_core_cap_apply(&rail->therm_cap_idx,
2112                                                      cur_state, cur_cap);
2113                 pr_err("tegra cooling device %s failed to register\n",
2114                        rail->vmax_cdev->cdev_type);
2115         }
2116 }
2117
2118 /* Cooling device to switch the cpu clock source between PLLX and DFLL */
2119 static int tegra_dvfs_rail_get_clk_switch_cdev_max_state(
2120         struct thermal_cooling_device *cdev, unsigned long *max_state)
2121 {
2122         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
2123         *max_state = rail->clk_switch_cdev->trip_temperatures_num;
2124         return 0;
2125 }
2126
2127 static int tegra_dvfs_rail_get_clk_switch_cdev_cur_state(
2128         struct thermal_cooling_device *cdev, unsigned long *cur_state)
2129 {
2130         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
2131         *cur_state = rail->therm_scale_idx;
2132         return 0;
2133 }
2134
2135 static int tegra_dvfs_rail_set_clk_switch_cdev_state(
2136         struct thermal_cooling_device *cdev, unsigned long cur_state)
2137 {
2138         int ret = 0;
2139         enum dfll_range use_dfll;
2140         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
2141
2142         if (CONFIG_TEGRA_USE_DFLL_RANGE == TEGRA_USE_DFLL_CDEV_CNTRL) {
2143                 if (rail->therm_scale_idx != cur_state) {
2144                         rail->therm_scale_idx = cur_state;
2145                         if (rail->therm_scale_idx == 0)
2146                                 use_dfll = DFLL_RANGE_NONE;
2147                         else
2148                                 use_dfll = DFLL_RANGE_ALL_RATES;
2149
2150                         ret = tegra_clk_dfll_range_control(use_dfll);
2151                 }
2152         } else {
2153                 pr_warn("\n%s: Not Allowed:", __func__);
2154                 pr_warn("DFLL is not under thermal cooling device control\n");
2155                 return -EACCES;
2156         }
2157         return ret;
2158 }
2159
2160 static struct thermal_cooling_device_ops tegra_dvfs_clk_cooling_ops = {
2161         .get_max_state = tegra_dvfs_rail_get_clk_switch_cdev_max_state,
2162         .get_cur_state = tegra_dvfs_rail_get_clk_switch_cdev_cur_state,
2163         .set_cur_state = tegra_dvfs_rail_set_clk_switch_cdev_state,
2164 };
2165
2166 static void tegra_dvfs_rail_register_clk_switch_cdev(struct dvfs_rail *rail)
2167 {
2168         struct thermal_cooling_device *dev;
2169
2170         if (!rail->clk_switch_cdev)
2171                 return;
2172
2173         dev = thermal_cooling_device_register(rail->clk_switch_cdev->cdev_type,
2174                 (void *)rail, &tegra_dvfs_clk_cooling_ops);
2175         /* report error & set max limits across thermal ranges as safe dvfs */
2176         if (IS_ERR_OR_NULL(dev) || list_empty(&dev->thermal_instances)) {
2177                 pr_err("tegra cooling device %s failed to register\n",
2178                        rail->clk_switch_cdev->cdev_type);
2179                 make_safe_thermal_dvfs(rail);
2180         }
2181 }
2182
2183
2184 /* Cooling device to scale voltage with temperature in pll mode */
2185 static int tegra_dvfs_rail_get_vts_cdev_max_state(
2186         struct thermal_cooling_device *cdev, unsigned long *max_state)
2187 {
2188         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
2189         *max_state = rail->vts_cdev->trip_temperatures_num;
2190         return 0;
2191 }
2192
2193 static int tegra_dvfs_rail_get_vts_cdev_cur_state(
2194         struct thermal_cooling_device *cdev, unsigned long *cur_state)
2195 {
2196         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
2197         *cur_state = rail->therm_scale_idx;
2198         return 0;
2199 }
2200
2201 static int tegra_dvfs_rail_set_vts_cdev_state(
2202         struct thermal_cooling_device *cdev, unsigned long cur_state)
2203 {
2204         struct dvfs_rail *rail = (struct dvfs_rail *)cdev->devdata;
2205         struct dvfs *d;
2206
2207         mutex_lock(&dvfs_lock);
2208         if (rail->therm_scale_idx != cur_state) {
2209                 rail->therm_scale_idx = cur_state;
2210                 list_for_each_entry(d, &rail->dvfs, reg_node) {
2211                         if (d->therm_dvfs)
2212                                 __tegra_dvfs_set_rate(d, d->cur_rate);
2213                 }
2214         }
2215         mutex_unlock(&dvfs_lock);
2216         return 0;
2217 }
2218
2219 static struct thermal_cooling_device_ops tegra_dvfs_vts_cooling_ops = {
2220         .get_max_state = tegra_dvfs_rail_get_vts_cdev_max_state,
2221         .get_cur_state = tegra_dvfs_rail_get_vts_cdev_cur_state,
2222         .set_cur_state = tegra_dvfs_rail_set_vts_cdev_state,
2223 };
2224
2225 static void tegra_dvfs_rail_register_vts_cdev(struct dvfs_rail *rail)
2226 {
2227         struct thermal_cooling_device *dev;
2228
2229         if (!rail->vts_cdev)
2230                 return;
2231
2232         dev = thermal_cooling_device_register(rail->vts_cdev->cdev_type,
2233                 (void *)rail, &tegra_dvfs_vts_cooling_ops);
2234         /* report error & set max limits across thermal ranges as safe dvfs */
2235         if (IS_ERR_OR_NULL(dev) || list_empty(&dev->thermal_instances)) {
2236                 pr_err("tegra cooling device %s failed to register\n",
2237                        rail->vts_cdev->cdev_type);
2238                 make_safe_thermal_dvfs(rail);
2239         }
2240 }
2241
2242 #else
2243 #define tegra_dvfs_rail_register_vmin_cdev(rail)
2244 void tegra_dvfs_rail_register_vmax_cdev(struct dvfs_rail *rail)
2245 { }
2246 static void tegra_dvfs_rail_register_vts_cdev(struct dvfs_rail *rail)
2247 {
2248         make_safe_thermal_dvfs(rail);
2249 }
2250 #endif
2251
2252 #ifdef CONFIG_TEGRA_USE_SIMON
2253 /*
2254  * Validate rail SiMon Vmin offsets. Valid offsets should be negative,
2255  * descending, starting from zero.
2256  */
2257 void __init tegra_dvfs_rail_init_simon_vmin_offsets(
2258         int *offsets, int offs_num, struct dvfs_rail *rail)
2259 {
2260         int i;
2261
2262         if (!offsets || !offs_num || offsets[0]) {
2263                 WARN(1, "%s: invalid initial SiMon offset\n", rail->reg_id);
2264                 return;
2265         }
2266
2267         for (i = 0; i < offs_num - 1; i++) {
2268                 if (offsets[i] < offsets[i+1]) {
2269                         WARN(1, "%s: SiMon offsets are not ordered\n",
2270                              rail->reg_id);
2271                         return;
2272                 }
2273         }
2274         rail->simon_vmin_offsets = offsets;
2275         rail->simon_vmin_offs_num = offs_num;
2276 }
2277 #endif
2278
2279 /*
2280  * Validate rail thermal profile, and get its size. Valid profile:
2281  * - voltage limits are descending with temperature increasing
2282  * - the lowest limit is above rail minimum voltage in pll and
2283  *   in dfll mode (if applicable)
2284  * - the highest limit is below rail nominal voltage (required only
2285  *   for Vmin profile)
2286  */
2287 static int __init get_thermal_profile_size(
2288         int *trips_table, int *limits_table,
2289         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
2290 {
2291         int i, min_mv;
2292
2293         for (i = 0; i < MAX_THERMAL_LIMITS - 1; i++) {
2294                 if (!limits_table[i+1])
2295                         break;
2296
2297                 if ((trips_table[i] >= trips_table[i+1]) ||
2298                     (limits_table[i] < limits_table[i+1])) {
2299                         pr_warn("%s: not ordered profile\n", rail->reg_id);
2300                         return -EINVAL;
2301                 }
2302         }
2303
2304         min_mv = max(rail->min_millivolts, d ? d->min_millivolts : 0);
2305         if (limits_table[i] < min_mv) {
2306                 pr_warn("%s: thermal profile below Vmin\n", rail->reg_id);
2307                 return -EINVAL;
2308         }
2309
2310         return i + 1;
2311 }
2312
2313 void __init tegra_dvfs_rail_init_vmax_thermal_profile(
2314         int *therm_trips_table, int *therm_caps_table,
2315         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
2316 {
2317         int i = get_thermal_profile_size(therm_trips_table,
2318                                          therm_caps_table, rail, d);
2319         if (i <= 0) {
2320                 rail->vmax_cdev = NULL;
2321                 WARN(1, "%s: invalid Vmax thermal profile\n", rail->reg_id);
2322                 return;
2323         }
2324
2325         /* Install validated thermal caps */
2326         rail->therm_mv_caps = therm_caps_table;
2327         rail->therm_mv_caps_num = i;
2328
2329         /* Setup trip-points if applicable */
2330         if (rail->vmax_cdev) {
2331                 rail->vmax_cdev->trip_temperatures_num = i;
2332                 rail->vmax_cdev->trip_temperatures = therm_trips_table;
2333         }
2334 }
2335
2336 int  __init tegra_dvfs_rail_init_clk_switch_thermal_profile(
2337         int *clk_switch_trips, struct dvfs_rail *rail)
2338 {
2339         int i;
2340
2341         if (!rail->clk_switch_cdev) {
2342                 WARN(1, "%s: missing thermal dvfs cooling device\n",
2343                         rail->reg_id);
2344                 return -ENOENT;
2345         }
2346
2347         for (i = 0; i < MAX_THERMAL_LIMITS - 1; i++) {
2348                 if (clk_switch_trips[i] >= clk_switch_trips[i+1])
2349                         break;
2350         }
2351
2352         /*Only one trip point is allowed for this cdev*/
2353         if (i != 0) {
2354                 WARN(1, "%s: Only one trip point allowed\n", __func__);
2355                 return -EINVAL;
2356         }
2357
2358         rail->clk_switch_cdev->trip_temperatures_num = i + 1;
2359         rail->clk_switch_cdev->trip_temperatures = clk_switch_trips;
2360         return 0;
2361 }
2362
2363 void __init tegra_dvfs_rail_init_vmin_thermal_profile(
2364         int *therm_trips_table, int *therm_floors_table,
2365         struct dvfs_rail *rail, struct dvfs_dfll_data *d)
2366 {
2367         int i = get_thermal_profile_size(therm_trips_table,
2368                                          therm_floors_table, rail, d);
2369
2370         if (i <= 0 || therm_floors_table[0] > rail->nominal_millivolts) {
2371                 rail->vmin_cdev = NULL;
2372                 WARN(1, "%s: invalid Vmin thermal profile\n", rail->reg_id);
2373                 return;
2374         }
2375
2376         /* Install validated thermal floors */
2377         rail->therm_mv_floors = therm_floors_table;
2378         rail->therm_mv_floors_num = i;
2379
2380         /* Setup trip-points if applicable */
2381         if (rail->vmin_cdev) {
2382                 rail->vmin_cdev->trip_temperatures_num = i;
2383                 rail->vmin_cdev->trip_temperatures = therm_trips_table;
2384         }
2385 }
2386
2387 /*
2388  * Validate thermal dvfs settings:
2389  * - trip-points are montonically increasing
2390  * - voltages in any temperature range are montonically increasing with
2391  *   frequency (can go up/down across ranges at iso frequency)
2392  * - voltage for any frequency/thermal range combination must be within
2393  *   rail minimum/maximum limits
2394  */
2395 int __init tegra_dvfs_rail_init_thermal_dvfs_trips(
2396         int *therm_trips_table, struct dvfs_rail *rail)
2397 {
2398         int i;
2399
2400         if (!rail->vts_cdev) {
2401                 WARN(1, "%s: missing thermal dvfs cooling device\n",
2402                      rail->reg_id);
2403                 return -ENOENT;
2404         }
2405
2406         for (i = 0; i < MAX_THERMAL_LIMITS - 1; i++) {
2407                 if (therm_trips_table[i] >= therm_trips_table[i+1])
2408                         break;
2409         }
2410
2411         rail->vts_cdev->trip_temperatures_num = i + 1;
2412         rail->vts_cdev->trip_temperatures = therm_trips_table;
2413         return 0;
2414 }
2415
2416 int __init tegra_dvfs_init_thermal_dvfs_voltages(int *therm_voltages,
2417         int *peak_voltages, int freqs_num, int ranges_num, struct dvfs *d)
2418 {
2419         int *millivolts;
2420         int freq_idx, therm_idx;
2421
2422         for (therm_idx = 0; therm_idx < ranges_num; therm_idx++) {
2423                 millivolts = therm_voltages + therm_idx * MAX_DVFS_FREQS;
2424                 for (freq_idx = 0; freq_idx < freqs_num; freq_idx++) {
2425                         int mv = millivolts[freq_idx];
2426                         if ((mv > d->dvfs_rail->max_millivolts) ||
2427                             (mv < d->dvfs_rail->min_millivolts) ||
2428                             (freq_idx && (mv < millivolts[freq_idx - 1]))) {
2429                                 WARN(1, "%s: invalid thermal dvfs entry %d(%d, %d)\n",
2430                                      d->clk_name, mv, freq_idx, therm_idx);
2431                                 return -EINVAL;
2432                         }
2433                         if (mv > peak_voltages[freq_idx])
2434                                 peak_voltages[freq_idx] = mv;
2435                 }
2436         }
2437
2438         d->millivolts = therm_voltages;
2439         d->peak_millivolts = peak_voltages;
2440         d->therm_dvfs = true;
2441         return 0;
2442 }
2443
2444 /* Directly set cold temperature limit in dfll mode */
2445 int tegra_dvfs_rail_dfll_mode_set_cold(struct dvfs_rail *rail,
2446                                        struct clk *dfll_clk)
2447 {
2448         int ret = 0;
2449
2450         /* No thermal floors - nothing to do */
2451         if (!rail || !rail->therm_mv_floors)
2452                 return ret;
2453
2454         /*
2455          * Compare last set Vmin with requirement based on current temperature,
2456          * and set cold limit at regulator only Vmin is below requirement.
2457          */
2458         mutex_lock(&dvfs_lock);
2459         if (rail->dfll_mode) {
2460                 int mv, cmp;
2461                 cmp = tegra_cl_dvfs_vmin_cmp_needed(
2462                         tegra_dfll_get_cl_dvfs_data(dfll_clk), &mv);
2463                 if (cmp < 0)
2464                         ret = dvfs_rail_set_voltage_reg(rail, mv);
2465         }
2466         mutex_unlock(&dvfs_lock);
2467
2468         return ret;
2469 }
2470
2471 /*
2472  * Get current thermal floor. Does not take DVFS lock. Should be called either
2473  * with lock already taken or in late suspend/early resume when DVFS operations
2474  * are suspended.
2475  */
2476 int tegra_dvfs_rail_get_thermal_floor(struct dvfs_rail *rail)
2477 {
2478         if (!rail)
2479                 return 0;
2480
2481         return rail->dfll_mode ? dvfs_rail_get_thermal_floor_dfll(rail) :
2482                 dvfs_rail_get_thermal_floor_pll(rail);
2483 }
2484
2485 /*
2486  * Iterate through all the dvfs regulators, finding the regulator exported
2487  * by the regulator api for each one.  Must be called in late init, after
2488  * all the regulator api's regulators are initialized.
2489  */
2490
2491 #ifdef CONFIG_TEGRA_DVFS_RAIL_CONNECT_ALL
2492 /*
2493  * Enable voltage scaling only if all the rails connect successfully
2494  */
2495 int __init tegra_dvfs_rail_connect_regulators(void)
2496 {
2497         bool connected = true;
2498         struct dvfs_rail *rail;
2499
2500         mutex_lock(&dvfs_lock);
2501
2502         list_for_each_entry(rail, &dvfs_rail_list, node)
2503                 if (dvfs_rail_connect_to_regulator(rail))
2504                         connected = false;
2505
2506         list_for_each_entry(rail, &dvfs_rail_list, node) {
2507                 if (connected) {
2508                         dvfs_rail_update(rail);
2509                         if (!rail->disabled)
2510                                 continue;
2511                         /* Don't rely on boot level - force disabled voltage */
2512                         rail->disabled = false;
2513                 }
2514                 __tegra_dvfs_rail_force_disable(rail);
2515         }
2516         mutex_unlock(&dvfs_lock);
2517
2518         if (!connected && tegra_platform_is_silicon()) {
2519                 pr_warn("tegra_dvfs: DVFS regulators connection failed\n"
2520                         "            !!!! voltage scaling is disabled !!!!\n");
2521                 return -ENODEV;
2522         }
2523
2524         return 0;
2525 }
2526 #else
2527 int __init tegra_dvfs_rail_connect_regulators(void)
2528 {
2529         struct dvfs_rail *rail;
2530
2531         mutex_lock(&dvfs_lock);
2532
2533         list_for_each_entry(rail, &dvfs_rail_list, node) {
2534                 if (!dvfs_rail_connect_to_regulator(rail)) {
2535                         dvfs_rail_update(rail);
2536                         if (!rail->disabled)
2537                                 continue;
2538                         /* Don't rely on boot level - force disabled voltage */
2539                         rail->disabled = false;
2540                 }
2541                 __tegra_dvfs_rail_force_disable(rail);
2542         }
2543
2544         mutex_unlock(&dvfs_lock);
2545
2546         return 0;
2547 }
2548 #endif
2549
2550 int __init tegra_dvfs_rail_register_notifiers(void)
2551 {
2552         struct dvfs_rail *rail;
2553
2554         register_pm_notifier(&tegra_dvfs_suspend_nb);
2555         register_pm_notifier(&tegra_dvfs_resume_nb);
2556         register_reboot_notifier(&tegra_dvfs_reboot_nb);
2557
2558         list_for_each_entry(rail, &dvfs_rail_list, node) {
2559                         tegra_dvfs_rail_register_vmin_cdev(rail);
2560                         tegra_dvfs_rail_register_vts_cdev(rail);
2561                         tegra_dvfs_rail_register_clk_switch_cdev(rail);
2562
2563         }
2564
2565         return 0;
2566 }
2567
2568 static int rail_stats_save_to_buf(char *buf, int len)
2569 {
2570         int i;
2571         struct dvfs_rail *rail;
2572         char *str = buf;
2573         char *end = buf + len;
2574
2575         str += scnprintf(str, end - str, "%-12s %-10s\n", "millivolts", "time");
2576
2577         mutex_lock(&dvfs_lock);
2578
2579         list_for_each_entry(rail, &dvfs_rail_list, node) {
2580                 str += scnprintf(str, end - str, "%s (bin: %d.%dmV)\n",
2581                            rail->reg_id,
2582                            rail->stats.bin_uV / 1000,
2583                            (rail->stats.bin_uV / 10) % 100);
2584
2585                 dvfs_rail_stats_update(rail, -1, ktime_get());
2586
2587                 str += scnprintf(str, end - str, "%-12d %-10llu\n", 0,
2588                         cputime64_to_clock_t(msecs_to_jiffies(
2589                                 ktime_to_ms(rail->stats.time_at_mv[0]))));
2590
2591                 for (i = 1; i <= DVFS_RAIL_STATS_TOP_BIN; i++) {
2592                         ktime_t ktime_zero = ktime_set(0, 0);
2593                         if (ktime_equal(rail->stats.time_at_mv[i], ktime_zero))
2594                                 continue;
2595                         str += scnprintf(str, end - str, "%-12d %-10llu\n",
2596                                 rail->min_millivolts +
2597                                 (i - 1) * rail->stats.bin_uV / 1000,
2598                                 cputime64_to_clock_t(msecs_to_jiffies(
2599                                         ktime_to_ms(rail->stats.time_at_mv[i])))
2600                         );
2601                 }
2602         }
2603         mutex_unlock(&dvfs_lock);
2604         return str - buf;
2605 }
2606
2607 #ifdef CONFIG_DEBUG_FS
2608 static int dvfs_tree_sort_cmp(void *p, struct list_head *a, struct list_head *b)
2609 {
2610         struct dvfs *da = list_entry(a, struct dvfs, reg_node);
2611         struct dvfs *db = list_entry(b, struct dvfs, reg_node);
2612         int ret;
2613
2614         ret = strcmp(da->dvfs_rail->reg_id, db->dvfs_rail->reg_id);
2615         if (ret != 0)
2616                 return ret;
2617
2618         if (da->cur_millivolts < db->cur_millivolts)
2619                 return 1;
2620         if (da->cur_millivolts > db->cur_millivolts)
2621                 return -1;
2622
2623         return strcmp(da->clk_name, db->clk_name);
2624 }
2625
2626 /* To emulate and show rail relations with 0 mV on dependent rail-to */
2627 static struct dvfs_rail show_to;
2628 static struct dvfs_relationship show_rel;
2629
2630 static int dvfs_tree_show(struct seq_file *s, void *data)
2631 {
2632         struct dvfs *d;
2633         struct dvfs_rail *rail;
2634         struct dvfs_relationship *rel;
2635
2636         seq_printf(s, "   clock      rate       mV\n");
2637         seq_printf(s, "--------------------------------\n");
2638
2639         mutex_lock(&dvfs_lock);
2640
2641         list_for_each_entry(rail, &dvfs_rail_list, node) {
2642                 int thermal_mv_floor = 0;
2643
2644                 seq_printf(s, "%s %d mV%s:\n", rail->reg_id,
2645                            rail->stats.off ? 0 : rail->millivolts,
2646                            rail->dfll_mode ? " dfll mode" :
2647                                 rail->disabled ? " disabled" : "");
2648                 list_for_each_entry(rel, &rail->relationships_from, from_node) {
2649                         show_rel = *rel;
2650                         show_rel.to = &show_to;
2651                         show_to = *rel->to;
2652                         show_to.millivolts = show_to.new_millivolts = 0;
2653                         seq_printf(s, "   %-10s %-7d mV %-4d mV .. %-4d mV\n",
2654                                 rel->from->reg_id, rel->from->millivolts,
2655                                 dvfs_solve_relationship(&show_rel),
2656                                 dvfs_solve_relationship(rel));
2657                 }
2658                 seq_printf(s, "   nominal    %-7d mV\n",
2659                            rail->nominal_millivolts);
2660                 seq_printf(s, "   minimum    %-7d mV\n", rail->min_millivolts);
2661                 seq_printf(s, "   offset     %-7d mV\n", rail->dbg_mv_offs);
2662
2663                 thermal_mv_floor = tegra_dvfs_rail_get_thermal_floor(rail);
2664                 seq_printf(s, "   thermal    %-7d mV\n", thermal_mv_floor);
2665
2666                 if (rail == tegra_core_rail) {
2667                         seq_printf(s, "   override   %-7d mV [%-4d...%-4d]",
2668                                    rail->override_millivolts,
2669                                    dvfs_rail_get_override_floor(rail),
2670                                    rail->nominal_millivolts);
2671                         if (rail->override_unresolved)
2672                                 seq_printf(s, " unresolved %d",
2673                                            rail->override_unresolved);
2674                         seq_putc(s, '\n');
2675                 }
2676
2677                 list_sort(NULL, &rail->dvfs, dvfs_tree_sort_cmp);
2678
2679                 list_for_each_entry(d, &rail->dvfs, reg_node) {
2680                         seq_printf(s, "   %-10s %-10lu %-4d mV\n", d->clk_name,
2681                                 d->cur_rate, d->cur_millivolts);
2682                 }
2683         }
2684
2685         mutex_unlock(&dvfs_lock);
2686
2687         return 0;
2688 }
2689
2690 static int dvfs_tree_open(struct inode *inode, struct file *file)
2691 {
2692         return single_open(file, dvfs_tree_show, inode->i_private);
2693 }
2694
2695 static const struct file_operations dvfs_tree_fops = {
2696         .open           = dvfs_tree_open,
2697         .read           = seq_read,
2698         .llseek         = seq_lseek,
2699         .release        = single_release,
2700 };
2701
2702 static int rail_stats_show(struct seq_file *s, void *data)
2703 {
2704         char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
2705         int size = 0;
2706
2707         if (!buf)
2708                 return -ENOMEM;
2709
2710         size = rail_stats_save_to_buf(buf, PAGE_SIZE);
2711         seq_write(s, buf, size);
2712         kfree(buf);
2713         return 0;
2714 }
2715
2716 static int rail_stats_open(struct inode *inode, struct file *file)
2717 {
2718         return single_open(file, rail_stats_show, inode->i_private);
2719 }
2720
2721 static const struct file_operations rail_stats_fops = {
2722         .open           = rail_stats_open,
2723         .read           = seq_read,
2724         .llseek         = seq_lseek,
2725         .release        = single_release,
2726 };
2727
2728 static int rail_offs_set(struct dvfs_rail *rail, int offs)
2729 {
2730         if (rail) {
2731                 mutex_lock(&dvfs_lock);
2732                 rail->dbg_mv_offs = offs;
2733                 dvfs_rail_update(rail);
2734                 mutex_unlock(&dvfs_lock);
2735                 return 0;
2736         }
2737         return -ENOENT;
2738 }
2739
2740 static int cpu_offs_get(void *data, u64 *val)
2741 {
2742         if (tegra_cpu_rail) {
2743                 *val = (u64)tegra_cpu_rail->dbg_mv_offs;
2744                 return 0;
2745         }
2746         *val = 0;
2747         return -ENOENT;
2748 }
2749 static int cpu_offs_set(void *data, u64 val)
2750 {
2751         return rail_offs_set(tegra_cpu_rail, (int)val);
2752 }
2753 DEFINE_SIMPLE_ATTRIBUTE(cpu_offs_fops, cpu_offs_get, cpu_offs_set, "%lld\n");
2754
2755 static int gpu_offs_get(void *data, u64 *val)
2756 {
2757         if (tegra_gpu_rail) {
2758                 *val = (u64)tegra_gpu_rail->dbg_mv_offs;
2759                 return 0;
2760         }
2761         *val = 0;
2762         return -ENOENT;
2763 }
2764 static int gpu_offs_set(void *data, u64 val)
2765 {
2766         return rail_offs_set(tegra_gpu_rail, (int)val);
2767 }
2768 DEFINE_SIMPLE_ATTRIBUTE(gpu_offs_fops, gpu_offs_get, gpu_offs_set, "%lld\n");
2769
2770 static int core_offs_get(void *data, u64 *val)
2771 {
2772         if (tegra_core_rail) {
2773                 *val = (u64)tegra_core_rail->dbg_mv_offs;
2774                 return 0;
2775         }
2776         *val = 0;
2777         return -ENOENT;
2778 }
2779 static int core_offs_set(void *data, u64 val)
2780 {
2781         return rail_offs_set(tegra_core_rail, (int)val);
2782 }
2783 DEFINE_SIMPLE_ATTRIBUTE(core_offs_fops, core_offs_get, core_offs_set, "%lld\n");
2784
2785 static int core_override_get(void *data, u64 *val)
2786 {
2787         if (tegra_core_rail) {
2788                 *val = (u64)tegra_core_rail->override_millivolts;
2789                 return 0;
2790         }
2791         *val = 0;
2792         return -ENOENT;
2793 }
2794 static int core_override_set(void *data, u64 val)
2795 {
2796         return dvfs_override_core_voltage((int)val);
2797 }
2798 DEFINE_SIMPLE_ATTRIBUTE(core_override_fops,
2799                         core_override_get, core_override_set, "%llu\n");
2800
2801 static int rail_mv_get(void *data, u64 *val)
2802 {
2803         struct dvfs_rail *rail = data;
2804         if (rail) {
2805                 *val = rail->stats.off ? 0 : rail->millivolts;
2806                 return 0;
2807         }
2808         *val = 0;
2809         return -ENOENT;
2810 }
2811 DEFINE_SIMPLE_ATTRIBUTE(rail_mv_fops, rail_mv_get, NULL, "%llu\n");
2812
2813 static int gpu_dvfs_t_show(struct seq_file *s, void *data)
2814 {
2815         int i, j;
2816         int num_ranges = 1;
2817         int *trips = NULL;
2818         struct dvfs *d;
2819         struct dvfs_rail *rail = tegra_gpu_rail;
2820         int max_mv[MAX_DVFS_FREQS] = {};
2821
2822         if (!tegra_gpu_rail) {
2823                 seq_printf(s, "Only supported for T124 or higher\n");
2824                 return -ENOSYS;
2825         }
2826
2827         mutex_lock(&dvfs_lock);
2828
2829         d = list_first_entry(&rail->dvfs, struct dvfs, reg_node);
2830         if (rail->vts_cdev && d->therm_dvfs) {
2831                 num_ranges = rail->vts_cdev->trip_temperatures_num + 1;
2832                 trips = rail->vts_cdev->trip_temperatures;
2833         }
2834
2835         seq_printf(s, "%-11s", "T(C)\\F(kHz)");
2836         for (i = 0; i < d->num_freqs; i++) {
2837                 unsigned int f = d->freqs[i]/1000;
2838                 seq_printf(s, " %7u", f);
2839         }
2840         seq_printf(s, "\n");
2841
2842         for (j = 0; j < num_ranges; j++) {
2843                 seq_printf(s, "%s", j == rail->therm_scale_idx ? ">" : " ");
2844
2845                 if (!trips || (num_ranges == 1))
2846                         seq_printf(s, "%4s..%-4s", "", "");
2847                 else if (j == 0)
2848                         seq_printf(s, "%4s..%-4d", "", trips[j]);
2849                 else if (j == num_ranges - 1)
2850                         seq_printf(s, "%4d..%-4s", trips[j], "");
2851                 else
2852                         seq_printf(s, "%4d..%-4d", trips[j-1], trips[j]);
2853
2854                 for (i = 0; i < d->num_freqs; i++) {
2855                         int mv = *(d->millivolts + j * MAX_DVFS_FREQS + i);
2856                         seq_printf(s, " %7d", mv);
2857                         max_mv[i] = max(max_mv[i], mv);
2858                 }
2859                 seq_printf(s, " mV\n");
2860         }
2861
2862         seq_printf(s, "%3s%-8s\n", "", "------");
2863         seq_printf(s, "%3s%-8s", "", "max(T)");
2864         for (i = 0; i < d->num_freqs; i++)
2865                 seq_printf(s, " %7d", max_mv[i]);
2866         seq_printf(s, " mV\n");
2867
2868         mutex_unlock(&dvfs_lock);
2869
2870         return 0;
2871 }
2872
2873 static int gpu_dvfs_t_open(struct inode *inode, struct file *file)
2874 {
2875         return single_open(file, gpu_dvfs_t_show, NULL);
2876 }
2877
2878 static const struct file_operations gpu_dvfs_t_fops = {
2879         .open           = gpu_dvfs_t_open,
2880         .read           = seq_read,
2881         .llseek         = seq_lseek,
2882         .release        = single_release,
2883 };
2884
2885 static int dvfs_table_show(struct seq_file *s, void *data)
2886 {
2887         int i;
2888         struct dvfs *d;
2889         struct dvfs_rail *rail;
2890         const int *v_pll, *last_v_pll = NULL;
2891         const int *v_dfll, *last_v_dfll = NULL;
2892
2893         seq_printf(s, "DVFS tables: units mV/MHz\n");
2894
2895         mutex_lock(&dvfs_lock);
2896
2897         list_for_each_entry(rail, &dvfs_rail_list, node) {
2898                 if (rail->version) {
2899                         seq_printf(s, "%-9s table version: ", rail->reg_id);
2900                         seq_printf(s, "%-16s\n", rail->version);
2901                 }
2902         }
2903
2904         list_for_each_entry(rail, &dvfs_rail_list, node) {
2905                 list_for_each_entry(d, &rail->dvfs, reg_node) {
2906                         bool mv_done = false;
2907                         v_pll = dvfs_get_millivolts_pll(d);
2908                         v_dfll = dvfs_get_millivolts_dfll(d);
2909
2910                         if (v_pll && (last_v_pll != v_pll)) {
2911                                 if (!mv_done) {
2912                                         seq_printf(s, "\n");
2913                                         mv_done = true;
2914                                 }
2915                                 last_v_pll = v_pll;
2916                                 seq_printf(s, "%-16s", rail->reg_id);
2917                                 for (i = 0; i < d->num_freqs; i++)
2918                                         seq_printf(s, "%7d", v_pll[i]);
2919                                 seq_printf(s, "\n");
2920                         }
2921
2922                         if (v_dfll && (last_v_dfll != v_dfll)) {
2923                                 if (!mv_done) {
2924                                         seq_printf(s, "\n");
2925                                         mv_done = true;
2926                                 }
2927                                 last_v_dfll = v_dfll;
2928                                 seq_printf(s, "%-8s (dfll) ", rail->reg_id);
2929                                 for (i = 0; i < d->num_freqs; i++)
2930                                         seq_printf(s, "%7d", v_dfll[i]);
2931                                 seq_printf(s, "\n");
2932                         }
2933
2934                         seq_printf(s, "%-16s", d->clk_name);
2935                         for (i = 0; i < d->num_freqs; i++) {
2936                                 unsigned long *freqs = dvfs_get_freqs(d);
2937                                 unsigned int f = freqs[i]/100000;
2938                                 seq_printf(s, " %4u.%u", f/10, f%10);
2939                         }
2940                         seq_printf(s, "\n");
2941                 }
2942         }
2943
2944         mutex_unlock(&dvfs_lock);
2945
2946         return 0;
2947 }
2948
2949 static int dvfs_table_open(struct inode *inode, struct file *file)
2950 {
2951         return single_open(file, dvfs_table_show, inode->i_private);
2952 }
2953
2954 static const struct file_operations dvfs_table_fops = {
2955         .open           = dvfs_table_open,
2956         .read           = seq_read,
2957         .llseek         = seq_lseek,
2958         .release        = single_release,
2959 };
2960
2961 int __init dvfs_debugfs_init(struct dentry *clk_debugfs_root)
2962 {
2963         struct dentry *d;
2964
2965         d = debugfs_create_file("dvfs", S_IRUGO, clk_debugfs_root, NULL,
2966                 &dvfs_tree_fops);
2967         if (!d)
2968                 return -ENOMEM;
2969
2970         d = debugfs_create_file("rails", S_IRUGO, clk_debugfs_root, NULL,
2971                 &rail_stats_fops);
2972         if (!d)
2973                 return -ENOMEM;
2974
2975         d = debugfs_create_file("vdd_cpu_offs", S_IRUGO | S_IWUSR,
2976                 clk_debugfs_root, NULL, &cpu_offs_fops);
2977         if (!d)
2978                 return -ENOMEM;
2979
2980         d = debugfs_create_file("vdd_gpu_offs", S_IRUGO | S_IWUSR,
2981                 clk_debugfs_root, NULL, &gpu_offs_fops);
2982         if (!d)
2983                 return -ENOMEM;
2984
2985         d = debugfs_create_file("vdd_core_offs", S_IRUGO | S_IWUSR,
2986                 clk_debugfs_root, NULL, &core_offs_fops);
2987         if (!d)
2988                 return -ENOMEM;
2989
2990         d = debugfs_create_file("vdd_core_override", S_IRUGO | S_IWUSR,
2991                 clk_debugfs_root, NULL, &core_override_fops);
2992         if (!d)
2993                 return -ENOMEM;
2994
2995         d = debugfs_create_file("vdd_cpu_mv", S_IRUGO, clk_debugfs_root,
2996                                 tegra_cpu_rail, &rail_mv_fops);
2997         if (!d)
2998                 return -ENOMEM;
2999
3000         d = debugfs_create_file("vdd_gpu_mv", S_IRUGO, clk_debugfs_root,
3001                                 tegra_gpu_rail, &rail_mv_fops);
3002         if (!d)
3003                 return -ENOMEM;
3004
3005         d = debugfs_create_file("vdd_core_mv", S_IRUGO, clk_debugfs_root,
3006                                 tegra_core_rail, &rail_mv_fops);
3007         if (!d)
3008                 return -ENOMEM;
3009
3010         d = debugfs_create_file("gpu_dvfs_t", S_IRUGO | S_IWUSR,
3011                 clk_debugfs_root, NULL, &gpu_dvfs_t_fops);
3012         if (!d)
3013                 return -ENOMEM;
3014
3015         d = debugfs_create_file("dvfs_table", S_IRUGO, clk_debugfs_root, NULL,
3016                 &dvfs_table_fops);
3017         if (!d)
3018                 return -ENOMEM;
3019
3020         return 0;
3021 }
3022
3023 #endif
3024
3025 #ifdef CONFIG_PM
3026 static ssize_t tegra_rail_stats_show(struct kobject *kobj,
3027                                         struct kobj_attribute *attr,
3028                                         char *buf)
3029 {
3030         return rail_stats_save_to_buf(buf, PAGE_SIZE);
3031 }
3032
3033 static struct kobj_attribute rail_stats_attr =
3034                 __ATTR_RO(tegra_rail_stats);
3035
3036 static int __init tegra_dvfs_sysfs_stats_init(void)
3037 {
3038         int error;
3039         error = sysfs_create_file(power_kobj, &rail_stats_attr.attr);
3040         return 0;
3041 }
3042 late_initcall(tegra_dvfs_sysfs_stats_init);
3043 #endif