drivers/soc/tegra/tegra210-dvfs.c

   1 /*
   2  * Copyright (c) 2014-2018, NVIDIA CORPORATION.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  15  */
  16
  17 #include <linux/kernel.h>
  18 #include <linux/module.h>
  19 #include <linux/clk.h>
  20 #include <linux/clk-provider.h>
  21 #include <linux/of_address.h>
  22 #include <linux/thermal.h>
  23 #include <linux/regulator/consumer.h>
  24
  25 #include <soc/tegra/cvb.h>
  26 #include <soc/tegra/tegra-dfll.h>
  27 #include <soc/tegra/tegra-dvfs.h>
  28 #include <soc/tegra/fuse.h>
  29 #include <soc/tegra/tegra_emc.h>
  30
  31 #include <dt-bindings/thermal/tegra210b01-trips.h>
  32
  33 #define KHZ             1000
  34 #define MHZ             1000000
  35 #define VDD_SAFE_STEP   100
  36
  37 /* Margin % applied to PLL CVB tables */
  38 #define CVB_PLL_MARGIN  30
  39
  40 #define VDD_CPU_INDEX   0
  41 #define VDD_CORE_INDEX  1
  42 #define VDD_GPU_INDEX   2
  43
  44 struct tegra_dvfs_data {
  45         struct dvfs_rail **rails;
  46         int rails_num;
  47         struct cpu_dvfs *cpu_fv_table;
  48         int cpu_fv_table_size;
  49         struct cvb_dvfs *gpu_cvb_table;
  50         int gpu_cvb_table_size;
  51         struct dvb_dvfs *emc_dvb_table;
  52         int emc_dvb_table_size;
  53
  54         const int *core_mv;
  55         struct dvfs *core_vf_table;
  56         int core_vf_table_size;
  57         struct dvfs *spi_vf_table;
  58         struct dvfs *spi_slave_vf_table;
  59         struct dvfs *qspi_sdr_vf_table;
  60         struct dvfs *qspi_ddr_vf_table;
  61         struct dvfs *sor1_dp_vf_table;
  62         int sor1_dp_vf_table_size;
  63         int (*get_core_min_mv)(void);
  64         int (*get_core_max_mv)(void);
  65
  66         struct dvfs_therm_limits *core_floors;
  67         struct dvfs_therm_limits *core_caps;
  68         struct dvfs_therm_limits *core_caps_ucm2;
  69         const char *core_dvfs_ver;
  70 };
  71
  72 static struct tegra_dvfs_data *dvfs_data;
  73
  74 static bool tegra_dvfs_cpu_disabled;
  75 static bool tegra_dvfs_core_disabled;
  76 static bool tegra_dvfs_gpu_disabled;
  77 static int cpu_millivolts[MAX_DVFS_FREQS];
  78 static int cpu_dfll_millivolts[MAX_DVFS_FREQS];
  79 static int cpu_lp_millivolts[MAX_DVFS_FREQS];
  80
  81 static struct dvfs_therm_limits
  82 tegra210_core_therm_floors[MAX_THERMAL_LIMITS] = {
  83         {15, 950},
  84         {0, 0},
  85 };
  86
  87 static struct dvfs_therm_limits
  88 tegra210_core_therm_caps[MAX_THERMAL_LIMITS] = {
  89         {86, 1132},
  90         {0, 0},
  91 };
  92
  93 static struct dvfs_therm_limits
  94 tegra210_core_therm_caps_ucm2[MAX_THERMAL_LIMITS] = {
  95         {86, 1090},
  96         {0, 0},
  97 };
  98
  99 static struct dvfs_therm_limits
 100 tegra210b01_core_therm_floors[MAX_THERMAL_LIMITS] = {
 101         {TEGRA210B01_SOC_THERMAL_FLOOR_0 / 1000, 800},
 102         {0, 0},
 103 };
 104
 105 static struct dvfs_therm_limits
 106 tegra210b01_core_therm_caps[MAX_THERMAL_LIMITS] = {
 107         {TEGRA210B01_SOC_THERMAL_CAP_0 / 1000, 1010},
 108         {0, 0},
 109 };
 110
 111 static struct dvfs_therm_limits
 112 tegra210b01_core_therm_caps_ucm2[MAX_THERMAL_LIMITS] = {
 113         {TEGRA210B01_SOC_THERMAL_CAP_0 / 1000, 1010},
 114         {0, 0},
 115 };
 116
 117
 118 static struct dvfs_rail tegra210_dvfs_rail_vdd_cpu = {
 119         .reg_id = "vdd-cpu",
 120         .max_millivolts = 1300,
 121         .min_millivolts = 800,
 122         .step = VDD_SAFE_STEP,
 123         .step_up = 1300,
 124         .jmp_to_zero = true,
 125         .dfll_mode = true,
 126         .alignment = {
 127                 .step_uv = 6250, /* 6.25mV */
 128         },
 129         .stats = {
 130                 .bin_uv = 6250, /* 6.25mV */
 131         },
 132         .is_ready = false,
 133 };
 134
 135 static struct dvfs_rail tegra210_dvfs_rail_vdd_core = {
 136         .reg_id = "vdd-core",
 137         .max_millivolts = 1300,
 138         .step = VDD_SAFE_STEP,
 139         .step_up = 1300,
 140         .alignment = {
 141                 .step_uv = 12500, /* 12.5mV */
 142         },
 143         .stats = {
 144                 .bin_uv = 12500, /* 12.5mV */
 145         },
 146         .is_ready = false,
 147 };
 148
 149 static struct dvfs_rail tegra210_dvfs_rail_vdd_gpu = {
 150         .reg_id = "vdd-gpu",
 151         .max_millivolts = 1300,
 152         .step = VDD_SAFE_STEP,
 153         .step_up = 1300,
 154         .alignment = {
 155                 .step_uv = 10000, /* 10mV */
 156         },
 157         .stats = {
 158                 .bin_uv = 10000, /* 10mV */
 159         },
 160         .in_band_pm = true,
 161 };
 162
 163 static struct dvfs_rail *tegra210_dvfs_rails[] = {
 164         [VDD_CPU_INDEX] = &tegra210_dvfs_rail_vdd_cpu,
 165         [VDD_CORE_INDEX] = &tegra210_dvfs_rail_vdd_core,
 166         [VDD_GPU_INDEX] = &tegra210_dvfs_rail_vdd_gpu,
 167 };
 168
 169 static struct dvfs_rail tegra210b01_dvfs_rail_vdd_core = {
 170         .reg_id = "vdd-core",
 171         .max_millivolts = 1300,
 172         .step = VDD_SAFE_STEP,
 173         .step_up = 1300,
 174         .alignment = {
 175                 .step_uv = 12500, /* 12.5mV */
 176         },
 177         .stats = {
 178                 .bin_uv = 12500, /* 12.5mV */
 179         },
 180 };
 181
 182 static struct dvfs_rail tegra210b01_dvfs_rail_vdd_cpu = {
 183         .reg_id = "vdd-cpu",
 184         .max_millivolts = 1125,
 185         .step = VDD_SAFE_STEP,
 186         .step_up = 1125,
 187         .jmp_to_zero = true,
 188         .dfll_mode = true,
 189         .alignment = {
 190                 .step_uv = 5000, /* 5.0mV */
 191         },
 192         .stats = {
 193                 .bin_uv = 5000, /* 5.0mV */
 194         },
 195 };
 196
 197 static struct dvfs_rail tegra210b01_dvfs_rail_vdd_gpu = {
 198         .reg_id = "vdd-gpu",
 199         .max_millivolts = 1125,
 200         .step = VDD_SAFE_STEP,
 201         .step_up = 1125,
 202         .alignment = {
 203                 .step_uv = 5000, /* 5mV */
 204         },
 205         .stats = {
 206                 .bin_uv = 5000, /* 10mV */
 207         },
 208         .vts_floors_table =             /* applied if no vts cdev */
 209                 { {TEGRA210B01_GPU_DVFS_THERMAL_MIN / 1000, 800 }, },
 210         .in_band_pm = true,
 211 };
 212
 213 static struct dvfs_rail *tegra210b01_dvfs_rails[] = {
 214         [VDD_CPU_INDEX] = &tegra210b01_dvfs_rail_vdd_cpu,
 215         [VDD_CORE_INDEX] = &tegra210b01_dvfs_rail_vdd_core,
 216         [VDD_GPU_INDEX] = &tegra210b01_dvfs_rail_vdd_gpu,
 217 };
 218
 219 static struct dvfs_rail vdd_cpu_rail;
 220 static struct dvfs_rail vdd_gpu_rail;
 221 static struct dvfs_rail vdd_core_rail;
 222
 223 static struct dvfs_rail *vdd_dvfs_rails[] = {
 224         [VDD_CPU_INDEX] = &vdd_cpu_rail,
 225         [VDD_CORE_INDEX] = &vdd_core_rail,
 226         [VDD_GPU_INDEX] = &vdd_gpu_rail,
 227 };
 228
 229 static struct dvfs cpu_dvfs = {
 230         .clk_name       = "cclk_g",
 231         .millivolts     = cpu_millivolts,
 232         .dfll_millivolts = cpu_dfll_millivolts,
 233         .auto_dvfs      = true,
 234         .dvfs_rail      = &vdd_cpu_rail,
 235 };
 236
 237 /* CPU DVFS tables */
 238 #define CPU_PLL_CVB_TABLE \
 239         .pll_min_millivolts = 950, \
 240         .speedo_scale = 100,    \
 241         .voltage_scale = 1000,  \
 242         .cvb_pll_table = {              \
 243                 {204000000UL,   {        0,        0,        0 } }, \
 244                 {306000000UL,   {        0,        0,        0 } }, \
 245                 {408000000UL,   {        0,        0,        0 } }, \
 246                 {510000000UL,   {        0,        0,        0 } }, \
 247                 {612000000UL,   {        0,        0,        0 } }, \
 248                 {714000000UL,   {        0,        0,        0 } }, \
 249                 {816000000UL,   {        0,        0,        0 } }, \
 250                 {918000000UL,   {        0,        0,        0 } }, \
 251                 {1020000000UL,  { -2875621,   358099,    -8585 } }, \
 252                 {1122000000UL,  {   -52225,   104159,    -2816 } }, \
 253                 {1224000000UL,  {  1076868,     8356,     -727 } }, \
 254                 {1326000000UL,  {  2208191,   -84659,     1240 } }, \
 255                 {1428000000UL,  {  2519460,  -105063,     1611 } }, \
 256                 {1530000000UL,  {  2639809,  -108729,     1626 } }, \
 257                 {1632000000UL,  {  2889664,  -122173,     1834 } }, \
 258                 {1734000000UL,  {  3386160,  -154021,     2393 } }, \
 259                 {1836000000UL,  {  5100873,  -279186,     4747 } }, \
 260                 {1912500000UL,  {  5100873,  -279186,     4747 } }, \
 261                 {2014500000UL,  {  5100873,  -279186,     4747 } }, \
 262                 {2218500000UL,  {  5100873,  -279186,     4747 } }, \
 263                 {0,             { } }, \
 264         }
 265
 266 #define CPU_PLL_CVB_TABLE_XA \
 267         .pll_min_millivolts = 950, \
 268         .speedo_scale = 100,    \
 269         .voltage_scale = 1000,  \
 270         .cvb_pll_table = {              \
 271                 {204000000UL,   {        0,        0,        0 } }, \
 272                 {306000000UL,   {        0,        0,        0 } }, \
 273                 {408000000UL,   {        0,        0,        0 } }, \
 274                 {510000000UL,   {        0,        0,        0 } }, \
 275                 {612000000UL,   {        0,        0,        0 } }, \
 276                 {714000000UL,   {        0,        0,        0 } }, \
 277                 {816000000UL,   {        0,        0,        0 } }, \
 278                 {918000000UL,   {        0,        0,        0 } }, \
 279                 {1020000000UL,  { -2875621,   358099,    -8585 } }, \
 280                 {1122000000UL,  {   -52225,   104159,    -2816 } }, \
 281                 {1224000000UL,  {  1076868,     8356,     -727 } }, \
 282                 {1326000000UL,  {  2208191,   -84659,     1240 } }, \
 283                 {1428000000UL,  {  2519460,  -105063,     1611 } }, \
 284                 {1530000000UL,  {  2639809,  -108729,     1626 } }, \
 285                 {1606500000UL,  {  2889664,  -122173,     1834 } }, \
 286                 {1632000000UL,  {  3386160,  -154021,     2393 } }, \
 287                 {0,             {      0,      0,   0} }, \
 288         }
 289
 290 #define CPU_PLL_CVB_TABLE_EUCM1 \
 291         .pll_min_millivolts = 950, \
 292         .speedo_scale = 100,    \
 293         .voltage_scale = 1000,  \
 294         .cvb_pll_table = {              \
 295                 {204000000UL,   {        0,        0,        0 } }, \
 296                 {306000000UL,   {        0,        0,        0 } }, \
 297                 {408000000UL,   {        0,        0,        0 } }, \
 298                 {510000000UL,   {        0,        0,        0 } }, \
 299                 {612000000UL,   {        0,        0,        0 } }, \
 300                 {714000000UL,   {        0,        0,        0 } }, \
 301                 {816000000UL,   {        0,        0,        0 } }, \
 302                 {918000000UL,   {        0,        0,        0 } }, \
 303                 {1020000000UL,  { -2875621,   358099,    -8585 } }, \
 304                 {1122000000UL,  {   -52225,   104159,    -2816 } }, \
 305                 {1224000000UL,  {  1076868,     8356,     -727 } }, \
 306                 {1326000000UL,  {  2208191,   -84659,     1240 } }, \
 307                 {1428000000UL,  {  2519460,  -105063,     1611 } }, \
 308                 {1555500000UL,  {  2639809,  -108729,     1626 } }, \
 309                 {1632000000UL,  {  2889664,  -122173,     1834 } }, \
 310                 {1734000000UL,  {  3386160,  -154021,     2393 } }, \
 311                 {0,             { } }, \
 312         }
 313
 314 #define CPU_PLL_CVB_TABLE_EUCM2 \
 315         .pll_min_millivolts = 950, \
 316         .speedo_scale = 100,    \
 317         .voltage_scale = 1000,  \
 318         .cvb_pll_table = {              \
 319                 {204000000UL,   {        0,        0,        0 } }, \
 320                 {306000000UL,   {        0,        0,        0 } }, \
 321                 {408000000UL,   {        0,        0,        0 } }, \
 322                 {510000000UL,   {        0,        0,        0 } }, \
 323                 {612000000UL,   {        0,        0,        0 } }, \
 324                 {714000000UL,   {        0,        0,        0 } }, \
 325                 {816000000UL,   {        0,        0,        0 } }, \
 326                 {918000000UL,   {        0,        0,        0 } }, \
 327                 {1020000000UL,  { -2875621,   358099,    -8585 } }, \
 328                 {1122000000UL,  {   -52225,   104159,    -2816 } }, \
 329                 {1224000000UL,  {  1076868,     8356,     -727 } }, \
 330                 {1326000000UL,  {  2208191,   -84659,     1240 } }, \
 331                 {1479000000UL,  {  2519460,  -105063,     1611 } }, \
 332                 {1555500000UL,  {  2639809,  -108729,     1626 } }, \
 333                 {1683000000UL,  {  2889664,  -122173,     1834 } }, \
 334                 {0,             { } }, \
 335         }
 336
 337 #define CPU_PLL_CVB_TABLE_EUCM2_JOINT_RAIL \
 338         .pll_min_millivolts = 950, \
 339         .speedo_scale = 100,    \
 340         .voltage_scale = 1000,  \
 341         .cvb_pll_table = {              \
 342                 {204000000UL,   {        0,        0,        0 } }, \
 343                 {306000000UL,   {        0,        0,        0 } }, \
 344                 {408000000UL,   {        0,        0,        0 } }, \
 345                 {510000000UL,   {        0,        0,        0 } }, \
 346                 {612000000UL,   {        0,        0,        0 } }, \
 347                 {714000000UL,   {        0,        0,        0 } }, \
 348                 {816000000UL,   {        0,        0,        0 } }, \
 349                 {918000000UL,   {        0,        0,        0 } }, \
 350                 {1020000000UL,  { -2875621,   358099,    -8585 } }, \
 351                 {1122000000UL,  {   -52225,   104159,    -2816 } }, \
 352                 {1224000000UL,  {  1076868,     8356,     -727 } }, \
 353                 {1326000000UL,  {  2208191,   -84659,     1240 } }, \
 354                 {1479000000UL,  {  2519460,  -105063,     1611 } }, \
 355                 {1504500000UL,  {  2639809,  -108729,     1626 } }, \
 356                 {0,             { } }, \
 357         }
 358
 359 #define CPU_PLL_CVB_TABLE_ODN \
 360         .pll_min_millivolts = 950, \
 361         .speedo_scale = 100,    \
 362         .voltage_scale = 1000,  \
 363         .cvb_pll_table = {              \
 364                 {204000000UL,   {        0,        0,        0 } }, \
 365                 {306000000UL,   {        0,        0,        0 } }, \
 366                 {408000000UL,   {        0,        0,        0 } }, \
 367                 {510000000UL,   {        0,        0,        0 } }, \
 368                 {612000000UL,   {        0,        0,        0 } }, \
 369                 {714000000UL,   {        0,        0,        0 } }, \
 370                 {816000000UL,   {        0,        0,        0 } }, \
 371                 {918000000UL,   {        0,        0,        0 } }, \
 372                 {1020000000UL,  { -2875621,   358099,    -8585 } }, \
 373                 {1122000000UL,  {   -52225,   104159,    -2816 } }, \
 374                 {1224000000UL,  {  1076868,     8356,     -727 } }, \
 375                 {1326000000UL,  {  2208191,   -84659,     1240 } }, \
 376                 {1428000000UL,  {  2519460,  -105063,     1611 } }, \
 377                 {1581000000UL,  {  2889664,  -122173,     1834 } }, \
 378                 {1683000000UL,  {  5100873,  -279186,     4747 } }, \
 379                 {1785000000UL,  {  5100873,  -279186,     4747 } }, \
 380                 {0,             { } }, \
 381         }
 382
 383 static struct cpu_dvfs cpu_fv_dvfs_table[] = {
 384         {
 385                 .speedo_id = 10,
 386                 .process_id = 0,
 387                 .min_mv = 840,
 388                 .max_mv = 1120,
 389                 CPU_PLL_CVB_TABLE_EUCM2_JOINT_RAIL,
 390         },
 391         {
 392                 .speedo_id = 10,
 393                 .process_id = 1,
 394                 .min_mv = 840,
 395                 .max_mv = 1120,
 396                 CPU_PLL_CVB_TABLE_EUCM2_JOINT_RAIL,
 397         },
 398         {
 399                 .speedo_id = 9,
 400                 .process_id = 0,
 401                 .min_mv = 900,
 402                 .max_mv = 1162,
 403                 CPU_PLL_CVB_TABLE_EUCM2,
 404         },
 405         {
 406                 .speedo_id = 9,
 407                 .process_id = 1,
 408                 .min_mv = 900,
 409                 .max_mv = 1162,
 410                 CPU_PLL_CVB_TABLE_EUCM2,
 411         },
 412         {
 413                 .speedo_id = 8,
 414                 .process_id = 0,
 415                 .min_mv = 900,
 416                 .max_mv = 1195,
 417                 CPU_PLL_CVB_TABLE_EUCM2,
 418         },
 419         {
 420                 .speedo_id = 8,
 421                 .process_id = 1,
 422                 .min_mv = 900,
 423                 .max_mv = 1195,
 424                 CPU_PLL_CVB_TABLE_EUCM2,
 425         },
 426         {
 427                 .speedo_id = 7,
 428                 .process_id = 0,
 429                 .min_mv = 841,
 430                 .max_mv = 1227,
 431                 CPU_PLL_CVB_TABLE_EUCM1,
 432         },
 433         {
 434                 .speedo_id = 7,
 435                 .process_id = 1,
 436                 .min_mv = 841,
 437                 .max_mv = 1227,
 438                 CPU_PLL_CVB_TABLE_EUCM1,
 439         },
 440         {
 441                 .speedo_id = 6,
 442                 .process_id = 0,
 443                 .min_mv = 870,
 444                 .max_mv = 1150,
 445                 CPU_PLL_CVB_TABLE,
 446         },
 447         {
 448                 .speedo_id = 6,
 449                 .process_id = 1,
 450                 .min_mv = 870,
 451                 .max_mv = 1150,
 452                 CPU_PLL_CVB_TABLE,
 453         },
 454         {
 455                 .speedo_id = 5,
 456                 .process_id = 0,
 457                 .min_mv = 818,
 458                 .max_mv = 1227,
 459                 CPU_PLL_CVB_TABLE,
 460         },
 461         {
 462                 .speedo_id = 5,
 463                 .process_id = 1,
 464                 .min_mv = 818,
 465                 .max_mv = 1227,
 466                 CPU_PLL_CVB_TABLE,
 467         },
 468         {
 469                 .speedo_id = 4,
 470                 .process_id = -1,
 471                 .min_mv = 918,
 472                 .max_mv = 1113,
 473                 CPU_PLL_CVB_TABLE_XA,
 474         },
 475         {
 476                 .speedo_id = 3,
 477                 .process_id = 0,
 478                 .min_mv = 825,
 479                 .max_mv = 1227,
 480                 CPU_PLL_CVB_TABLE_ODN,
 481         },
 482         {
 483                 .speedo_id = 3,
 484                 .process_id = 1,
 485                 .min_mv = 825,
 486                 .max_mv = 1227,
 487                 CPU_PLL_CVB_TABLE_ODN,
 488         },
 489         {
 490                 .speedo_id = 2,
 491                 .process_id = 0,
 492                 .min_mv = 870,
 493                 .max_mv = 1227,
 494                 CPU_PLL_CVB_TABLE,
 495         },
 496         {
 497                 .speedo_id = 2,
 498                 .process_id = 1,
 499                 .min_mv = 870,
 500                 .max_mv = 1227,
 501                 CPU_PLL_CVB_TABLE,
 502         },
 503         {
 504                 .speedo_id = 1,
 505                 .process_id = 0,
 506                 .min_mv = 837,
 507                 .max_mv = 1227,
 508                 CPU_PLL_CVB_TABLE,
 509         },
 510         {
 511                 .speedo_id = 1,
 512                 .process_id = 1,
 513                 .min_mv = 837,
 514                 .max_mv = 1227,
 515                 CPU_PLL_CVB_TABLE,
 516         },
 517         {
 518                 .speedo_id = 0,
 519                 .process_id = 0,
 520                 .min_mv = 850,
 521                 .max_mv = 1170,
 522                 CPU_PLL_CVB_TABLE,
 523         },
 524         {
 525                 .speedo_id = 0,
 526                 .process_id = 1,
 527                 .min_mv = 850,
 528                 .max_mv = 1170,
 529                 CPU_PLL_CVB_TABLE,
 530         },
 531 };
 532
 533 #define CPUB01_PLL_CVB_TABLE_SLT        \
 534         .speedo_scale = 100,    \
 535         .voltage_scale = 1000,  \
 536         .cvb_pll_table = {      \
 537                 /* f                    c0,       c1,       c2 */   \
 538                 {  204000000UL, {        0,        0,        0 } }, \
 539                 {  306000000UL, {        0,        0,        0 } }, \
 540                 {  408000000UL, {        0,        0,        0 } }, \
 541                 {  510000000UL, {        0,        0,        0 } }, \
 542                 {  612000000UL, {        0,        0,        0 } }, \
 543                 {  714000000UL, {        0,        0,        0 } }, \
 544                 {  816000000UL, {        0,        0,        0 } }, \
 545                 {  918000000UL, {        0,        0,        0 } }, \
 546                 { 1020000000UL, {  1120000,        0,        0 } }, \
 547                 { 1122000000UL, {  1120000,        0,        0 } }, \
 548                 { 1224000000UL, {  1120000,        0,        0 } }, \
 549                 { 1326000000UL, {  1120000,        0,        0 } }, \
 550                 { 1428000000UL, {  1120000,        0,        0 } }, \
 551                 { 1581000000UL, {  1120000,        0,        0 } }, \
 552                 { 1683000000UL, {  1120000,        0,        0 } }, \
 553                 { 1785000000UL, {  1120000,        0,        0 } }, \
 554                 { 1887000000UL, {  1120000,        0,        0 } }, \
 555                 { 1963500000UL, {  1120000,        0,        0 } }, \
 556                 { 2091000000UL, {  1120000,        0,        0 } }, \
 557                 { 0,            { } }, \
 558         }, \
 559         .pll_min_millivolts = 800
 560
 561 #define CPUB01_PLL_CVB_TABLE    \
 562         .speedo_scale = 100,    \
 563         .voltage_scale = 1000,  \
 564         .cvb_pll_table = {      \
 565                 /* f                    c0,       c1,       c2 */   \
 566                 {  204000000UL, {        0,        0,        0 } }, \
 567                 {  306000000UL, {        0,        0,        0 } }, \
 568                 {  408000000UL, {        0,        0,        0 } }, \
 569                 {  510000000UL, {        0,        0,        0 } }, \
 570                 {  612000000UL, {        0,        0,        0 } }, \
 571                 {  714000000UL, {        0,        0,        0 } }, \
 572                 {  816000000UL, {        0,        0,        0 } }, \
 573                 {  918000000UL, {        0,        0,        0 } }, \
 574                 { 1020000000UL, {  1120000,        0,        0 } }, \
 575                 { 1122000000UL, {  1120000,        0,        0 } }, \
 576                 { 1224000000UL, {  1120000,        0,        0 } }, \
 577                 { 1326000000UL, {  1120000,        0,        0 } }, \
 578                 { 1428000000UL, {  1120000,        0,        0 } }, \
 579                 { 1581000000UL, {  1120000,        0,        0 } }, \
 580                 { 1683000000UL, {  1120000,        0,        0 } }, \
 581                 { 1785000000UL, {  1120000,        0,        0 } }, \
 582                 { 1887000000UL, {  1120000,        0,        0 } }, \
 583                 { 1963500000UL, {  1120000,        0,        0 } }, \
 584                 { 2014500000UL, {  1120000,        0,        0 } }, \
 585                 { 0,            { } }, \
 586         }, \
 587         .pll_min_millivolts = 800
 588
 589 static struct cpu_dvfs cpub01_fv_dvfs_table[] = {
 590         {
 591                 .speedo_id = 2,
 592                 .process_id = -1,
 593                 .max_mv = 1120,
 594                 CPUB01_PLL_CVB_TABLE_SLT,
 595         },
 596         {
 597                 .speedo_id = -1,
 598                 .process_id = -1,
 599                 .max_mv = 1120,
 600                 CPUB01_PLL_CVB_TABLE,
 601         },
 602 };
 603
 604
 605 /* CPU LP DVFS tables */
 606 static unsigned long cpu_lp_max_freq[] = {
 607 /* speedo_id    0        1        2        3        4       5 */
 608                 1132800, 1132800, 1132800, 1132800, 940800, 1132800
 609 };
 610
 611 #define CPU_LP_FV_TABLE          \
 612         .fv_table = {            \
 613                 {51000,   850},  \
 614                 {102000,  850},  \
 615                 {204000,  850},  \
 616                 {307200,  850},  \
 617                 {403200,  850},  \
 618                 {518400,  850},  \
 619                 {614400,  868},  \
 620                 {710400,  912},  \
 621                 {825600,  962},  \
 622                 {921600,  1006}, \
 623                 {1036800, 1062}, \
 624                 {1132800, 1118}, \
 625                 {1228800, 1168}, \
 626         }
 627
 628 static struct cpu_dvfs cpu_lp_fv_dvfs_table[] = {
 629         {
 630                 .speedo_id = 5,
 631                 .process_id = -1,
 632                 .min_mv = 818,
 633                 .max_mv = 1227,
 634                 CPU_LP_FV_TABLE,
 635         },
 636         {
 637                 .speedo_id = 2,
 638                 .process_id = -1,
 639                 .min_mv = 804,
 640                 .max_mv = 1170,
 641                 CPU_LP_FV_TABLE,
 642         },
 643         {
 644                 .speedo_id = 1,
 645                 .process_id = -1,
 646                 .min_mv = 837,
 647                 .max_mv = 1227,
 648                 CPU_LP_FV_TABLE,
 649         },
 650         {
 651                 .speedo_id = -1,
 652                 .process_id = -1,
 653                 .min_mv = 850,
 654                 .max_mv = 1170,
 655                 CPU_LP_FV_TABLE,
 656         },
 657 };
 658
 659 static struct dvfs cpu_lp_dvfs = {
 660         .clk_name       = "cclk_lp",
 661         .millivolts     = cpu_lp_millivolts,
 662         .auto_dvfs      = true,
 663         .dvfs_rail      = &vdd_cpu_rail,
 664         .freqs_mult     = KHZ,
 665 };
 666
 667 /* GPU DVFS tables */
 668 #define NA_FREQ_CVB_TABLE       \
 669         .freqs_mult = KHZ,      \
 670         .speedo_scale = 100,    \
 671         .thermal_scale = 10,    \
 672         .voltage_scale = 1000,  \
 673         .cvb_table = {          \
 674                 /* f       dfll pll:    c0,       c1,       c2,       c3,       c4,       c5 */    \
 675                 {   76800, { }, {   814294,     8144,     -940,      808,   -21583,      226 }, }, \
 676                 {  153600, { }, {   856185,     8144,     -940,      808,   -21583,      226 }, }, \
 677                 {  230400, { }, {   898077,     8144,     -940,      808,   -21583,      226 }, }, \
 678                 {  307200, { }, {   939968,     8144,     -940,      808,   -21583,      226 }, }, \
 679                 {  384000, { }, {   981860,     8144,     -940,      808,   -21583,      226 }, }, \
 680                 {  460800, { }, {  1023751,     8144,     -940,      808,   -21583,      226 }, }, \
 681                 {  537600, { }, {  1065642,     8144,     -940,      808,   -21583,      226 }, }, \
 682                 {  614400, { }, {  1107534,     8144,     -940,      808,   -21583,      226 }, }, \
 683                 {  691200, { }, {  1149425,     8144,     -940,      808,   -21583,      226 }, }, \
 684                 {  768000, { }, {  1191317,     8144,     -940,      808,   -21583,      226 }, }, \
 685                 {  844800, { }, {  1233208,     8144,     -940,      808,   -21583,      226 }, }, \
 686                 {  921600, { }, {  1275100,     8144,     -940,      808,   -21583,      226 }, }, \
 687                 {  998400, { }, {  1316991,     8144,     -940,      808,   -21583,      226 }, }, \
 688                 { 0,       { }, { }, }, \
 689         }
 690
 691 #define NA_FREQ_CVB_TABLE_XA    \
 692         .freqs_mult = KHZ,      \
 693         .speedo_scale = 100,    \
 694         .thermal_scale = 10,    \
 695         .voltage_scale = 1000,  \
 696         .cvb_table = {          \
 697                 /* f       dfll pll:    c0,       c1,       c2,       c3,       c4,       c5 */    \
 698                 {   76800, { }, {  1526811,   -59106,      963,      238,   -11292,      185 }, }, \
 699                 {  153600, { }, {  1543573,   -57798,      910,      179,    -9918,      191 }, }, \
 700                 {  230400, { }, {  1567838,   -56991,      869,       60,    -8545,      203 }, }, \
 701                 {  307200, { }, {  1600241,   -56742,      841,        0,    -7019,      209 }, }, \
 702                 {  384000, { }, {  1635184,   -56501,      813,        0,    -5493,      221 }, }, \
 703                 {  460800, { }, {  1672308,   -56300,      787,     -119,    -3662,      226 }, }, \
 704                 {  537600, { }, {  1712114,   -56093,      759,     -179,    -1526,      238 }, }, \
 705                 {  614400, { }, {  1756009,   -56048,      737,     -298,      610,      244 }, }, \
 706                 {  691200, { }, {  1790251,   -54860,      687,     -358,     3204,      238 }, }, \
 707                 {  768000, { }, {  1783830,   -49449,      532,     -477,     6714,      197 }, }, \
 708                 {  844800, { }, {  1819706,   -45928,      379,     -358,     7019,       89 }, }, \
 709                 { 0,       { }, { }, }, \
 710         }
 711
 712 #define FIXED_FREQ_CVB_TABLE    \
 713         .freqs_mult = KHZ,      \
 714         .speedo_scale = 100,    \
 715         .thermal_scale = 10,    \
 716         .voltage_scale = 1000,  \
 717         .cvb_table = {          \
 718                 /* f       dfll pll:    c0,       c1,       c2 */    \
 719                 {   76800, { }, {  1786666,   -85625,     1632 }, }, \
 720                 {  153600, { }, {  1846729,   -87525,     1632 }, }, \
 721                 {  230400, { }, {  1910480,   -89425,     1632 }, }, \
 722                 {  307200, { }, {  1977920,   -91325,     1632 }, }, \
 723                 {  384000, { }, {  2049049,   -93215,     1632 }, }, \
 724                 {  460800, { }, {  2122872,   -95095,     1632 }, }, \
 725                 {  537600, { }, {  2201331,   -96985,     1632 }, }, \
 726                 {  614400, { }, {  2283479,   -98885,     1632 }, }, \
 727                 {  691200, { }, {  2369315,  -100785,     1632 }, }, \
 728                 {  768000, { }, {  2458841,  -102685,     1632 }, }, \
 729                 {  844800, { }, {  2550821,  -104555,     1632 }, }, \
 730                 {  921600, { }, {  2647676,  -106455,     1632 }, }, \
 731                 { 0,       { }, { }, }, \
 732         }
 733
 734 static struct dvfs gpu_dvfs = {
 735         .clk_name       = "gbus",
 736         .auto_dvfs      = true,
 737         .dvfs_rail      = &vdd_gpu_rail,
 738 };
 739
 740 static struct cvb_dvfs gpu_cvb_dvfs_table[] = {
 741         {
 742                 .speedo_id = 4,
 743                 .process_id = -1,
 744                 .pll_min_millivolts = 918,
 745                 .max_mv = 1113,
 746                 .max_freq = 844800,
 747 #ifdef CONFIG_TEGRA_USE_NA_GPCPLL
 748                 NA_FREQ_CVB_TABLE_XA,
 749 #else
 750                 FIXED_FREQ_CVB_TABLE,
 751 #endif
 752         },
 753
 754         {
 755                 .speedo_id = 3,
 756                 .process_id = -1,
 757                 .pll_min_millivolts = 810,
 758                 .max_mv = 1150,
 759                 .max_freq = 921600,
 760 #ifdef CONFIG_TEGRA_USE_NA_GPCPLL
 761                 NA_FREQ_CVB_TABLE,
 762 #else
 763                 FIXED_FREQ_CVB_TABLE,
 764 #endif
 765         },
 766
 767         {
 768                 .speedo_id = 2,
 769                 .process_id = -1,
 770                 .pll_min_millivolts = 818,
 771                 .max_mv = 1150,
 772                 .max_freq = 998400,
 773 #ifdef CONFIG_TEGRA_USE_NA_GPCPLL
 774                 NA_FREQ_CVB_TABLE,
 775 #else
 776                 FIXED_FREQ_CVB_TABLE,
 777 #endif
 778         },
 779
 780         {
 781                 .speedo_id = 1,
 782                 .process_id = -1,
 783                 .pll_min_millivolts = 840,
 784                 .max_mv = 1150,
 785                 .max_freq = 998400,
 786 #ifdef CONFIG_TEGRA_USE_NA_GPCPLL
 787                 NA_FREQ_CVB_TABLE,
 788 #else
 789                 FIXED_FREQ_CVB_TABLE,
 790 #endif
 791         },
 792
 793         {
 794                 .speedo_id = 0,
 795                 .process_id = -1,
 796                 .pll_min_millivolts = 950,
 797 #ifdef CONFIG_TEGRA_GPU_DVFS
 798                 .max_mv = 1150,
 799 #else
 800                 .max_mv = 1000,
 801 #endif
 802                 .max_freq = 921600,
 803                 FIXED_FREQ_CVB_TABLE,
 804         },
 805 };
 806
 807 #define GPUB01_NA_CVB_TABLE_SLT \
 808         .freqs_mult = KHZ,      \
 809         .speedo_scale = 100,    \
 810         .thermal_scale = 10,    \
 811         .voltage_scale = 1000,  \
 812         .cvb_table = {          \
 813                 /* f       dfll pll:    c0,       c1,       c2,       c3,       c4,       c5 */    \
 814                 {   76800, { }, {   590000,        0,        0,        0,        0,        0 }, }, \
 815                 {  153600, { }, {   590000,        0,        0,        0,        0,        0 }, }, \
 816                 {  230400, { }, {   590000,        0,        0,        0,        0,        0 }, }, \
 817                 {  307200, { }, {   590000,        0,        0,        0,        0,        0 }, }, \
 818                 {  384000, { }, {   590000,        0,        0,        0,        0,        0 }, }, \
 819                 {  460800, { }, {   795089,   -11096,     -163,      298,   -10421,      162 }, }, \
 820                 {  537600, { }, {   795089,   -11096,     -163,      298,   -10421,      162 }, }, \
 821                 {  614400, { }, {   820606,    -6285,     -452,      238,    -6182,       81 }, }, \
 822                 {  691200, { }, {   846289,    -4565,     -552,      119,    -3958,       -2 }, }, \
 823                 {  768000, { }, {   888720,    -5110,     -584,        0,    -2849,       39 }, }, \
 824                 {  844800, { }, {   936634,    -6089,     -602,      -60,      -99,      -93 }, }, \
 825                 {  921600, { }, {   982562,    -7373,     -614,     -179,     1797,      -13 }, }, \
 826                 {  998400, { }, {  1090179,   -14125,     -497,     -179,     3518,        9 }, }, \
 827                 { 1075200, { }, {  1155798,   -13465,     -648,        0,     1077,       40 }, }, \
 828                 { 1152000, { }, {  1198568,   -10904,     -830,        0,     1469,      110 }, }, \
 829                 { 1228800, { }, {  1269988,   -12707,     -859,        0,     3722,      313 }, }, \
 830                 { 1267200, { }, {  1308155,   -13694,     -867,        0,     3681,      559 }, }, \
 831                 { 0,       { }, { }, }, \
 832         }, \
 833         .cvb_vmin = {   0, { }, {   590000,        0,        0 }, }, \
 834         .cvb_version = "NAPLL En - p4v2-AggressiveSLT"
 835
 836 #define GPUB01_NA_CVB_TABLE     \
 837         .freqs_mult = KHZ,      \
 838         .speedo_scale = 100,    \
 839         .thermal_scale = 10,    \
 840         .voltage_scale = 1000,  \
 841         .cvb_table = {          \
 842                 /* f       dfll pll:    c0,       c1,       c2,       c3,       c4,       c5 */    \
 843                 {   76800, { }, {   610000,        0,        0,        0,        0,        0 }, }, \
 844                 {  153600, { }, {   610000,        0,        0,        0,        0,        0 }, }, \
 845                 {  230400, { }, {   610000,        0,        0,        0,        0,        0 }, }, \
 846                 {  307200, { }, {   610000,        0,        0,        0,        0,        0 }, }, \
 847                 {  384000, { }, {   610000,        0,        0,        0,        0,        0 }, }, \
 848                 {  460800, { }, {   610000,        0,        0,        0,        0,        0 }, }, \
 849                 {  537600, { }, {   801688,   -10900,     -163,      298,   -10599,      162 }, }, \
 850                 {  614400, { }, {   824214,    -5743,     -452,      238,    -6325,       81 }, }, \
 851                 {  691200, { }, {   848830,    -3903,     -552,      119,    -4030,       -2 }, }, \
 852                 {  768000, { }, {   891575,    -4409,     -584,        0,    -2849,       39 }, }, \
 853                 {  844800, { }, {   940071,    -5367,     -602,      -60,      -63,      -93 }, }, \
 854                 {  921600, { }, {   986765,    -6637,     -614,     -179,     1905,      -13 }, }, \
 855                 {  998400, { }, {  1098475,   -13529,     -497,     -179,     3626,        9 }, }, \
 856                 { 1075200, { }, {  1163644,   -12688,     -648,        0,     1077,       40 }, }, \
 857                 { 1152000, { }, {  1204812,    -9908,     -830,        0,     1469,      110 }, }, \
 858                 { 1228800, { }, {  1277303,   -11675,     -859,        0,     3722,      313 }, }, \
 859                 { 1267200, { }, {  1335531,   -12567,     -867,        0,     3681,      559 }, }, \
 860                 { 0,       { }, { }, }, \
 861         }, \
 862         .cvb_vmin = {   0, { }, {   610000,        0,        0 }, }, \
 863         .cvb_version = "NAPLL En - p4v3"
 864
 865 static struct cvb_dvfs gpub01_cvb_dvfs_table[] = {
 866         {
 867                 .speedo_id = 2,
 868                 .process_id = -1,
 869                 .max_mv = 1050,
 870                 .max_freq = 1267200,
 871                 GPUB01_NA_CVB_TABLE_SLT,
 872         },
 873         {
 874                 .speedo_id = -1,
 875                 .process_id = -1,
 876                 .max_mv = 1050,
 877                 .max_freq = 1267200,
 878                 GPUB01_NA_CVB_TABLE,
 879         },
 880 };
 881
 882 static int gpu_vmin[MAX_THERMAL_RANGES];
 883 static int gpu_peak_millivolts[MAX_DVFS_FREQS];
 884 static int gpu_millivolts[MAX_THERMAL_RANGES][MAX_DVFS_FREQS];
 885 static struct dvfs_therm_limits vdd_gpu_therm_caps_table[MAX_THERMAL_LIMITS];
 886 static struct dvfs_therm_limits vdd_gpu_therm_caps_ucm2_table[MAX_THERMAL_LIMITS];
 887 static unsigned long gpu_cap_rates[MAX_THERMAL_LIMITS];
 888 static struct clk *vgpu_cap_clk;
 889
 890 /* Core DVFS tables */
 891 static int core_millivolts[MAX_DVFS_FREQS];
 892
 893 #define CORE_DVFS(_clk_name, _speedo_id, _process_id, _auto, _mult, _freqs...) \
 894         {                                                       \
 895                 .clk_name       = _clk_name,                    \
 896                 .speedo_id      = _speedo_id,                   \
 897                 .process_id     = _process_id,                  \
 898                 .freqs          = {_freqs},                     \
 899                 .freqs_mult     = _mult,                        \
 900                 .millivolts     = core_millivolts,              \
 901                 .auto_dvfs      = _auto,                        \
 902                 .dvfs_rail      = &vdd_core_rail,       \
 903         }
 904
 905
 906 /* Include T210 core DVFS tables generated from characterization data */
 907 #include "tegra210-core-dvfs.c"
 908
 909 /* Include T210b01 core DVFS tables generated from characterization data */
 910 #include "tegra210b01-core-dvfs.c"
 911 #include "tegra210b01-slt-core-dvfs.c"
 912
 913 int tegra_dvfs_disable_core_set(const char *arg, const struct kernel_param *kp)
 914 {
 915         int ret;
 916
 917         ret = param_set_bool(arg, kp);
 918         if (ret)
 919                 return ret;
 920
 921         if (tegra_dvfs_core_disabled)
 922                 tegra_dvfs_rail_disable(&vdd_core_rail);
 923         else
 924                 tegra_dvfs_rail_enable(&vdd_core_rail);
 925
 926         return 0;
 927 }
 928
 929 int tegra_dvfs_disable_cpu_set(const char *arg, const struct kernel_param *kp)
 930 {
 931         int ret;
 932
 933         ret = param_set_bool(arg, kp);
 934         if (ret)
 935                 return ret;
 936
 937         if (tegra_dvfs_cpu_disabled)
 938                 tegra_dvfs_rail_disable(&vdd_cpu_rail);
 939         else
 940                 tegra_dvfs_rail_enable(&vdd_cpu_rail);
 941
 942         return 0;
 943 }
 944
 945 int tegra_dvfs_disable_gpu_set(const char *arg, const struct kernel_param *kp)
 946 {
 947         int ret;
 948
 949         ret = param_set_bool(arg, kp);
 950         if (ret)
 951                 return ret;
 952
 953         if (tegra_dvfs_gpu_disabled)
 954                 tegra_dvfs_rail_disable(&vdd_gpu_rail);
 955         else
 956                 tegra_dvfs_rail_enable(&vdd_gpu_rail);
 957
 958         return 0;
 959 }
 960
 961 int tegra_dvfs_disable_get(char *buffer, const struct kernel_param *kp)
 962 {
 963         return param_get_bool(buffer, kp);
 964 }
 965
 966 static struct kernel_param_ops tegra_dvfs_disable_core_ops = {
 967         .set = tegra_dvfs_disable_core_set,
 968         .get = tegra_dvfs_disable_get,
 969 };
 970
 971 static struct kernel_param_ops tegra_dvfs_disable_cpu_ops = {
 972         .set = tegra_dvfs_disable_cpu_set,
 973         .get = tegra_dvfs_disable_get,
 974 };
 975
 976 static struct kernel_param_ops tegra_dvfs_disable_gpu_ops = {
 977         .set = tegra_dvfs_disable_gpu_set,
 978         .get = tegra_dvfs_disable_get,
 979 };
 980
 981 module_param_cb(disable_core, &tegra_dvfs_disable_core_ops,
 982         &tegra_dvfs_core_disabled, 0644);
 983 module_param_cb(disable_cpu, &tegra_dvfs_disable_cpu_ops,
 984         &tegra_dvfs_cpu_disabled, 0644);
 985 module_param_cb(disable_gpu, &tegra_dvfs_disable_gpu_ops,
 986         &tegra_dvfs_gpu_disabled, 0644);
 987
 988 static void init_dvfs_one(struct dvfs *d, int max_freq_index)
 989 {
 990         int ret;
 991         struct clk *c = clk_get_sys(d->clk_name, d->clk_name);
 992
 993         if (IS_ERR(c)) {
 994                 pr_info("tegra210_dvfs: no clock found for %s\n",
 995                         d->clk_name);
 996                 return;
 997         }
 998
 999         d->max_millivolts = d->dvfs_rail->nominal_millivolts;
1000         d->num_freqs = max_freq_index + 1;
1001
1002         ret = tegra_setup_dvfs(c, d);
1003         if (ret)
1004                 pr_err("tegra210_dvfs: failed to enable dvfs on %s\n",
1005                                 __clk_get_name(c));
1006 }
1007
1008 static bool match_dvfs_one(const char *name, int dvfs_speedo_id,
1009                            int dvfs_process_id, int speedo_id, int process_id)
1010 {
1011         if ((dvfs_process_id != -1 && dvfs_process_id != process_id) ||
1012                 (dvfs_speedo_id != -1 && dvfs_speedo_id != speedo_id)) {
1013                 pr_debug("tegra210_dvfs: rejected %s speedo %d, process %d\n",
1014                          name, dvfs_speedo_id, dvfs_process_id);
1015                 return false;
1016         }
1017         return true;
1018 }
1019
1020 static int set_cpu_dvfs_data(struct cpu_dvfs *d,
1021                              struct dvfs *cpu_dvfs, int *max_freq_index)
1022 {
1023         int i, mv, dfll_mv, min_mv, min_dfll_mv, num_freqs;
1024         unsigned long fmax_at_vmin = 0;
1025         unsigned long fmin_use_dfll = 0;
1026         unsigned long *freqs;
1027         int *dfll_millivolts;
1028         struct rail_alignment *align = tegra_dfll_get_alignment();
1029         const char *version = tegra_dfll_get_cvb_version();
1030         int speedo = tegra_sku_info.cpu_speedo_value;
1031
1032         if (align == ERR_PTR(-EPROBE_DEFER))
1033                 return -EPROBE_DEFER;
1034
1035         vdd_cpu_rail.nvver = version;
1036
1037         min_dfll_mv = d->min_mv;
1038         if (min_dfll_mv < vdd_cpu_rail.min_millivolts) {
1039                 pr_debug("tegra210_dvfs: dfll min %dmV below rail min %dmV\n",
1040                          min_dfll_mv, vdd_cpu_rail.min_millivolts);
1041                 min_dfll_mv = vdd_cpu_rail.min_millivolts;
1042         }
1043         min_dfll_mv = tegra_round_voltage(min_dfll_mv, align, true);
1044         d->max_mv = tegra_round_voltage(d->max_mv, align, false);
1045
1046         min_mv = d->pll_min_millivolts;
1047         if (min_mv < vdd_cpu_rail.min_millivolts) {
1048                 pr_debug("tegra210_dvfs: pll min %dmV below rail min %dmV\n",
1049                          min_mv, vdd_cpu_rail.min_millivolts);
1050                 min_mv = vdd_cpu_rail.min_millivolts;
1051         }
1052         min_mv = tegra_round_voltage(min_mv, align, true);
1053
1054         if (tegra_get_cpu_fv_table(&num_freqs, &freqs, &dfll_millivolts))
1055                 return -EPROBE_DEFER;
1056
1057         for (i = 0; i < num_freqs; i++) {
1058                 if (freqs[i] != d->cvb_pll_table[i].freq) {
1059                         pr_err("Err: DFLL freq ladder does not match PLL's\n");
1060                         return -EINVAL;
1061                 }
1062
1063                 /*
1064                  * Check maximum frequency at minimum voltage for dfll source;
1065                  * round down unless all table entries are above Vmin, then use
1066                  * the 1st entry as is.
1067                  */
1068                 dfll_mv = max(dfll_millivolts[i] / 1000, min_dfll_mv);
1069                 if (dfll_mv > min_dfll_mv) {
1070                         if (!i)
1071                                 fmax_at_vmin = freqs[i];
1072                         if (!fmax_at_vmin)
1073                                 fmax_at_vmin = freqs[i - 1];
1074                 }
1075
1076                 /* Clip maximum frequency at maximum voltage for pll source */
1077                 mv = tegra_get_cvb_voltage(speedo, d->speedo_scale,
1078                                            &d->cvb_pll_table[i].coefficients);
1079                 mv = (100 + CVB_PLL_MARGIN) * mv / 100;
1080                 mv = tegra_round_cvb_voltage(mv, d->voltage_scale, align);
1081                 mv = max(mv, min_mv);
1082                 if ((mv > d->max_mv) && !i) {
1083                         pr_err("Err: volt of 1st entry is higher than Vmax\n");
1084                         return -EINVAL;
1085                 }
1086
1087                 /* Minimum rate with pll source voltage above dfll Vmin */
1088                 if ((mv >= min_dfll_mv) && !fmin_use_dfll)
1089                         fmin_use_dfll = freqs[i];
1090
1091                 /* fill in dvfs tables */
1092                 cpu_dvfs->freqs[i] = freqs[i];
1093                 cpu_millivolts[i] = mv;
1094                 cpu_dfll_millivolts[i] = min(dfll_mv, d->max_mv);
1095         }
1096
1097         /*
1098          * In the dfll operating range dfll voltage at any rate should be
1099          * better (below) than pll voltage
1100          */
1101         if (!fmin_use_dfll || (fmin_use_dfll > fmax_at_vmin))
1102                 fmin_use_dfll = fmax_at_vmin;
1103
1104         /* dvfs tables are successfully populated - fill in the rest */
1105         cpu_dvfs->speedo_id = d->speedo_id;
1106         cpu_dvfs->process_id = d->process_id;
1107         cpu_dvfs->dvfs_rail->nominal_millivolts = min(d->max_mv,
1108                 max(cpu_millivolts[i - 1], cpu_dfll_millivolts[i - 1]));
1109         *max_freq_index = i - 1;
1110
1111         cpu_dvfs->use_dfll_rate_min = fmin_use_dfll;
1112
1113         return 0;
1114 }
1115
1116 /*
1117  * Setup slow CPU (a.k.a LP CPU) DVFS table from FV data. Only PLL is used as
1118  * a clock source for slow CPU. Its maximum frequency must be reached within
1119  * nominal voltage -- FV frequency list is cut off at rate that exceeds either
1120  * sku-based maximum limit or requires voltage above nominal. Error when DVFS
1121  * table can not be constructed must never happen.
1122  *
1123  * Final CPU rail nominal voltage is set as maximum of fast and slow CPUs
1124  * nominal voltages.
1125  */
1126 static int set_cpu_lp_dvfs_data(unsigned long max_freq, struct cpu_dvfs *d,
1127                                 struct dvfs *cpu_lp_dvfs, int *max_freq_index)
1128 {
1129         int i, mv, min_mv;
1130         struct rail_alignment *align = &vdd_cpu_rail.alignment;
1131
1132         min_mv = d->min_mv;
1133         if (min_mv < vdd_cpu_rail.min_millivolts) {
1134                 pr_debug("tegra210_dvfs: scpu min %dmV below rail min %dmV\n",
1135                          min_mv, vdd_cpu_rail.min_millivolts);
1136                 min_mv = vdd_cpu_rail.min_millivolts;
1137         }
1138         min_mv = tegra_round_voltage(min_mv, align, true);
1139
1140         d->max_mv = tegra_round_voltage(d->max_mv, align, false);
1141         BUG_ON(d->max_mv > vdd_cpu_rail.max_millivolts);
1142         cpu_lp_dvfs->dvfs_rail->nominal_millivolts =
1143                 max(cpu_lp_dvfs->dvfs_rail->nominal_millivolts, d->max_mv);
1144
1145         for (i = 0; i < MAX_DVFS_FREQS; i++) {
1146                 struct cpu_pll_fv_table *t = &d->fv_table[i];
1147                 if (!t->freq || t->freq > max_freq)
1148                         break;
1149
1150                 mv = t->volt;
1151                 mv = max(mv, min_mv);
1152                 if (mv > d->max_mv) {
1153                         pr_warn("tegra210_dvfs: %dmV for %s rate %d above limit %dmV\n",
1154                              mv, cpu_lp_dvfs->clk_name, t->freq, d->max_mv);
1155                         break;
1156                 }
1157
1158                 /* fill in dvfs tables */
1159                 cpu_lp_dvfs->freqs[i] = t->freq;
1160                 cpu_lp_millivolts[i] = mv;
1161         }
1162
1163         /* Table must not be empty */
1164         if (!i) {
1165                 pr_err("tegra210_dvfs: invalid cpu lp dvfs table\n");
1166                 return -ENOENT;
1167         }
1168
1169         /* dvfs tables are successfully populated - fill in the rest */
1170         cpu_lp_dvfs->speedo_id = d->speedo_id;
1171         cpu_lp_dvfs->process_id = d->process_id;
1172         *max_freq_index = i - 1;
1173
1174         return 0;
1175 }
1176
1177 int of_tegra_dvfs_init(const struct of_device_id *matches)
1178 {
1179         int ret;
1180         struct device_node *np;
1181
1182         for_each_matching_node(np, matches) {
1183                 const struct of_device_id *match = of_match_node(matches, np);
1184                 int (*dvfs_init_cb)(struct device_node *) = match->data;
1185
1186                 ret = dvfs_init_cb(np);
1187                 if (ret) {
1188                         pr_err("dt: Failed to read %s data from DT\n",
1189                                match->compatible);
1190                         return ret;
1191                 }
1192         }
1193         return 0;
1194 }
1195
1196 /*
1197  * QSPI DVFS tables are different in SDR and DDR modes. Use SDR tables by
1198  * default. Check if DDR mode is specified for enabled QSPI device in DT,
1199  * and overwrite DVFS table, respectively.
1200  */
1201
1202 static struct dvfs *qspi_dvfs;
1203
1204 static int of_update_qspi_dvfs(struct device_node *dn)
1205 {
1206         if (of_device_is_available(dn)) {
1207                 if (of_get_property(dn, "nvidia,x4-is-ddr", NULL))
1208                         qspi_dvfs = &dvfs_data->qspi_ddr_vf_table[0];
1209         }
1210         return 0;
1211 }
1212
1213 static struct of_device_id tegra210_dvfs_qspi_of_match[] = {
1214         { .compatible = "nvidia,tegra210-qspi", .data = of_update_qspi_dvfs, },
1215         { },
1216 };
1217
1218 static void init_qspi_dvfs(int soc_speedo_id, int core_process_id,
1219                                   int core_nominal_mv_index)
1220 {
1221         qspi_dvfs = &dvfs_data->qspi_sdr_vf_table[0];
1222
1223         of_tegra_dvfs_init(tegra210_dvfs_qspi_of_match);
1224
1225         if (match_dvfs_one(qspi_dvfs->clk_name, qspi_dvfs->speedo_id,
1226                 qspi_dvfs->process_id, soc_speedo_id, core_process_id))
1227                 init_dvfs_one(qspi_dvfs, core_nominal_mv_index);
1228 }
1229
1230 /*
1231  * SPI DVFS tables are different in master and in slave mode. Use master tables
1232  * by default. Check if slave mode is specified for enabled SPI devices in DT,
1233  * and overwrite master table for the respective SPI controller.
1234  */
1235
1236 static struct {
1237         u64 address;
1238         struct dvfs *d;
1239 } spi_map[] = {
1240         { 0x7000d400, },
1241         { 0x7000d600, },
1242         { 0x7000d800, },
1243         { 0x7000da00, },
1244 };
1245
1246 static int of_update_spi_slave_dvfs(struct device_node *dn)
1247 {
1248         int i;
1249         u64 addr = 0;
1250         const __be32 *reg;
1251
1252         if (!of_device_is_available(dn))
1253                 return 0;
1254
1255         reg = of_get_property(dn, "reg", NULL);
1256         if (reg)
1257                 addr = of_translate_address(dn, reg);
1258
1259         for (i = 0; i < ARRAY_SIZE(spi_map); i++) {
1260                 if (spi_map[i].address == addr) {
1261                         spi_map[i].d = &dvfs_data->spi_slave_vf_table[i];
1262                         break;
1263                 }
1264         }
1265         return 0;
1266 }
1267
1268 static struct of_device_id tegra21_dvfs_spi_slave_of_match[] = {
1269         { .compatible = "nvidia,tegra210-spi-slave",
1270           .data = of_update_spi_slave_dvfs, },
1271         { },
1272 };
1273
1274 static void init_spi_dvfs(int soc_speedo_id, int core_process_id,
1275                           int core_nominal_mv_index)
1276 {
1277         int i;
1278
1279         for (i = 0; i <  ARRAY_SIZE(spi_map); i++)
1280                 spi_map[i].d = &dvfs_data->spi_vf_table[i];
1281
1282         of_tegra_dvfs_init(tegra21_dvfs_spi_slave_of_match);
1283
1284         for (i = 0; i <  ARRAY_SIZE(spi_map); i++) {
1285                 struct dvfs *d = spi_map[i].d;
1286                 if (!match_dvfs_one(d->clk_name, d->speedo_id,
1287                         d->process_id, soc_speedo_id, core_process_id))
1288                         continue;
1289                 init_dvfs_one(d, core_nominal_mv_index);
1290         }
1291 }
1292
1293 static void init_sor1_dvfs(int soc_speedo_id, int core_process_id,
1294                            int core_nominal_mv_index)
1295 {
1296         struct dvfs *sor1_dp_dvfs = &dvfs_data->sor1_dp_vf_table[0];
1297         struct clk *c;
1298         int i, n = dvfs_data->sor1_dp_vf_table_size;
1299
1300         c = clk_get_sys(sor1_dp_dvfs->clk_name, sor1_dp_dvfs->clk_name);
1301         if (IS_ERR(c)) {
1302                 pr_debug("init_sor1_dvfs: no clock found for %s\n",
1303                         sor1_dp_dvfs->clk_name);
1304                 return;
1305         }
1306
1307         for (i = 0; i < n; i++, sor1_dp_dvfs++) {
1308                 if (match_dvfs_one(sor1_dp_dvfs->clk_name,
1309                         sor1_dp_dvfs->speedo_id, sor1_dp_dvfs->process_id,
1310                         soc_speedo_id, core_process_id)) {
1311                         tegra_dvfs_add_alt_freqs(c, sor1_dp_dvfs);
1312                         break;
1313                 }
1314         }
1315
1316         return;
1317
1318 }
1319
1320 static int get_core_sku_max_mv(void)
1321 {
1322         int speedo_rev = tegra_sku_info.speedo_rev;
1323
1324         switch (tegra_sku_info.soc_process_id) {
1325         case 0:
1326                 if (tegra_sku_info.soc_speedo_id == 1)
1327                         return 1100;
1328                 if (speedo_rev <= 1)
1329                         return 1000;
1330
1331                 return 1125;
1332         case 1:
1333                 return 1075;
1334         case 2:
1335                 return 1000;
1336         default:
1337                 pr_err("Un-supported Tegra210 speedo %d\n",
1338                                 tegra_sku_info.soc_speedo_id);
1339                 return -EINVAL;
1340         }
1341 }
1342
1343 static int get_core_sku_min_mv(void)
1344 {
1345         int rev = tegra_sku_info.revision;
1346         bool a02 = (rev == TEGRA_REVISION_A02) || (rev == TEGRA_REVISION_A02p);
1347
1348         if (tegra_sku_info.soc_speedo_id == 1)
1349                 return 1100;
1350
1351         switch (tegra_sku_info.sku_id) {
1352         case 0x0:
1353         case 0x1:
1354         case 0x7:
1355         case 0x17:
1356         case 0x13:
1357                 if (!a02)
1358                         return 825;
1359                 return 800;
1360         case 0x87:
1361                 return 825;
1362         case 0x83:
1363         case 0x8f:
1364                 return 800;
1365         default:
1366                 return 950;
1367         }
1368 }
1369
1370 static int get_coreb01_sku_max_mv(void)
1371 {
1372         switch (tegra_sku_info.soc_process_id) {
1373         case 0:
1374                 return 1050;
1375         case 1:
1376                 return 1025;
1377         case 2:
1378                 return 1000;
1379         default:
1380                 pr_err("Un-supported Tegra210b01 process id %d\n",
1381                                 tegra_sku_info.soc_process_id);
1382                 return -EINVAL;
1383         }
1384 }
1385
1386 static int get_coreb01slt_sku_max_mv(void)
1387 {
1388         return get_coreb01_sku_max_mv();
1389 }
1390
1391 static int get_coreb01_sku_min_mv(void)
1392 {
1393         return 637;
1394 }
1395
1396 static int get_coreb01slt_sku_min_mv(void)
1397 {
1398         return 600;
1399 }
1400
1401 static int get_core_nominal_mv_index(int speedo_id)
1402 {
1403         int i;
1404         int mv = dvfs_data->get_core_max_mv();
1405
1406         if (mv < 0)
1407                 return mv;
1408
1409         /* Round nominal level down to the nearest core scaling step */
1410         for (i = 0; i < MAX_DVFS_FREQS; i++) {
1411                 if ((core_millivolts[i] == 0) || (mv < core_millivolts[i]))
1412                         break;
1413         }
1414
1415         if (i == 0) {
1416                 pr_err("tegra210-dvfs: failed to get nominal idx at volt %d\n",
1417                        mv);
1418                 return -ENOSYS;
1419         }
1420
1421         return i - 1;
1422 }
1423
1424 static int get_core_min_mv_index(void)
1425 {
1426         int i;
1427         int mv = dvfs_data->get_core_min_mv();
1428
1429         if (mv < 0)
1430                 return mv;
1431
1432         /* Round minimum level up to the nearest core scaling step */
1433         for (i = 0; i < MAX_DVFS_FREQS - 1; i++) {
1434                 if ((core_millivolts[i+1] == 0) || (mv <= core_millivolts[i]))
1435                         break;
1436         }
1437
1438         return i;
1439 }
1440
1441 static int init_cpu_dvfs_table(struct cpu_dvfs *fv_dvfs_table,
1442                                int table_size, int *cpu_max_freq_index)
1443 {
1444         int i, ret;
1445         int cpu_speedo_id = tegra_sku_info.cpu_speedo_id;
1446         int cpu_process_id = tegra_sku_info.cpu_process_id;
1447
1448         for (ret = 0, i = 0; i < table_size; i++) {
1449                 struct cpu_dvfs *d = &fv_dvfs_table[i];
1450
1451                 if (match_dvfs_one("cpu dvfs", d->speedo_id, d->process_id,
1452                                    cpu_speedo_id, cpu_process_id)) {
1453                         ret = set_cpu_dvfs_data(
1454                                 d, &cpu_dvfs, cpu_max_freq_index);
1455                         if (ret)
1456                                 return ret;
1457                         break;
1458                 }
1459         }
1460         BUG_ON(i == table_size);
1461
1462         return ret;
1463 }
1464
1465 static int init_cpu_lp_dvfs_table(int *cpu_lp_max_freq_index)
1466 {
1467         int i, ret;
1468         unsigned int cpu_lp_speedo_id = tegra_sku_info.cpu_speedo_id;
1469         int cpu_lp_process_id = tegra_sku_info.cpu_process_id;
1470         unsigned long max_freq;
1471
1472         if (cpu_lp_speedo_id >= ARRAY_SIZE(cpu_lp_max_freq))
1473                 max_freq = cpu_lp_max_freq[0];
1474         else
1475                 max_freq = cpu_lp_max_freq[cpu_lp_speedo_id];
1476
1477         for (ret = 0, i = 0; i <  ARRAY_SIZE(cpu_lp_fv_dvfs_table); i++) {
1478                 struct cpu_dvfs *d = &cpu_lp_fv_dvfs_table[i];
1479                 if (match_dvfs_one("cpu lp dvfs", d->speedo_id, d->process_id,
1480                                    cpu_lp_speedo_id, cpu_lp_process_id)) {
1481                         ret = set_cpu_lp_dvfs_data(max_freq,
1482                                 d, &cpu_lp_dvfs, cpu_lp_max_freq_index);
1483                         break;
1484                 }
1485         }
1486
1487         return ret;
1488 }
1489
1490 static void adjust_emc_dvfs_from_timing_table(struct dvfs *d)
1491 {
1492         unsigned long rate;
1493         int i;
1494
1495         for (i = 0; i < ARRAY_SIZE(core_millivolts); i++) {
1496                 if (core_millivolts[i] == 0)
1497                         return;
1498
1499                 rate = tegra210_predict_emc_rate(core_millivolts[i]);
1500                 if (IS_ERR_VALUE(rate))
1501                         return;
1502
1503                 if (rate)
1504                         d->freqs[i] = rate;
1505         }
1506 }
1507
1508 static unsigned long dvb_predict_rate(
1509         int process_id, struct dvb_dvfs *dvbd, int mv)
1510 {
1511         int i;
1512         unsigned long rate = 0;
1513
1514         for (i = 0; i < MAX_DVFS_FREQS; i++) {
1515                 if (!dvbd->dvb_table[i].freq)
1516                         break;
1517                 if (dvbd->dvb_table[i].mvolts[process_id] > mv)
1518                         break;
1519                 rate = dvbd->dvb_table[i].freq * dvbd->freqs_mult;
1520         }
1521
1522         return rate;
1523 }
1524
1525 static void adjust_emc_dvfs_from_dvb_table(
1526         int process_id, struct dvb_dvfs *dvbd, struct dvfs *d)
1527 {
1528         unsigned long rate;
1529         int i;
1530
1531         if (process_id > MAX_PROCESS_ID) {
1532                 WARN(1, "Process id %d above emc dvb table max\n", process_id);
1533                 return;
1534         }
1535
1536         for (i = 0; i < ARRAY_SIZE(core_millivolts); i++) {
1537                 if (core_millivolts[i] == 0)
1538                         return;
1539
1540                 rate = dvb_predict_rate(process_id, dvbd, core_millivolts[i]);
1541                 if (rate)
1542                         d->freqs[i] = rate;
1543         }
1544 }
1545
1546 static struct dvb_dvfs *get_emc_dvb_dvfs(int speedo_id)
1547 {
1548         int i;
1549
1550         if (dvfs_data->emc_dvb_table) {
1551                 for (i = 0; i < dvfs_data->emc_dvb_table_size; i++) {
1552                         struct dvb_dvfs *dvbd = &dvfs_data->emc_dvb_table[i];
1553
1554                         if (dvbd->speedo_id == -1 ||
1555                             dvbd->speedo_id == speedo_id)
1556                                 return dvbd;
1557                 }
1558         }
1559         return NULL;
1560 }
1561
1562 /*
1563  * Find maximum GPU frequency that can be reached at minimum voltage across all
1564  * temperature ranges.
1565  */
1566 static unsigned long find_gpu_fmax_at_vmin(
1567         struct dvfs *gpu_dvfs, int thermal_ranges, int freqs_num)
1568 {
1569         int i, j;
1570         unsigned long fmax = ULONG_MAX;
1571
1572         /*
1573          * For voltage scaling row in each temperature range, as well as peak
1574          * voltage row find maximum frequency at lowest voltage, and return
1575          * minimax. On Tegra21 all GPU DVFS thermal dependencies are integrated
1576          * into thermal DVFS table (i.e., there is no separate thermal floors
1577          * applied in the rail level). Hence, returned frequency specifies max
1578          * frequency safe at minimum voltage across all temperature ranges.
1579          */
1580         for (j = 0; j < thermal_ranges; j++) {
1581                 for (i = 1; i < freqs_num; i++) {
1582                         if (gpu_millivolts[j][i] > gpu_millivolts[j][0])
1583                                 break;
1584                 }
1585                 fmax = min(fmax, gpu_dvfs->freqs[i - 1]);
1586         }
1587
1588         for (i = 1; i < freqs_num; i++) {
1589                 if (gpu_peak_millivolts[i] > gpu_peak_millivolts[0])
1590                         break;
1591         }
1592         fmax = min(fmax, gpu_dvfs->freqs[i - 1]);
1593
1594         return fmax;
1595 }
1596
1597 /*
1598  * Determine minimum voltage safe at maximum frequency across all temperature
1599  * ranges.
1600  */
1601 static int find_gpu_vmin_at_fmax(
1602         struct dvfs *gpu_dvfs, int thermal_ranges, int freqs_num)
1603 {
1604         int j, vmin;
1605
1606         /*
1607          * For voltage scaling row in each temperature range find minimum
1608          * voltage at maximum frequency and return max Vmin across ranges.
1609          */
1610         for (vmin = 0, j = 0; j < thermal_ranges; j++)
1611                 vmin = max(vmin, gpu_millivolts[j][freqs_num-1]);
1612
1613         return vmin;
1614 }
1615
1616 static int of_parse_dvfs_rail_cdev_trips(struct device_node *node,
1617                 int *therm_trips_table,
1618                 struct dvfs_therm_limits *therm_limits_table,
1619                 struct dvfs_therm_limits *therm_limits_ucm2_table,
1620                 struct rail_alignment *align, bool up)
1621 {
1622         struct of_phandle_iter iter;
1623         int cells_num, i = 0, t;
1624
1625         /* 1 cell per trip-point, if constraint is specified */
1626         cells_num = of_property_read_bool(node, "nvidia,constraint-ucm2") ? 2 :
1627                 of_property_read_bool(node, "nvidia,constraint") ? 1 : 0;
1628
1629         if (of_phandle_iterator_init(&iter, node, "nvidia,trips",
1630                                      NULL, cells_num))
1631                 return -ENOENT;
1632
1633         while (!of_phandle_iterator_next(&iter)) {
1634                 struct device_node *trip_dn = iter.node;
1635
1636                 if (i >= MAX_THERMAL_LIMITS) {
1637                         pr_err("tegra_dvfs: list of scaling cdev trips exceeds max limit\n");
1638                         return -EINVAL;
1639                 }
1640
1641                 if (of_property_read_s32(trip_dn, "temperature", &t)) {
1642                         pr_err("tegra_dvfs: failed to read scalings cdev trip %d\n", i);
1643                         return -ENODATA;
1644                 }
1645
1646                 if (therm_trips_table)
1647                         therm_trips_table[i] = t / 1000; /* convert mC to C */
1648
1649                 if (cells_num && therm_limits_table) {
1650                         int mv = 0;
1651                         struct of_phandle_args dvfs_args;
1652                         dvfs_args.args_count = of_phandle_iterator_args(&iter,
1653                                                 dvfs_args.args, MAX_PHANDLE_ARGS);
1654
1655                         mv = dvfs_args.args[0];
1656                         mv = tegra_round_voltage(mv, align, up);
1657                         therm_limits_table[i].temperature = t / 1000;
1658                         therm_limits_table[i].mv = mv;
1659                         if (cells_num == 2 && therm_limits_ucm2_table) {
1660                                 mv = dvfs_args.args[1];
1661                                 mv = tegra_round_voltage(mv, align, up);
1662                                 therm_limits_ucm2_table[i].temperature = t/1000;
1663                                 therm_limits_ucm2_table[i].mv = mv;
1664                         }
1665                 }
1666                 i++;
1667         }
1668
1669         return i;
1670 }
1671
1672 static int init_gpu_rail_thermal_scaling(struct device_node *node,
1673                                          struct dvfs_rail *rail,
1674                                          struct cvb_dvfs *d)
1675 {
1676         int thermal_ranges;
1677         struct device_node *cdev_node;
1678
1679         cdev_node = of_find_compatible_node(NULL, NULL,
1680                                 "nvidia,tegra210-rail-scaling-cdev");
1681         if (!cdev_node || !of_device_is_available(cdev_node))
1682                 return 1;
1683
1684         rail->vts_of_node = cdev_node;
1685
1686         thermal_ranges = of_parse_dvfs_rail_cdev_trips(cdev_node,
1687                 &rail->vts_trips_table[0], &rail->vts_floors_table[0],
1688                 NULL, &rail->alignment, true);
1689
1690         if (thermal_ranges <= 0)
1691                 return 1;
1692
1693         rail->vts_number_of_trips = thermal_ranges - 1;
1694
1695         return thermal_ranges;
1696 }
1697
1698 /* cooling device to limit GPU frequenct based on the vmax thermal profile */
1699 #define GPU_MAX_RATE 1300000000UL
1700 static int gpu_dvfs_rail_get_vmax_cdev_max_state(
1701         struct thermal_cooling_device *cdev, unsigned long *max_state)
1702 {
1703         struct dvfs_rail *rail = cdev->devdata;
1704
1705         *max_state = rail->therm_caps_size;
1706
1707         return 0;
1708 }
1709
1710 static int gpu_dvfs_rail_get_vmax_cdev_cur_state(
1711         struct thermal_cooling_device *cdev, unsigned long *cur_state)
1712 {
1713         struct dvfs_rail *rail = cdev->devdata;
1714
1715         *cur_state = rail->therm_cap_idx;
1716
1717         return 0;
1718 }
1719
1720 static int gpu_dvfs_rail_set_vmax_cdev_cur_state(
1721         struct thermal_cooling_device *cdev, unsigned long cur_state)
1722 {
1723         struct dvfs_rail *rail = cdev->devdata;
1724         int level = 0, err = -EINVAL;
1725         unsigned long cap_rate = GPU_MAX_RATE;
1726
1727         if (cur_state)
1728                 level = rail->therm_caps[cur_state - 1].mv;
1729
1730         if (level) {
1731                 if (rail->vts_cdev && gpu_dvfs.therm_dvfs)
1732                         cap_rate = gpu_cap_rates[cur_state - 1];
1733                 else
1734                         cap_rate = tegra_dvfs_predict_hz_at_mv_max_tfloor(
1735                                 gpu_dvfs.clk, level);
1736         }
1737
1738         if (!IS_ERR_VALUE(cap_rate)) {
1739                 err = clk_set_rate(vgpu_cap_clk, cap_rate);
1740                 if (err)
1741                         pr_err("tegra_dvfs: Failed to set GPU cap rate %lu\n",
1742                                cap_rate);
1743         } else {
1744                 pr_err("tegra_dvfs: Failed to find GPU cap rate for %dmV\n",
1745                        level);
1746         }
1747
1748         rail->therm_cap_idx = cur_state;
1749
1750         return err;
1751 }
1752
1753 static struct thermal_cooling_device_ops gpu_dvfs_rail_vmax_cooling_ops = {
1754         .get_max_state = gpu_dvfs_rail_get_vmax_cdev_max_state,
1755         .get_cur_state = gpu_dvfs_rail_get_vmax_cdev_cur_state,
1756         .set_cur_state = gpu_dvfs_rail_set_vmax_cdev_cur_state,
1757 };
1758
1759 #define CAP_TRIP_ON_SCALING_MARGIN 5
1760
1761 static int init_gpu_rail_thermal_caps(struct device_node *node,
1762                 struct dvfs *dvfs, struct dvfs_rail *rail, int thermal_ranges,
1763                 int freqs_num)
1764 {
1765         struct device_node *cdev_node;
1766         int num_trips, i, j, k;
1767         bool ucm2 = tegra_sku_info.ucm == TEGRA_UCM2;
1768
1769         if (thermal_ranges <= 1 )
1770                 return 0;
1771
1772         cdev_node = of_find_compatible_node(NULL, NULL,
1773                                 "nvidia,tegra210-rail-vmax-cdev");
1774         if (!cdev_node || !of_device_is_available(cdev_node))
1775                 return 0;
1776
1777         rail->vmax_of_node = cdev_node;
1778
1779         vgpu_cap_clk = of_clk_get_by_name(cdev_node, "cap-clk");
1780         if (IS_ERR(vgpu_cap_clk))
1781                 return 0;
1782
1783         num_trips = of_parse_dvfs_rail_cdev_trips(cdev_node,
1784                 NULL, vdd_gpu_therm_caps_table, vdd_gpu_therm_caps_ucm2_table,
1785                 &rail->alignment, false);
1786         if (num_trips <= 0)
1787                 return 0;
1788
1789         rail->therm_caps =
1790                 ucm2 ? vdd_gpu_therm_caps_ucm2_table : vdd_gpu_therm_caps_table;
1791         if (!rail->therm_caps[0].mv) {
1792                 pr_err("tegra_dvfs: invalid gpu cap table\n");
1793                 rail->therm_caps = NULL;
1794                 return 0;
1795         }
1796         rail->therm_caps_size = num_trips;
1797         rail->therm_cap_idx = num_trips;
1798
1799         for (k = 0; k < num_trips; k++) {
1800                 int cap_tempr = rail->therm_caps[k].temperature;
1801                 int cap_level = rail->therm_caps[k].mv;
1802                 unsigned long cap_freq = GPU_MAX_RATE;
1803
1804                 for (j = 0; j < thermal_ranges; j++) {
1805                         if ((j < thermal_ranges - 1) && /* vts trips=ranges-1 */
1806                             (rail->vts_trips_table[j] +
1807                             CAP_TRIP_ON_SCALING_MARGIN < cap_tempr))
1808                                 continue;
1809
1810                         for (i = 1; i < freqs_num; i++) {
1811                                 if (gpu_millivolts[j][i] > cap_level)
1812                                         break;
1813                         }
1814                         cap_freq = min(cap_freq, dvfs->freqs[i - 1]);
1815                 }
1816                 gpu_cap_rates[k] = cap_freq * dvfs->freqs_mult;
1817         }
1818
1819
1820         clk_set_rate(vgpu_cap_clk, gpu_cap_rates[num_trips - 1]);
1821         rail->vmax_cdev = thermal_of_cooling_device_register(rail->vmax_of_node,
1822                 "GPU-cap", rail, &gpu_dvfs_rail_vmax_cooling_ops);
1823         pr_info("tegra_dvfs: GPU-cap: %sregistered\n",
1824                 IS_ERR_OR_NULL(rail->vmax_cdev) ? "not " : "");
1825
1826         return num_trips;
1827 }
1828
1829 /*
1830  * Setup gpu dvfs tables from cvb data, determine nominal voltage for gpu rail,
1831  * and gpu maximum frequency. Error when gpu dvfs table can not be constructed
1832  * must never happen.
1833  */
1834 static int set_gpu_dvfs_data(struct device_node *node, unsigned long max_freq,
1835         struct cvb_dvfs *d, struct dvfs *gpu_dvfs, int *max_freq_index)
1836 {
1837         int i, j, thermal_ranges, mv, min_mv, err;
1838         struct cvb_dvfs_table *table = NULL;
1839         int speedo = tegra_sku_info.gpu_speedo_value;
1840         struct dvfs_rail *rail = &vdd_gpu_rail;
1841         struct rail_alignment *align = &rail->alignment;
1842
1843         rail->nvver = d->cvb_version;
1844
1845         d->max_mv = tegra_round_voltage(d->max_mv, align, false);
1846         min_mv = d->pll_min_millivolts;
1847         mv = tegra_get_cvb_voltage(
1848                 speedo, d->speedo_scale, &d->cvb_vmin.cvb_pll_param);
1849         mv = tegra_round_cvb_voltage(mv, d->voltage_scale, align);
1850         min_mv = max(min_mv, mv);
1851         if (min_mv < rail->min_millivolts) {
1852                 pr_debug("tegra21_dvfs: gpu min %dmV below rail min %dmV\n",
1853                          min_mv, rail->min_millivolts);
1854                 min_mv = rail->min_millivolts;
1855         }
1856
1857         /*
1858          * Get scaling thermal ranges; 1 range implies no thermal dependency.
1859          * Invalidate scaling cooling device in the latter case.
1860          */
1861         thermal_ranges = init_gpu_rail_thermal_scaling(node, rail, d);
1862         if (thermal_ranges == 1)
1863                 rail->vts_cdev = NULL;
1864
1865         /*
1866          * Apply fixed thermal floor for each temperature range
1867          */
1868         for (j = 0; j < thermal_ranges; j++) {
1869                 mv = max(min_mv, (int)rail->vts_floors_table[j].mv);
1870                 gpu_vmin[j] = tegra_round_voltage(mv, align, true);
1871         }
1872
1873         /*
1874          * Use CVB table to fill in gpu dvfs frequencies and voltages. Each
1875          * CVB entry specifies gpu frequency and CVB coefficients to calculate
1876          * the respective voltage.
1877          */
1878         for (i = 0; i < MAX_DVFS_FREQS; i++) {
1879                 table = &d->cvb_table[i];
1880                 if (!table->freq || (table->freq > max_freq))
1881                         break;
1882
1883                 mv = tegra_get_cvb_voltage(
1884                         speedo, d->speedo_scale, &table->cvb_pll_param);
1885
1886                 for (j = 0; j < thermal_ranges; j++) {
1887                         int mvj = mv;
1888                         int t = 0;
1889
1890                         if (thermal_ranges > 1)
1891                                 t = rail->vts_trips_table[j];
1892
1893                         /* get thermal offset for this trip-point */
1894                         mvj += tegra_get_cvb_t_voltage(speedo, d->speedo_scale,
1895                                 t, d->thermal_scale, &table->cvb_pll_param);
1896                         mvj = tegra_round_cvb_voltage(mvj, d->voltage_scale, align);
1897
1898                         /* clip to minimum, abort if above maximum */
1899                         mvj = max(mvj, gpu_vmin[j]);
1900                         if (mvj > d->max_mv)
1901                                 break;
1902
1903                         /*
1904                          * Update voltage for adjacent ranges bounded by this
1905                          * trip-point (cvb & dvfs are transpose matrices, and
1906                          * cvb freq row index is column index for dvfs matrix)
1907                          */
1908                         gpu_millivolts[j][i] = mvj;
1909                         if (j && (gpu_millivolts[j-1][i] < mvj))
1910                                 gpu_millivolts[j-1][i] = mvj;
1911                 }
1912                 /* Make sure all voltages for this frequency are below max */
1913                 if (j < thermal_ranges)
1914                         break;
1915
1916                 /* fill in gpu dvfs tables */
1917                 gpu_dvfs->freqs[i] = table->freq;
1918         }
1919
1920         gpu_dvfs->millivolts = &gpu_millivolts[0][0];
1921
1922         /*
1923          * Table must not be empty, must have at least one entry in range, and
1924          * must specify monotonically increasing voltage on frequency dependency
1925          * in each temperature range.
1926          */
1927         err = tegra_dvfs_init_thermal_dvfs_voltages(&gpu_millivolts[0][0],
1928                         gpu_peak_millivolts, i, thermal_ranges, gpu_dvfs);
1929
1930         if (err || !i) {
1931                 pr_err("tegra21_dvfs: invalid gpu dvfs table\n");
1932                 return -ENOENT;
1933         }
1934
1935         /* Shift out the 1st trip-point */
1936         for (j = 1; j < thermal_ranges; j++)
1937                 rail->vts_trips_table[j - 1] = rail->vts_trips_table[j];
1938
1939         /* dvfs tables are successfully populated - fill in the gpu dvfs */
1940         gpu_dvfs->speedo_id = d->speedo_id;
1941         gpu_dvfs->process_id = d->process_id;
1942         gpu_dvfs->freqs_mult = d->freqs_mult;
1943
1944         *max_freq_index = i - 1;
1945
1946         gpu_dvfs->dvfs_rail->nominal_millivolts = min(d->max_mv,
1947                 find_gpu_vmin_at_fmax(gpu_dvfs, thermal_ranges, i));
1948
1949         gpu_dvfs->fmax_at_vmin_safe_t = d->freqs_mult *
1950                 find_gpu_fmax_at_vmin(gpu_dvfs, thermal_ranges, i);
1951
1952         /* Initialize thermal capping */
1953         init_gpu_rail_thermal_caps(node, gpu_dvfs, rail, thermal_ranges, i);
1954
1955 #ifdef CONFIG_TEGRA_USE_NA_GPCPLL
1956         /*
1957          * Set NA DVFS flag, if GPCPLL NA mode is enabled. This is necessary to
1958          * make sure that GPCPLL configuration is updated by tegra core DVFS
1959          * when thermal DVFS cooling device state is changed. Since tegra core
1960          * DVFS does not support NA operations for Vmin cooling device, GPU Vmin
1961          * thermal floors have been integrated with thermal DVFS, and no Vmin
1962          * cooling device is installed.
1963          */
1964         if (tegra_sku_info.gpu_speedo_id)
1965                 gpu_dvfs->na_dvfs = 1;
1966 #else
1967         if (of_device_is_compatible(node, "nvidia,tegra210b01-dvfs"))
1968                 WARN(1, "TEGRA_USE_NA_GPCPLL must be set on T210b01\n");
1969 #endif
1970         return 0;
1971 }
1972
1973 static void init_gpu_dvfs_table(struct device_node *node,
1974         struct cvb_dvfs *cvb_dvfs_table, int table_size,
1975         int *gpu_max_freq_index)
1976 {
1977         int i, ret;
1978         int gpu_speedo_id = tegra_sku_info.gpu_speedo_id;
1979         int gpu_process_id = tegra_sku_info.gpu_process_id;
1980
1981         for (ret = 0, i = 0; i < table_size; i++) {
1982                 struct cvb_dvfs *d = &cvb_dvfs_table[i];
1983                 unsigned long max_freq = d->max_freq;
1984                 u32 f;
1985
1986                 if (!of_property_read_u32(node, "nvidia,gpu-max-freq-khz", &f))
1987                         max_freq = min(max_freq, (unsigned long)f);
1988
1989                 if (match_dvfs_one("gpu cvb", d->speedo_id, d->process_id,
1990                                    gpu_speedo_id, gpu_process_id)) {
1991                         ret = set_gpu_dvfs_data(node, max_freq,
1992                                 d, &gpu_dvfs, gpu_max_freq_index);
1993                         break;
1994                 }
1995         }
1996         BUG_ON((i == table_size) || ret);
1997 }
1998
1999 static void init_core_dvfs_table(int soc_speedo_id, int core_process_id)
2000 {
2001         int core_nominal_mv_index;
2002         int core_min_mv_index;
2003         int i;
2004         static bool initialized;
2005
2006         if (initialized)
2007                 return;
2008
2009         initialized = true;
2010
2011         /*
2012          * Find nominal voltages for core (1st) and cpu rails before rail
2013          * init. Nominal voltage index in core scaling ladder can also be
2014          * used to determine max dvfs frequencies for all core clocks. In
2015          * case of error disable core scaling and set index to 0, so that
2016          * core clocks would not exceed rates allowed at minimum voltage.
2017          */
2018         core_nominal_mv_index = get_core_nominal_mv_index(soc_speedo_id);
2019         if (core_nominal_mv_index < 0) {
2020                 vdd_core_rail.disabled = true;
2021                 tegra_dvfs_core_disabled = true;
2022                 core_nominal_mv_index = 0;
2023         }
2024
2025         core_min_mv_index = get_core_min_mv_index();
2026         if (core_min_mv_index < 0) {
2027                 vdd_core_rail.disabled = true;
2028                 tegra_dvfs_core_disabled = true;
2029                 core_min_mv_index = 0;
2030         }
2031
2032         vdd_core_rail.nominal_millivolts =
2033                 core_millivolts[core_nominal_mv_index];
2034         vdd_core_rail.min_millivolts =
2035                 core_millivolts[core_min_mv_index];
2036         vdd_core_rail.nvver = dvfs_data->core_dvfs_ver;
2037
2038         /*
2039          * Search core dvfs table for speedo/process matching entries and
2040          * initialize dvfs-ed clocks
2041          */
2042         for (i = 0; i < dvfs_data->core_vf_table_size; i++) {
2043                 struct dvfs *d = &dvfs_data->core_vf_table[i];
2044                 if (!match_dvfs_one(d->clk_name, d->speedo_id,
2045                         d->process_id, soc_speedo_id, core_process_id))
2046                         continue;
2047                 init_dvfs_one(d, core_nominal_mv_index);
2048
2049                 /*
2050                  * If EMC DVB table is installed, use it to set EMC VF opps.
2051                  * Otherwise, EMC dvfs is board dependent, EMC frequencies are
2052                  * determined by the Tegra BCT and the board specific EMC DFS
2053                  * table owned by EMC driver.
2054                  */
2055                 if (!strcmp(d->clk_name, "emc") && tegra210_emc_is_ready()) {
2056                         struct dvb_dvfs *dvbd = get_emc_dvb_dvfs(soc_speedo_id);
2057
2058                         if (dvbd)
2059                                 adjust_emc_dvfs_from_dvb_table(
2060                                         core_process_id, dvbd, d);
2061                         else
2062                                 adjust_emc_dvfs_from_timing_table(d);
2063                 }
2064         }
2065
2066         init_qspi_dvfs(soc_speedo_id, core_process_id, core_nominal_mv_index);
2067         init_sor1_dvfs(soc_speedo_id, core_process_id, core_nominal_mv_index);
2068         init_spi_dvfs(soc_speedo_id, core_process_id, core_nominal_mv_index);
2069 }
2070
2071 static void init_dvfs_data(struct tegra_dvfs_data *data)
2072 {
2073         int i;
2074         static bool initialized;
2075
2076         if (initialized)
2077                 return;
2078
2079         initialized = true;
2080
2081         dvfs_data = data;
2082
2083         BUG_ON(dvfs_data->rails_num != ARRAY_SIZE(vdd_dvfs_rails));
2084         vdd_cpu_rail = *dvfs_data->rails[VDD_CPU_INDEX];
2085         vdd_core_rail = *dvfs_data->rails[VDD_CORE_INDEX];
2086         vdd_gpu_rail = *dvfs_data->rails[VDD_GPU_INDEX];
2087
2088         for (i = 0; i < MAX_DVFS_FREQS; i++)
2089                 core_millivolts[i] = dvfs_data->core_mv[i];
2090 }
2091
2092 static struct tegra_dvfs_data tegra210_dvfs_data = {
2093         .rails = tegra210_dvfs_rails,
2094         .rails_num = ARRAY_SIZE(tegra210_dvfs_rails),
2095         .cpu_fv_table = cpu_fv_dvfs_table,
2096         .cpu_fv_table_size = ARRAY_SIZE(cpu_fv_dvfs_table),
2097         .gpu_cvb_table = gpu_cvb_dvfs_table,
2098         .gpu_cvb_table_size = ARRAY_SIZE(gpu_cvb_dvfs_table),
2099
2100         .core_mv = core_voltages_mv,
2101         .core_vf_table = core_dvfs_table,
2102         .core_vf_table_size = ARRAY_SIZE(core_dvfs_table),
2103         .spi_vf_table = spi_dvfs_table,
2104         .spi_slave_vf_table = spi_slave_dvfs_table,
2105         .qspi_sdr_vf_table = qspi_sdr_dvfs_table,
2106         .qspi_ddr_vf_table = qspi_ddr_dvfs_table,
2107         .sor1_dp_vf_table = sor1_dp_dvfs_table,
2108         .sor1_dp_vf_table_size = ARRAY_SIZE(sor1_dp_dvfs_table),
2109         .get_core_min_mv = get_core_sku_min_mv,
2110         .get_core_max_mv = get_core_sku_max_mv,
2111
2112         .core_floors = tegra210_core_therm_floors,
2113         .core_caps = tegra210_core_therm_caps,
2114         .core_caps_ucm2 = tegra210_core_therm_caps_ucm2,
2115 };
2116
2117 static struct tegra_dvfs_data tegra210b01_dvfs_data = {
2118         .rails = tegra210b01_dvfs_rails,
2119         .rails_num = ARRAY_SIZE(tegra210b01_dvfs_rails),
2120         .cpu_fv_table = cpub01_fv_dvfs_table,
2121         .cpu_fv_table_size = ARRAY_SIZE(cpub01_fv_dvfs_table),
2122         .gpu_cvb_table = gpub01_cvb_dvfs_table,
2123         .gpu_cvb_table_size = ARRAY_SIZE(gpub01_cvb_dvfs_table),
2124
2125         .emc_dvb_table = emcb01_dvb_dvfs_table,
2126         .emc_dvb_table_size = ARRAY_SIZE(emcb01_dvb_dvfs_table),
2127
2128         .core_mv = coreb01_voltages_mv,
2129         .core_vf_table = coreb01_dvfs_table,
2130         .core_vf_table_size = ARRAY_SIZE(coreb01_dvfs_table),
2131         .spi_vf_table = spib01_dvfs_table,
2132         .spi_slave_vf_table = spi_slaveb01_dvfs_table,
2133         .qspi_sdr_vf_table = qspi_sdrb01_dvfs_table,
2134         .qspi_ddr_vf_table = qspi_ddrb01_dvfs_table,
2135         .sor1_dp_vf_table = sor1_dpb01_dvfs_table,
2136         .sor1_dp_vf_table_size = ARRAY_SIZE(sor1_dpb01_dvfs_table),
2137         .get_core_min_mv = get_coreb01_sku_min_mv,
2138         .get_core_max_mv = get_coreb01_sku_max_mv,
2139
2140         .core_floors = tegra210b01_core_therm_floors,
2141         .core_caps = tegra210b01_core_therm_caps,
2142         .core_caps_ucm2 = tegra210b01_core_therm_caps_ucm2,
2143
2144         .core_dvfs_ver = coreb01_dvfs_table_ver,
2145 };
2146
2147 static struct tegra_dvfs_data tegra210b01slt_dvfs_data = {
2148         .rails = tegra210b01_dvfs_rails,
2149         .rails_num = ARRAY_SIZE(tegra210b01_dvfs_rails),
2150         .cpu_fv_table = cpub01_fv_dvfs_table,
2151         .cpu_fv_table_size = ARRAY_SIZE(cpub01_fv_dvfs_table),
2152         .gpu_cvb_table = gpub01_cvb_dvfs_table,
2153         .gpu_cvb_table_size = ARRAY_SIZE(gpub01_cvb_dvfs_table),
2154
2155         .emc_dvb_table = emcb01slt_dvb_dvfs_table,
2156         .emc_dvb_table_size = ARRAY_SIZE(emcb01slt_dvb_dvfs_table),
2157
2158         .core_mv = coreb01slt_voltages_mv,
2159         .core_vf_table = coreb01slt_dvfs_table,
2160         .core_vf_table_size = ARRAY_SIZE(coreb01slt_dvfs_table),
2161         .spi_vf_table = spib01slt_dvfs_table,
2162         .spi_slave_vf_table = spi_slaveb01slt_dvfs_table,
2163         .qspi_sdr_vf_table = qspi_sdrb01slt_dvfs_table,
2164         .qspi_ddr_vf_table = qspi_ddrb01slt_dvfs_table,
2165         .sor1_dp_vf_table = sor1_dpb01slt_dvfs_table,
2166         .sor1_dp_vf_table_size = ARRAY_SIZE(sor1_dpb01slt_dvfs_table),
2167         .get_core_min_mv = get_coreb01slt_sku_min_mv,
2168         .get_core_max_mv = get_coreb01slt_sku_max_mv,
2169
2170         .core_floors = tegra210b01_core_therm_floors,
2171         .core_caps = tegra210b01_core_therm_caps,
2172         .core_caps_ucm2 = tegra210b01_core_therm_caps_ucm2,
2173
2174         .core_dvfs_ver = coreb01slt_dvfs_table_ver,
2175 };
2176
2177 static void disable_rail_scaling(struct device_node *np)
2178 {
2179         /* With DFLL as clock source CPU rail scaling cannot be disabled */
2180
2181         if (tegra_dvfs_core_disabled ||
2182             of_property_read_bool(np, "nvidia,core-rail-scaling-disabled")) {
2183                 vdd_dvfs_rails[VDD_CORE_INDEX]->disabled = true;
2184         }
2185         if (tegra_dvfs_gpu_disabled ||
2186             of_property_read_bool(np, "nvidia,gpu-rail-scaling-disabled")) {
2187                 vdd_dvfs_rails[VDD_GPU_INDEX]->disabled = true;
2188         }
2189 }
2190
2191 static int tegra210x_init_dvfs(struct device *dev, bool cpu_lp_init)
2192 {
2193         int soc_speedo_id = tegra_sku_info.soc_speedo_id;
2194         int core_process_id = tegra_sku_info.soc_process_id;
2195         bool ucm2 = tegra_sku_info.ucm == TEGRA_UCM2;
2196         int i, ret;
2197         int cpu_max_freq_index = 0;
2198         int cpu_lp_max_freq_index = 0;
2199         int gpu_max_freq_index = 0;
2200         struct device_node *node = dev->of_node;
2201
2202         tegra_dvfs_init_rails_lists(vdd_dvfs_rails, dvfs_data->rails_num);
2203         init_core_dvfs_table(soc_speedo_id, core_process_id);
2204
2205         /* Get rails alignment, defer probe if regulators are not ready */
2206         for (i = 0; i <  dvfs_data->rails_num; i++) {
2207                 struct regulator *reg;
2208                 unsigned int step_uv;
2209                 int min_uV, max_uV, ret;
2210
2211                 reg = regulator_get(dev, vdd_dvfs_rails[i]->reg_id);
2212                 if (IS_ERR(reg)) {
2213                         pr_info("tegra_dvfs: Unable to get %s rail for step info, defering probe\n",
2214                                         vdd_dvfs_rails[i]->reg_id);
2215                         return -EPROBE_DEFER;
2216                 }
2217
2218                 ret = regulator_get_constraint_voltages(reg, &min_uV, &max_uV);
2219                 if (!ret)
2220                         vdd_dvfs_rails[i]->alignment.offset_uv = min_uV;
2221
2222                 step_uv = regulator_get_linear_step(reg); /* 1st try get step */
2223                 if (!step_uv && !ret) {    /* if no step, try to calculate it */
2224                         int n_voltages = regulator_count_voltages(reg);
2225
2226                         if (n_voltages > 1)
2227                                 step_uv = (max_uV - min_uV) / (n_voltages - 1);
2228                 }
2229                 if (step_uv) {
2230                         vdd_dvfs_rails[i]->alignment.step_uv = step_uv;
2231                         vdd_dvfs_rails[i]->stats.bin_uv = step_uv;
2232                 }
2233                 regulator_put(reg);
2234         }
2235
2236         /*
2237          * Construct fast and slow CPU DVFS tables from FV data; find maximum
2238          * frequency, minimum  and nominal voltage for each CPU cluster, and
2239          * combined rail limits (fast CPU should be initialized 1st).
2240          */
2241         ret = init_cpu_dvfs_table(dvfs_data->cpu_fv_table,
2242                 dvfs_data->cpu_fv_table_size, &cpu_max_freq_index);
2243         if (ret)
2244                 goto out;
2245
2246         if (cpu_lp_init) {
2247                 ret = init_cpu_lp_dvfs_table(&cpu_lp_max_freq_index);
2248                 if (ret)
2249                         goto out;
2250         }
2251
2252         /*
2253          * Construct GPU DVFS table from CVB data; find GPU maximum frequency,
2254          * and nominal voltage.
2255          */
2256         init_gpu_dvfs_table(node, dvfs_data->gpu_cvb_table,
2257                 dvfs_data->gpu_cvb_table_size, &gpu_max_freq_index);
2258
2259         /* Init core thermal floors abd caps */
2260         vdd_core_rail.therm_floors = dvfs_data->core_floors;
2261         vdd_core_rail.therm_caps =
2262                 ucm2 ? dvfs_data->core_caps_ucm2 : dvfs_data->core_caps;
2263         tegra_dvfs_core_init_therm_limits(&vdd_core_rail);
2264
2265         /* Init rail structures and dependencies */
2266         tegra_dvfs_init_rails(vdd_dvfs_rails, dvfs_data->rails_num);
2267
2268         if (of_property_read_bool(of_chosen, "nvidia,tegra-joint_xpu_rail"))
2269                 vdd_dvfs_rails[VDD_GPU_INDEX]->joint_rail_with_dfll = true;
2270
2271         /*
2272          * Initialize matching cpu dvfs entry already found when nominal
2273          * voltage was determined
2274          */
2275         init_dvfs_one(&cpu_dvfs, cpu_max_freq_index);
2276         if (cpu_lp_init)
2277                 init_dvfs_one(&cpu_lp_dvfs, cpu_lp_max_freq_index);
2278         init_dvfs_one(&gpu_dvfs, gpu_max_freq_index);
2279
2280         disable_rail_scaling(node);
2281
2282         for (i = 0; i < dvfs_data->rails_num; i++) {
2283                 struct dvfs_rail *rail = vdd_dvfs_rails[i];
2284                 pr_info("tegra dvfs: %s: nominal %dmV, offset %duV, step %duV, scaling %s\n",
2285                         rail->reg_id, rail->nominal_millivolts,
2286                         rail->alignment.offset_uv, rail->alignment.step_uv,
2287                         rail->disabled ? "disabled" : "enabled");
2288         }
2289         return 0;
2290 out:
2291         return ret;
2292 }
2293
2294 int tegra210_init_dvfs(struct device *dev)
2295 {
2296         init_dvfs_data(&tegra210_dvfs_data);
2297         return tegra210x_init_dvfs(dev, true);
2298 }
2299
2300 int tegra210b01_init_dvfs(struct device *dev)
2301 {
2302         if (tegra_sku_info.soc_speedo_id == 2)
2303                 init_dvfs_data(&tegra210b01slt_dvfs_data);
2304         else
2305                 init_dvfs_data(&tegra210b01_dvfs_data);
2306
2307         return tegra210x_init_dvfs(dev, false);
2308 }