video: tegra: host: add 3dfs user space control
[linux-3.10.git] / drivers / video / tegra / host / gr3d / pod_scaling.c
1 /*
2  * drivers/video/tegra/host/gr3d/pod_scaling.c
3  *
4  * Tegra Graphics Host 3D clock scaling
5  *
6  * Copyright (c) 2013, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms and conditions of the GNU General Public License,
10  * version 2, as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope it will be useful, but WITHOUT
13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15  * more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
19  */
20
21 /*
22  * Power-on-demand clock scaling for nvhost devices
23  *
24  * devfreq calls nvhost_pod_estimate_freq() for estimating the new
25  * frequency for the device. The clocking is done using two properties:
26  *
27  *  (1) Usually the governor receives actively throughput hints that indicate
28  *      whether scaling up or down is required.
29  *  (2) The load of the device is estimated using the busy times from the
30  *      device profile. This information indicates if the device frequency
31  *      should be altered.
32  *
33  */
34
35 #include <linux/devfreq.h>
36 #include <linux/debugfs.h>
37 #include <linux/types.h>
38 #include <linux/clk.h>
39 #include <linux/export.h>
40 #include <linux/slab.h>
41
42 #define CREATE_TRACE_POINTS
43 #include <trace/events/nvhost_podgov.h>
44
45 #include <mach/clk.h>
46 #include <mach/hardware.h>
47
48 #include <governor.h>
49
50 #include "scale3d.h"
51 #include "pod_scaling.h"
52 #include "dev.h"
53
54 /* time frame for load and hint tracking - when events come in at a larger
55  * interval, this probably indicates the current estimates are stale
56  */
57 #define GR3D_TIMEFRAME 1000000 /* 1 sec */
58
59 /* the number of frames to use in the running average of load estimates and
60  * throughput hints. Choosing 6 frames targets a window of about 100 msec.
61  * Large flucutuations in frame times require a window that's large enough to
62  * prevent spiky scaling behavior, which in turn exacerbates frame rate
63  * instability.
64  */
65
66 static int podgov_is_enabled(struct device *dev);
67 static void podgov_enable(struct device *dev, int enable);
68 static int podgov_user_ctl(struct device *dev);
69 static void podgov_set_user_ctl(struct device *dev, int enable);
70
71 /*******************************************************************************
72  * podgov_info_rec - gr3d scaling governor specific parameters
73  ******************************************************************************/
74
75 struct podgov_info_rec {
76
77         int                     enable;
78         int                     init;
79         int                     fast_up_count;
80         int                     slow_down_count;
81         int                     is_scaled;
82
83         unsigned long           idle_total;
84         unsigned long           idle_short_term_total;
85
86         ktime_t                 last_throughput_hint;
87         ktime_t                 last_scale;
88         ktime_t                 last_adjust;
89         ktime_t                 last_estimation_window;
90         ktime_t                 estimation_window;
91         ktime_t                 last_notification;
92
93         struct work_struct      work;
94         struct delayed_work     idle_timer;
95
96         unsigned int            scale;
97         unsigned int            p_estimation_window;
98         unsigned int            p_use_throughput_hint;
99         unsigned int            p_hint_lo_limit;
100         unsigned int            p_hint_hi_limit;
101         unsigned int            p_scaleup_limit;
102         unsigned int            p_scaledown_limit;
103         unsigned int            p_smooth;
104         unsigned int            p_idle_min;
105         unsigned int            idle_min;
106         unsigned int            p_idle_max;
107         unsigned int            idle_max;
108         unsigned int            p_adjust;
109         unsigned int            p_user;
110         unsigned int            p_freq_request;
111
112         long                    last_total_idle;
113         long                    total_idle;
114         long                    idle_estimate;
115
116         int                     adjustment_type;
117         unsigned long           adjustment_frequency;
118
119         int                     last_event_type;
120
121         struct devfreq          *power_manager;
122         struct dentry           *debugdir;
123
124         int                     *freqlist;
125         int                     freq_count;
126
127         unsigned int            idle_avg;
128         unsigned int            hint_avg;
129         int                     block;
130
131 };
132
133 /*******************************************************************************
134  * Adjustment type is used to tell the source that requested frequency re-
135  * estimation. Type ADJUSTMENT_LOCAL indicates that the re-estimation was
136  * initiated by the governor itself. This happens when one of the worker
137  * threads want to adjust the frequency.
138  *
139  * ADJUSTMENT_DEVICE_REQ (default value) indicates that the adjustment was
140  * initiated by a device event.
141  ******************************************************************************/
142
143 enum podgov_adjustment_type {
144         ADJUSTMENT_LOCAL = 0,
145         ADJUSTMENT_DEVICE_REQ = 1
146 };
147
148
149 /* Some functions cannot get pointer to podgov anywhere :-(
150  * Yes, this should be fixed */
151 struct podgov_info_rec *local_podgov;
152
153 /*******************************************************************************
154  * scaling_limit(df, freq)
155  *
156  * Limit the given frequency
157  ******************************************************************************/
158
159 static void scaling_limit(struct devfreq *df, unsigned long *freq)
160 {
161         if (*freq < df->min_freq)
162                 *freq = df->min_freq;
163         else if (*freq > df->max_freq)
164                 *freq = df->max_freq;
165 }
166
167 /*******************************************************************************
168  * podgov_clocks_handler(work)
169  *
170  * This handler is called periodically to re-estimate the frequency using the
171  * relation between "fast scale ups" and "slow scale downs".
172  ******************************************************************************/
173
174 static void podgov_clocks_handler(struct work_struct *work)
175 {
176         struct podgov_info_rec *podgov =
177                 container_of(work, struct podgov_info_rec, work);
178         struct devfreq *df = podgov->power_manager;
179         unsigned long freq;
180
181         mutex_lock(&df->lock);
182
183         if (!podgov->enable) {
184                 mutex_unlock(&df->lock);
185                 return;
186         }
187         if (podgov->scale == 0) {
188                 mutex_unlock(&df->lock);
189                 return;
190         }
191
192         freq = podgov->scale * (df->previous_freq / 100);
193         scaling_limit(df, &freq);
194
195         if (df->previous_freq != freq) {
196                 trace_podgov_clocks_handler(df->previous_freq, freq);
197                 podgov->adjustment_frequency = freq;
198                 podgov->adjustment_type = ADJUSTMENT_LOCAL;
199                 update_devfreq(df);
200         }
201
202         mutex_unlock(&df->lock);
203 }
204
205 /*******************************************************************************
206  * nvhost_scale3d_suspend(dev)
207  *
208  * Prepare the device for suspend
209  ******************************************************************************/
210
211 void nvhost_scale3d_suspend(struct platform_device *dev)
212 {
213         struct nvhost_device_data *pdata = platform_get_drvdata(dev);
214         struct devfreq *df = pdata->power_manager;
215         struct podgov_info_rec *podgov;
216
217         if (!df)
218                 return;
219
220         mutex_lock(&df->lock);
221         podgov = df->data;
222         if (!podgov->enable) {
223                 mutex_unlock(&df->lock);
224                 return;
225         }
226
227         cancel_work_sync(&podgov->work);
228         cancel_delayed_work(&podgov->idle_timer);
229
230         mutex_unlock(&df->lock);
231 }
232
233 /*******************************************************************************
234  * podgov_is_enabled(dev)
235  *
236  * Check whether the device is enabled or not.
237  ******************************************************************************/
238
239 static int podgov_is_enabled(struct device *dev)
240 {
241         struct platform_device *d = to_platform_device(dev);
242         struct nvhost_device_data *pdata = platform_get_drvdata(d);
243         struct devfreq *df = pdata->power_manager;
244         struct podgov_info_rec *podgov;
245         int enable;
246
247         if (!df)
248                 return 0;
249
250         mutex_lock(&df->lock);
251         podgov = df->data;
252         enable = podgov->enable;
253         mutex_unlock(&df->lock);
254
255         return enable;
256 }
257
258 /*******************************************************************************
259  * podgov_enable(dev, enable)
260  *
261  * This function enables (enable=1) or disables (enable=0) the automatic scaling
262  * of the device. If the device is disabled, the device's clock is set to its
263  * maximum.
264  ******************************************************************************/
265
266 static void podgov_enable(struct device *dev, int enable)
267 {
268         struct platform_device *d = to_platform_device(dev);
269         struct nvhost_device_data *pdata = platform_get_drvdata(d);
270         struct devfreq *df = pdata->power_manager;
271         struct podgov_info_rec *podgov;
272
273         if (!df)
274                 return;
275
276         mutex_lock(&df->lock);
277         podgov = df->data;
278
279         trace_podgov_enabled(enable);
280
281         if (enable && df->min_freq != df->max_freq) {
282                 podgov->enable = 1;
283         } else {
284                 cancel_work_sync(&podgov->work);
285                 cancel_delayed_work(&podgov->idle_timer);
286
287                 podgov->adjustment_frequency = df->max_freq;
288                 podgov->adjustment_type = ADJUSTMENT_LOCAL;
289                 update_devfreq(df);
290
291                 podgov->enable = 0;
292
293         }
294         mutex_unlock(&df->lock);
295 }
296
297 /*******************************************************************************
298  * podgov_user_ctl(dev)
299  *
300  * Check whether the gpu scaling is set to user space control.
301  ******************************************************************************/
302
303 static int podgov_user_ctl(struct device *dev)
304 {
305         struct platform_device *d = to_platform_device(dev);
306         struct nvhost_device_data *pdata = platform_get_drvdata(d);
307         struct devfreq *df = pdata->power_manager;
308         struct podgov_info_rec *podgov;
309         int user;
310
311         if (!df)
312                 return 0;
313
314         mutex_lock(&df->lock);
315         podgov = df->data;
316         user = podgov->p_user;
317         mutex_unlock(&df->lock);
318
319         return user;
320 }
321
322 /*****************************************************************************
323  * podgov_set_user_ctl(dev, user)
324  *
325  * This function enables or disables user control of the gpu. If user control
326  * is enabled, setting the freq_request controls the gpu frequency, and other
327  * gpu scaling mechanisms are disabled.
328  ******************************************************************************/
329
330 static void podgov_set_user_ctl(struct device *dev, int user)
331 {
332         struct platform_device *d = to_platform_device(dev);
333         struct nvhost_device_data *pdata = platform_get_drvdata(d);
334         struct devfreq *df = pdata->power_manager;
335         struct podgov_info_rec *podgov;
336
337         if (!df)
338                 return;
339
340         mutex_lock(&df->lock);
341         podgov = df->data;
342
343         trace_podgov_set_user_ctl(user);
344
345         if (podgov->enable && user && !podgov->p_user) {
346                 cancel_work_sync(&podgov->work);
347                 cancel_delayed_work(&podgov->idle_timer);
348
349                 podgov->adjustment_frequency = podgov->p_freq_request;
350                 podgov->adjustment_type = ADJUSTMENT_LOCAL;
351                 update_devfreq(df);
352         }
353
354         podgov->p_user = user;
355
356         mutex_unlock(&df->lock);
357 }
358
359 /*******************************************************************************
360  * podgov_get_freq_request(dev)
361  *
362  * return the latest freq request if anybody asks
363  ******************************************************************************/
364
365 static int podgov_get_freq_request(struct device *dev)
366 {
367         struct platform_device *d = to_platform_device(dev);
368         struct nvhost_device_data *pdata = platform_get_drvdata(d);
369         struct devfreq *df = pdata->power_manager;
370         struct podgov_info_rec *podgov;
371         int freq_request;
372
373         if (!df)
374                 return 0;
375
376         mutex_lock(&df->lock);
377         podgov = df->data;
378         freq_request = podgov->p_freq_request;
379         mutex_unlock(&df->lock);
380
381         return freq_request;
382 }
383
384 /*****************************************************************************
385  * podgov_set_freq_request(dev, user)
386  *
387  * Set the current freq request. If scaling is enabled, and podgov user space
388  * control is enabled, this will set the gpu frequency.
389  ******************************************************************************/
390
391 static void podgov_set_freq_request(struct device *dev, int freq_request)
392 {
393         struct platform_device *d = to_platform_device(dev);
394         struct nvhost_device_data *pdata = platform_get_drvdata(d);
395         struct devfreq *df = pdata->power_manager;
396         struct podgov_info_rec *podgov;
397
398         if (!df)
399                 return;
400
401         mutex_lock(&df->lock);
402         podgov = df->data;
403
404         trace_podgov_set_freq_request(freq_request);
405
406         podgov->p_freq_request = freq_request;
407
408         if (podgov->enable && podgov->p_user) {
409
410                 podgov->adjustment_frequency = freq_request;
411                 podgov->adjustment_type = ADJUSTMENT_LOCAL;
412                 update_devfreq(df);
413         }
414
415         mutex_unlock(&df->lock);
416 }
417
418 /*******************************************************************************
419  * scaling_adjust(podgov, time)
420  *
421  * use scale up / scale down hint counts to adjust scaling parameters.
422  *
423  * hint_ratio is 100 x the ratio of scale up to scale down hints. Three cases
424  * are distinguished:
425  *
426  * hint_ratio < HINT_RATIO_MIN - set parameters to maximize scaling effect
427  * hint_ratio > HINT_RATIO_MAX - set parameters to minimize scaling effect
428  * hint_ratio between limits - scale parameters linearly
429  *
430  * the parameters adjusted are
431  *
432  *  - fast_response time
433  *  - period - time for scaling down estimate
434  *  - idle_min percentage
435  *  - idle_max percentage
436  *
437  ******************************************************************************/
438
439 #define SCALING_ADJUST_PERIOD   1000000
440 #define HINT_RATIO_MAX          400
441 #define HINT_RATIO_MIN          100
442 #define HINT_RATIO_MID          ((HINT_RATIO_MAX + HINT_RATIO_MIN) / 2)
443 #define HINT_RATIO_DIFF         (HINT_RATIO_MAX - HINT_RATIO_MIN)
444
445 static void scaling_adjust(struct podgov_info_rec *podgov, ktime_t time)
446 {
447         long hint_ratio;
448         int idle_min_adjustment;
449         int idle_max_adjustment;
450         unsigned long dt;
451
452         dt = (unsigned long) ktime_us_delta(time, podgov->last_adjust);
453         if (dt < SCALING_ADJUST_PERIOD)
454                 return;
455
456         hint_ratio = (100 * (podgov->fast_up_count + 1)) /
457                                  (podgov->slow_down_count + 1);
458
459         if (hint_ratio > HINT_RATIO_MAX) {
460                 idle_min_adjustment = podgov->p_idle_min;
461                 idle_max_adjustment = podgov->p_idle_max;
462         } else if (hint_ratio < HINT_RATIO_MIN) {
463                 idle_min_adjustment = -((int) podgov->p_idle_min) / 2;
464                 idle_max_adjustment = -((int) podgov->p_idle_max) / 2;
465         } else {
466                 int diff;
467                 int factor;
468
469                 diff = HINT_RATIO_MID - hint_ratio;
470                 if (diff < 0)
471                         factor = -diff * 2;
472                 else {
473                         factor = -diff;
474                         diff *= 2;
475                 }
476
477                 idle_min_adjustment =
478                         (factor * (int) podgov->p_idle_min) / HINT_RATIO_DIFF;
479                 idle_max_adjustment =
480                         (factor * (int) podgov->p_idle_max) / HINT_RATIO_DIFF;
481         }
482
483         podgov->idle_min = podgov->p_idle_min + idle_min_adjustment;
484         podgov->idle_max = podgov->p_idle_max + idle_max_adjustment;
485
486         trace_podgov_stats(podgov->fast_up_count, podgov->slow_down_count,
487                 podgov->idle_min, podgov->idle_max);
488
489         podgov->fast_up_count = 0;
490         podgov->slow_down_count = 0;
491         podgov->last_adjust = time;
492 }
493
494 #undef SCALING_ADJUST_PERIOD
495 #undef HINT_RATIO_MAX
496 #undef HINT_RATIO_MIN
497 #undef HINT_RATIO_MID
498 #undef HINT_RATIO_DIFF
499
500 /*******************************************************************************
501  * freq = scaling_state_check(df, time)
502  *
503  * This handler is called to adjust the frequency of the device. The function
504  * returns the desired frequency for the clock. If there is no need to tune the
505  * clock immediately, 0 is returned.
506  ******************************************************************************/
507
508 static unsigned long scaling_state_check(struct devfreq *df, ktime_t time)
509 {
510         struct podgov_info_rec *podgov = df->data;
511         unsigned long dt;
512
513         /* adjustment: set scale parameters (idle_min, idle_max) +/- 25%
514          * based on ratio of scale up to scale down hints
515          */
516         if (podgov->p_adjust)
517                 scaling_adjust(podgov, time);
518         else {
519                 podgov->idle_min = podgov->p_idle_min;
520                 podgov->idle_max = podgov->p_idle_max;
521         }
522
523         dt = (unsigned long) ktime_us_delta(time, podgov->last_scale);
524         if (dt < podgov->p_estimation_window)
525                 return 0;
526
527         podgov->last_scale = time;
528
529         /* if too busy, scale up */
530         if (podgov->idle_estimate < podgov->idle_min) {
531                 podgov->is_scaled = 0;
532                 podgov->fast_up_count++;
533
534                 trace_podgov_busy(1000 - podgov->idle_estimate);
535                 trace_podgov_scaling_state_check(df->previous_freq,
536                         df->max_freq);
537
538                 return df->max_freq;
539
540         }
541
542         trace_podgov_idle(podgov->idle_estimate);
543
544         if (podgov->idle_estimate > podgov->idle_max) {
545                 if (!podgov->is_scaled)
546                         podgov->is_scaled = 1;
547
548                 podgov->slow_down_count++;
549                 /* if idle time is high, clock down */
550                 podgov->scale =
551                         100 - (podgov->idle_estimate - podgov->idle_min) / 10;
552                 schedule_work(&podgov->work);
553         }
554
555         return 0;
556 }
557
558 /*******************************************************************************
559  * update_load_estimate(df)
560  *
561  * The idle estimate is done by keeping 2 time stamps, initially set to the
562  * same time. Once the estimation_window time has been exceeded, one time
563  * stamp is moved up to the current time. The idle estimate is calculated
564  * based on the idle time percentage from the earlier estimate. The next time
565  * an estimation_window time is exceeded, the previous idle time and estimates
566  * are moved up - this is intended to prevent abrupt changes to the idle
567  * estimate.
568  ******************************************************************************/
569
570 static void update_load_estimate(struct devfreq *df)
571 {
572         struct podgov_info_rec *podgov = df->data;
573         unsigned long window;
574         unsigned long t;
575
576         ktime_t now = ktime_get();
577         t = ktime_us_delta(now, podgov->last_notification);
578
579         /* if the last event was over GR3D_TIMEFRAME usec ago (1 sec), the
580          * current load tracking data is probably stale
581          */
582         if (t > GR3D_TIMEFRAME) {
583                 podgov->last_notification = now;
584                 podgov->estimation_window = now;
585                 podgov->last_estimation_window = now;
586                 podgov->total_idle = 0;
587                 podgov->last_total_idle = 0;
588                 podgov->idle_estimate =
589                         (podgov->last_event_type == DEVICE_IDLE) ? 1000 : 0;
590                 return;
591         }
592
593         podgov->last_notification = now;
594
595         window = ktime_us_delta(now, podgov->last_estimation_window);
596         /* prevent division by 0 if events come in less than 1 usec apart */
597         if (window > 0)
598                 podgov->idle_estimate =
599                         (1000 * podgov->last_total_idle) / window;
600
601         /* move up to the last estimation window */
602         if (ktime_us_delta(now, podgov->estimation_window) >
603                 podgov->p_estimation_window) {
604                 podgov->last_estimation_window = podgov->estimation_window;
605                 podgov->last_total_idle = podgov->total_idle;
606                 podgov->total_idle = 0;
607                 podgov->estimation_window = now;
608         }
609 }
610
611 /*******************************************************************************
612  * freqlist_up(podgov, target, steps)
613  *
614  * This function determines the frequency that is "steps" frequency steps
615  * higher compared to the target frequency.
616  ******************************************************************************/
617
618 int freqlist_up(struct podgov_info_rec *podgov, long target, int steps)
619 {
620         int i, pos;
621
622         for (i = 0; i < podgov->freq_count; i++)
623                 if (podgov->freqlist[i] >= target)
624                         break;
625
626         pos = min(podgov->freq_count - 1, i + steps);
627         return podgov->freqlist[pos];
628 }
629
630 /*******************************************************************************
631  * freqlist_down(podgov, target, steps)
632  *
633  * This function determines the frequency that is "steps" frequency steps
634  * lower compared to the target frequency.
635  ******************************************************************************/
636
637 int freqlist_down(struct podgov_info_rec *podgov, long target, int steps)
638 {
639         int i, pos;
640
641         for (i = podgov->freq_count - 1; i >= 0; i--)
642                 if (podgov->freqlist[i] <= target)
643                         break;
644
645         pos = max(0, i - steps);
646         return podgov->freqlist[pos];
647 }
648
649 /*******************************************************************************
650  * podgov_idle_handler(work)
651  *
652  * This handler is called after the device has been idle long enough. This
653  * handler forms a (positive) feedback loop by notifying idle to the device.
654  ******************************************************************************/
655
656 static void podgov_idle_handler(struct work_struct *work)
657 {
658         struct delayed_work *idle_timer =
659                 container_of(work, struct delayed_work, work);
660         struct podgov_info_rec *podgov =
661                 container_of(idle_timer, struct podgov_info_rec, idle_timer);
662         struct devfreq *df = podgov->power_manager;
663
664         /* Retrieve device driver ops and the device struct */
665         struct device *d = df->dev.parent;
666         struct platform_device *dev = to_platform_device(d);
667         struct nvhost_device_data *pdata = platform_get_drvdata(dev);
668
669         int notify_idle = 0;
670
671         mutex_lock(&df->lock);
672
673         if (!podgov->enable) {
674                 mutex_unlock(&df->lock);
675                 return;
676         }
677
678         if (podgov->last_event_type == DEVICE_IDLE &&
679                 df->previous_freq > df->min_freq)
680                 notify_idle = 1;
681
682         mutex_unlock(&df->lock);
683
684         if (pdata->idle && notify_idle)
685                 pdata->idle(dev);
686 }
687
688 /*******************************************************************************
689  * nvhost_scale3d_set_throughput_hint(hint)
690  *
691  * This function can be used to request scaling up or down based on the
692  * required throughput
693  ******************************************************************************/
694
695 void nvhost_scale3d_set_throughput_hint(int hint)
696 {
697         struct podgov_info_rec *podgov = local_podgov;
698         struct devfreq *df;
699
700         long idle;
701         long curr, target;
702         int avg_idle, avg_hint, scale_score;
703         unsigned int smooth;
704
705         if (!podgov)
706                 return;
707         df = podgov->power_manager;
708         if (!df)
709                 return;
710
711         mutex_lock(&podgov->power_manager->lock);
712
713         podgov->block--;
714
715         if (!podgov->enable ||
716                 !podgov->p_use_throughput_hint ||
717                 podgov->block > 0) {
718                 mutex_unlock(&podgov->power_manager->lock);
719                 return;
720         }
721
722         trace_podgov_hint(podgov->idle_estimate, hint);
723         podgov->last_throughput_hint = ktime_get();
724
725         curr = podgov->power_manager->previous_freq;
726         idle = podgov->idle_estimate;
727         smooth = podgov->p_smooth;
728
729         /* compute averages usings exponential-moving-average */
730         avg_idle = ((smooth*podgov->idle_avg + idle)/(smooth+1));
731         podgov->idle_avg = avg_idle;
732         avg_hint = ((smooth*podgov->hint_avg + hint)/(smooth+1));
733         podgov->hint_avg = avg_hint;
734
735         /* set the target using avg_hint and avg_idle */
736         target = curr;
737         if (avg_hint < podgov->p_hint_lo_limit) {
738                 target = freqlist_up(podgov, curr, 1);
739         } else {
740                 scale_score = avg_idle + avg_hint;
741                 if (scale_score > podgov->p_scaledown_limit)
742                         target = freqlist_down(podgov, curr, 1);
743                 else if (scale_score < podgov->p_scaleup_limit
744                                 && hint < podgov->p_hint_hi_limit)
745                         target = freqlist_up(podgov, curr, 1);
746         }
747
748         /* clamp and apply target */
749         scaling_limit(df, &target);
750         if (target != curr) {
751                 podgov->block = podgov->p_smooth;
752                 trace_podgov_do_scale(df->previous_freq, target);
753                 podgov->adjustment_frequency = target;
754                 podgov->adjustment_type = ADJUSTMENT_LOCAL;
755                 update_devfreq(df);
756         }
757
758         trace_podgov_print_target(idle, avg_idle, curr / 1000000, target, hint,
759                 avg_hint);
760
761         mutex_unlock(&podgov->power_manager->lock);
762 }
763 EXPORT_SYMBOL(nvhost_scale3d_set_throughput_hint);
764
765 /*******************************************************************************
766  * debugfs interface for controlling 3d clock scaling on the fly
767  ******************************************************************************/
768
769 #ifdef CONFIG_DEBUG_FS
770
771 static void nvhost_scale3d_debug_init(struct devfreq *df)
772 {
773         struct podgov_info_rec *podgov = df->data;
774         struct platform_device *dev = to_platform_device(df->dev.parent);
775         struct nvhost_device_data *pdata = platform_get_drvdata(dev);
776         struct dentry *f;
777
778         if (!podgov)
779                 return;
780
781         podgov->debugdir = debugfs_create_dir("scaling", pdata->debugfs);
782         if (!podgov->debugdir) {
783                 pr_err("podgov: can\'t create debugfs directory\n");
784                 return;
785         }
786
787 #define CREATE_PODGOV_FILE(fname) \
788         do {\
789                 f = debugfs_create_u32(#fname, S_IRUGO | S_IWUSR, \
790                         podgov->debugdir, &podgov->p_##fname); \
791                 if (NULL == f) { \
792                         pr_err("podgov: can\'t create file " #fname "\n"); \
793                         return; \
794                 } \
795         } while (0)
796
797         CREATE_PODGOV_FILE(estimation_window);
798         CREATE_PODGOV_FILE(idle_min);
799         CREATE_PODGOV_FILE(idle_max);
800         CREATE_PODGOV_FILE(adjust);
801         CREATE_PODGOV_FILE(use_throughput_hint);
802         CREATE_PODGOV_FILE(hint_hi_limit);
803         CREATE_PODGOV_FILE(hint_lo_limit);
804         CREATE_PODGOV_FILE(scaleup_limit);
805         CREATE_PODGOV_FILE(scaledown_limit);
806         CREATE_PODGOV_FILE(smooth);
807 #undef CREATE_PODGOV_FILE
808 }
809
810 static void nvhost_scale3d_debug_deinit(struct devfreq *df)
811 {
812         struct podgov_info_rec *podgov = df->data;
813
814         debugfs_remove_recursive(podgov->debugdir);
815 }
816
817 #else
818 static void nvhost_scale3d_debug_init(struct devfreq *df)
819 {
820         (void)df;
821 }
822
823 static void nvhost_scale3d_debug_deinit(struct devfreq *df)
824 {
825         (void)df;
826 }
827 #endif
828
829 /*******************************************************************************
830  * sysfs interface for enabling/disabling 3d scaling
831  ******************************************************************************/
832
833 static ssize_t enable_3d_scaling_show(struct device *dev,
834         struct device_attribute *attr, char *buf)
835 {
836         ssize_t res;
837
838         res = snprintf(buf, PAGE_SIZE, "%d\n", podgov_is_enabled(dev));
839
840         return res;
841 }
842
843 static ssize_t enable_3d_scaling_store(struct device *dev,
844         struct device_attribute *attr, const char *buf, size_t count)
845 {
846         unsigned long val = 0;
847
848         if (kstrtoul(buf, 10, &val) < 0)
849                 return -EINVAL;
850
851         podgov_enable(dev, val);
852
853         return count;
854 }
855
856 static DEVICE_ATTR(enable_3d_scaling, S_IRUGO | S_IWUSR,
857         enable_3d_scaling_show, enable_3d_scaling_store);
858
859 /*******************************************************************************
860  * sysfs interface for user space control
861  * user = [0,1] disables / enabled user space control
862  * freq_request is the sysfs node user space writes frequency requests to
863  ******************************************************************************/
864
865 static ssize_t user_show(struct device *dev,
866         struct device_attribute *attr, char *buf)
867 {
868         ssize_t res;
869
870         res = snprintf(buf, PAGE_SIZE, "%d\n", podgov_user_ctl(dev));
871
872         return res;
873 }
874
875 static ssize_t user_store(struct device *dev,
876         struct device_attribute *attr, const char *buf, size_t count)
877 {
878         unsigned long val = 0;
879
880         if (kstrtoul(buf, 10, &val) < 0)
881                 return -EINVAL;
882
883         podgov_set_user_ctl(dev, val);
884
885         return count;
886 }
887
888 static DEVICE_ATTR(user, S_IRUGO | S_IWUSR,
889         user_show, user_store);
890
891 static ssize_t freq_request_show(struct device *dev,
892         struct device_attribute *attr, char *buf)
893 {
894         ssize_t res;
895
896         res = snprintf(buf, PAGE_SIZE, "%d\n", podgov_get_freq_request(dev));
897
898         return res;
899 }
900
901 static ssize_t freq_request_store(struct device *dev,
902         struct device_attribute *attr, const char *buf, size_t count)
903 {
904         unsigned long val = 0;
905
906         if (kstrtoul(buf, 10, &val) < 0)
907                 return -EINVAL;
908
909         podgov_set_freq_request(dev, val);
910
911         return count;
912 }
913
914 static DEVICE_ATTR(freq_request, S_IRUGO | S_IWUSR,
915         freq_request_show, freq_request_store);
916
917 /*******************************************************************************
918  * nvhost_pod_estimate_freq(df, freq)
919  *
920  * This function is called for re-estimating the frequency. The function is
921  * called in three conditions:
922  *
923  *  (1) Internal request to change the frequency. In this case a new clock
924  *      target is immediately set for the device.
925  *  (2) Call from the client (something has happened and re-estimation
926  *      is required).
927  *  (3) Some other reason (i.e. periodic call)
928  *
929  ******************************************************************************/
930
931 static int nvhost_pod_estimate_freq(struct devfreq *df,
932                                     unsigned long *freq)
933 {
934         struct podgov_info_rec *podgov = df->data;
935         struct devfreq_dev_status dev_stat;
936         struct nvhost_devfreq_ext_stat *ext_stat;
937         long delay;
938         int current_event;
939         int stat;
940         ktime_t now;
941
942         /* Ensure maximal clock when scaling is disabled */
943         if (!podgov->enable) {
944                 *freq = df->max_freq;
945                 return 0;
946         }
947
948         if (podgov->p_user) {
949                 *freq = podgov->p_freq_request;
950                 return 0;
951         }
952
953         current_event = DEVICE_IDLE;
954         stat = 0;
955         now = ktime_get();
956
957         /* Local adjustments (i.e. requests from kernel threads) are
958          * handled here */
959
960         if (podgov->adjustment_type == ADJUSTMENT_LOCAL) {
961
962                 podgov->adjustment_type = ADJUSTMENT_DEVICE_REQ;
963
964                 /* Do not do unnecessary scaling */
965                 scaling_limit(df, &podgov->adjustment_frequency);
966                 if (df->previous_freq == podgov->adjustment_frequency)
967                         return GET_TARGET_FREQ_DONTSCALE;
968
969                 trace_podgov_estimate_freq(df->previous_freq,
970                         podgov->adjustment_frequency);
971
972                 *freq = podgov->adjustment_frequency;
973                 return 0;
974         }
975
976         stat = df->profile->get_dev_status(df->dev.parent, &dev_stat);
977         if (stat < 0)
978                 return stat;
979
980         /* Retrieve extended data */
981         ext_stat = dev_stat.private_data;
982         if (!ext_stat)
983                 return -EINVAL;
984
985         current_event = ext_stat->busy;
986         *freq = dev_stat.current_frequency;
987         df->min_freq = ext_stat->min_freq;
988         df->max_freq = ext_stat->max_freq;
989
990         /* Sustain local variables */
991         podgov->last_event_type = current_event;
992         podgov->total_idle += (dev_stat.total_time - dev_stat.busy_time);
993         podgov->last_total_idle += (dev_stat.total_time - dev_stat.busy_time);
994
995         /* update the load estimate based on idle time */
996         update_load_estimate(df);
997
998         /* if throughput hint enabled, and last hint is recent enough, return */
999         if (podgov->p_use_throughput_hint &&
1000                 ktime_us_delta(now, podgov->last_throughput_hint) < 1000000)
1001                 return GET_TARGET_FREQ_DONTSCALE;
1002
1003         switch (current_event) {
1004
1005         case DEVICE_IDLE:
1006                 /* delay idle_max % of 2 * fast_response time (given in
1007                  * microseconds) */
1008                 *freq = scaling_state_check(df, now);
1009                 delay = (podgov->idle_max * podgov->p_estimation_window)
1010                         / 500000;
1011                 schedule_delayed_work(&podgov->idle_timer,
1012                         msecs_to_jiffies(delay));
1013                 break;
1014         case DEVICE_BUSY:
1015                 cancel_delayed_work(&podgov->idle_timer);
1016                 *freq = scaling_state_check(df, now);
1017                 break;
1018         case DEVICE_UNKNOWN:
1019                 *freq = scaling_state_check(df, now);
1020                 break;
1021
1022         }
1023
1024         if (!(*freq) || (*freq == df->previous_freq))
1025                 return GET_TARGET_FREQ_DONTSCALE;
1026
1027         trace_podgov_estimate_freq(df->previous_freq, *freq);
1028
1029
1030         return 0;
1031 }
1032
1033 /*******************************************************************************
1034  * nvhost_pod_init(struct devfreq *df)
1035  *
1036  * Governor initialisation.
1037  ******************************************************************************/
1038
1039 #define MAX_FREQ_COUNT 0x40 /* 64 frequencies should be enough for anyone */
1040
1041 static int nvhost_pod_init(struct devfreq *df)
1042 {
1043         struct podgov_info_rec *podgov;
1044         struct platform_device *d = to_platform_device(df->dev.parent);
1045         struct nvhost_device_data *pdata = platform_get_drvdata(d);
1046         ktime_t now = ktime_get();
1047         int error = 0;
1048
1049         struct nvhost_devfreq_ext_stat *ext_stat;
1050         struct devfreq_dev_status dev_stat;
1051         int stat = 0;
1052
1053         long rate;
1054         int freqs[MAX_FREQ_COUNT];
1055
1056         podgov = kzalloc(sizeof(struct podgov_info_rec), GFP_KERNEL);
1057         if (!podgov)
1058                 goto err_alloc_podgov;
1059         df->data = (void *)podgov;
1060
1061         /* This should be removed after the governor include also the hint
1062          * interface */
1063         local_podgov = podgov;
1064
1065         /* Initialise workers */
1066         INIT_WORK(&podgov->work, podgov_clocks_handler);
1067         INIT_DELAYED_WORK(&podgov->idle_timer, podgov_idle_handler);
1068
1069         /* Set scaling parameter defaults */
1070         podgov->enable = 1;
1071         podgov->p_adjust = 0;
1072         podgov->block = 0;
1073         podgov->p_use_throughput_hint = 1;
1074         if (tegra_get_chipid() == TEGRA_CHIPID_TEGRA11) {
1075                 podgov->idle_min = podgov->p_idle_min = 400;
1076                 podgov->idle_max = podgov->p_idle_max = 500;
1077                 podgov->p_hint_lo_limit = 500;
1078                 podgov->p_hint_hi_limit = 997;
1079                 podgov->p_scaleup_limit = 1400;
1080                 podgov->p_scaledown_limit = 1600;
1081                 podgov->p_smooth = 3;
1082         } else {
1083                 podgov->idle_min = podgov->p_idle_min = 100;
1084                 podgov->idle_max = podgov->p_idle_max = 150;
1085                 podgov->p_hint_lo_limit = 800;
1086                 podgov->p_hint_hi_limit = 1015;
1087                 podgov->p_scaleup_limit = 1275;
1088                 podgov->p_scaledown_limit = 1475;
1089                 podgov->p_smooth = 7;
1090         }
1091         podgov->p_estimation_window = 10000;
1092         podgov->adjustment_type = ADJUSTMENT_DEVICE_REQ;
1093         podgov->p_user = 0;
1094
1095         /* Reset clock counters */
1096         podgov->last_throughput_hint = now;
1097         podgov->last_scale = now;
1098         podgov->last_adjust = now;
1099         podgov->last_estimation_window = now;
1100         podgov->estimation_window = now;
1101         podgov->last_notification = now;
1102
1103         podgov->power_manager = df;
1104
1105         /* Get the current status of the device */
1106         stat = df->profile->get_dev_status(df->dev.parent, &dev_stat);
1107         if (!dev_stat.private_data) {
1108                 pr_err("podgov: device does not support ext_stat.\n");
1109                 goto err_get_current_status;
1110         }
1111         ext_stat = dev_stat.private_data;
1112         df->previous_freq = dev_stat.current_frequency;
1113         df->min_freq = ext_stat->min_freq;
1114         df->max_freq = ext_stat->max_freq;
1115
1116         podgov->p_freq_request = ext_stat->max_freq;
1117
1118         /* Create sysfs entries for controlling this governor */
1119         error = device_create_file(&d->dev,
1120                         &dev_attr_enable_3d_scaling);
1121         if (error)
1122                 goto err_create_sysfs_entry;
1123
1124         error = device_create_file(&d->dev,
1125                         &dev_attr_user);
1126         if (error)
1127                 goto err_create_sysfs_entry;
1128
1129         error = device_create_file(&d->dev,
1130                         &dev_attr_freq_request);
1131         if (error)
1132                 goto err_create_sysfs_entry;
1133
1134         rate = 0;
1135         podgov->freq_count = 0;
1136         while (rate <= df->max_freq) {
1137                 long rounded_rate;
1138                 if (unlikely(podgov->freq_count == MAX_FREQ_COUNT)) {
1139                         pr_err("%s: too many frequencies\n", __func__);
1140                         break;
1141                 }
1142                 rounded_rate =
1143                         clk_round_rate(clk_get_parent(pdata->clk[0]), rate);
1144                 freqs[podgov->freq_count++] = rounded_rate;
1145                 rate = rounded_rate + 2000;
1146         }
1147
1148         podgov->freqlist =
1149                 kmalloc(podgov->freq_count * sizeof(int), GFP_KERNEL);
1150         if (podgov->freqlist == NULL)
1151                 goto err_allocate_freq_list;
1152
1153         memcpy(podgov->freqlist, freqs,
1154                 podgov->freq_count * sizeof(int));
1155
1156         podgov->idle_avg = 0;
1157         podgov->hint_avg = 0;
1158
1159         nvhost_scale3d_debug_init(df);
1160
1161         return 0;
1162
1163 err_allocate_freq_list:
1164         device_remove_file(&d->dev, &dev_attr_enable_3d_scaling);
1165         device_remove_file(&d->dev, &dev_attr_user);
1166         device_remove_file(&d->dev, &dev_attr_freq_request);
1167 err_create_sysfs_entry:
1168         dev_err(&d->dev, "failed to create sysfs attributes");
1169 err_get_current_status:
1170         kfree(podgov);
1171 err_alloc_podgov:
1172         return -ENOMEM;
1173 }
1174
1175 /*******************************************************************************
1176  * nvhost_pod_exit(struct devfreq *df)
1177  *
1178  * Clean up governor data structures
1179  ******************************************************************************/
1180
1181 static void nvhost_pod_exit(struct devfreq *df)
1182 {
1183         struct podgov_info_rec *podgov = df->data;
1184         struct platform_device *d = to_platform_device(df->dev.parent);
1185
1186         cancel_work_sync(&podgov->work);
1187         cancel_delayed_work(&podgov->idle_timer);
1188
1189         kfree(podgov->freqlist);
1190
1191         device_remove_file(&d->dev, &dev_attr_enable_3d_scaling);
1192         device_remove_file(&d->dev, &dev_attr_user);
1193         device_remove_file(&d->dev, &dev_attr_freq_request);
1194
1195         nvhost_scale3d_debug_deinit(df);
1196
1197         kfree(podgov);
1198         local_podgov = NULL;
1199 }
1200
1201 static int nvhost_pod_handler(struct devfreq *devfreq, unsigned int event,
1202                                 void *data)
1203 {
1204         int ret = 0;
1205
1206         switch(event) {
1207         case DEVFREQ_GOV_START:
1208                 ret = nvhost_pod_init(devfreq);
1209                 break;
1210         
1211         case DEVFREQ_GOV_STOP:
1212                 nvhost_pod_exit(devfreq);
1213                 break;
1214         default:
1215                 break;
1216         }
1217
1218         return ret;
1219 }
1220
1221 const struct devfreq_governor nvhost_podgov = {
1222         .name = "pod",
1223         .get_target_freq = nvhost_pod_estimate_freq,
1224         .event_handler = nvhost_pod_handler,
1225 };
1226