599b76a74b6df8a89fbcb65d38259ce126015718
[linux-3.10.git] / drivers / video / tegra / nvmap / nvmap_pp.c
1 /*
2  * drivers/video/tegra/nvmap/nvmap_pp.c
3  *
4  * Manage page pools to speed up page allocation.
5  *
6  * Copyright (c) 2009-2014, NVIDIA CORPORATION. All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #define pr_fmt(fmt) "%s: " fmt, __func__
24
25 #include <linux/kernel.h>
26 #include <linux/vmalloc.h>
27 #include <linux/moduleparam.h>
28 #include <linux/shrinker.h>
29 #include <linux/kthread.h>
30 #include <linux/debugfs.h>
31 #include <linux/freezer.h>
32 #include <linux/highmem.h>
33
34 #include "nvmap_priv.h"
35
36 #define NVMAP_TEST_PAGE_POOL_SHRINKER     1
37 #define PENDING_PAGES_SIZE                32
38
39 static bool enable_pp = 1;
40 static int pool_size;
41
42 static struct task_struct *background_allocator;
43 static DECLARE_WAIT_QUEUE_HEAD(nvmap_bg_wait);
44 static struct page *pending_pages[PENDING_PAGES_SIZE];
45
46 #ifdef CONFIG_NVMAP_PAGE_POOL_DEBUG
47 static inline void __pp_dbg_var_add(u64 *dbg_var, u32 nr)
48 {
49         *dbg_var += nr;
50 }
51 #else
52 #define __pp_dbg_var_add(dbg_var, nr)
53 #endif
54
55 #define pp_alloc_add(pool, nr) __pp_dbg_var_add(&(pool)->allocs, nr)
56 #define pp_fill_add(pool, nr)  __pp_dbg_var_add(&(pool)->fills, nr)
57 #define pp_hit_add(pool, nr)   __pp_dbg_var_add(&(pool)->hits, nr)
58 #define pp_miss_add(pool, nr)  __pp_dbg_var_add(&(pool)->misses, nr)
59
60 static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool,
61                                        struct page **pages, u32 nr);
62
63 static inline struct page *get_zero_list_page(struct nvmap_page_pool *pool)
64 {
65         struct page *page;
66
67         if (list_empty(&pool->zero_list))
68                 return NULL;
69
70         page = list_first_entry(&pool->zero_list, struct page, lru);
71         list_del(&page->lru);
72
73         pool->to_zero--;
74
75         return page;
76 }
77
78 static inline struct page *get_page_list_page(struct nvmap_page_pool *pool)
79 {
80         struct page *page;
81
82         if (list_empty(&pool->page_list))
83                 return NULL;
84
85         page = list_first_entry(&pool->page_list, struct page, lru);
86         list_del(&page->lru);
87
88         pool->count--;
89
90         return page;
91 }
92
93 static inline bool nvmap_bg_should_run(struct nvmap_page_pool *pool)
94 {
95         bool ret;
96
97         mutex_lock(&pool->lock);
98         ret = (pool->to_zero > 0);
99         mutex_unlock(&pool->lock);
100
101         return ret;
102 }
103
104 static int nvmap_pp_zero_pages(struct page **pages, int nr)
105 {
106         int i;
107
108         for (i = 0; i < nr; i++)
109                 clear_highpage(pages[i]);
110
111         return 0;
112 }
113
114 static void nvmap_pp_do_background_zero_pages(struct nvmap_page_pool *pool)
115 {
116         int i;
117         struct page *page;
118         int ret;
119
120         mutex_lock(&pool->lock);
121         for (i = 0; i < PENDING_PAGES_SIZE; i++) {
122                 page = get_zero_list_page(pool);
123                 if (page == NULL)
124                         break;
125                 pending_pages[i] = page;
126         }
127         mutex_unlock(&pool->lock);
128
129         ret = nvmap_pp_zero_pages(pending_pages, i);
130         if (ret < 0) {
131                 ret = 0;
132                 goto out;
133         }
134
135         mutex_lock(&pool->lock);
136         ret = __nvmap_page_pool_fill_lots_locked(pool, pending_pages, i);
137         mutex_unlock(&pool->lock);
138
139 out:
140         for (; ret < i; ret++)
141                 __free_page(pending_pages[ret]);
142 }
143
144 /*
145  * This thread fills the page pools with zeroed pages. We avoid releasing the
146  * pages directly back into the page pools since we would then have to zero
147  * them ourselves. Instead it is easier to just reallocate zeroed pages. This
148  * happens in the background so that the overhead of allocating zeroed pages is
149  * not directly seen by userspace. Of course if the page pools are empty user
150  * space will suffer.
151  */
152 static int nvmap_background_zero_thread(void *arg)
153 {
154         struct nvmap_page_pool *pool = &nvmap_dev->pool;
155         struct sched_param param = { .sched_priority = 0 };
156
157         pr_info("PP zeroing thread starting.\n");
158
159         set_freezable();
160         sched_setscheduler(current, SCHED_IDLE, &param);
161
162         while (!kthread_should_stop()) {
163                 while (nvmap_bg_should_run(pool)) {
164                         nvmap_pp_do_background_zero_pages(pool);
165                 }
166
167                 wait_event_freezable(nvmap_bg_wait,
168                                 nvmap_bg_should_run(pool) ||
169                                 kthread_should_stop());
170         }
171
172         return 0;
173 }
174
175 /*
176  * This removes a page from the page pool. If ignore_disable is set, then
177  * the enable_pp flag is ignored.
178  */
179 static struct page *nvmap_page_pool_alloc_locked(struct nvmap_page_pool *pool,
180                                                  int force_alloc)
181 {
182         struct page *page;
183
184         if (!force_alloc && !enable_pp)
185                 return NULL;
186
187         if (list_empty(&pool->page_list)) {
188                 pp_miss_add(pool, 1);
189                 return NULL;
190         }
191
192         if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG))
193                 BUG_ON(pool->count == 0);
194
195         page = get_page_list_page(pool);
196         if (!page)
197                 return NULL;
198
199         /* Sanity check. */
200         if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG)) {
201                 atomic_dec(&page->_count);
202                 BUG_ON(atomic_read(&page->_count) != 1);
203         }
204
205         pp_alloc_add(pool, 1);
206         pp_hit_add(pool, 1);
207
208         return page;
209 }
210
211 /*
212  * Alloc a bunch of pages from the page pool. This will alloc as many as it can
213  * and return the number of pages allocated. Pages are placed into the passed
214  * array in a linear fashion starting from index 0.
215  *
216  * You must lock the page pool before using this.
217  */
218 static int __nvmap_page_pool_alloc_lots_locked(struct nvmap_page_pool *pool,
219                                         struct page **pages, u32 nr)
220 {
221         u32 real_nr;
222         u32 ind = 0;
223
224         if (!enable_pp)
225                 return 0;
226
227         real_nr = min_t(u32, nr, pool->count);
228
229         while (real_nr--) {
230                 struct page *page;
231                 if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG))
232                         BUG_ON(list_empty(&pool->page_list));
233                 page = get_page_list_page(pool);
234                 pages[ind++] = page;
235                 if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG)) {
236                         atomic_dec(&page->_count);
237                         BUG_ON(atomic_read(&page->_count) != 1);
238                 }
239         }
240
241         pp_alloc_add(pool, ind);
242         pp_hit_add(pool, ind);
243         pp_miss_add(pool, nr - ind);
244
245         return ind;
246 }
247
248 int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool,
249                                 struct page **pages, u32 nr)
250 {
251         int ret;
252
253         mutex_lock(&pool->lock);
254         ret = __nvmap_page_pool_alloc_lots_locked(pool, pages, nr);
255         mutex_unlock(&pool->lock);
256
257         return ret;
258 }
259
260 /*
261  * Fill a bunch of pages into the page pool. This will fill as many as it can
262  * and return the number of pages filled. Pages are used from the start of the
263  * passed page pointer array in a linear fashion.
264  *
265  * You must lock the page pool before using this.
266  */
267 static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool,
268                                        struct page **pages, u32 nr)
269 {
270         u32 real_nr;
271         u32 ind = 0;
272
273         if (!enable_pp)
274                 return 0;
275
276         real_nr = min_t(u32, pool->max - pool->count, nr);
277         if (real_nr == 0)
278                 return 0;
279
280         while (real_nr--) {
281                 if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG)) {
282                         atomic_inc(&pages[ind]->_count);
283                         BUG_ON(atomic_read(&pages[ind]->_count) != 2);
284                 }
285                 list_add_tail(&pages[ind++]->lru, &pool->page_list);
286         }
287
288         pool->count += ind;
289         pp_fill_add(pool, ind);
290
291         return ind;
292 }
293
294 int nvmap_page_pool_fill_lots(struct nvmap_page_pool *pool,
295                                        struct page **pages, u32 nr)
296 {
297         int ret;
298
299         mutex_lock(&pool->lock);
300         if (zero_memory) {
301                 int i;
302
303                 nr = min(nr, pool->max - pool->count - pool->to_zero);
304
305                 for (i = 0; i < nr; i++) {
306                         list_add_tail(&pages[i]->lru, &pool->zero_list);
307                         pool->to_zero++;
308                 }
309
310                 wake_up_interruptible(&nvmap_bg_wait);
311
312                 ret = i;
313         } else {
314                 ret = __nvmap_page_pool_fill_lots_locked(pool, pages, nr);
315         }
316         mutex_unlock(&pool->lock);
317
318         return ret;
319 }
320
321 /*
322  * Free the passed number of pages from the page pool. This happen irregardless
323  * of whether ther page pools are enabled. This lets one disable the page pools
324  * and then free all the memory therein.
325  */
326 static int nvmap_page_pool_free(struct nvmap_page_pool *pool, int nr_free)
327 {
328         int i = nr_free;
329         struct page *page;
330
331         if (!nr_free)
332                 return nr_free;
333
334         mutex_lock(&pool->lock);
335         while (i) {
336                 page = get_zero_list_page(pool);
337                 if (!page)
338                         page = nvmap_page_pool_alloc_locked(pool, 1);
339                 if (!page)
340                         break;
341                 __free_page(page);
342                 i--;
343         }
344         mutex_unlock(&pool->lock);
345
346         return i;
347 }
348
349 ulong nvmap_page_pool_get_unused_pages(void)
350 {
351         int total = 0;
352
353         if (!nvmap_dev)
354                 return 0;
355
356         total = nvmap_dev->pool.count + nvmap_dev->pool.to_zero;
357
358         return total;
359 }
360
361 /*
362  * Remove and free to the system all the pages currently in the page
363  * pool. This operation will happen even if the page pools are disabled.
364  */
365 int nvmap_page_pool_clear(void)
366 {
367         struct page *page;
368         struct nvmap_page_pool *pool = &nvmap_dev->pool;
369
370         mutex_lock(&pool->lock);
371
372         while ((page = nvmap_page_pool_alloc_locked(pool, 1)) != NULL)
373                 __free_page(page);
374
375         while (!list_empty(&pool->zero_list)) {
376                 page = get_zero_list_page(pool);
377                 __free_page(page);
378         }
379
380         /* For some reason, if an error occured... */
381         if (!list_empty(&pool->page_list) || !list_empty(&pool->zero_list)) {
382                 mutex_unlock(&pool->lock);
383                 return -ENOMEM;
384         }
385
386         mutex_unlock(&pool->lock);
387
388         return 0;
389 }
390
391 /*
392  * Resizes the page pool to the passed size. If the passed size is 0 then
393  * all associated resources are released back to the system. This operation
394  * will only occur if the page pools are enabled.
395  */
396 static void nvmap_page_pool_resize(struct nvmap_page_pool *pool, int size)
397 {
398         if (!enable_pp || size == pool->max || size < 0)
399                 return;
400
401         mutex_lock(&pool->lock);
402
403         while (pool->count > size)
404                 __free_page(nvmap_page_pool_alloc_locked(pool, 0));
405
406         pool->max = size;
407
408         pr_debug("page pool resized to %d from %d pages\n", size, pool->max);
409         mutex_unlock(&pool->lock);
410 }
411
412 static int nvmap_page_pool_shrink(struct shrinker *shrinker,
413                                   struct shrink_control *sc)
414 {
415         int shrink_pages = sc->nr_to_scan;
416
417         if (!shrink_pages)
418                 goto out;
419
420         pr_debug("sh_pages=%d", shrink_pages);
421
422         shrink_pages = nvmap_page_pool_free(&nvmap_dev->pool, shrink_pages);
423 out:
424         return nvmap_page_pool_get_unused_pages();
425 }
426
427 static struct shrinker nvmap_page_pool_shrinker = {
428         .shrink = nvmap_page_pool_shrink,
429         .seeks = 1,
430 };
431
432 static void shrink_page_pools(int *total_pages, int *available_pages)
433 {
434         struct shrink_control sc;
435
436         if (*total_pages == 0) {
437                 sc.gfp_mask = GFP_KERNEL;
438                 sc.nr_to_scan = 0;
439                 *total_pages = nvmap_page_pool_shrink(NULL, &sc);
440         }
441         sc.nr_to_scan = *total_pages;
442         *available_pages = nvmap_page_pool_shrink(NULL, &sc);
443 }
444
445 #if NVMAP_TEST_PAGE_POOL_SHRINKER
446 static int shrink_pp;
447 static int shrink_set(const char *arg, const struct kernel_param *kp)
448 {
449         int cpu = smp_processor_id();
450         unsigned long long t1, t2;
451         int total_pages, available_pages;
452
453         param_set_int(arg, kp);
454
455         if (shrink_pp) {
456                 total_pages = shrink_pp;
457                 t1 = cpu_clock(cpu);
458                 shrink_page_pools(&total_pages, &available_pages);
459                 t2 = cpu_clock(cpu);
460                 pr_debug("shrink page pools: time=%lldns, "
461                         "total_pages_released=%d, free_pages_available=%d",
462                         t2-t1, total_pages, available_pages);
463         }
464         return 0;
465 }
466
467 static int shrink_get(char *buff, const struct kernel_param *kp)
468 {
469         return param_get_int(buff, kp);
470 }
471
472 static struct kernel_param_ops shrink_ops = {
473         .get = shrink_get,
474         .set = shrink_set,
475 };
476
477 module_param_cb(shrink_page_pools, &shrink_ops, &shrink_pp, 0644);
478 #endif
479
480 static int enable_pp_set(const char *arg, const struct kernel_param *kp)
481 {
482         int ret;
483
484         ret = param_set_bool(arg, kp);
485         if (ret)
486                 return ret;
487
488         if (!enable_pp)
489                 nvmap_page_pool_clear();
490
491         return 0;
492 }
493
494 static int enable_pp_get(char *buff, const struct kernel_param *kp)
495 {
496         return param_get_int(buff, kp);
497 }
498
499 static struct kernel_param_ops enable_pp_ops = {
500         .get = enable_pp_get,
501         .set = enable_pp_set,
502 };
503
504 module_param_cb(enable_page_pools, &enable_pp_ops, &enable_pp, 0644);
505
506 static int pool_size_set(const char *arg, const struct kernel_param *kp)
507 {
508         param_set_int(arg, kp);
509         nvmap_page_pool_resize(&nvmap_dev->pool, pool_size);
510         return 0;
511 }
512
513 static int pool_size_get(char *buff, const struct kernel_param *kp)
514 {
515         return param_get_int(buff, kp);
516 }
517
518 static struct kernel_param_ops pool_size_ops = {
519         .get = pool_size_get,
520         .set = pool_size_set,
521 };
522
523 module_param_cb(pool_size, &pool_size_ops, &pool_size, 0644);
524
525 int nvmap_page_pool_debugfs_init(struct dentry *nvmap_root)
526 {
527         struct dentry *pp_root;
528
529         if (!nvmap_root)
530                 return -ENODEV;
531
532         pp_root = debugfs_create_dir("pagepool", nvmap_root);
533         if (!pp_root)
534                 return -ENODEV;
535
536         debugfs_create_u32("page_pool_available_pages",
537                            S_IRUGO, pp_root,
538                            &nvmap_dev->pool.count);
539         debugfs_create_u32("page_pool_pages_to_zero",
540                            S_IRUGO, pp_root,
541                            &nvmap_dev->pool.to_zero);
542 #ifdef CONFIG_NVMAP_PAGE_POOL_DEBUG
543         debugfs_create_u64("page_pool_allocs",
544                            S_IRUGO, pp_root,
545                            &nvmap_dev->pool.allocs);
546         debugfs_create_u64("page_pool_fills",
547                            S_IRUGO, pp_root,
548                            &nvmap_dev->pool.fills);
549         debugfs_create_u64("page_pool_hits",
550                            S_IRUGO, pp_root,
551                            &nvmap_dev->pool.hits);
552         debugfs_create_u64("page_pool_misses",
553                            S_IRUGO, pp_root,
554                            &nvmap_dev->pool.misses);
555 #endif
556
557         return 0;
558 }
559
560 int nvmap_page_pool_init(struct nvmap_device *dev)
561 {
562         unsigned long totalram_mb;
563         struct sysinfo info;
564         struct nvmap_page_pool *pool = &dev->pool;
565
566         memset(pool, 0x0, sizeof(*pool));
567         mutex_init(&pool->lock);
568         INIT_LIST_HEAD(&pool->page_list);
569         INIT_LIST_HEAD(&pool->zero_list);
570
571         si_meminfo(&info);
572         totalram_mb = (info.totalram * info.mem_unit) >> 20;
573         pr_info("Total MB RAM: %lu\n", totalram_mb);
574
575         if (!CONFIG_NVMAP_PAGE_POOL_SIZE)
576                 /* The ratio is pool pages per 1K ram pages. So, the >> 10. */
577                 pool->max = (info.totalram * NVMAP_PP_POOL_SIZE) >> 10;
578         else
579                 pool->max = CONFIG_NVMAP_PAGE_POOL_SIZE;
580
581         if (pool->max >= info.totalram)
582                 goto fail;
583         pool_size = pool->max;
584
585         pr_info("nvmap page pool size: %u pages (%u MB)\n", pool->max,
586                 (pool->max * info.mem_unit) >> 20);
587
588         background_allocator = kthread_run(nvmap_background_zero_thread,
589                                             NULL, "nvmap-bz");
590         if (IS_ERR_OR_NULL(background_allocator))
591                 goto fail;
592
593         register_shrinker(&nvmap_page_pool_shrinker);
594
595         return 0;
596 fail:
597         nvmap_page_pool_fini(dev);
598         return -ENOMEM;
599 }
600
601 int nvmap_page_pool_fini(struct nvmap_device *dev)
602 {
603         struct nvmap_page_pool *pool = &dev->pool;
604
605         if (!IS_ERR_OR_NULL(background_allocator))
606                 kthread_stop(background_allocator);
607
608         WARN_ON(!list_empty(&pool->page_list));
609
610         return 0;
611 }