nvmap: page pools: fix lockdep splat
[linux-3.10.git] / drivers / video / tegra / nvmap / nvmap_pp.c
1 /*
2  * drivers/video/tegra/nvmap/nvmap_pp.c
3  *
4  * Manage page pools to speed up page allocation.
5  *
6  * Copyright (c) 2009-2014, NVIDIA CORPORATION. All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #define pr_fmt(fmt) "%s: " fmt, __func__
24
25 #include <linux/kernel.h>
26 #include <linux/vmalloc.h>
27 #include <linux/moduleparam.h>
28 #include <linux/shrinker.h>
29 #include <linux/kthread.h>
30 #include <linux/debugfs.h>
31 #include <linux/freezer.h>
32 #include <linux/highmem.h>
33
34 #include "nvmap_priv.h"
35
36 #define NVMAP_TEST_PAGE_POOL_SHRINKER     1
37 #define PENDING_PAGES_SIZE                32
38
39 static bool enable_pp = 1;
40 static int pool_size;
41
42 static struct task_struct *background_allocator;
43 static DECLARE_WAIT_QUEUE_HEAD(nvmap_bg_wait);
44
45 #ifdef CONFIG_NVMAP_PAGE_POOL_DEBUG
46 static inline void __pp_dbg_var_add(u64 *dbg_var, u32 nr)
47 {
48         *dbg_var += nr;
49 }
50 #else
51 #define __pp_dbg_var_add(dbg_var, nr)
52 #endif
53
54 #define pp_alloc_add(pool, nr) __pp_dbg_var_add(&(pool)->allocs, nr)
55 #define pp_fill_add(pool, nr)  __pp_dbg_var_add(&(pool)->fills, nr)
56 #define pp_hit_add(pool, nr)   __pp_dbg_var_add(&(pool)->hits, nr)
57 #define pp_miss_add(pool, nr)  __pp_dbg_var_add(&(pool)->misses, nr)
58
59 static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool,
60                                        struct page **pages, u32 nr);
61
62 static inline struct page *get_zero_list_page(struct nvmap_page_pool *pool)
63 {
64         struct page *page;
65
66         if (list_empty(&pool->zero_list))
67                 return NULL;
68
69         page = list_first_entry(&pool->zero_list, struct page, lru);
70         list_del(&page->lru);
71
72         pool->to_zero--;
73
74         return page;
75 }
76
77 static inline struct page *get_page_list_page(struct nvmap_page_pool *pool)
78 {
79         struct page *page;
80
81         if (list_empty(&pool->page_list))
82                 return NULL;
83
84         page = list_first_entry(&pool->page_list, struct page, lru);
85         list_del(&page->lru);
86
87         pool->count--;
88
89         return page;
90 }
91
92 static inline bool nvmap_bg_should_run(struct nvmap_page_pool *pool)
93 {
94         bool ret;
95
96         mutex_lock(&pool->lock);
97         ret = (pool->to_zero > 0);
98         mutex_unlock(&pool->lock);
99
100         return ret;
101 }
102
103 static int nvmap_pp_zero_pages(struct page **pages, int nr)
104 {
105         int i;
106
107         for (i = 0; i < nr; i++)
108                 clear_highpage(pages[i]);
109
110         return 0;
111 }
112
113 static void nvmap_pp_do_background_zero_pages(struct nvmap_page_pool *pool)
114 {
115         int i;
116         struct page *page;
117         int ret;
118
119         /*
120          * Statically declared array of pages to be zeroed in a batch,
121          * local to this thread but too big for the stack.
122          */
123         static struct page *pending_zero_pages[PENDING_PAGES_SIZE];
124
125         mutex_lock(&pool->lock);
126         for (i = 0; i < PENDING_PAGES_SIZE; i++) {
127                 page = get_zero_list_page(pool);
128                 if (page == NULL)
129                         break;
130                 pending_zero_pages[i] = page;
131         }
132         mutex_unlock(&pool->lock);
133
134         ret = nvmap_pp_zero_pages(pending_zero_pages, i);
135         if (ret < 0) {
136                 ret = 0;
137                 goto out;
138         }
139
140         mutex_lock(&pool->lock);
141         ret = __nvmap_page_pool_fill_lots_locked(pool, pending_zero_pages, i);
142         mutex_unlock(&pool->lock);
143
144 out:
145         for (; ret < i; ret++)
146                 __free_page(pending_zero_pages[ret]);
147 }
148
149 /*
150  * This thread fills the page pools with zeroed pages. We avoid releasing the
151  * pages directly back into the page pools since we would then have to zero
152  * them ourselves. Instead it is easier to just reallocate zeroed pages. This
153  * happens in the background so that the overhead of allocating zeroed pages is
154  * not directly seen by userspace. Of course if the page pools are empty user
155  * space will suffer.
156  */
157 static int nvmap_background_zero_thread(void *arg)
158 {
159         struct nvmap_page_pool *pool = &nvmap_dev->pool;
160         struct sched_param param = { .sched_priority = 0 };
161
162         pr_info("PP zeroing thread starting.\n");
163
164         set_freezable();
165         sched_setscheduler(current, SCHED_IDLE, &param);
166
167         while (!kthread_should_stop()) {
168                 while (nvmap_bg_should_run(pool)) {
169                         nvmap_pp_do_background_zero_pages(pool);
170                 }
171
172                 wait_event_freezable(nvmap_bg_wait,
173                                 nvmap_bg_should_run(pool) ||
174                                 kthread_should_stop());
175         }
176
177         return 0;
178 }
179
180 /*
181  * This removes a page from the page pool. If ignore_disable is set, then
182  * the enable_pp flag is ignored.
183  */
184 static struct page *nvmap_page_pool_alloc_locked(struct nvmap_page_pool *pool,
185                                                  int force_alloc)
186 {
187         struct page *page;
188
189         if (!force_alloc && !enable_pp)
190                 return NULL;
191
192         if (list_empty(&pool->page_list)) {
193                 pp_miss_add(pool, 1);
194                 return NULL;
195         }
196
197         if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG))
198                 BUG_ON(pool->count == 0);
199
200         page = get_page_list_page(pool);
201         if (!page)
202                 return NULL;
203
204         /* Sanity check. */
205         if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG)) {
206                 atomic_dec(&page->_count);
207                 BUG_ON(atomic_read(&page->_count) != 1);
208         }
209
210         pp_alloc_add(pool, 1);
211         pp_hit_add(pool, 1);
212
213         return page;
214 }
215
216 /*
217  * Alloc a bunch of pages from the page pool. This will alloc as many as it can
218  * and return the number of pages allocated. Pages are placed into the passed
219  * array in a linear fashion starting from index 0.
220  */
221 int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool,
222                                 struct page **pages, u32 nr)
223 {
224         u32 real_nr;
225         u32 ind = 0;
226
227         if (!enable_pp)
228                 return 0;
229
230         mutex_lock(&pool->lock);
231
232         real_nr = min_t(u32, nr, pool->count);
233
234         while (real_nr--) {
235                 struct page *page;
236                 if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG))
237                         BUG_ON(list_empty(&pool->page_list));
238                 page = get_page_list_page(pool);
239                 pages[ind++] = page;
240                 if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG)) {
241                         atomic_dec(&page->_count);
242                         BUG_ON(atomic_read(&page->_count) != 1);
243                 }
244         }
245         mutex_unlock(&pool->lock);
246
247         pp_alloc_add(pool, ind);
248         pp_hit_add(pool, ind);
249         pp_miss_add(pool, nr - ind);
250
251         return ind;
252 }
253
254 /*
255  * Fill a bunch of pages into the page pool. This will fill as many as it can
256  * and return the number of pages filled. Pages are used from the start of the
257  * passed page pointer array in a linear fashion.
258  *
259  * You must lock the page pool before using this.
260  */
261 static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool,
262                                        struct page **pages, u32 nr)
263 {
264         u32 real_nr;
265         u32 ind = 0;
266
267         if (!enable_pp)
268                 return 0;
269
270         real_nr = min_t(u32, pool->max - pool->count, nr);
271         if (real_nr == 0)
272                 return 0;
273
274         while (real_nr--) {
275                 if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG)) {
276                         atomic_inc(&pages[ind]->_count);
277                         BUG_ON(atomic_read(&pages[ind]->_count) != 2);
278                 }
279                 list_add_tail(&pages[ind++]->lru, &pool->page_list);
280         }
281
282         pool->count += ind;
283         pp_fill_add(pool, ind);
284
285         return ind;
286 }
287
288 int nvmap_page_pool_fill_lots(struct nvmap_page_pool *pool,
289                                        struct page **pages, u32 nr)
290 {
291         int ret;
292
293         mutex_lock(&pool->lock);
294         if (zero_memory) {
295                 int i;
296
297                 nr = min(nr, pool->max - pool->count - pool->to_zero);
298
299                 for (i = 0; i < nr; i++) {
300                         list_add_tail(&pages[i]->lru, &pool->zero_list);
301                         pool->to_zero++;
302                 }
303
304                 wake_up_interruptible(&nvmap_bg_wait);
305
306                 ret = i;
307         } else {
308                 ret = __nvmap_page_pool_fill_lots_locked(pool, pages, nr);
309         }
310         mutex_unlock(&pool->lock);
311
312         return ret;
313 }
314
315 /*
316  * Free the passed number of pages from the page pool. This happen irregardless
317  * of whether ther page pools are enabled. This lets one disable the page pools
318  * and then free all the memory therein.
319  */
320 static int nvmap_page_pool_free(struct nvmap_page_pool *pool, int nr_free)
321 {
322         int i = nr_free;
323         struct page *page;
324
325         if (!nr_free)
326                 return nr_free;
327
328         mutex_lock(&pool->lock);
329         while (i) {
330                 page = get_zero_list_page(pool);
331                 if (!page)
332                         page = nvmap_page_pool_alloc_locked(pool, 1);
333                 if (!page)
334                         break;
335                 __free_page(page);
336                 i--;
337         }
338         mutex_unlock(&pool->lock);
339
340         return i;
341 }
342
343 ulong nvmap_page_pool_get_unused_pages(void)
344 {
345         int total = 0;
346
347         if (!nvmap_dev)
348                 return 0;
349
350         total = nvmap_dev->pool.count + nvmap_dev->pool.to_zero;
351
352         return total;
353 }
354
355 /*
356  * Remove and free to the system all the pages currently in the page
357  * pool. This operation will happen even if the page pools are disabled.
358  */
359 int nvmap_page_pool_clear(void)
360 {
361         struct page *page;
362         struct nvmap_page_pool *pool = &nvmap_dev->pool;
363
364         mutex_lock(&pool->lock);
365
366         while ((page = nvmap_page_pool_alloc_locked(pool, 1)) != NULL)
367                 __free_page(page);
368
369         while (!list_empty(&pool->zero_list)) {
370                 page = get_zero_list_page(pool);
371                 __free_page(page);
372         }
373
374         /* For some reason, if an error occured... */
375         if (!list_empty(&pool->page_list) || !list_empty(&pool->zero_list)) {
376                 mutex_unlock(&pool->lock);
377                 return -ENOMEM;
378         }
379
380         mutex_unlock(&pool->lock);
381
382         return 0;
383 }
384
385 /*
386  * Resizes the page pool to the passed size. If the passed size is 0 then
387  * all associated resources are released back to the system. This operation
388  * will only occur if the page pools are enabled.
389  */
390 static void nvmap_page_pool_resize(struct nvmap_page_pool *pool, int size)
391 {
392         if (!enable_pp || size == pool->max || size < 0)
393                 return;
394
395         mutex_lock(&pool->lock);
396
397         while (pool->count > size)
398                 __free_page(nvmap_page_pool_alloc_locked(pool, 0));
399
400         pool->max = size;
401
402         pr_debug("page pool resized to %d from %d pages\n", size, pool->max);
403         mutex_unlock(&pool->lock);
404 }
405
406 static int nvmap_page_pool_shrink(struct shrinker *shrinker,
407                                   struct shrink_control *sc)
408 {
409         int shrink_pages = sc->nr_to_scan;
410
411         if (!shrink_pages)
412                 goto out;
413
414         pr_debug("sh_pages=%d", shrink_pages);
415
416         shrink_pages = nvmap_page_pool_free(&nvmap_dev->pool, shrink_pages);
417 out:
418         return nvmap_page_pool_get_unused_pages();
419 }
420
421 static struct shrinker nvmap_page_pool_shrinker = {
422         .shrink = nvmap_page_pool_shrink,
423         .seeks = 1,
424 };
425
426 static void shrink_page_pools(int *total_pages, int *available_pages)
427 {
428         struct shrink_control sc;
429
430         if (*total_pages == 0) {
431                 sc.gfp_mask = GFP_KERNEL;
432                 sc.nr_to_scan = 0;
433                 *total_pages = nvmap_page_pool_shrink(NULL, &sc);
434         }
435         sc.nr_to_scan = *total_pages;
436         *available_pages = nvmap_page_pool_shrink(NULL, &sc);
437 }
438
439 #if NVMAP_TEST_PAGE_POOL_SHRINKER
440 static int shrink_pp;
441 static int shrink_set(const char *arg, const struct kernel_param *kp)
442 {
443         int cpu = smp_processor_id();
444         unsigned long long t1, t2;
445         int total_pages, available_pages;
446
447         param_set_int(arg, kp);
448
449         if (shrink_pp) {
450                 total_pages = shrink_pp;
451                 t1 = cpu_clock(cpu);
452                 shrink_page_pools(&total_pages, &available_pages);
453                 t2 = cpu_clock(cpu);
454                 pr_debug("shrink page pools: time=%lldns, "
455                         "total_pages_released=%d, free_pages_available=%d",
456                         t2-t1, total_pages, available_pages);
457         }
458         return 0;
459 }
460
461 static int shrink_get(char *buff, const struct kernel_param *kp)
462 {
463         return param_get_int(buff, kp);
464 }
465
466 static struct kernel_param_ops shrink_ops = {
467         .get = shrink_get,
468         .set = shrink_set,
469 };
470
471 module_param_cb(shrink_page_pools, &shrink_ops, &shrink_pp, 0644);
472 #endif
473
474 static int enable_pp_set(const char *arg, const struct kernel_param *kp)
475 {
476         int ret;
477
478         ret = param_set_bool(arg, kp);
479         if (ret)
480                 return ret;
481
482         if (!enable_pp)
483                 nvmap_page_pool_clear();
484
485         return 0;
486 }
487
488 static int enable_pp_get(char *buff, const struct kernel_param *kp)
489 {
490         return param_get_int(buff, kp);
491 }
492
493 static struct kernel_param_ops enable_pp_ops = {
494         .get = enable_pp_get,
495         .set = enable_pp_set,
496 };
497
498 module_param_cb(enable_page_pools, &enable_pp_ops, &enable_pp, 0644);
499
500 static int pool_size_set(const char *arg, const struct kernel_param *kp)
501 {
502         param_set_int(arg, kp);
503         nvmap_page_pool_resize(&nvmap_dev->pool, pool_size);
504         return 0;
505 }
506
507 static int pool_size_get(char *buff, const struct kernel_param *kp)
508 {
509         return param_get_int(buff, kp);
510 }
511
512 static struct kernel_param_ops pool_size_ops = {
513         .get = pool_size_get,
514         .set = pool_size_set,
515 };
516
517 module_param_cb(pool_size, &pool_size_ops, &pool_size, 0644);
518
519 int nvmap_page_pool_debugfs_init(struct dentry *nvmap_root)
520 {
521         struct dentry *pp_root;
522
523         if (!nvmap_root)
524                 return -ENODEV;
525
526         pp_root = debugfs_create_dir("pagepool", nvmap_root);
527         if (!pp_root)
528                 return -ENODEV;
529
530         debugfs_create_u32("page_pool_available_pages",
531                            S_IRUGO, pp_root,
532                            &nvmap_dev->pool.count);
533         debugfs_create_u32("page_pool_pages_to_zero",
534                            S_IRUGO, pp_root,
535                            &nvmap_dev->pool.to_zero);
536 #ifdef CONFIG_NVMAP_PAGE_POOL_DEBUG
537         debugfs_create_u64("page_pool_allocs",
538                            S_IRUGO, pp_root,
539                            &nvmap_dev->pool.allocs);
540         debugfs_create_u64("page_pool_fills",
541                            S_IRUGO, pp_root,
542                            &nvmap_dev->pool.fills);
543         debugfs_create_u64("page_pool_hits",
544                            S_IRUGO, pp_root,
545                            &nvmap_dev->pool.hits);
546         debugfs_create_u64("page_pool_misses",
547                            S_IRUGO, pp_root,
548                            &nvmap_dev->pool.misses);
549 #endif
550
551         return 0;
552 }
553
554 int nvmap_page_pool_init(struct nvmap_device *dev)
555 {
556         unsigned long totalram_mb;
557         struct sysinfo info;
558         struct nvmap_page_pool *pool = &dev->pool;
559
560         memset(pool, 0x0, sizeof(*pool));
561         mutex_init(&pool->lock);
562         INIT_LIST_HEAD(&pool->page_list);
563         INIT_LIST_HEAD(&pool->zero_list);
564
565         si_meminfo(&info);
566         totalram_mb = (info.totalram * info.mem_unit) >> 20;
567         pr_info("Total MB RAM: %lu\n", totalram_mb);
568
569         if (!CONFIG_NVMAP_PAGE_POOL_SIZE)
570                 /* The ratio is pool pages per 1K ram pages. So, the >> 10. */
571                 pool->max = (info.totalram * NVMAP_PP_POOL_SIZE) >> 10;
572         else
573                 pool->max = CONFIG_NVMAP_PAGE_POOL_SIZE;
574
575         if (pool->max >= info.totalram)
576                 goto fail;
577         pool_size = pool->max;
578
579         pr_info("nvmap page pool size: %u pages (%u MB)\n", pool->max,
580                 (pool->max * info.mem_unit) >> 20);
581
582         background_allocator = kthread_run(nvmap_background_zero_thread,
583                                             NULL, "nvmap-bz");
584         if (IS_ERR_OR_NULL(background_allocator))
585                 goto fail;
586
587         register_shrinker(&nvmap_page_pool_shrinker);
588
589         return 0;
590 fail:
591         nvmap_page_pool_fini(dev);
592         return -ENOMEM;
593 }
594
595 int nvmap_page_pool_fini(struct nvmap_device *dev)
596 {
597         struct nvmap_page_pool *pool = &dev->pool;
598
599         if (!IS_ERR_OR_NULL(background_allocator))
600                 kthread_stop(background_allocator);
601
602         WARN_ON(!list_empty(&pool->page_list));
603
604         return 0;
605 }