thp: improve order in lru list for split huge page
[linux-2.6.git] / mm / swap.c
1 /*
2  *  linux/mm/swap.c
3  *
4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
5  */
6
7 /*
8  * This file contains the default values for the operation of the
9  * Linux VM subsystem. Fine-tuning documentation can be found in
10  * Documentation/sysctl/vm.txt.
11  * Started 18.12.91
12  * Swap aging added 23.2.95, Stephen Tweedie.
13  * Buffermem limits added 12.3.98, Rik van Riel.
14  */
15
16 #include <linux/mm.h>
17 #include <linux/sched.h>
18 #include <linux/kernel_stat.h>
19 #include <linux/swap.h>
20 #include <linux/mman.h>
21 #include <linux/pagemap.h>
22 #include <linux/pagevec.h>
23 #include <linux/init.h>
24 #include <linux/export.h>
25 #include <linux/mm_inline.h>
26 #include <linux/buffer_head.h>  /* for try_to_release_page() */
27 #include <linux/percpu_counter.h>
28 #include <linux/percpu.h>
29 #include <linux/cpu.h>
30 #include <linux/notifier.h>
31 #include <linux/backing-dev.h>
32 #include <linux/memcontrol.h>
33 #include <linux/gfp.h>
34
35 #include "internal.h"
36
37 /* How many pages do we try to swap or page in/out together? */
38 int page_cluster;
39
40 static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs);
41 static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
42 static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
43
44 /*
45  * This path almost never happens for VM activity - pages are normally
46  * freed via pagevecs.  But it gets used by networking.
47  */
48 static void __page_cache_release(struct page *page)
49 {
50         if (PageLRU(page)) {
51                 unsigned long flags;
52                 struct zone *zone = page_zone(page);
53
54                 spin_lock_irqsave(&zone->lru_lock, flags);
55                 VM_BUG_ON(!PageLRU(page));
56                 __ClearPageLRU(page);
57                 del_page_from_lru(zone, page);
58                 spin_unlock_irqrestore(&zone->lru_lock, flags);
59         }
60 }
61
62 static void __put_single_page(struct page *page)
63 {
64         __page_cache_release(page);
65         free_hot_cold_page(page, 0);
66 }
67
68 static void __put_compound_page(struct page *page)
69 {
70         compound_page_dtor *dtor;
71
72         __page_cache_release(page);
73         dtor = get_compound_page_dtor(page);
74         (*dtor)(page);
75 }
76
77 static void put_compound_page(struct page *page)
78 {
79         if (unlikely(PageTail(page))) {
80                 /* __split_huge_page_refcount can run under us */
81                 struct page *page_head = compound_trans_head(page);
82
83                 if (likely(page != page_head &&
84                            get_page_unless_zero(page_head))) {
85                         unsigned long flags;
86                         /*
87                          * page_head wasn't a dangling pointer but it
88                          * may not be a head page anymore by the time
89                          * we obtain the lock. That is ok as long as it
90                          * can't be freed from under us.
91                          */
92                         flags = compound_lock_irqsave(page_head);
93                         if (unlikely(!PageTail(page))) {
94                                 /* __split_huge_page_refcount run before us */
95                                 compound_unlock_irqrestore(page_head, flags);
96                                 VM_BUG_ON(PageHead(page_head));
97                                 if (put_page_testzero(page_head))
98                                         __put_single_page(page_head);
99                         out_put_single:
100                                 if (put_page_testzero(page))
101                                         __put_single_page(page);
102                                 return;
103                         }
104                         VM_BUG_ON(page_head != page->first_page);
105                         /*
106                          * We can release the refcount taken by
107                          * get_page_unless_zero() now that
108                          * __split_huge_page_refcount() is blocked on
109                          * the compound_lock.
110                          */
111                         if (put_page_testzero(page_head))
112                                 VM_BUG_ON(1);
113                         /* __split_huge_page_refcount will wait now */
114                         VM_BUG_ON(page_mapcount(page) <= 0);
115                         atomic_dec(&page->_mapcount);
116                         VM_BUG_ON(atomic_read(&page_head->_count) <= 0);
117                         VM_BUG_ON(atomic_read(&page->_count) != 0);
118                         compound_unlock_irqrestore(page_head, flags);
119                         if (put_page_testzero(page_head)) {
120                                 if (PageHead(page_head))
121                                         __put_compound_page(page_head);
122                                 else
123                                         __put_single_page(page_head);
124                         }
125                 } else {
126                         /* page_head is a dangling pointer */
127                         VM_BUG_ON(PageTail(page));
128                         goto out_put_single;
129                 }
130         } else if (put_page_testzero(page)) {
131                 if (PageHead(page))
132                         __put_compound_page(page);
133                 else
134                         __put_single_page(page);
135         }
136 }
137
138 void put_page(struct page *page)
139 {
140         if (unlikely(PageCompound(page)))
141                 put_compound_page(page);
142         else if (put_page_testzero(page))
143                 __put_single_page(page);
144 }
145 EXPORT_SYMBOL(put_page);
146
147 /*
148  * This function is exported but must not be called by anything other
149  * than get_page(). It implements the slow path of get_page().
150  */
151 bool __get_page_tail(struct page *page)
152 {
153         /*
154          * This takes care of get_page() if run on a tail page
155          * returned by one of the get_user_pages/follow_page variants.
156          * get_user_pages/follow_page itself doesn't need the compound
157          * lock because it runs __get_page_tail_foll() under the
158          * proper PT lock that already serializes against
159          * split_huge_page().
160          */
161         unsigned long flags;
162         bool got = false;
163         struct page *page_head = compound_trans_head(page);
164
165         if (likely(page != page_head && get_page_unless_zero(page_head))) {
166                 /*
167                  * page_head wasn't a dangling pointer but it
168                  * may not be a head page anymore by the time
169                  * we obtain the lock. That is ok as long as it
170                  * can't be freed from under us.
171                  */
172                 flags = compound_lock_irqsave(page_head);
173                 /* here __split_huge_page_refcount won't run anymore */
174                 if (likely(PageTail(page))) {
175                         __get_page_tail_foll(page, false);
176                         got = true;
177                 }
178                 compound_unlock_irqrestore(page_head, flags);
179                 if (unlikely(!got))
180                         put_page(page_head);
181         }
182         return got;
183 }
184 EXPORT_SYMBOL(__get_page_tail);
185
186 /**
187  * put_pages_list() - release a list of pages
188  * @pages: list of pages threaded on page->lru
189  *
190  * Release a list of pages which are strung together on page.lru.  Currently
191  * used by read_cache_pages() and related error recovery code.
192  */
193 void put_pages_list(struct list_head *pages)
194 {
195         while (!list_empty(pages)) {
196                 struct page *victim;
197
198                 victim = list_entry(pages->prev, struct page, lru);
199                 list_del(&victim->lru);
200                 page_cache_release(victim);
201         }
202 }
203 EXPORT_SYMBOL(put_pages_list);
204
205 static void pagevec_lru_move_fn(struct pagevec *pvec,
206                                 void (*move_fn)(struct page *page, void *arg),
207                                 void *arg)
208 {
209         int i;
210         struct zone *zone = NULL;
211         unsigned long flags = 0;
212
213         for (i = 0; i < pagevec_count(pvec); i++) {
214                 struct page *page = pvec->pages[i];
215                 struct zone *pagezone = page_zone(page);
216
217                 if (pagezone != zone) {
218                         if (zone)
219                                 spin_unlock_irqrestore(&zone->lru_lock, flags);
220                         zone = pagezone;
221                         spin_lock_irqsave(&zone->lru_lock, flags);
222                 }
223
224                 (*move_fn)(page, arg);
225         }
226         if (zone)
227                 spin_unlock_irqrestore(&zone->lru_lock, flags);
228         release_pages(pvec->pages, pvec->nr, pvec->cold);
229         pagevec_reinit(pvec);
230 }
231
232 static void pagevec_move_tail_fn(struct page *page, void *arg)
233 {
234         int *pgmoved = arg;
235
236         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
237                 enum lru_list lru = page_lru_base_type(page);
238                 struct lruvec *lruvec;
239
240                 lruvec = mem_cgroup_lru_move_lists(page_zone(page),
241                                                    page, lru, lru);
242                 list_move_tail(&page->lru, &lruvec->lists[lru]);
243                 (*pgmoved)++;
244         }
245 }
246
247 /*
248  * pagevec_move_tail() must be called with IRQ disabled.
249  * Otherwise this may cause nasty races.
250  */
251 static void pagevec_move_tail(struct pagevec *pvec)
252 {
253         int pgmoved = 0;
254
255         pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved);
256         __count_vm_events(PGROTATED, pgmoved);
257 }
258
259 /*
260  * Writeback is about to end against a page which has been marked for immediate
261  * reclaim.  If it still appears to be reclaimable, move it to the tail of the
262  * inactive list.
263  */
264 void rotate_reclaimable_page(struct page *page)
265 {
266         if (!PageLocked(page) && !PageDirty(page) && !PageActive(page) &&
267             !PageUnevictable(page) && PageLRU(page)) {
268                 struct pagevec *pvec;
269                 unsigned long flags;
270
271                 page_cache_get(page);
272                 local_irq_save(flags);
273                 pvec = &__get_cpu_var(lru_rotate_pvecs);
274                 if (!pagevec_add(pvec, page))
275                         pagevec_move_tail(pvec);
276                 local_irq_restore(flags);
277         }
278 }
279
280 static void update_page_reclaim_stat(struct zone *zone, struct page *page,
281                                      int file, int rotated)
282 {
283         struct zone_reclaim_stat *reclaim_stat = &zone->reclaim_stat;
284         struct zone_reclaim_stat *memcg_reclaim_stat;
285
286         memcg_reclaim_stat = mem_cgroup_get_reclaim_stat_from_page(page);
287
288         reclaim_stat->recent_scanned[file]++;
289         if (rotated)
290                 reclaim_stat->recent_rotated[file]++;
291
292         if (!memcg_reclaim_stat)
293                 return;
294
295         memcg_reclaim_stat->recent_scanned[file]++;
296         if (rotated)
297                 memcg_reclaim_stat->recent_rotated[file]++;
298 }
299
300 static void __activate_page(struct page *page, void *arg)
301 {
302         struct zone *zone = page_zone(page);
303
304         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
305                 int file = page_is_file_cache(page);
306                 int lru = page_lru_base_type(page);
307                 del_page_from_lru_list(zone, page, lru);
308
309                 SetPageActive(page);
310                 lru += LRU_ACTIVE;
311                 add_page_to_lru_list(zone, page, lru);
312                 __count_vm_event(PGACTIVATE);
313
314                 update_page_reclaim_stat(zone, page, file, 1);
315         }
316 }
317
318 #ifdef CONFIG_SMP
319 static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
320
321 static void activate_page_drain(int cpu)
322 {
323         struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu);
324
325         if (pagevec_count(pvec))
326                 pagevec_lru_move_fn(pvec, __activate_page, NULL);
327 }
328
329 void activate_page(struct page *page)
330 {
331         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
332                 struct pagevec *pvec = &get_cpu_var(activate_page_pvecs);
333
334                 page_cache_get(page);
335                 if (!pagevec_add(pvec, page))
336                         pagevec_lru_move_fn(pvec, __activate_page, NULL);
337                 put_cpu_var(activate_page_pvecs);
338         }
339 }
340
341 #else
342 static inline void activate_page_drain(int cpu)
343 {
344 }
345
346 void activate_page(struct page *page)
347 {
348         struct zone *zone = page_zone(page);
349
350         spin_lock_irq(&zone->lru_lock);
351         __activate_page(page, NULL);
352         spin_unlock_irq(&zone->lru_lock);
353 }
354 #endif
355
356 /*
357  * Mark a page as having seen activity.
358  *
359  * inactive,unreferenced        ->      inactive,referenced
360  * inactive,referenced          ->      active,unreferenced
361  * active,unreferenced          ->      active,referenced
362  */
363 void mark_page_accessed(struct page *page)
364 {
365         if (!PageActive(page) && !PageUnevictable(page) &&
366                         PageReferenced(page) && PageLRU(page)) {
367                 activate_page(page);
368                 ClearPageReferenced(page);
369         } else if (!PageReferenced(page)) {
370                 SetPageReferenced(page);
371         }
372 }
373
374 EXPORT_SYMBOL(mark_page_accessed);
375
376 void __lru_cache_add(struct page *page, enum lru_list lru)
377 {
378         struct pagevec *pvec = &get_cpu_var(lru_add_pvecs)[lru];
379
380         page_cache_get(page);
381         if (!pagevec_add(pvec, page))
382                 ____pagevec_lru_add(pvec, lru);
383         put_cpu_var(lru_add_pvecs);
384 }
385 EXPORT_SYMBOL(__lru_cache_add);
386
387 /**
388  * lru_cache_add_lru - add a page to a page list
389  * @page: the page to be added to the LRU.
390  * @lru: the LRU list to which the page is added.
391  */
392 void lru_cache_add_lru(struct page *page, enum lru_list lru)
393 {
394         if (PageActive(page)) {
395                 VM_BUG_ON(PageUnevictable(page));
396                 ClearPageActive(page);
397         } else if (PageUnevictable(page)) {
398                 VM_BUG_ON(PageActive(page));
399                 ClearPageUnevictable(page);
400         }
401
402         VM_BUG_ON(PageLRU(page) || PageActive(page) || PageUnevictable(page));
403         __lru_cache_add(page, lru);
404 }
405
406 /**
407  * add_page_to_unevictable_list - add a page to the unevictable list
408  * @page:  the page to be added to the unevictable list
409  *
410  * Add page directly to its zone's unevictable list.  To avoid races with
411  * tasks that might be making the page evictable, through eg. munlock,
412  * munmap or exit, while it's not on the lru, we want to add the page
413  * while it's locked or otherwise "invisible" to other tasks.  This is
414  * difficult to do when using the pagevec cache, so bypass that.
415  */
416 void add_page_to_unevictable_list(struct page *page)
417 {
418         struct zone *zone = page_zone(page);
419
420         spin_lock_irq(&zone->lru_lock);
421         SetPageUnevictable(page);
422         SetPageLRU(page);
423         add_page_to_lru_list(zone, page, LRU_UNEVICTABLE);
424         spin_unlock_irq(&zone->lru_lock);
425 }
426
427 /*
428  * If the page can not be invalidated, it is moved to the
429  * inactive list to speed up its reclaim.  It is moved to the
430  * head of the list, rather than the tail, to give the flusher
431  * threads some time to write it out, as this is much more
432  * effective than the single-page writeout from reclaim.
433  *
434  * If the page isn't page_mapped and dirty/writeback, the page
435  * could reclaim asap using PG_reclaim.
436  *
437  * 1. active, mapped page -> none
438  * 2. active, dirty/writeback page -> inactive, head, PG_reclaim
439  * 3. inactive, mapped page -> none
440  * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim
441  * 5. inactive, clean -> inactive, tail
442  * 6. Others -> none
443  *
444  * In 4, why it moves inactive's head, the VM expects the page would
445  * be write it out by flusher threads as this is much more effective
446  * than the single-page writeout from reclaim.
447  */
448 static void lru_deactivate_fn(struct page *page, void *arg)
449 {
450         int lru, file;
451         bool active;
452         struct zone *zone = page_zone(page);
453
454         if (!PageLRU(page))
455                 return;
456
457         if (PageUnevictable(page))
458                 return;
459
460         /* Some processes are using the page */
461         if (page_mapped(page))
462                 return;
463
464         active = PageActive(page);
465
466         file = page_is_file_cache(page);
467         lru = page_lru_base_type(page);
468         del_page_from_lru_list(zone, page, lru + active);
469         ClearPageActive(page);
470         ClearPageReferenced(page);
471         add_page_to_lru_list(zone, page, lru);
472
473         if (PageWriteback(page) || PageDirty(page)) {
474                 /*
475                  * PG_reclaim could be raced with end_page_writeback
476                  * It can make readahead confusing.  But race window
477                  * is _really_ small and  it's non-critical problem.
478                  */
479                 SetPageReclaim(page);
480         } else {
481                 struct lruvec *lruvec;
482                 /*
483                  * The page's writeback ends up during pagevec
484                  * We moves tha page into tail of inactive.
485                  */
486                 lruvec = mem_cgroup_lru_move_lists(zone, page, lru, lru);
487                 list_move_tail(&page->lru, &lruvec->lists[lru]);
488                 __count_vm_event(PGROTATED);
489         }
490
491         if (active)
492                 __count_vm_event(PGDEACTIVATE);
493         update_page_reclaim_stat(zone, page, file, 0);
494 }
495
496 /*
497  * Drain pages out of the cpu's pagevecs.
498  * Either "cpu" is the current CPU, and preemption has already been
499  * disabled; or "cpu" is being hot-unplugged, and is already dead.
500  */
501 static void drain_cpu_pagevecs(int cpu)
502 {
503         struct pagevec *pvecs = per_cpu(lru_add_pvecs, cpu);
504         struct pagevec *pvec;
505         int lru;
506
507         for_each_lru(lru) {
508                 pvec = &pvecs[lru - LRU_BASE];
509                 if (pagevec_count(pvec))
510                         ____pagevec_lru_add(pvec, lru);
511         }
512
513         pvec = &per_cpu(lru_rotate_pvecs, cpu);
514         if (pagevec_count(pvec)) {
515                 unsigned long flags;
516
517                 /* No harm done if a racing interrupt already did this */
518                 local_irq_save(flags);
519                 pagevec_move_tail(pvec);
520                 local_irq_restore(flags);
521         }
522
523         pvec = &per_cpu(lru_deactivate_pvecs, cpu);
524         if (pagevec_count(pvec))
525                 pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
526
527         activate_page_drain(cpu);
528 }
529
530 /**
531  * deactivate_page - forcefully deactivate a page
532  * @page: page to deactivate
533  *
534  * This function hints the VM that @page is a good reclaim candidate,
535  * for example if its invalidation fails due to the page being dirty
536  * or under writeback.
537  */
538 void deactivate_page(struct page *page)
539 {
540         /*
541          * In a workload with many unevictable page such as mprotect, unevictable
542          * page deactivation for accelerating reclaim is pointless.
543          */
544         if (PageUnevictable(page))
545                 return;
546
547         if (likely(get_page_unless_zero(page))) {
548                 struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
549
550                 if (!pagevec_add(pvec, page))
551                         pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
552                 put_cpu_var(lru_deactivate_pvecs);
553         }
554 }
555
556 void lru_add_drain(void)
557 {
558         drain_cpu_pagevecs(get_cpu());
559         put_cpu();
560 }
561
562 static void lru_add_drain_per_cpu(struct work_struct *dummy)
563 {
564         lru_add_drain();
565 }
566
567 /*
568  * Returns 0 for success
569  */
570 int lru_add_drain_all(void)
571 {
572         return schedule_on_each_cpu(lru_add_drain_per_cpu);
573 }
574
575 /*
576  * Batched page_cache_release().  Decrement the reference count on all the
577  * passed pages.  If it fell to zero then remove the page from the LRU and
578  * free it.
579  *
580  * Avoid taking zone->lru_lock if possible, but if it is taken, retain it
581  * for the remainder of the operation.
582  *
583  * The locking in this function is against shrink_inactive_list(): we recheck
584  * the page count inside the lock to see whether shrink_inactive_list()
585  * grabbed the page via the LRU.  If it did, give up: shrink_inactive_list()
586  * will free it.
587  */
588 void release_pages(struct page **pages, int nr, int cold)
589 {
590         int i;
591         LIST_HEAD(pages_to_free);
592         struct zone *zone = NULL;
593         unsigned long uninitialized_var(flags);
594
595         for (i = 0; i < nr; i++) {
596                 struct page *page = pages[i];
597
598                 if (unlikely(PageCompound(page))) {
599                         if (zone) {
600                                 spin_unlock_irqrestore(&zone->lru_lock, flags);
601                                 zone = NULL;
602                         }
603                         put_compound_page(page);
604                         continue;
605                 }
606
607                 if (!put_page_testzero(page))
608                         continue;
609
610                 if (PageLRU(page)) {
611                         struct zone *pagezone = page_zone(page);
612
613                         if (pagezone != zone) {
614                                 if (zone)
615                                         spin_unlock_irqrestore(&zone->lru_lock,
616                                                                         flags);
617                                 zone = pagezone;
618                                 spin_lock_irqsave(&zone->lru_lock, flags);
619                         }
620                         VM_BUG_ON(!PageLRU(page));
621                         __ClearPageLRU(page);
622                         del_page_from_lru(zone, page);
623                 }
624
625                 list_add(&page->lru, &pages_to_free);
626         }
627         if (zone)
628                 spin_unlock_irqrestore(&zone->lru_lock, flags);
629
630         free_hot_cold_page_list(&pages_to_free, cold);
631 }
632 EXPORT_SYMBOL(release_pages);
633
634 /*
635  * The pages which we're about to release may be in the deferred lru-addition
636  * queues.  That would prevent them from really being freed right now.  That's
637  * OK from a correctness point of view but is inefficient - those pages may be
638  * cache-warm and we want to give them back to the page allocator ASAP.
639  *
640  * So __pagevec_release() will drain those queues here.  __pagevec_lru_add()
641  * and __pagevec_lru_add_active() call release_pages() directly to avoid
642  * mutual recursion.
643  */
644 void __pagevec_release(struct pagevec *pvec)
645 {
646         lru_add_drain();
647         release_pages(pvec->pages, pagevec_count(pvec), pvec->cold);
648         pagevec_reinit(pvec);
649 }
650
651 EXPORT_SYMBOL(__pagevec_release);
652
653 /* used by __split_huge_page_refcount() */
654 void lru_add_page_tail(struct zone* zone,
655                        struct page *page, struct page *page_tail)
656 {
657         int active;
658         enum lru_list lru;
659         const int file = 0;
660
661         VM_BUG_ON(!PageHead(page));
662         VM_BUG_ON(PageCompound(page_tail));
663         VM_BUG_ON(PageLRU(page_tail));
664         VM_BUG_ON(!spin_is_locked(&zone->lru_lock));
665
666         SetPageLRU(page_tail);
667
668         if (page_evictable(page_tail, NULL)) {
669                 struct lruvec *lruvec;
670
671                 if (PageActive(page)) {
672                         SetPageActive(page_tail);
673                         active = 1;
674                         lru = LRU_ACTIVE_ANON;
675                 } else {
676                         active = 0;
677                         lru = LRU_INACTIVE_ANON;
678                 }
679                 update_page_reclaim_stat(zone, page_tail, file, active);
680                 lruvec = mem_cgroup_lru_add_list(zone, page_tail, lru);
681                 if (likely(PageLRU(page)))
682                         list_add(&page_tail->lru, page->lru.prev);
683                 else
684                         list_add(&page_tail->lru, lruvec->lists[lru].prev);
685                 __mod_zone_page_state(zone, NR_LRU_BASE + lru,
686                                       hpage_nr_pages(page_tail));
687         } else {
688                 SetPageUnevictable(page_tail);
689                 add_page_to_lru_list(zone, page_tail, LRU_UNEVICTABLE);
690         }
691 }
692
693 static void ____pagevec_lru_add_fn(struct page *page, void *arg)
694 {
695         enum lru_list lru = (enum lru_list)arg;
696         struct zone *zone = page_zone(page);
697         int file = is_file_lru(lru);
698         int active = is_active_lru(lru);
699
700         VM_BUG_ON(PageActive(page));
701         VM_BUG_ON(PageUnevictable(page));
702         VM_BUG_ON(PageLRU(page));
703
704         SetPageLRU(page);
705         if (active)
706                 SetPageActive(page);
707         update_page_reclaim_stat(zone, page, file, active);
708         add_page_to_lru_list(zone, page, lru);
709 }
710
711 /*
712  * Add the passed pages to the LRU, then drop the caller's refcount
713  * on them.  Reinitialises the caller's pagevec.
714  */
715 void ____pagevec_lru_add(struct pagevec *pvec, enum lru_list lru)
716 {
717         VM_BUG_ON(is_unevictable_lru(lru));
718
719         pagevec_lru_move_fn(pvec, ____pagevec_lru_add_fn, (void *)lru);
720 }
721
722 EXPORT_SYMBOL(____pagevec_lru_add);
723
724 /*
725  * Try to drop buffers from the pages in a pagevec
726  */
727 void pagevec_strip(struct pagevec *pvec)
728 {
729         int i;
730
731         for (i = 0; i < pagevec_count(pvec); i++) {
732                 struct page *page = pvec->pages[i];
733
734                 if (page_has_private(page) && trylock_page(page)) {
735                         if (page_has_private(page))
736                                 try_to_release_page(page, 0);
737                         unlock_page(page);
738                 }
739         }
740 }
741
742 /**
743  * pagevec_lookup - gang pagecache lookup
744  * @pvec:       Where the resulting pages are placed
745  * @mapping:    The address_space to search
746  * @start:      The starting page index
747  * @nr_pages:   The maximum number of pages
748  *
749  * pagevec_lookup() will search for and return a group of up to @nr_pages pages
750  * in the mapping.  The pages are placed in @pvec.  pagevec_lookup() takes a
751  * reference against the pages in @pvec.
752  *
753  * The search returns a group of mapping-contiguous pages with ascending
754  * indexes.  There may be holes in the indices due to not-present pages.
755  *
756  * pagevec_lookup() returns the number of pages which were found.
757  */
758 unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping,
759                 pgoff_t start, unsigned nr_pages)
760 {
761         pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages);
762         return pagevec_count(pvec);
763 }
764
765 EXPORT_SYMBOL(pagevec_lookup);
766
767 unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping,
768                 pgoff_t *index, int tag, unsigned nr_pages)
769 {
770         pvec->nr = find_get_pages_tag(mapping, index, tag,
771                                         nr_pages, pvec->pages);
772         return pagevec_count(pvec);
773 }
774
775 EXPORT_SYMBOL(pagevec_lookup_tag);
776
777 /*
778  * Perform any setup for the swap system
779  */
780 void __init swap_setup(void)
781 {
782         unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT);
783
784 #ifdef CONFIG_SWAP
785         bdi_init(swapper_space.backing_dev_info);
786 #endif
787
788         /* Use a smaller cluster for small-memory machines */
789         if (megs < 16)
790                 page_cluster = 2;
791         else
792                 page_cluster = 3;
793         /*
794          * Right now other parts of the system means that we
795          * _really_ don't want to cluster much more
796          */
797 }