drm/ttm: fix two regressions since move_notify changes
[linux-2.6.git] / mm / swap.c
1 /*
2  *  linux/mm/swap.c
3  *
4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
5  */
6
7 /*
8  * This file contains the default values for the operation of the
9  * Linux VM subsystem. Fine-tuning documentation can be found in
10  * Documentation/sysctl/vm.txt.
11  * Started 18.12.91
12  * Swap aging added 23.2.95, Stephen Tweedie.
13  * Buffermem limits added 12.3.98, Rik van Riel.
14  */
15
16 #include <linux/mm.h>
17 #include <linux/sched.h>
18 #include <linux/kernel_stat.h>
19 #include <linux/swap.h>
20 #include <linux/mman.h>
21 #include <linux/pagemap.h>
22 #include <linux/pagevec.h>
23 #include <linux/init.h>
24 #include <linux/export.h>
25 #include <linux/mm_inline.h>
26 #include <linux/percpu_counter.h>
27 #include <linux/percpu.h>
28 #include <linux/cpu.h>
29 #include <linux/notifier.h>
30 #include <linux/backing-dev.h>
31 #include <linux/memcontrol.h>
32 #include <linux/gfp.h>
33
34 #include "internal.h"
35
36 /* How many pages do we try to swap or page in/out together? */
37 int page_cluster;
38
39 static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs);
40 static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
41 static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
42
43 /*
44  * This path almost never happens for VM activity - pages are normally
45  * freed via pagevecs.  But it gets used by networking.
46  */
47 static void __page_cache_release(struct page *page)
48 {
49         if (PageLRU(page)) {
50                 unsigned long flags;
51                 struct zone *zone = page_zone(page);
52
53                 spin_lock_irqsave(&zone->lru_lock, flags);
54                 VM_BUG_ON(!PageLRU(page));
55                 __ClearPageLRU(page);
56                 del_page_from_lru_list(zone, page, page_off_lru(page));
57                 spin_unlock_irqrestore(&zone->lru_lock, flags);
58         }
59 }
60
61 static void __put_single_page(struct page *page)
62 {
63         __page_cache_release(page);
64         free_hot_cold_page(page, 0);
65 }
66
67 static void __put_compound_page(struct page *page)
68 {
69         compound_page_dtor *dtor;
70
71         __page_cache_release(page);
72         dtor = get_compound_page_dtor(page);
73         (*dtor)(page);
74 }
75
76 static void put_compound_page(struct page *page)
77 {
78         if (unlikely(PageTail(page))) {
79                 /* __split_huge_page_refcount can run under us */
80                 struct page *page_head = compound_trans_head(page);
81
82                 if (likely(page != page_head &&
83                            get_page_unless_zero(page_head))) {
84                         unsigned long flags;
85                         /*
86                          * page_head wasn't a dangling pointer but it
87                          * may not be a head page anymore by the time
88                          * we obtain the lock. That is ok as long as it
89                          * can't be freed from under us.
90                          */
91                         flags = compound_lock_irqsave(page_head);
92                         if (unlikely(!PageTail(page))) {
93                                 /* __split_huge_page_refcount run before us */
94                                 compound_unlock_irqrestore(page_head, flags);
95                                 VM_BUG_ON(PageHead(page_head));
96                                 if (put_page_testzero(page_head))
97                                         __put_single_page(page_head);
98                         out_put_single:
99                                 if (put_page_testzero(page))
100                                         __put_single_page(page);
101                                 return;
102                         }
103                         VM_BUG_ON(page_head != page->first_page);
104                         /*
105                          * We can release the refcount taken by
106                          * get_page_unless_zero() now that
107                          * __split_huge_page_refcount() is blocked on
108                          * the compound_lock.
109                          */
110                         if (put_page_testzero(page_head))
111                                 VM_BUG_ON(1);
112                         /* __split_huge_page_refcount will wait now */
113                         VM_BUG_ON(page_mapcount(page) <= 0);
114                         atomic_dec(&page->_mapcount);
115                         VM_BUG_ON(atomic_read(&page_head->_count) <= 0);
116                         VM_BUG_ON(atomic_read(&page->_count) != 0);
117                         compound_unlock_irqrestore(page_head, flags);
118                         if (put_page_testzero(page_head)) {
119                                 if (PageHead(page_head))
120                                         __put_compound_page(page_head);
121                                 else
122                                         __put_single_page(page_head);
123                         }
124                 } else {
125                         /* page_head is a dangling pointer */
126                         VM_BUG_ON(PageTail(page));
127                         goto out_put_single;
128                 }
129         } else if (put_page_testzero(page)) {
130                 if (PageHead(page))
131                         __put_compound_page(page);
132                 else
133                         __put_single_page(page);
134         }
135 }
136
137 void put_page(struct page *page)
138 {
139         if (unlikely(PageCompound(page)))
140                 put_compound_page(page);
141         else if (put_page_testzero(page))
142                 __put_single_page(page);
143 }
144 EXPORT_SYMBOL(put_page);
145
146 /*
147  * This function is exported but must not be called by anything other
148  * than get_page(). It implements the slow path of get_page().
149  */
150 bool __get_page_tail(struct page *page)
151 {
152         /*
153          * This takes care of get_page() if run on a tail page
154          * returned by one of the get_user_pages/follow_page variants.
155          * get_user_pages/follow_page itself doesn't need the compound
156          * lock because it runs __get_page_tail_foll() under the
157          * proper PT lock that already serializes against
158          * split_huge_page().
159          */
160         unsigned long flags;
161         bool got = false;
162         struct page *page_head = compound_trans_head(page);
163
164         if (likely(page != page_head && get_page_unless_zero(page_head))) {
165                 /*
166                  * page_head wasn't a dangling pointer but it
167                  * may not be a head page anymore by the time
168                  * we obtain the lock. That is ok as long as it
169                  * can't be freed from under us.
170                  */
171                 flags = compound_lock_irqsave(page_head);
172                 /* here __split_huge_page_refcount won't run anymore */
173                 if (likely(PageTail(page))) {
174                         __get_page_tail_foll(page, false);
175                         got = true;
176                 }
177                 compound_unlock_irqrestore(page_head, flags);
178                 if (unlikely(!got))
179                         put_page(page_head);
180         }
181         return got;
182 }
183 EXPORT_SYMBOL(__get_page_tail);
184
185 /**
186  * put_pages_list() - release a list of pages
187  * @pages: list of pages threaded on page->lru
188  *
189  * Release a list of pages which are strung together on page.lru.  Currently
190  * used by read_cache_pages() and related error recovery code.
191  */
192 void put_pages_list(struct list_head *pages)
193 {
194         while (!list_empty(pages)) {
195                 struct page *victim;
196
197                 victim = list_entry(pages->prev, struct page, lru);
198                 list_del(&victim->lru);
199                 page_cache_release(victim);
200         }
201 }
202 EXPORT_SYMBOL(put_pages_list);
203
204 static void pagevec_lru_move_fn(struct pagevec *pvec,
205                                 void (*move_fn)(struct page *page, void *arg),
206                                 void *arg)
207 {
208         int i;
209         struct zone *zone = NULL;
210         unsigned long flags = 0;
211
212         for (i = 0; i < pagevec_count(pvec); i++) {
213                 struct page *page = pvec->pages[i];
214                 struct zone *pagezone = page_zone(page);
215
216                 if (pagezone != zone) {
217                         if (zone)
218                                 spin_unlock_irqrestore(&zone->lru_lock, flags);
219                         zone = pagezone;
220                         spin_lock_irqsave(&zone->lru_lock, flags);
221                 }
222
223                 (*move_fn)(page, arg);
224         }
225         if (zone)
226                 spin_unlock_irqrestore(&zone->lru_lock, flags);
227         release_pages(pvec->pages, pvec->nr, pvec->cold);
228         pagevec_reinit(pvec);
229 }
230
231 static void pagevec_move_tail_fn(struct page *page, void *arg)
232 {
233         int *pgmoved = arg;
234
235         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
236                 enum lru_list lru = page_lru_base_type(page);
237                 struct lruvec *lruvec;
238
239                 lruvec = mem_cgroup_lru_move_lists(page_zone(page),
240                                                    page, lru, lru);
241                 list_move_tail(&page->lru, &lruvec->lists[lru]);
242                 (*pgmoved)++;
243         }
244 }
245
246 /*
247  * pagevec_move_tail() must be called with IRQ disabled.
248  * Otherwise this may cause nasty races.
249  */
250 static void pagevec_move_tail(struct pagevec *pvec)
251 {
252         int pgmoved = 0;
253
254         pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved);
255         __count_vm_events(PGROTATED, pgmoved);
256 }
257
258 /*
259  * Writeback is about to end against a page which has been marked for immediate
260  * reclaim.  If it still appears to be reclaimable, move it to the tail of the
261  * inactive list.
262  */
263 void rotate_reclaimable_page(struct page *page)
264 {
265         if (!PageLocked(page) && !PageDirty(page) && !PageActive(page) &&
266             !PageUnevictable(page) && PageLRU(page)) {
267                 struct pagevec *pvec;
268                 unsigned long flags;
269
270                 page_cache_get(page);
271                 local_irq_save(flags);
272                 pvec = &__get_cpu_var(lru_rotate_pvecs);
273                 if (!pagevec_add(pvec, page))
274                         pagevec_move_tail(pvec);
275                 local_irq_restore(flags);
276         }
277 }
278
279 static void update_page_reclaim_stat(struct zone *zone, struct page *page,
280                                      int file, int rotated)
281 {
282         struct zone_reclaim_stat *reclaim_stat = &zone->reclaim_stat;
283         struct zone_reclaim_stat *memcg_reclaim_stat;
284
285         memcg_reclaim_stat = mem_cgroup_get_reclaim_stat_from_page(page);
286
287         reclaim_stat->recent_scanned[file]++;
288         if (rotated)
289                 reclaim_stat->recent_rotated[file]++;
290
291         if (!memcg_reclaim_stat)
292                 return;
293
294         memcg_reclaim_stat->recent_scanned[file]++;
295         if (rotated)
296                 memcg_reclaim_stat->recent_rotated[file]++;
297 }
298
299 static void __activate_page(struct page *page, void *arg)
300 {
301         struct zone *zone = page_zone(page);
302
303         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
304                 int file = page_is_file_cache(page);
305                 int lru = page_lru_base_type(page);
306                 del_page_from_lru_list(zone, page, lru);
307
308                 SetPageActive(page);
309                 lru += LRU_ACTIVE;
310                 add_page_to_lru_list(zone, page, lru);
311                 __count_vm_event(PGACTIVATE);
312
313                 update_page_reclaim_stat(zone, page, file, 1);
314         }
315 }
316
317 #ifdef CONFIG_SMP
318 static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
319
320 static void activate_page_drain(int cpu)
321 {
322         struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu);
323
324         if (pagevec_count(pvec))
325                 pagevec_lru_move_fn(pvec, __activate_page, NULL);
326 }
327
328 void activate_page(struct page *page)
329 {
330         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
331                 struct pagevec *pvec = &get_cpu_var(activate_page_pvecs);
332
333                 page_cache_get(page);
334                 if (!pagevec_add(pvec, page))
335                         pagevec_lru_move_fn(pvec, __activate_page, NULL);
336                 put_cpu_var(activate_page_pvecs);
337         }
338 }
339
340 #else
341 static inline void activate_page_drain(int cpu)
342 {
343 }
344
345 void activate_page(struct page *page)
346 {
347         struct zone *zone = page_zone(page);
348
349         spin_lock_irq(&zone->lru_lock);
350         __activate_page(page, NULL);
351         spin_unlock_irq(&zone->lru_lock);
352 }
353 #endif
354
355 /*
356  * Mark a page as having seen activity.
357  *
358  * inactive,unreferenced        ->      inactive,referenced
359  * inactive,referenced          ->      active,unreferenced
360  * active,unreferenced          ->      active,referenced
361  */
362 void mark_page_accessed(struct page *page)
363 {
364         if (!PageActive(page) && !PageUnevictable(page) &&
365                         PageReferenced(page) && PageLRU(page)) {
366                 activate_page(page);
367                 ClearPageReferenced(page);
368         } else if (!PageReferenced(page)) {
369                 SetPageReferenced(page);
370         }
371 }
372 EXPORT_SYMBOL(mark_page_accessed);
373
374 void __lru_cache_add(struct page *page, enum lru_list lru)
375 {
376         struct pagevec *pvec = &get_cpu_var(lru_add_pvecs)[lru];
377
378         page_cache_get(page);
379         if (!pagevec_add(pvec, page))
380                 __pagevec_lru_add(pvec, lru);
381         put_cpu_var(lru_add_pvecs);
382 }
383 EXPORT_SYMBOL(__lru_cache_add);
384
385 /**
386  * lru_cache_add_lru - add a page to a page list
387  * @page: the page to be added to the LRU.
388  * @lru: the LRU list to which the page is added.
389  */
390 void lru_cache_add_lru(struct page *page, enum lru_list lru)
391 {
392         if (PageActive(page)) {
393                 VM_BUG_ON(PageUnevictable(page));
394                 ClearPageActive(page);
395         } else if (PageUnevictable(page)) {
396                 VM_BUG_ON(PageActive(page));
397                 ClearPageUnevictable(page);
398         }
399
400         VM_BUG_ON(PageLRU(page) || PageActive(page) || PageUnevictable(page));
401         __lru_cache_add(page, lru);
402 }
403
404 /**
405  * add_page_to_unevictable_list - add a page to the unevictable list
406  * @page:  the page to be added to the unevictable list
407  *
408  * Add page directly to its zone's unevictable list.  To avoid races with
409  * tasks that might be making the page evictable, through eg. munlock,
410  * munmap or exit, while it's not on the lru, we want to add the page
411  * while it's locked or otherwise "invisible" to other tasks.  This is
412  * difficult to do when using the pagevec cache, so bypass that.
413  */
414 void add_page_to_unevictable_list(struct page *page)
415 {
416         struct zone *zone = page_zone(page);
417
418         spin_lock_irq(&zone->lru_lock);
419         SetPageUnevictable(page);
420         SetPageLRU(page);
421         add_page_to_lru_list(zone, page, LRU_UNEVICTABLE);
422         spin_unlock_irq(&zone->lru_lock);
423 }
424
425 /*
426  * If the page can not be invalidated, it is moved to the
427  * inactive list to speed up its reclaim.  It is moved to the
428  * head of the list, rather than the tail, to give the flusher
429  * threads some time to write it out, as this is much more
430  * effective than the single-page writeout from reclaim.
431  *
432  * If the page isn't page_mapped and dirty/writeback, the page
433  * could reclaim asap using PG_reclaim.
434  *
435  * 1. active, mapped page -> none
436  * 2. active, dirty/writeback page -> inactive, head, PG_reclaim
437  * 3. inactive, mapped page -> none
438  * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim
439  * 5. inactive, clean -> inactive, tail
440  * 6. Others -> none
441  *
442  * In 4, why it moves inactive's head, the VM expects the page would
443  * be write it out by flusher threads as this is much more effective
444  * than the single-page writeout from reclaim.
445  */
446 static void lru_deactivate_fn(struct page *page, void *arg)
447 {
448         int lru, file;
449         bool active;
450         struct zone *zone = page_zone(page);
451
452         if (!PageLRU(page))
453                 return;
454
455         if (PageUnevictable(page))
456                 return;
457
458         /* Some processes are using the page */
459         if (page_mapped(page))
460                 return;
461
462         active = PageActive(page);
463
464         file = page_is_file_cache(page);
465         lru = page_lru_base_type(page);
466         del_page_from_lru_list(zone, page, lru + active);
467         ClearPageActive(page);
468         ClearPageReferenced(page);
469         add_page_to_lru_list(zone, page, lru);
470
471         if (PageWriteback(page) || PageDirty(page)) {
472                 /*
473                  * PG_reclaim could be raced with end_page_writeback
474                  * It can make readahead confusing.  But race window
475                  * is _really_ small and  it's non-critical problem.
476                  */
477                 SetPageReclaim(page);
478         } else {
479                 struct lruvec *lruvec;
480                 /*
481                  * The page's writeback ends up during pagevec
482                  * We moves tha page into tail of inactive.
483                  */
484                 lruvec = mem_cgroup_lru_move_lists(zone, page, lru, lru);
485                 list_move_tail(&page->lru, &lruvec->lists[lru]);
486                 __count_vm_event(PGROTATED);
487         }
488
489         if (active)
490                 __count_vm_event(PGDEACTIVATE);
491         update_page_reclaim_stat(zone, page, file, 0);
492 }
493
494 /*
495  * Drain pages out of the cpu's pagevecs.
496  * Either "cpu" is the current CPU, and preemption has already been
497  * disabled; or "cpu" is being hot-unplugged, and is already dead.
498  */
499 static void drain_cpu_pagevecs(int cpu)
500 {
501         struct pagevec *pvecs = per_cpu(lru_add_pvecs, cpu);
502         struct pagevec *pvec;
503         int lru;
504
505         for_each_lru(lru) {
506                 pvec = &pvecs[lru - LRU_BASE];
507                 if (pagevec_count(pvec))
508                         __pagevec_lru_add(pvec, lru);
509         }
510
511         pvec = &per_cpu(lru_rotate_pvecs, cpu);
512         if (pagevec_count(pvec)) {
513                 unsigned long flags;
514
515                 /* No harm done if a racing interrupt already did this */
516                 local_irq_save(flags);
517                 pagevec_move_tail(pvec);
518                 local_irq_restore(flags);
519         }
520
521         pvec = &per_cpu(lru_deactivate_pvecs, cpu);
522         if (pagevec_count(pvec))
523                 pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
524
525         activate_page_drain(cpu);
526 }
527
528 /**
529  * deactivate_page - forcefully deactivate a page
530  * @page: page to deactivate
531  *
532  * This function hints the VM that @page is a good reclaim candidate,
533  * for example if its invalidation fails due to the page being dirty
534  * or under writeback.
535  */
536 void deactivate_page(struct page *page)
537 {
538         /*
539          * In a workload with many unevictable page such as mprotect, unevictable
540          * page deactivation for accelerating reclaim is pointless.
541          */
542         if (PageUnevictable(page))
543                 return;
544
545         if (likely(get_page_unless_zero(page))) {
546                 struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
547
548                 if (!pagevec_add(pvec, page))
549                         pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
550                 put_cpu_var(lru_deactivate_pvecs);
551         }
552 }
553
554 void lru_add_drain(void)
555 {
556         drain_cpu_pagevecs(get_cpu());
557         put_cpu();
558 }
559
560 static void lru_add_drain_per_cpu(struct work_struct *dummy)
561 {
562         lru_add_drain();
563 }
564
565 /*
566  * Returns 0 for success
567  */
568 int lru_add_drain_all(void)
569 {
570         return schedule_on_each_cpu(lru_add_drain_per_cpu);
571 }
572
573 /*
574  * Batched page_cache_release().  Decrement the reference count on all the
575  * passed pages.  If it fell to zero then remove the page from the LRU and
576  * free it.
577  *
578  * Avoid taking zone->lru_lock if possible, but if it is taken, retain it
579  * for the remainder of the operation.
580  *
581  * The locking in this function is against shrink_inactive_list(): we recheck
582  * the page count inside the lock to see whether shrink_inactive_list()
583  * grabbed the page via the LRU.  If it did, give up: shrink_inactive_list()
584  * will free it.
585  */
586 void release_pages(struct page **pages, int nr, int cold)
587 {
588         int i;
589         LIST_HEAD(pages_to_free);
590         struct zone *zone = NULL;
591         unsigned long uninitialized_var(flags);
592
593         for (i = 0; i < nr; i++) {
594                 struct page *page = pages[i];
595
596                 if (unlikely(PageCompound(page))) {
597                         if (zone) {
598                                 spin_unlock_irqrestore(&zone->lru_lock, flags);
599                                 zone = NULL;
600                         }
601                         put_compound_page(page);
602                         continue;
603                 }
604
605                 if (!put_page_testzero(page))
606                         continue;
607
608                 if (PageLRU(page)) {
609                         struct zone *pagezone = page_zone(page);
610
611                         if (pagezone != zone) {
612                                 if (zone)
613                                         spin_unlock_irqrestore(&zone->lru_lock,
614                                                                         flags);
615                                 zone = pagezone;
616                                 spin_lock_irqsave(&zone->lru_lock, flags);
617                         }
618                         VM_BUG_ON(!PageLRU(page));
619                         __ClearPageLRU(page);
620                         del_page_from_lru_list(zone, page, page_off_lru(page));
621                 }
622
623                 list_add(&page->lru, &pages_to_free);
624         }
625         if (zone)
626                 spin_unlock_irqrestore(&zone->lru_lock, flags);
627
628         free_hot_cold_page_list(&pages_to_free, cold);
629 }
630 EXPORT_SYMBOL(release_pages);
631
632 /*
633  * The pages which we're about to release may be in the deferred lru-addition
634  * queues.  That would prevent them from really being freed right now.  That's
635  * OK from a correctness point of view but is inefficient - those pages may be
636  * cache-warm and we want to give them back to the page allocator ASAP.
637  *
638  * So __pagevec_release() will drain those queues here.  __pagevec_lru_add()
639  * and __pagevec_lru_add_active() call release_pages() directly to avoid
640  * mutual recursion.
641  */
642 void __pagevec_release(struct pagevec *pvec)
643 {
644         lru_add_drain();
645         release_pages(pvec->pages, pagevec_count(pvec), pvec->cold);
646         pagevec_reinit(pvec);
647 }
648 EXPORT_SYMBOL(__pagevec_release);
649
650 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
651 /* used by __split_huge_page_refcount() */
652 void lru_add_page_tail(struct zone* zone,
653                        struct page *page, struct page *page_tail)
654 {
655         int active;
656         enum lru_list lru;
657         const int file = 0;
658
659         VM_BUG_ON(!PageHead(page));
660         VM_BUG_ON(PageCompound(page_tail));
661         VM_BUG_ON(PageLRU(page_tail));
662         VM_BUG_ON(!spin_is_locked(&zone->lru_lock));
663
664         SetPageLRU(page_tail);
665
666         if (page_evictable(page_tail, NULL)) {
667                 if (PageActive(page)) {
668                         SetPageActive(page_tail);
669                         active = 1;
670                         lru = LRU_ACTIVE_ANON;
671                 } else {
672                         active = 0;
673                         lru = LRU_INACTIVE_ANON;
674                 }
675                 update_page_reclaim_stat(zone, page_tail, file, active);
676         } else {
677                 SetPageUnevictable(page_tail);
678                 lru = LRU_UNEVICTABLE;
679         }
680
681         if (likely(PageLRU(page)))
682                 list_add_tail(&page_tail->lru, &page->lru);
683         else {
684                 struct list_head *list_head;
685                 /*
686                  * Head page has not yet been counted, as an hpage,
687                  * so we must account for each subpage individually.
688                  *
689                  * Use the standard add function to put page_tail on the list,
690                  * but then correct its position so they all end up in order.
691                  */
692                 add_page_to_lru_list(zone, page_tail, lru);
693                 list_head = page_tail->lru.prev;
694                 list_move_tail(&page_tail->lru, list_head);
695         }
696 }
697 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
698
699 static void __pagevec_lru_add_fn(struct page *page, void *arg)
700 {
701         enum lru_list lru = (enum lru_list)arg;
702         struct zone *zone = page_zone(page);
703         int file = is_file_lru(lru);
704         int active = is_active_lru(lru);
705
706         VM_BUG_ON(PageActive(page));
707         VM_BUG_ON(PageUnevictable(page));
708         VM_BUG_ON(PageLRU(page));
709
710         SetPageLRU(page);
711         if (active)
712                 SetPageActive(page);
713         update_page_reclaim_stat(zone, page, file, active);
714         add_page_to_lru_list(zone, page, lru);
715 }
716
717 /*
718  * Add the passed pages to the LRU, then drop the caller's refcount
719  * on them.  Reinitialises the caller's pagevec.
720  */
721 void __pagevec_lru_add(struct pagevec *pvec, enum lru_list lru)
722 {
723         VM_BUG_ON(is_unevictable_lru(lru));
724
725         pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, (void *)lru);
726 }
727 EXPORT_SYMBOL(__pagevec_lru_add);
728
729 /**
730  * pagevec_lookup - gang pagecache lookup
731  * @pvec:       Where the resulting pages are placed
732  * @mapping:    The address_space to search
733  * @start:      The starting page index
734  * @nr_pages:   The maximum number of pages
735  *
736  * pagevec_lookup() will search for and return a group of up to @nr_pages pages
737  * in the mapping.  The pages are placed in @pvec.  pagevec_lookup() takes a
738  * reference against the pages in @pvec.
739  *
740  * The search returns a group of mapping-contiguous pages with ascending
741  * indexes.  There may be holes in the indices due to not-present pages.
742  *
743  * pagevec_lookup() returns the number of pages which were found.
744  */
745 unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping,
746                 pgoff_t start, unsigned nr_pages)
747 {
748         pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages);
749         return pagevec_count(pvec);
750 }
751 EXPORT_SYMBOL(pagevec_lookup);
752
753 unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping,
754                 pgoff_t *index, int tag, unsigned nr_pages)
755 {
756         pvec->nr = find_get_pages_tag(mapping, index, tag,
757                                         nr_pages, pvec->pages);
758         return pagevec_count(pvec);
759 }
760 EXPORT_SYMBOL(pagevec_lookup_tag);
761
762 /*
763  * Perform any setup for the swap system
764  */
765 void __init swap_setup(void)
766 {
767         unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT);
768
769 #ifdef CONFIG_SWAP
770         bdi_init(swapper_space.backing_dev_info);
771 #endif
772
773         /* Use a smaller cluster for small-memory machines */
774         if (megs < 16)
775                 page_cluster = 2;
776         else
777                 page_cluster = 3;
778         /*
779          * Right now other parts of the system means that we
780          * _really_ don't want to cluster much more
781          */
782 }