Fix typo in include/linux/clk.h .
[linux-3.10.git] / mm / vmscan.c
1 /*
2  *  linux/mm/vmscan.c
3  *
4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
5  *
6  *  Swap reorganised 29.12.95, Stephen Tweedie.
7  *  kswapd added: 7.1.96  sct
8  *  Removed kswapd_ctl limits, and swap out as many pages as needed
9  *  to bring the system back to freepages.high: 2.4.97, Rik van Riel.
10  *  Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
11  *  Multiqueue VM started 5.8.00, Rik van Riel.
12  */
13
14 #include <linux/mm.h>
15 #include <linux/module.h>
16 #include <linux/gfp.h>
17 #include <linux/kernel_stat.h>
18 #include <linux/swap.h>
19 #include <linux/pagemap.h>
20 #include <linux/init.h>
21 #include <linux/highmem.h>
22 #include <linux/vmstat.h>
23 #include <linux/file.h>
24 #include <linux/writeback.h>
25 #include <linux/blkdev.h>
26 #include <linux/buffer_head.h>  /* for try_to_release_page(),
27                                         buffer_heads_over_limit */
28 #include <linux/mm_inline.h>
29 #include <linux/backing-dev.h>
30 #include <linux/rmap.h>
31 #include <linux/topology.h>
32 #include <linux/cpu.h>
33 #include <linux/cpuset.h>
34 #include <linux/compaction.h>
35 #include <linux/notifier.h>
36 #include <linux/rwsem.h>
37 #include <linux/delay.h>
38 #include <linux/kthread.h>
39 #include <linux/freezer.h>
40 #include <linux/memcontrol.h>
41 #include <linux/delayacct.h>
42 #include <linux/sysctl.h>
43 #include <linux/oom.h>
44 #include <linux/prefetch.h>
45
46 #include <asm/tlbflush.h>
47 #include <asm/div64.h>
48
49 #include <linux/swapops.h>
50
51 #include "internal.h"
52
53 #define CREATE_TRACE_POINTS
54 #include <trace/events/vmscan.h>
55
56 struct scan_control {
57         /* Incremented by the number of inactive pages that were scanned */
58         unsigned long nr_scanned;
59
60         /* Number of pages freed so far during a call to shrink_zones() */
61         unsigned long nr_reclaimed;
62
63         /* How many pages shrink_list() should reclaim */
64         unsigned long nr_to_reclaim;
65
66         unsigned long hibernation_mode;
67
68         /* This context's GFP mask */
69         gfp_t gfp_mask;
70
71         int may_writepage;
72
73         /* Can mapped pages be reclaimed? */
74         int may_unmap;
75
76         /* Can pages be swapped as part of reclaim? */
77         int may_swap;
78
79         int order;
80
81         /*
82          * The memory cgroup that hit its limit and as a result is the
83          * primary target of this reclaim invocation.
84          */
85         struct mem_cgroup *target_mem_cgroup;
86
87         /*
88          * Nodemask of nodes allowed by the caller. If NULL, all nodes
89          * are scanned.
90          */
91         nodemask_t      *nodemask;
92 };
93
94 struct mem_cgroup_zone {
95         struct mem_cgroup *mem_cgroup;
96         struct zone *zone;
97 };
98
99 #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
100
101 #ifdef ARCH_HAS_PREFETCH
102 #define prefetch_prev_lru_page(_page, _base, _field)                    \
103         do {                                                            \
104                 if ((_page)->lru.prev != _base) {                       \
105                         struct page *prev;                              \
106                                                                         \
107                         prev = lru_to_page(&(_page->lru));              \
108                         prefetch(&prev->_field);                        \
109                 }                                                       \
110         } while (0)
111 #else
112 #define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
113 #endif
114
115 #ifdef ARCH_HAS_PREFETCHW
116 #define prefetchw_prev_lru_page(_page, _base, _field)                   \
117         do {                                                            \
118                 if ((_page)->lru.prev != _base) {                       \
119                         struct page *prev;                              \
120                                                                         \
121                         prev = lru_to_page(&(_page->lru));              \
122                         prefetchw(&prev->_field);                       \
123                 }                                                       \
124         } while (0)
125 #else
126 #define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
127 #endif
128
129 /*
130  * From 0 .. 100.  Higher means more swappy.
131  */
132 int vm_swappiness = 60;
133 long vm_total_pages;    /* The total number of pages which the VM controls */
134
135 static LIST_HEAD(shrinker_list);
136 static DECLARE_RWSEM(shrinker_rwsem);
137
138 #ifdef CONFIG_CGROUP_MEM_RES_CTLR
139 static bool global_reclaim(struct scan_control *sc)
140 {
141         return !sc->target_mem_cgroup;
142 }
143 #else
144 static bool global_reclaim(struct scan_control *sc)
145 {
146         return true;
147 }
148 #endif
149
150 static struct zone_reclaim_stat *get_reclaim_stat(struct mem_cgroup_zone *mz)
151 {
152         return &mem_cgroup_zone_lruvec(mz->zone, mz->mem_cgroup)->reclaim_stat;
153 }
154
155 static unsigned long zone_nr_lru_pages(struct mem_cgroup_zone *mz,
156                                        enum lru_list lru)
157 {
158         if (!mem_cgroup_disabled())
159                 return mem_cgroup_zone_nr_lru_pages(mz->mem_cgroup,
160                                                     zone_to_nid(mz->zone),
161                                                     zone_idx(mz->zone),
162                                                     BIT(lru));
163
164         return zone_page_state(mz->zone, NR_LRU_BASE + lru);
165 }
166
167
168 /*
169  * Add a shrinker callback to be called from the vm
170  */
171 void register_shrinker(struct shrinker *shrinker)
172 {
173         atomic_long_set(&shrinker->nr_in_batch, 0);
174         down_write(&shrinker_rwsem);
175         list_add_tail(&shrinker->list, &shrinker_list);
176         up_write(&shrinker_rwsem);
177 }
178 EXPORT_SYMBOL(register_shrinker);
179
180 /*
181  * Remove one
182  */
183 void unregister_shrinker(struct shrinker *shrinker)
184 {
185         down_write(&shrinker_rwsem);
186         list_del(&shrinker->list);
187         up_write(&shrinker_rwsem);
188 }
189 EXPORT_SYMBOL(unregister_shrinker);
190
191 static inline int do_shrinker_shrink(struct shrinker *shrinker,
192                                      struct shrink_control *sc,
193                                      unsigned long nr_to_scan)
194 {
195         sc->nr_to_scan = nr_to_scan;
196         return (*shrinker->shrink)(shrinker, sc);
197 }
198
199 #define SHRINK_BATCH 128
200 /*
201  * Call the shrink functions to age shrinkable caches
202  *
203  * Here we assume it costs one seek to replace a lru page and that it also
204  * takes a seek to recreate a cache object.  With this in mind we age equal
205  * percentages of the lru and ageable caches.  This should balance the seeks
206  * generated by these structures.
207  *
208  * If the vm encountered mapped pages on the LRU it increase the pressure on
209  * slab to avoid swapping.
210  *
211  * We do weird things to avoid (scanned*seeks*entries) overflowing 32 bits.
212  *
213  * `lru_pages' represents the number of on-LRU pages in all the zones which
214  * are eligible for the caller's allocation attempt.  It is used for balancing
215  * slab reclaim versus page reclaim.
216  *
217  * Returns the number of slab objects which we shrunk.
218  */
219 unsigned long shrink_slab(struct shrink_control *shrink,
220                           unsigned long nr_pages_scanned,
221                           unsigned long lru_pages)
222 {
223         struct shrinker *shrinker;
224         unsigned long ret = 0;
225
226         if (nr_pages_scanned == 0)
227                 nr_pages_scanned = SWAP_CLUSTER_MAX;
228
229         if (!down_read_trylock(&shrinker_rwsem)) {
230                 /* Assume we'll be able to shrink next time */
231                 ret = 1;
232                 goto out;
233         }
234
235         list_for_each_entry(shrinker, &shrinker_list, list) {
236                 unsigned long long delta;
237                 long total_scan;
238                 long max_pass;
239                 int shrink_ret = 0;
240                 long nr;
241                 long new_nr;
242                 long batch_size = shrinker->batch ? shrinker->batch
243                                                   : SHRINK_BATCH;
244
245                 max_pass = do_shrinker_shrink(shrinker, shrink, 0);
246                 if (max_pass <= 0)
247                         continue;
248
249                 /*
250                  * copy the current shrinker scan count into a local variable
251                  * and zero it so that other concurrent shrinker invocations
252                  * don't also do this scanning work.
253                  */
254                 nr = atomic_long_xchg(&shrinker->nr_in_batch, 0);
255
256                 total_scan = nr;
257                 delta = (4 * nr_pages_scanned) / shrinker->seeks;
258                 delta *= max_pass;
259                 do_div(delta, lru_pages + 1);
260                 total_scan += delta;
261                 if (total_scan < 0) {
262                         printk(KERN_ERR "shrink_slab: %pF negative objects to "
263                                "delete nr=%ld\n",
264                                shrinker->shrink, total_scan);
265                         total_scan = max_pass;
266                 }
267
268                 /*
269                  * We need to avoid excessive windup on filesystem shrinkers
270                  * due to large numbers of GFP_NOFS allocations causing the
271                  * shrinkers to return -1 all the time. This results in a large
272                  * nr being built up so when a shrink that can do some work
273                  * comes along it empties the entire cache due to nr >>>
274                  * max_pass.  This is bad for sustaining a working set in
275                  * memory.
276                  *
277                  * Hence only allow the shrinker to scan the entire cache when
278                  * a large delta change is calculated directly.
279                  */
280                 if (delta < max_pass / 4)
281                         total_scan = min(total_scan, max_pass / 2);
282
283                 /*
284                  * Avoid risking looping forever due to too large nr value:
285                  * never try to free more than twice the estimate number of
286                  * freeable entries.
287                  */
288                 if (total_scan > max_pass * 2)
289                         total_scan = max_pass * 2;
290
291                 trace_mm_shrink_slab_start(shrinker, shrink, nr,
292                                         nr_pages_scanned, lru_pages,
293                                         max_pass, delta, total_scan);
294
295                 while (total_scan >= batch_size) {
296                         int nr_before;
297
298                         nr_before = do_shrinker_shrink(shrinker, shrink, 0);
299                         shrink_ret = do_shrinker_shrink(shrinker, shrink,
300                                                         batch_size);
301                         if (shrink_ret == -1)
302                                 break;
303                         if (shrink_ret < nr_before)
304                                 ret += nr_before - shrink_ret;
305                         count_vm_events(SLABS_SCANNED, batch_size);
306                         total_scan -= batch_size;
307
308                         cond_resched();
309                 }
310
311                 /*
312                  * move the unused scan count back into the shrinker in a
313                  * manner that handles concurrent updates. If we exhausted the
314                  * scan, there is no need to do an update.
315                  */
316                 if (total_scan > 0)
317                         new_nr = atomic_long_add_return(total_scan,
318                                         &shrinker->nr_in_batch);
319                 else
320                         new_nr = atomic_long_read(&shrinker->nr_in_batch);
321
322                 trace_mm_shrink_slab_end(shrinker, shrink_ret, nr, new_nr);
323         }
324         up_read(&shrinker_rwsem);
325 out:
326         cond_resched();
327         return ret;
328 }
329
330 static inline int is_page_cache_freeable(struct page *page)
331 {
332         /*
333          * A freeable page cache page is referenced only by the caller
334          * that isolated the page, the page cache radix tree and
335          * optional buffer heads at page->private.
336          */
337         return page_count(page) - page_has_private(page) == 2;
338 }
339
340 static int may_write_to_queue(struct backing_dev_info *bdi,
341                               struct scan_control *sc)
342 {
343         if (current->flags & PF_SWAPWRITE)
344                 return 1;
345         if (!bdi_write_congested(bdi))
346                 return 1;
347         if (bdi == current->backing_dev_info)
348                 return 1;
349         return 0;
350 }
351
352 /*
353  * We detected a synchronous write error writing a page out.  Probably
354  * -ENOSPC.  We need to propagate that into the address_space for a subsequent
355  * fsync(), msync() or close().
356  *
357  * The tricky part is that after writepage we cannot touch the mapping: nothing
358  * prevents it from being freed up.  But we have a ref on the page and once
359  * that page is locked, the mapping is pinned.
360  *
361  * We're allowed to run sleeping lock_page() here because we know the caller has
362  * __GFP_FS.
363  */
364 static void handle_write_error(struct address_space *mapping,
365                                 struct page *page, int error)
366 {
367         lock_page(page);
368         if (page_mapping(page) == mapping)
369                 mapping_set_error(mapping, error);
370         unlock_page(page);
371 }
372
373 /* possible outcome of pageout() */
374 typedef enum {
375         /* failed to write page out, page is locked */
376         PAGE_KEEP,
377         /* move page to the active list, page is locked */
378         PAGE_ACTIVATE,
379         /* page has been sent to the disk successfully, page is unlocked */
380         PAGE_SUCCESS,
381         /* page is clean and locked */
382         PAGE_CLEAN,
383 } pageout_t;
384
385 /*
386  * pageout is called by shrink_page_list() for each dirty page.
387  * Calls ->writepage().
388  */
389 static pageout_t pageout(struct page *page, struct address_space *mapping,
390                          struct scan_control *sc)
391 {
392         /*
393          * If the page is dirty, only perform writeback if that write
394          * will be non-blocking.  To prevent this allocation from being
395          * stalled by pagecache activity.  But note that there may be
396          * stalls if we need to run get_block().  We could test
397          * PagePrivate for that.
398          *
399          * If this process is currently in __generic_file_aio_write() against
400          * this page's queue, we can perform writeback even if that
401          * will block.
402          *
403          * If the page is swapcache, write it back even if that would
404          * block, for some throttling. This happens by accident, because
405          * swap_backing_dev_info is bust: it doesn't reflect the
406          * congestion state of the swapdevs.  Easy to fix, if needed.
407          */
408         if (!is_page_cache_freeable(page))
409                 return PAGE_KEEP;
410         if (!mapping) {
411                 /*
412                  * Some data journaling orphaned pages can have
413                  * page->mapping == NULL while being dirty with clean buffers.
414                  */
415                 if (page_has_private(page)) {
416                         if (try_to_free_buffers(page)) {
417                                 ClearPageDirty(page);
418                                 printk("%s: orphaned page\n", __func__);
419                                 return PAGE_CLEAN;
420                         }
421                 }
422                 return PAGE_KEEP;
423         }
424         if (mapping->a_ops->writepage == NULL)
425                 return PAGE_ACTIVATE;
426         if (!may_write_to_queue(mapping->backing_dev_info, sc))
427                 return PAGE_KEEP;
428
429         if (clear_page_dirty_for_io(page)) {
430                 int res;
431                 struct writeback_control wbc = {
432                         .sync_mode = WB_SYNC_NONE,
433                         .nr_to_write = SWAP_CLUSTER_MAX,
434                         .range_start = 0,
435                         .range_end = LLONG_MAX,
436                         .for_reclaim = 1,
437                 };
438
439                 SetPageReclaim(page);
440                 res = mapping->a_ops->writepage(page, &wbc);
441                 if (res < 0)
442                         handle_write_error(mapping, page, res);
443                 if (res == AOP_WRITEPAGE_ACTIVATE) {
444                         ClearPageReclaim(page);
445                         return PAGE_ACTIVATE;
446                 }
447
448                 if (!PageWriteback(page)) {
449                         /* synchronous write or broken a_ops? */
450                         ClearPageReclaim(page);
451                 }
452                 trace_mm_vmscan_writepage(page, trace_reclaim_flags(page));
453                 inc_zone_page_state(page, NR_VMSCAN_WRITE);
454                 return PAGE_SUCCESS;
455         }
456
457         return PAGE_CLEAN;
458 }
459
460 /*
461  * Same as remove_mapping, but if the page is removed from the mapping, it
462  * gets returned with a refcount of 0.
463  */
464 static int __remove_mapping(struct address_space *mapping, struct page *page)
465 {
466         BUG_ON(!PageLocked(page));
467         BUG_ON(mapping != page_mapping(page));
468
469         spin_lock_irq(&mapping->tree_lock);
470         /*
471          * The non racy check for a busy page.
472          *
473          * Must be careful with the order of the tests. When someone has
474          * a ref to the page, it may be possible that they dirty it then
475          * drop the reference. So if PageDirty is tested before page_count
476          * here, then the following race may occur:
477          *
478          * get_user_pages(&page);
479          * [user mapping goes away]
480          * write_to(page);
481          *                              !PageDirty(page)    [good]
482          * SetPageDirty(page);
483          * put_page(page);
484          *                              !page_count(page)   [good, discard it]
485          *
486          * [oops, our write_to data is lost]
487          *
488          * Reversing the order of the tests ensures such a situation cannot
489          * escape unnoticed. The smp_rmb is needed to ensure the page->flags
490          * load is not satisfied before that of page->_count.
491          *
492          * Note that if SetPageDirty is always performed via set_page_dirty,
493          * and thus under tree_lock, then this ordering is not required.
494          */
495         if (!page_freeze_refs(page, 2))
496                 goto cannot_free;
497         /* note: atomic_cmpxchg in page_freeze_refs provides the smp_rmb */
498         if (unlikely(PageDirty(page))) {
499                 page_unfreeze_refs(page, 2);
500                 goto cannot_free;
501         }
502
503         if (PageSwapCache(page)) {
504                 swp_entry_t swap = { .val = page_private(page) };
505                 __delete_from_swap_cache(page);
506                 spin_unlock_irq(&mapping->tree_lock);
507                 swapcache_free(swap, page);
508         } else {
509                 void (*freepage)(struct page *);
510
511                 freepage = mapping->a_ops->freepage;
512
513                 __delete_from_page_cache(page);
514                 spin_unlock_irq(&mapping->tree_lock);
515                 mem_cgroup_uncharge_cache_page(page);
516
517                 if (freepage != NULL)
518                         freepage(page);
519         }
520
521         return 1;
522
523 cannot_free:
524         spin_unlock_irq(&mapping->tree_lock);
525         return 0;
526 }
527
528 /*
529  * Attempt to detach a locked page from its ->mapping.  If it is dirty or if
530  * someone else has a ref on the page, abort and return 0.  If it was
531  * successfully detached, return 1.  Assumes the caller has a single ref on
532  * this page.
533  */
534 int remove_mapping(struct address_space *mapping, struct page *page)
535 {
536         if (__remove_mapping(mapping, page)) {
537                 /*
538                  * Unfreezing the refcount with 1 rather than 2 effectively
539                  * drops the pagecache ref for us without requiring another
540                  * atomic operation.
541                  */
542                 page_unfreeze_refs(page, 1);
543                 return 1;
544         }
545         return 0;
546 }
547
548 /**
549  * putback_lru_page - put previously isolated page onto appropriate LRU list
550  * @page: page to be put back to appropriate lru list
551  *
552  * Add previously isolated @page to appropriate LRU list.
553  * Page may still be unevictable for other reasons.
554  *
555  * lru_lock must not be held, interrupts must be enabled.
556  */
557 void putback_lru_page(struct page *page)
558 {
559         int lru;
560         int active = !!TestClearPageActive(page);
561         int was_unevictable = PageUnevictable(page);
562
563         VM_BUG_ON(PageLRU(page));
564
565 redo:
566         ClearPageUnevictable(page);
567
568         if (page_evictable(page, NULL)) {
569                 /*
570                  * For evictable pages, we can use the cache.
571                  * In event of a race, worst case is we end up with an
572                  * unevictable page on [in]active list.
573                  * We know how to handle that.
574                  */
575                 lru = active + page_lru_base_type(page);
576                 lru_cache_add_lru(page, lru);
577         } else {
578                 /*
579                  * Put unevictable pages directly on zone's unevictable
580                  * list.
581                  */
582                 lru = LRU_UNEVICTABLE;
583                 add_page_to_unevictable_list(page);
584                 /*
585                  * When racing with an mlock or AS_UNEVICTABLE clearing
586                  * (page is unlocked) make sure that if the other thread
587                  * does not observe our setting of PG_lru and fails
588                  * isolation/check_move_unevictable_pages,
589                  * we see PG_mlocked/AS_UNEVICTABLE cleared below and move
590                  * the page back to the evictable list.
591                  *
592                  * The other side is TestClearPageMlocked() or shmem_lock().
593                  */
594                 smp_mb();
595         }
596
597         /*
598          * page's status can change while we move it among lru. If an evictable
599          * page is on unevictable list, it never be freed. To avoid that,
600          * check after we added it to the list, again.
601          */
602         if (lru == LRU_UNEVICTABLE && page_evictable(page, NULL)) {
603                 if (!isolate_lru_page(page)) {
604                         put_page(page);
605                         goto redo;
606                 }
607                 /* This means someone else dropped this page from LRU
608                  * So, it will be freed or putback to LRU again. There is
609                  * nothing to do here.
610                  */
611         }
612
613         if (was_unevictable && lru != LRU_UNEVICTABLE)
614                 count_vm_event(UNEVICTABLE_PGRESCUED);
615         else if (!was_unevictable && lru == LRU_UNEVICTABLE)
616                 count_vm_event(UNEVICTABLE_PGCULLED);
617
618         put_page(page);         /* drop ref from isolate */
619 }
620
621 enum page_references {
622         PAGEREF_RECLAIM,
623         PAGEREF_RECLAIM_CLEAN,
624         PAGEREF_KEEP,
625         PAGEREF_ACTIVATE,
626 };
627
628 static enum page_references page_check_references(struct page *page,
629                                                   struct mem_cgroup_zone *mz,
630                                                   struct scan_control *sc)
631 {
632         int referenced_ptes, referenced_page;
633         unsigned long vm_flags;
634
635         referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup,
636                                           &vm_flags);
637         referenced_page = TestClearPageReferenced(page);
638
639         /*
640          * Mlock lost the isolation race with us.  Let try_to_unmap()
641          * move the page to the unevictable list.
642          */
643         if (vm_flags & VM_LOCKED)
644                 return PAGEREF_RECLAIM;
645
646         if (referenced_ptes) {
647                 if (PageSwapBacked(page))
648                         return PAGEREF_ACTIVATE;
649                 /*
650                  * All mapped pages start out with page table
651                  * references from the instantiating fault, so we need
652                  * to look twice if a mapped file page is used more
653                  * than once.
654                  *
655                  * Mark it and spare it for another trip around the
656                  * inactive list.  Another page table reference will
657                  * lead to its activation.
658                  *
659                  * Note: the mark is set for activated pages as well
660                  * so that recently deactivated but used pages are
661                  * quickly recovered.
662                  */
663                 SetPageReferenced(page);
664
665                 if (referenced_page || referenced_ptes > 1)
666                         return PAGEREF_ACTIVATE;
667
668                 /*
669                  * Activate file-backed executable pages after first usage.
670                  */
671                 if (vm_flags & VM_EXEC)
672                         return PAGEREF_ACTIVATE;
673
674                 return PAGEREF_KEEP;
675         }
676
677         /* Reclaim if clean, defer dirty pages to writeback */
678         if (referenced_page && !PageSwapBacked(page))
679                 return PAGEREF_RECLAIM_CLEAN;
680
681         return PAGEREF_RECLAIM;
682 }
683
684 /*
685  * shrink_page_list() returns the number of reclaimed pages
686  */
687 static unsigned long shrink_page_list(struct list_head *page_list,
688                                       struct mem_cgroup_zone *mz,
689                                       struct scan_control *sc,
690                                       int priority,
691                                       unsigned long *ret_nr_dirty,
692                                       unsigned long *ret_nr_writeback)
693 {
694         LIST_HEAD(ret_pages);
695         LIST_HEAD(free_pages);
696         int pgactivate = 0;
697         unsigned long nr_dirty = 0;
698         unsigned long nr_congested = 0;
699         unsigned long nr_reclaimed = 0;
700         unsigned long nr_writeback = 0;
701
702         cond_resched();
703
704         while (!list_empty(page_list)) {
705                 enum page_references references;
706                 struct address_space *mapping;
707                 struct page *page;
708                 int may_enter_fs;
709
710                 cond_resched();
711
712                 page = lru_to_page(page_list);
713                 list_del(&page->lru);
714
715                 if (!trylock_page(page))
716                         goto keep;
717
718                 VM_BUG_ON(PageActive(page));
719                 VM_BUG_ON(page_zone(page) != mz->zone);
720
721                 sc->nr_scanned++;
722
723                 if (unlikely(!page_evictable(page, NULL)))
724                         goto cull_mlocked;
725
726                 if (!sc->may_unmap && page_mapped(page))
727                         goto keep_locked;
728
729                 /* Double the slab pressure for mapped and swapcache pages */
730                 if (page_mapped(page) || PageSwapCache(page))
731                         sc->nr_scanned++;
732
733                 may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
734                         (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
735
736                 if (PageWriteback(page)) {
737                         nr_writeback++;
738                         unlock_page(page);
739                         goto keep;
740                 }
741
742                 references = page_check_references(page, mz, sc);
743                 switch (references) {
744                 case PAGEREF_ACTIVATE:
745                         goto activate_locked;
746                 case PAGEREF_KEEP:
747                         goto keep_locked;
748                 case PAGEREF_RECLAIM:
749                 case PAGEREF_RECLAIM_CLEAN:
750                         ; /* try to reclaim the page below */
751                 }
752
753                 /*
754                  * Anonymous process memory has backing store?
755                  * Try to allocate it some swap space here.
756                  */
757                 if (PageAnon(page) && !PageSwapCache(page)) {
758                         if (!(sc->gfp_mask & __GFP_IO))
759                                 goto keep_locked;
760                         if (!add_to_swap(page))
761                                 goto activate_locked;
762                         may_enter_fs = 1;
763                 }
764
765                 mapping = page_mapping(page);
766
767                 /*
768                  * The page is mapped into the page tables of one or more
769                  * processes. Try to unmap it here.
770                  */
771                 if (page_mapped(page) && mapping) {
772                         switch (try_to_unmap(page, TTU_UNMAP)) {
773                         case SWAP_FAIL:
774                                 goto activate_locked;
775                         case SWAP_AGAIN:
776                                 goto keep_locked;
777                         case SWAP_MLOCK:
778                                 goto cull_mlocked;
779                         case SWAP_SUCCESS:
780                                 ; /* try to free the page below */
781                         }
782                 }
783
784                 if (PageDirty(page)) {
785                         nr_dirty++;
786
787                         /*
788                          * Only kswapd can writeback filesystem pages to
789                          * avoid risk of stack overflow but do not writeback
790                          * unless under significant pressure.
791                          */
792                         if (page_is_file_cache(page) &&
793                                         (!current_is_kswapd() || priority >= DEF_PRIORITY - 2)) {
794                                 /*
795                                  * Immediately reclaim when written back.
796                                  * Similar in principal to deactivate_page()
797                                  * except we already have the page isolated
798                                  * and know it's dirty
799                                  */
800                                 inc_zone_page_state(page, NR_VMSCAN_IMMEDIATE);
801                                 SetPageReclaim(page);
802
803                                 goto keep_locked;
804                         }
805
806                         if (references == PAGEREF_RECLAIM_CLEAN)
807                                 goto keep_locked;
808                         if (!may_enter_fs)
809                                 goto keep_locked;
810                         if (!sc->may_writepage)
811                                 goto keep_locked;
812
813                         /* Page is dirty, try to write it out here */
814                         switch (pageout(page, mapping, sc)) {
815                         case PAGE_KEEP:
816                                 nr_congested++;
817                                 goto keep_locked;
818                         case PAGE_ACTIVATE:
819                                 goto activate_locked;
820                         case PAGE_SUCCESS:
821                                 if (PageWriteback(page))
822                                         goto keep;
823                                 if (PageDirty(page))
824                                         goto keep;
825
826                                 /*
827                                  * A synchronous write - probably a ramdisk.  Go
828                                  * ahead and try to reclaim the page.
829                                  */
830                                 if (!trylock_page(page))
831                                         goto keep;
832                                 if (PageDirty(page) || PageWriteback(page))
833                                         goto keep_locked;
834                                 mapping = page_mapping(page);
835                         case PAGE_CLEAN:
836                                 ; /* try to free the page below */
837                         }
838                 }
839
840                 /*
841                  * If the page has buffers, try to free the buffer mappings
842                  * associated with this page. If we succeed we try to free
843                  * the page as well.
844                  *
845                  * We do this even if the page is PageDirty().
846                  * try_to_release_page() does not perform I/O, but it is
847                  * possible for a page to have PageDirty set, but it is actually
848                  * clean (all its buffers are clean).  This happens if the
849                  * buffers were written out directly, with submit_bh(). ext3
850                  * will do this, as well as the blockdev mapping.
851                  * try_to_release_page() will discover that cleanness and will
852                  * drop the buffers and mark the page clean - it can be freed.
853                  *
854                  * Rarely, pages can have buffers and no ->mapping.  These are
855                  * the pages which were not successfully invalidated in
856                  * truncate_complete_page().  We try to drop those buffers here
857                  * and if that worked, and the page is no longer mapped into
858                  * process address space (page_count == 1) it can be freed.
859                  * Otherwise, leave the page on the LRU so it is swappable.
860                  */
861                 if (page_has_private(page)) {
862                         if (!try_to_release_page(page, sc->gfp_mask))
863                                 goto activate_locked;
864                         if (!mapping && page_count(page) == 1) {
865                                 unlock_page(page);
866                                 if (put_page_testzero(page))
867                                         goto free_it;
868                                 else {
869                                         /*
870                                          * rare race with speculative reference.
871                                          * the speculative reference will free
872                                          * this page shortly, so we may
873                                          * increment nr_reclaimed here (and
874                                          * leave it off the LRU).
875                                          */
876                                         nr_reclaimed++;
877                                         continue;
878                                 }
879                         }
880                 }
881
882                 if (!mapping || !__remove_mapping(mapping, page))
883                         goto keep_locked;
884
885                 /*
886                  * At this point, we have no other references and there is
887                  * no way to pick any more up (removed from LRU, removed
888                  * from pagecache). Can use non-atomic bitops now (and
889                  * we obviously don't have to worry about waking up a process
890                  * waiting on the page lock, because there are no references.
891                  */
892                 __clear_page_locked(page);
893 free_it:
894                 nr_reclaimed++;
895
896                 /*
897                  * Is there need to periodically free_page_list? It would
898                  * appear not as the counts should be low
899                  */
900                 list_add(&page->lru, &free_pages);
901                 continue;
902
903 cull_mlocked:
904                 if (PageSwapCache(page))
905                         try_to_free_swap(page);
906                 unlock_page(page);
907                 putback_lru_page(page);
908                 continue;
909
910 activate_locked:
911                 /* Not a candidate for swapping, so reclaim swap space. */
912                 if (PageSwapCache(page) && vm_swap_full())
913                         try_to_free_swap(page);
914                 VM_BUG_ON(PageActive(page));
915                 SetPageActive(page);
916                 pgactivate++;
917 keep_locked:
918                 unlock_page(page);
919 keep:
920                 list_add(&page->lru, &ret_pages);
921                 VM_BUG_ON(PageLRU(page) || PageUnevictable(page));
922         }
923
924         /*
925          * Tag a zone as congested if all the dirty pages encountered were
926          * backed by a congested BDI. In this case, reclaimers should just
927          * back off and wait for congestion to clear because further reclaim
928          * will encounter the same problem
929          */
930         if (nr_dirty && nr_dirty == nr_congested && global_reclaim(sc))
931                 zone_set_flag(mz->zone, ZONE_CONGESTED);
932
933         free_hot_cold_page_list(&free_pages, 1);
934
935         list_splice(&ret_pages, page_list);
936         count_vm_events(PGACTIVATE, pgactivate);
937         *ret_nr_dirty += nr_dirty;
938         *ret_nr_writeback += nr_writeback;
939         return nr_reclaimed;
940 }
941
942 /*
943  * Attempt to remove the specified page from its LRU.  Only take this page
944  * if it is of the appropriate PageActive status.  Pages which are being
945  * freed elsewhere are also ignored.
946  *
947  * page:        page to consider
948  * mode:        one of the LRU isolation modes defined above
949  *
950  * returns 0 on success, -ve errno on failure.
951  */
952 int __isolate_lru_page(struct page *page, isolate_mode_t mode, int file)
953 {
954         bool all_lru_mode;
955         int ret = -EINVAL;
956
957         /* Only take pages on the LRU. */
958         if (!PageLRU(page))
959                 return ret;
960
961         all_lru_mode = (mode & (ISOLATE_ACTIVE|ISOLATE_INACTIVE)) ==
962                 (ISOLATE_ACTIVE|ISOLATE_INACTIVE);
963
964         /*
965          * When checking the active state, we need to be sure we are
966          * dealing with comparible boolean values.  Take the logical not
967          * of each.
968          */
969         if (!all_lru_mode && !PageActive(page) != !(mode & ISOLATE_ACTIVE))
970                 return ret;
971
972         if (!all_lru_mode && !!page_is_file_cache(page) != file)
973                 return ret;
974
975         /* Do not give back unevictable pages for compaction */
976         if (PageUnevictable(page))
977                 return ret;
978
979         ret = -EBUSY;
980
981         /*
982          * To minimise LRU disruption, the caller can indicate that it only
983          * wants to isolate pages it will be able to operate on without
984          * blocking - clean pages for the most part.
985          *
986          * ISOLATE_CLEAN means that only clean pages should be isolated. This
987          * is used by reclaim when it is cannot write to backing storage
988          *
989          * ISOLATE_ASYNC_MIGRATE is used to indicate that it only wants to pages
990          * that it is possible to migrate without blocking
991          */
992         if (mode & (ISOLATE_CLEAN|ISOLATE_ASYNC_MIGRATE)) {
993                 /* All the caller can do on PageWriteback is block */
994                 if (PageWriteback(page))
995                         return ret;
996
997                 if (PageDirty(page)) {
998                         struct address_space *mapping;
999
1000                         /* ISOLATE_CLEAN means only clean pages */
1001                         if (mode & ISOLATE_CLEAN)
1002                                 return ret;
1003
1004                         /*
1005                          * Only pages without mappings or that have a
1006                          * ->migratepage callback are possible to migrate
1007                          * without blocking
1008                          */
1009                         mapping = page_mapping(page);
1010                         if (mapping && !mapping->a_ops->migratepage)
1011                                 return ret;
1012                 }
1013         }
1014
1015         if ((mode & ISOLATE_UNMAPPED) && page_mapped(page))
1016                 return ret;
1017
1018         if (likely(get_page_unless_zero(page))) {
1019                 /*
1020                  * Be careful not to clear PageLRU until after we're
1021                  * sure the page is not being freed elsewhere -- the
1022                  * page release code relies on it.
1023                  */
1024                 ClearPageLRU(page);
1025                 ret = 0;
1026         }
1027
1028         return ret;
1029 }
1030
1031 /*
1032  * zone->lru_lock is heavily contended.  Some of the functions that
1033  * shrink the lists perform better by taking out a batch of pages
1034  * and working on them outside the LRU lock.
1035  *
1036  * For pagecache intensive workloads, this function is the hottest
1037  * spot in the kernel (apart from copy_*_user functions).
1038  *
1039  * Appropriate locks must be held before calling this function.
1040  *
1041  * @nr_to_scan: The number of pages to look through on the list.
1042  * @mz:         The mem_cgroup_zone to pull pages from.
1043  * @dst:        The temp list to put pages on to.
1044  * @nr_scanned: The number of pages that were scanned.
1045  * @sc:         The scan_control struct for this reclaim session
1046  * @mode:       One of the LRU isolation modes
1047  * @active:     True [1] if isolating active pages
1048  * @file:       True [1] if isolating file [!anon] pages
1049  *
1050  * returns how many pages were moved onto *@dst.
1051  */
1052 static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
1053                 struct mem_cgroup_zone *mz, struct list_head *dst,
1054                 unsigned long *nr_scanned, struct scan_control *sc,
1055                 isolate_mode_t mode, int active, int file)
1056 {
1057         struct lruvec *lruvec;
1058         struct list_head *src;
1059         unsigned long nr_taken = 0;
1060         unsigned long scan;
1061         int lru = LRU_BASE;
1062
1063         lruvec = mem_cgroup_zone_lruvec(mz->zone, mz->mem_cgroup);
1064         if (active)
1065                 lru += LRU_ACTIVE;
1066         if (file)
1067                 lru += LRU_FILE;
1068         src = &lruvec->lists[lru];
1069
1070         for (scan = 0; scan < nr_to_scan && !list_empty(src); scan++) {
1071                 struct page *page;
1072
1073                 page = lru_to_page(src);
1074                 prefetchw_prev_lru_page(page, src, flags);
1075
1076                 VM_BUG_ON(!PageLRU(page));
1077
1078                 switch (__isolate_lru_page(page, mode, file)) {
1079                 case 0:
1080                         mem_cgroup_lru_del(page);
1081                         list_move(&page->lru, dst);
1082                         nr_taken += hpage_nr_pages(page);
1083                         break;
1084
1085                 case -EBUSY:
1086                         /* else it is being freed elsewhere */
1087                         list_move(&page->lru, src);
1088                         continue;
1089
1090                 default:
1091                         BUG();
1092                 }
1093         }
1094
1095         *nr_scanned = scan;
1096
1097         trace_mm_vmscan_lru_isolate(sc->order,
1098                         nr_to_scan, scan,
1099                         nr_taken,
1100                         mode, file);
1101         return nr_taken;
1102 }
1103
1104 /**
1105  * isolate_lru_page - tries to isolate a page from its LRU list
1106  * @page: page to isolate from its LRU list
1107  *
1108  * Isolates a @page from an LRU list, clears PageLRU and adjusts the
1109  * vmstat statistic corresponding to whatever LRU list the page was on.
1110  *
1111  * Returns 0 if the page was removed from an LRU list.
1112  * Returns -EBUSY if the page was not on an LRU list.
1113  *
1114  * The returned page will have PageLRU() cleared.  If it was found on
1115  * the active list, it will have PageActive set.  If it was found on
1116  * the unevictable list, it will have the PageUnevictable bit set. That flag
1117  * may need to be cleared by the caller before letting the page go.
1118  *
1119  * The vmstat statistic corresponding to the list on which the page was
1120  * found will be decremented.
1121  *
1122  * Restrictions:
1123  * (1) Must be called with an elevated refcount on the page. This is a
1124  *     fundamentnal difference from isolate_lru_pages (which is called
1125  *     without a stable reference).
1126  * (2) the lru_lock must not be held.
1127  * (3) interrupts must be enabled.
1128  */
1129 int isolate_lru_page(struct page *page)
1130 {
1131         int ret = -EBUSY;
1132
1133         VM_BUG_ON(!page_count(page));
1134
1135         if (PageLRU(page)) {
1136                 struct zone *zone = page_zone(page);
1137
1138                 spin_lock_irq(&zone->lru_lock);
1139                 if (PageLRU(page)) {
1140                         int lru = page_lru(page);
1141                         ret = 0;
1142                         get_page(page);
1143                         ClearPageLRU(page);
1144
1145                         del_page_from_lru_list(zone, page, lru);
1146                 }
1147                 spin_unlock_irq(&zone->lru_lock);
1148         }
1149         return ret;
1150 }
1151
1152 /*
1153  * Are there way too many processes in the direct reclaim path already?
1154  */
1155 static int too_many_isolated(struct zone *zone, int file,
1156                 struct scan_control *sc)
1157 {
1158         unsigned long inactive, isolated;
1159
1160         if (current_is_kswapd())
1161                 return 0;
1162
1163         if (!global_reclaim(sc))
1164                 return 0;
1165
1166         if (file) {
1167                 inactive = zone_page_state(zone, NR_INACTIVE_FILE);
1168                 isolated = zone_page_state(zone, NR_ISOLATED_FILE);
1169         } else {
1170                 inactive = zone_page_state(zone, NR_INACTIVE_ANON);
1171                 isolated = zone_page_state(zone, NR_ISOLATED_ANON);
1172         }
1173
1174         return isolated > inactive;
1175 }
1176
1177 static noinline_for_stack void
1178 putback_inactive_pages(struct mem_cgroup_zone *mz,
1179                        struct list_head *page_list)
1180 {
1181         struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(mz);
1182         struct zone *zone = mz->zone;
1183         LIST_HEAD(pages_to_free);
1184
1185         /*
1186          * Put back any unfreeable pages.
1187          */
1188         while (!list_empty(page_list)) {
1189                 struct page *page = lru_to_page(page_list);
1190                 int lru;
1191
1192                 VM_BUG_ON(PageLRU(page));
1193                 list_del(&page->lru);
1194                 if (unlikely(!page_evictable(page, NULL))) {
1195                         spin_unlock_irq(&zone->lru_lock);
1196                         putback_lru_page(page);
1197                         spin_lock_irq(&zone->lru_lock);
1198                         continue;
1199                 }
1200                 SetPageLRU(page);
1201                 lru = page_lru(page);
1202                 add_page_to_lru_list(zone, page, lru);
1203                 if (is_active_lru(lru)) {
1204                         int file = is_file_lru(lru);
1205                         int numpages = hpage_nr_pages(page);
1206                         reclaim_stat->recent_rotated[file] += numpages;
1207                 }
1208                 if (put_page_testzero(page)) {
1209                         __ClearPageLRU(page);
1210                         __ClearPageActive(page);
1211                         del_page_from_lru_list(zone, page, lru);
1212
1213                         if (unlikely(PageCompound(page))) {
1214                                 spin_unlock_irq(&zone->lru_lock);
1215                                 (*get_compound_page_dtor(page))(page);
1216                                 spin_lock_irq(&zone->lru_lock);
1217                         } else
1218                                 list_add(&page->lru, &pages_to_free);
1219                 }
1220         }
1221
1222         /*
1223          * To save our caller's stack, now use input list for pages to free.
1224          */
1225         list_splice(&pages_to_free, page_list);
1226 }
1227
1228 static noinline_for_stack void
1229 update_isolated_counts(struct mem_cgroup_zone *mz,
1230                        struct list_head *page_list,
1231                        unsigned long *nr_anon,
1232                        unsigned long *nr_file)
1233 {
1234         struct zone *zone = mz->zone;
1235         unsigned int count[NR_LRU_LISTS] = { 0, };
1236         unsigned long nr_active = 0;
1237         struct page *page;
1238         int lru;
1239
1240         /*
1241          * Count pages and clear active flags
1242          */
1243         list_for_each_entry(page, page_list, lru) {
1244                 int numpages = hpage_nr_pages(page);
1245                 lru = page_lru_base_type(page);
1246                 if (PageActive(page)) {
1247                         lru += LRU_ACTIVE;
1248                         ClearPageActive(page);
1249                         nr_active += numpages;
1250                 }
1251                 count[lru] += numpages;
1252         }
1253
1254         preempt_disable();
1255         __count_vm_events(PGDEACTIVATE, nr_active);
1256
1257         __mod_zone_page_state(zone, NR_ACTIVE_FILE,
1258                               -count[LRU_ACTIVE_FILE]);
1259         __mod_zone_page_state(zone, NR_INACTIVE_FILE,
1260                               -count[LRU_INACTIVE_FILE]);
1261         __mod_zone_page_state(zone, NR_ACTIVE_ANON,
1262                               -count[LRU_ACTIVE_ANON]);
1263         __mod_zone_page_state(zone, NR_INACTIVE_ANON,
1264                               -count[LRU_INACTIVE_ANON]);
1265
1266         *nr_anon = count[LRU_ACTIVE_ANON] + count[LRU_INACTIVE_ANON];
1267         *nr_file = count[LRU_ACTIVE_FILE] + count[LRU_INACTIVE_FILE];
1268
1269         __mod_zone_page_state(zone, NR_ISOLATED_ANON, *nr_anon);
1270         __mod_zone_page_state(zone, NR_ISOLATED_FILE, *nr_file);
1271         preempt_enable();
1272 }
1273
1274 /*
1275  * shrink_inactive_list() is a helper for shrink_zone().  It returns the number
1276  * of reclaimed pages
1277  */
1278 static noinline_for_stack unsigned long
1279 shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz,
1280                      struct scan_control *sc, int priority, int file)
1281 {
1282         LIST_HEAD(page_list);
1283         unsigned long nr_scanned;
1284         unsigned long nr_reclaimed = 0;
1285         unsigned long nr_taken;
1286         unsigned long nr_anon;
1287         unsigned long nr_file;
1288         unsigned long nr_dirty = 0;
1289         unsigned long nr_writeback = 0;
1290         isolate_mode_t isolate_mode = ISOLATE_INACTIVE;
1291         struct zone *zone = mz->zone;
1292         struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(mz);
1293
1294         while (unlikely(too_many_isolated(zone, file, sc))) {
1295                 congestion_wait(BLK_RW_ASYNC, HZ/10);
1296
1297                 /* We are about to die and free our memory. Return now. */
1298                 if (fatal_signal_pending(current))
1299                         return SWAP_CLUSTER_MAX;
1300         }
1301
1302         lru_add_drain();
1303
1304         if (!sc->may_unmap)
1305                 isolate_mode |= ISOLATE_UNMAPPED;
1306         if (!sc->may_writepage)
1307                 isolate_mode |= ISOLATE_CLEAN;
1308
1309         spin_lock_irq(&zone->lru_lock);
1310
1311         nr_taken = isolate_lru_pages(nr_to_scan, mz, &page_list, &nr_scanned,
1312                                      sc, isolate_mode, 0, file);
1313         if (global_reclaim(sc)) {
1314                 zone->pages_scanned += nr_scanned;
1315                 if (current_is_kswapd())
1316                         __count_zone_vm_events(PGSCAN_KSWAPD, zone,
1317                                                nr_scanned);
1318                 else
1319                         __count_zone_vm_events(PGSCAN_DIRECT, zone,
1320                                                nr_scanned);
1321         }
1322         spin_unlock_irq(&zone->lru_lock);
1323
1324         if (nr_taken == 0)
1325                 return 0;
1326
1327         update_isolated_counts(mz, &page_list, &nr_anon, &nr_file);
1328
1329         nr_reclaimed = shrink_page_list(&page_list, mz, sc, priority,
1330                                                 &nr_dirty, &nr_writeback);
1331
1332         spin_lock_irq(&zone->lru_lock);
1333
1334         reclaim_stat->recent_scanned[0] += nr_anon;
1335         reclaim_stat->recent_scanned[1] += nr_file;
1336
1337         if (global_reclaim(sc)) {
1338                 if (current_is_kswapd())
1339                         __count_zone_vm_events(PGSTEAL_KSWAPD, zone,
1340                                                nr_reclaimed);
1341                 else
1342                         __count_zone_vm_events(PGSTEAL_DIRECT, zone,
1343                                                nr_reclaimed);
1344         }
1345
1346         putback_inactive_pages(mz, &page_list);
1347
1348         __mod_zone_page_state(zone, NR_ISOLATED_ANON, -nr_anon);
1349         __mod_zone_page_state(zone, NR_ISOLATED_FILE, -nr_file);
1350
1351         spin_unlock_irq(&zone->lru_lock);
1352
1353         free_hot_cold_page_list(&page_list, 1);
1354
1355         /*
1356          * If reclaim is isolating dirty pages under writeback, it implies
1357          * that the long-lived page allocation rate is exceeding the page
1358          * laundering rate. Either the global limits are not being effective
1359          * at throttling processes due to the page distribution throughout
1360          * zones or there is heavy usage of a slow backing device. The
1361          * only option is to throttle from reclaim context which is not ideal
1362          * as there is no guarantee the dirtying process is throttled in the
1363          * same way balance_dirty_pages() manages.
1364          *
1365          * This scales the number of dirty pages that must be under writeback
1366          * before throttling depending on priority. It is a simple backoff
1367          * function that has the most effect in the range DEF_PRIORITY to
1368          * DEF_PRIORITY-2 which is the priority reclaim is considered to be
1369          * in trouble and reclaim is considered to be in trouble.
1370          *
1371          * DEF_PRIORITY   100% isolated pages must be PageWriteback to throttle
1372          * DEF_PRIORITY-1  50% must be PageWriteback
1373          * DEF_PRIORITY-2  25% must be PageWriteback, kswapd in trouble
1374          * ...
1375          * DEF_PRIORITY-6 For SWAP_CLUSTER_MAX isolated pages, throttle if any
1376          *                     isolated page is PageWriteback
1377          */
1378         if (nr_writeback && nr_writeback >= (nr_taken >> (DEF_PRIORITY-priority)))
1379                 wait_iff_congested(zone, BLK_RW_ASYNC, HZ/10);
1380
1381         trace_mm_vmscan_lru_shrink_inactive(zone->zone_pgdat->node_id,
1382                 zone_idx(zone),
1383                 nr_scanned, nr_reclaimed,
1384                 priority,
1385                 trace_shrink_flags(file));
1386         return nr_reclaimed;
1387 }
1388
1389 /*
1390  * This moves pages from the active list to the inactive list.
1391  *
1392  * We move them the other way if the page is referenced by one or more
1393  * processes, from rmap.
1394  *
1395  * If the pages are mostly unmapped, the processing is fast and it is
1396  * appropriate to hold zone->lru_lock across the whole operation.  But if
1397  * the pages are mapped, the processing is slow (page_referenced()) so we
1398  * should drop zone->lru_lock around each page.  It's impossible to balance
1399  * this, so instead we remove the pages from the LRU while processing them.
1400  * It is safe to rely on PG_active against the non-LRU pages in here because
1401  * nobody will play with that bit on a non-LRU page.
1402  *
1403  * The downside is that we have to touch page->_count against each page.
1404  * But we had to alter page->flags anyway.
1405  */
1406
1407 static void move_active_pages_to_lru(struct zone *zone,
1408                                      struct list_head *list,
1409                                      struct list_head *pages_to_free,
1410                                      enum lru_list lru)
1411 {
1412         unsigned long pgmoved = 0;
1413         struct page *page;
1414
1415         while (!list_empty(list)) {
1416                 struct lruvec *lruvec;
1417
1418                 page = lru_to_page(list);
1419
1420                 VM_BUG_ON(PageLRU(page));
1421                 SetPageLRU(page);
1422
1423                 lruvec = mem_cgroup_lru_add_list(zone, page, lru);
1424                 list_move(&page->lru, &lruvec->lists[lru]);
1425                 pgmoved += hpage_nr_pages(page);
1426
1427                 if (put_page_testzero(page)) {
1428                         __ClearPageLRU(page);
1429                         __ClearPageActive(page);
1430                         del_page_from_lru_list(zone, page, lru);
1431
1432                         if (unlikely(PageCompound(page))) {
1433                                 spin_unlock_irq(&zone->lru_lock);
1434                                 (*get_compound_page_dtor(page))(page);
1435                                 spin_lock_irq(&zone->lru_lock);
1436                         } else
1437                                 list_add(&page->lru, pages_to_free);
1438                 }
1439         }
1440         __mod_zone_page_state(zone, NR_LRU_BASE + lru, pgmoved);
1441         if (!is_active_lru(lru))
1442                 __count_vm_events(PGDEACTIVATE, pgmoved);
1443 }
1444
1445 static void shrink_active_list(unsigned long nr_to_scan,
1446                                struct mem_cgroup_zone *mz,
1447                                struct scan_control *sc,
1448                                int priority, int file)
1449 {
1450         unsigned long nr_taken;
1451         unsigned long nr_scanned;
1452         unsigned long vm_flags;
1453         LIST_HEAD(l_hold);      /* The pages which were snipped off */
1454         LIST_HEAD(l_active);
1455         LIST_HEAD(l_inactive);
1456         struct page *page;
1457         struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(mz);
1458         unsigned long nr_rotated = 0;
1459         isolate_mode_t isolate_mode = ISOLATE_ACTIVE;
1460         struct zone *zone = mz->zone;
1461
1462         lru_add_drain();
1463
1464         if (!sc->may_unmap)
1465                 isolate_mode |= ISOLATE_UNMAPPED;
1466         if (!sc->may_writepage)
1467                 isolate_mode |= ISOLATE_CLEAN;
1468
1469         spin_lock_irq(&zone->lru_lock);
1470
1471         nr_taken = isolate_lru_pages(nr_to_scan, mz, &l_hold, &nr_scanned, sc,
1472                                      isolate_mode, 1, file);
1473         if (global_reclaim(sc))
1474                 zone->pages_scanned += nr_scanned;
1475
1476         reclaim_stat->recent_scanned[file] += nr_taken;
1477
1478         __count_zone_vm_events(PGREFILL, zone, nr_scanned);
1479         if (file)
1480                 __mod_zone_page_state(zone, NR_ACTIVE_FILE, -nr_taken);
1481         else
1482                 __mod_zone_page_state(zone, NR_ACTIVE_ANON, -nr_taken);
1483         __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, nr_taken);
1484         spin_unlock_irq(&zone->lru_lock);
1485
1486         while (!list_empty(&l_hold)) {
1487                 cond_resched();
1488                 page = lru_to_page(&l_hold);
1489                 list_del(&page->lru);
1490
1491                 if (unlikely(!page_evictable(page, NULL))) {
1492                         putback_lru_page(page);
1493                         continue;
1494                 }
1495
1496                 if (unlikely(buffer_heads_over_limit)) {
1497                         if (page_has_private(page) && trylock_page(page)) {
1498                                 if (page_has_private(page))
1499                                         try_to_release_page(page, 0);
1500                                 unlock_page(page);
1501                         }
1502                 }
1503
1504                 if (page_referenced(page, 0, sc->target_mem_cgroup,
1505                                     &vm_flags)) {
1506                         nr_rotated += hpage_nr_pages(page);
1507                         /*
1508                          * Identify referenced, file-backed active pages and
1509                          * give them one more trip around the active list. So
1510                          * that executable code get better chances to stay in
1511                          * memory under moderate memory pressure.  Anon pages
1512                          * are not likely to be evicted by use-once streaming
1513                          * IO, plus JVM can create lots of anon VM_EXEC pages,
1514                          * so we ignore them here.
1515                          */
1516                         if ((vm_flags & VM_EXEC) && page_is_file_cache(page)) {
1517                                 list_add(&page->lru, &l_active);
1518                                 continue;
1519                         }
1520                 }
1521
1522                 ClearPageActive(page);  /* we are de-activating */
1523                 list_add(&page->lru, &l_inactive);
1524         }
1525
1526         /*
1527          * Move pages back to the lru list.
1528          */
1529         spin_lock_irq(&zone->lru_lock);
1530         /*
1531          * Count referenced pages from currently used mappings as rotated,
1532          * even though only some of them are actually re-activated.  This
1533          * helps balance scan pressure between file and anonymous pages in
1534          * get_scan_ratio.
1535          */
1536         reclaim_stat->recent_rotated[file] += nr_rotated;
1537
1538         move_active_pages_to_lru(zone, &l_active, &l_hold,
1539                                                 LRU_ACTIVE + file * LRU_FILE);
1540         move_active_pages_to_lru(zone, &l_inactive, &l_hold,
1541                                                 LRU_BASE   + file * LRU_FILE);
1542         __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, -nr_taken);
1543         spin_unlock_irq(&zone->lru_lock);
1544
1545         free_hot_cold_page_list(&l_hold, 1);
1546 }
1547
1548 #ifdef CONFIG_SWAP
1549 static int inactive_anon_is_low_global(struct zone *zone)
1550 {
1551         unsigned long active, inactive;
1552
1553         active = zone_page_state(zone, NR_ACTIVE_ANON);
1554         inactive = zone_page_state(zone, NR_INACTIVE_ANON);
1555
1556         if (inactive * zone->inactive_ratio < active)
1557                 return 1;
1558
1559         return 0;
1560 }
1561
1562 /**
1563  * inactive_anon_is_low - check if anonymous pages need to be deactivated
1564  * @zone: zone to check
1565  * @sc:   scan control of this context
1566  *
1567  * Returns true if the zone does not have enough inactive anon pages,
1568  * meaning some active anon pages need to be deactivated.
1569  */
1570 static int inactive_anon_is_low(struct mem_cgroup_zone *mz)
1571 {
1572         /*
1573          * If we don't have swap space, anonymous page deactivation
1574          * is pointless.
1575          */
1576         if (!total_swap_pages)
1577                 return 0;
1578
1579         if (!mem_cgroup_disabled())
1580                 return mem_cgroup_inactive_anon_is_low(mz->mem_cgroup,
1581                                                        mz->zone);
1582
1583         return inactive_anon_is_low_global(mz->zone);
1584 }
1585 #else
1586 static inline int inactive_anon_is_low(struct mem_cgroup_zone *mz)
1587 {
1588         return 0;
1589 }
1590 #endif
1591
1592 static int inactive_file_is_low_global(struct zone *zone)
1593 {
1594         unsigned long active, inactive;
1595
1596         active = zone_page_state(zone, NR_ACTIVE_FILE);
1597         inactive = zone_page_state(zone, NR_INACTIVE_FILE);
1598
1599         return (active > inactive);
1600 }
1601
1602 /**
1603  * inactive_file_is_low - check if file pages need to be deactivated
1604  * @mz: memory cgroup and zone to check
1605  *
1606  * When the system is doing streaming IO, memory pressure here
1607  * ensures that active file pages get deactivated, until more
1608  * than half of the file pages are on the inactive list.
1609  *
1610  * Once we get to that situation, protect the system's working
1611  * set from being evicted by disabling active file page aging.
1612  *
1613  * This uses a different ratio than the anonymous pages, because
1614  * the page cache uses a use-once replacement algorithm.
1615  */
1616 static int inactive_file_is_low(struct mem_cgroup_zone *mz)
1617 {
1618         if (!mem_cgroup_disabled())
1619                 return mem_cgroup_inactive_file_is_low(mz->mem_cgroup,
1620                                                        mz->zone);
1621
1622         return inactive_file_is_low_global(mz->zone);
1623 }
1624
1625 static int inactive_list_is_low(struct mem_cgroup_zone *mz, int file)
1626 {
1627         if (file)
1628                 return inactive_file_is_low(mz);
1629         else
1630                 return inactive_anon_is_low(mz);
1631 }
1632
1633 static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
1634                                  struct mem_cgroup_zone *mz,
1635                                  struct scan_control *sc, int priority)
1636 {
1637         int file = is_file_lru(lru);
1638
1639         if (is_active_lru(lru)) {
1640                 if (inactive_list_is_low(mz, file))
1641                         shrink_active_list(nr_to_scan, mz, sc, priority, file);
1642                 return 0;
1643         }
1644
1645         return shrink_inactive_list(nr_to_scan, mz, sc, priority, file);
1646 }
1647
1648 static int vmscan_swappiness(struct mem_cgroup_zone *mz,
1649                              struct scan_control *sc)
1650 {
1651         if (global_reclaim(sc))
1652                 return vm_swappiness;
1653         return mem_cgroup_swappiness(mz->mem_cgroup);
1654 }
1655
1656 /*
1657  * Determine how aggressively the anon and file LRU lists should be
1658  * scanned.  The relative value of each set of LRU lists is determined
1659  * by looking at the fraction of the pages scanned we did rotate back
1660  * onto the active list instead of evict.
1661  *
1662  * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
1663  * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
1664  */
1665 static void get_scan_count(struct mem_cgroup_zone *mz, struct scan_control *sc,
1666                            unsigned long *nr, int priority)
1667 {
1668         unsigned long anon, file, free;
1669         unsigned long anon_prio, file_prio;
1670         unsigned long ap, fp;
1671         struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(mz);
1672         u64 fraction[2], denominator;
1673         enum lru_list lru;
1674         int noswap = 0;
1675         bool force_scan = false;
1676
1677         /*
1678          * If the zone or memcg is small, nr[l] can be 0.  This
1679          * results in no scanning on this priority and a potential
1680          * priority drop.  Global direct reclaim can go to the next
1681          * zone and tends to have no problems. Global kswapd is for
1682          * zone balancing and it needs to scan a minimum amount. When
1683          * reclaiming for a memcg, a priority drop can cause high
1684          * latencies, so it's better to scan a minimum amount there as
1685          * well.
1686          */
1687         if (current_is_kswapd() && mz->zone->all_unreclaimable)
1688                 force_scan = true;
1689         if (!global_reclaim(sc))
1690                 force_scan = true;
1691
1692         /* If we have no swap space, do not bother scanning anon pages. */
1693         if (!sc->may_swap || (nr_swap_pages <= 0)) {
1694                 noswap = 1;
1695                 fraction[0] = 0;
1696                 fraction[1] = 1;
1697                 denominator = 1;
1698                 goto out;
1699         }
1700
1701         anon  = zone_nr_lru_pages(mz, LRU_ACTIVE_ANON) +
1702                 zone_nr_lru_pages(mz, LRU_INACTIVE_ANON);
1703         file  = zone_nr_lru_pages(mz, LRU_ACTIVE_FILE) +
1704                 zone_nr_lru_pages(mz, LRU_INACTIVE_FILE);
1705
1706         if (global_reclaim(sc)) {
1707                 free  = zone_page_state(mz->zone, NR_FREE_PAGES);
1708                 /* If we have very few page cache pages,
1709                    force-scan anon pages. */
1710                 if (unlikely(file + free <= high_wmark_pages(mz->zone))) {
1711                         fraction[0] = 1;
1712                         fraction[1] = 0;
1713                         denominator = 1;
1714                         goto out;
1715                 }
1716         }
1717
1718         /*
1719          * With swappiness at 100, anonymous and file have the same priority.
1720          * This scanning priority is essentially the inverse of IO cost.
1721          */
1722         anon_prio = vmscan_swappiness(mz, sc);
1723         file_prio = 200 - vmscan_swappiness(mz, sc);
1724
1725         /*
1726          * OK, so we have swap space and a fair amount of page cache
1727          * pages.  We use the recently rotated / recently scanned
1728          * ratios to determine how valuable each cache is.
1729          *
1730          * Because workloads change over time (and to avoid overflow)
1731          * we keep these statistics as a floating average, which ends
1732          * up weighing recent references more than old ones.
1733          *
1734          * anon in [0], file in [1]
1735          */
1736         spin_lock_irq(&mz->zone->lru_lock);
1737         if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) {
1738                 reclaim_stat->recent_scanned[0] /= 2;
1739                 reclaim_stat->recent_rotated[0] /= 2;
1740         }
1741
1742         if (unlikely(reclaim_stat->recent_scanned[1] > file / 4)) {
1743                 reclaim_stat->recent_scanned[1] /= 2;
1744                 reclaim_stat->recent_rotated[1] /= 2;
1745         }
1746
1747         /*
1748          * The amount of pressure on anon vs file pages is inversely
1749          * proportional to the fraction of recently scanned pages on
1750          * each list that were recently referenced and in active use.
1751          */
1752         ap = anon_prio * (reclaim_stat->recent_scanned[0] + 1);
1753         ap /= reclaim_stat->recent_rotated[0] + 1;
1754
1755         fp = file_prio * (reclaim_stat->recent_scanned[1] + 1);
1756         fp /= reclaim_stat->recent_rotated[1] + 1;
1757         spin_unlock_irq(&mz->zone->lru_lock);
1758
1759         fraction[0] = ap;
1760         fraction[1] = fp;
1761         denominator = ap + fp + 1;
1762 out:
1763         for_each_evictable_lru(lru) {
1764                 int file = is_file_lru(lru);
1765                 unsigned long scan;
1766
1767                 scan = zone_nr_lru_pages(mz, lru);
1768                 if (priority || noswap || !vmscan_swappiness(mz, sc)) {
1769                         scan >>= priority;
1770                         if (!scan && force_scan)
1771                                 scan = SWAP_CLUSTER_MAX;
1772                         scan = div64_u64(scan * fraction[file], denominator);
1773                 }
1774                 nr[lru] = scan;
1775         }
1776 }
1777
1778 /* Use reclaim/compaction for costly allocs or under memory pressure */
1779 static bool in_reclaim_compaction(int priority, struct scan_control *sc)
1780 {
1781         if (COMPACTION_BUILD && sc->order &&
1782                         (sc->order > PAGE_ALLOC_COSTLY_ORDER ||
1783                          priority < DEF_PRIORITY - 2))
1784                 return true;
1785
1786         return false;
1787 }
1788
1789 /*
1790  * Reclaim/compaction is used for high-order allocation requests. It reclaims
1791  * order-0 pages before compacting the zone. should_continue_reclaim() returns
1792  * true if more pages should be reclaimed such that when the page allocator
1793  * calls try_to_compact_zone() that it will have enough free pages to succeed.
1794  * It will give up earlier than that if there is difficulty reclaiming pages.
1795  */
1796 static inline bool should_continue_reclaim(struct mem_cgroup_zone *mz,
1797                                         unsigned long nr_reclaimed,
1798                                         unsigned long nr_scanned,
1799                                         int priority,
1800                                         struct scan_control *sc)
1801 {
1802         unsigned long pages_for_compaction;
1803         unsigned long inactive_lru_pages;
1804
1805         /* If not in reclaim/compaction mode, stop */
1806         if (!in_reclaim_compaction(priority, sc))
1807                 return false;
1808
1809         /* Consider stopping depending on scan and reclaim activity */
1810         if (sc->gfp_mask & __GFP_REPEAT) {
1811                 /*
1812                  * For __GFP_REPEAT allocations, stop reclaiming if the
1813                  * full LRU list has been scanned and we are still failing
1814                  * to reclaim pages. This full LRU scan is potentially
1815                  * expensive but a __GFP_REPEAT caller really wants to succeed
1816                  */
1817                 if (!nr_reclaimed && !nr_scanned)
1818                         return false;
1819         } else {
1820                 /*
1821                  * For non-__GFP_REPEAT allocations which can presumably
1822                  * fail without consequence, stop if we failed to reclaim
1823                  * any pages from the last SWAP_CLUSTER_MAX number of
1824                  * pages that were scanned. This will return to the
1825                  * caller faster at the risk reclaim/compaction and
1826                  * the resulting allocation attempt fails
1827                  */
1828                 if (!nr_reclaimed)
1829                         return false;
1830         }
1831
1832         /*
1833          * If we have not reclaimed enough pages for compaction and the
1834          * inactive lists are large enough, continue reclaiming
1835          */
1836         pages_for_compaction = (2UL << sc->order);
1837         inactive_lru_pages = zone_nr_lru_pages(mz, LRU_INACTIVE_FILE);
1838         if (nr_swap_pages > 0)
1839                 inactive_lru_pages += zone_nr_lru_pages(mz, LRU_INACTIVE_ANON);
1840         if (sc->nr_reclaimed < pages_for_compaction &&
1841                         inactive_lru_pages > pages_for_compaction)
1842                 return true;
1843
1844         /* If compaction would go ahead or the allocation would succeed, stop */
1845         switch (compaction_suitable(mz->zone, sc->order)) {
1846         case COMPACT_PARTIAL:
1847         case COMPACT_CONTINUE:
1848                 return false;
1849         default:
1850                 return true;
1851         }
1852 }
1853
1854 /*
1855  * This is a basic per-zone page freer.  Used by both kswapd and direct reclaim.
1856  */
1857 static void shrink_mem_cgroup_zone(int priority, struct mem_cgroup_zone *mz,
1858                                    struct scan_control *sc)
1859 {
1860         unsigned long nr[NR_LRU_LISTS];
1861         unsigned long nr_to_scan;
1862         enum lru_list lru;
1863         unsigned long nr_reclaimed, nr_scanned;
1864         unsigned long nr_to_reclaim = sc->nr_to_reclaim;
1865         struct blk_plug plug;
1866
1867 restart:
1868         nr_reclaimed = 0;
1869         nr_scanned = sc->nr_scanned;
1870         get_scan_count(mz, sc, nr, priority);
1871
1872         blk_start_plug(&plug);
1873         while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
1874                                         nr[LRU_INACTIVE_FILE]) {
1875                 for_each_evictable_lru(lru) {
1876                         if (nr[lru]) {
1877                                 nr_to_scan = min_t(unsigned long,
1878                                                    nr[lru], SWAP_CLUSTER_MAX);
1879                                 nr[lru] -= nr_to_scan;
1880
1881                                 nr_reclaimed += shrink_list(lru, nr_to_scan,
1882                                                             mz, sc, priority);
1883                         }
1884                 }
1885                 /*
1886                  * On large memory systems, scan >> priority can become
1887                  * really large. This is fine for the starting priority;
1888                  * we want to put equal scanning pressure on each zone.
1889                  * However, if the VM has a harder time of freeing pages,
1890                  * with multiple processes reclaiming pages, the total
1891                  * freeing target can get unreasonably large.
1892                  */
1893                 if (nr_reclaimed >= nr_to_reclaim && priority < DEF_PRIORITY)
1894                         break;
1895         }
1896         blk_finish_plug(&plug);
1897         sc->nr_reclaimed += nr_reclaimed;
1898
1899         /*
1900          * Even if we did not try to evict anon pages at all, we want to
1901          * rebalance the anon lru active/inactive ratio.
1902          */
1903         if (inactive_anon_is_low(mz))
1904                 shrink_active_list(SWAP_CLUSTER_MAX, mz, sc, priority, 0);
1905
1906         /* reclaim/compaction might need reclaim to continue */
1907         if (should_continue_reclaim(mz, nr_reclaimed,
1908                                         sc->nr_scanned - nr_scanned,
1909                                         priority, sc))
1910                 goto restart;
1911
1912         throttle_vm_writeout(sc->gfp_mask);
1913 }
1914
1915 static void shrink_zone(int priority, struct zone *zone,
1916                         struct scan_control *sc)
1917 {
1918         struct mem_cgroup *root = sc->target_mem_cgroup;
1919         struct mem_cgroup_reclaim_cookie reclaim = {
1920                 .zone = zone,
1921                 .priority = priority,
1922         };
1923         struct mem_cgroup *memcg;
1924
1925         memcg = mem_cgroup_iter(root, NULL, &reclaim);
1926         do {
1927                 struct mem_cgroup_zone mz = {
1928                         .mem_cgroup = memcg,
1929                         .zone = zone,
1930                 };
1931
1932                 shrink_mem_cgroup_zone(priority, &mz, sc);
1933                 /*
1934                  * Limit reclaim has historically picked one memcg and
1935                  * scanned it with decreasing priority levels until
1936                  * nr_to_reclaim had been reclaimed.  This priority
1937                  * cycle is thus over after a single memcg.
1938                  *
1939                  * Direct reclaim and kswapd, on the other hand, have
1940                  * to scan all memory cgroups to fulfill the overall
1941                  * scan target for the zone.
1942                  */
1943                 if (!global_reclaim(sc)) {
1944                         mem_cgroup_iter_break(root, memcg);
1945                         break;
1946                 }
1947                 memcg = mem_cgroup_iter(root, memcg, &reclaim);
1948         } while (memcg);
1949 }
1950
1951 /* Returns true if compaction should go ahead for a high-order request */
1952 static inline bool compaction_ready(struct zone *zone, struct scan_control *sc)
1953 {
1954         unsigned long balance_gap, watermark;
1955         bool watermark_ok;
1956
1957         /* Do not consider compaction for orders reclaim is meant to satisfy */
1958         if (sc->order <= PAGE_ALLOC_COSTLY_ORDER)
1959                 return false;
1960
1961         /*
1962          * Compaction takes time to run and there are potentially other
1963          * callers using the pages just freed. Continue reclaiming until
1964          * there is a buffer of free pages available to give compaction
1965          * a reasonable chance of completing and allocating the page
1966          */
1967         balance_gap = min(low_wmark_pages(zone),
1968                 (zone->present_pages + KSWAPD_ZONE_BALANCE_GAP_RATIO-1) /
1969                         KSWAPD_ZONE_BALANCE_GAP_RATIO);
1970         watermark = high_wmark_pages(zone) + balance_gap + (2UL << sc->order);
1971         watermark_ok = zone_watermark_ok_safe(zone, 0, watermark, 0, 0);
1972
1973         /*
1974          * If compaction is deferred, reclaim up to a point where
1975          * compaction will have a chance of success when re-enabled
1976          */
1977         if (compaction_deferred(zone, sc->order))
1978                 return watermark_ok;
1979
1980         /* If compaction is not ready to start, keep reclaiming */
1981         if (!compaction_suitable(zone, sc->order))
1982                 return false;
1983
1984         return watermark_ok;
1985 }
1986
1987 /*
1988  * This is the direct reclaim path, for page-allocating processes.  We only
1989  * try to reclaim pages from zones which will satisfy the caller's allocation
1990  * request.
1991  *
1992  * We reclaim from a zone even if that zone is over high_wmark_pages(zone).
1993  * Because:
1994  * a) The caller may be trying to free *extra* pages to satisfy a higher-order
1995  *    allocation or
1996  * b) The target zone may be at high_wmark_pages(zone) but the lower zones
1997  *    must go *over* high_wmark_pages(zone) to satisfy the `incremental min'
1998  *    zone defense algorithm.
1999  *
2000  * If a zone is deemed to be full of pinned pages then just give it a light
2001  * scan then give up on it.
2002  *
2003  * This function returns true if a zone is being reclaimed for a costly
2004  * high-order allocation and compaction is ready to begin. This indicates to
2005  * the caller that it should consider retrying the allocation instead of
2006  * further reclaim.
2007  */
2008 static bool shrink_zones(int priority, struct zonelist *zonelist,
2009                                         struct scan_control *sc)
2010 {
2011         struct zoneref *z;
2012         struct zone *zone;
2013         unsigned long nr_soft_reclaimed;
2014         unsigned long nr_soft_scanned;
2015         bool aborted_reclaim = false;
2016
2017         /*
2018          * If the number of buffer_heads in the machine exceeds the maximum
2019          * allowed level, force direct reclaim to scan the highmem zone as
2020          * highmem pages could be pinning lowmem pages storing buffer_heads
2021          */
2022         if (buffer_heads_over_limit)
2023                 sc->gfp_mask |= __GFP_HIGHMEM;
2024
2025         for_each_zone_zonelist_nodemask(zone, z, zonelist,
2026                                         gfp_zone(sc->gfp_mask), sc->nodemask) {
2027                 if (!populated_zone(zone))
2028                         continue;
2029                 /*
2030                  * Take care memory controller reclaiming has small influence
2031                  * to global LRU.
2032                  */
2033                 if (global_reclaim(sc)) {
2034                         if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
2035                                 continue;
2036                         if (zone->all_unreclaimable && priority != DEF_PRIORITY)
2037                                 continue;       /* Let kswapd poll it */
2038                         if (COMPACTION_BUILD) {
2039                                 /*
2040                                  * If we already have plenty of memory free for
2041                                  * compaction in this zone, don't free any more.
2042                                  * Even though compaction is invoked for any
2043                                  * non-zero order, only frequent costly order
2044                                  * reclamation is disruptive enough to become a
2045                                  * noticeable problem, like transparent huge
2046                                  * page allocations.
2047                                  */
2048                                 if (compaction_ready(zone, sc)) {
2049                                         aborted_reclaim = true;
2050                                         continue;
2051                                 }
2052                         }
2053                         /*
2054                          * This steals pages from memory cgroups over softlimit
2055                          * and returns the number of reclaimed pages and
2056                          * scanned pages. This works for global memory pressure
2057                          * and balancing, not for a memcg's limit.
2058                          */
2059                         nr_soft_scanned = 0;
2060                         nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone,
2061                                                 sc->order, sc->gfp_mask,
2062                                                 &nr_soft_scanned);
2063                         sc->nr_reclaimed += nr_soft_reclaimed;
2064                         sc->nr_scanned += nr_soft_scanned;
2065                         /* need some check for avoid more shrink_zone() */
2066                 }
2067
2068                 shrink_zone(priority, zone, sc);
2069         }
2070
2071         return aborted_reclaim;
2072 }
2073
2074 static bool zone_reclaimable(struct zone *zone)
2075 {
2076         return zone->pages_scanned < zone_reclaimable_pages(zone) * 6;
2077 }
2078
2079 /* All zones in zonelist are unreclaimable? */
2080 static bool all_unreclaimable(struct zonelist *zonelist,
2081                 struct scan_control *sc)
2082 {
2083         struct zoneref *z;
2084         struct zone *zone;
2085
2086         for_each_zone_zonelist_nodemask(zone, z, zonelist,
2087                         gfp_zone(sc->gfp_mask), sc->nodemask) {
2088                 if (!populated_zone(zone))
2089                         continue;
2090                 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
2091                         continue;
2092                 if (!zone->all_unreclaimable)
2093                         return false;
2094         }
2095
2096         return true;
2097 }
2098
2099 /*
2100  * This is the main entry point to direct page reclaim.
2101  *
2102  * If a full scan of the inactive list fails to free enough memory then we
2103  * are "out of memory" and something needs to be killed.
2104  *
2105  * If the caller is !__GFP_FS then the probability of a failure is reasonably
2106  * high - the zone may be full of dirty or under-writeback pages, which this
2107  * caller can't do much about.  We kick the writeback threads and take explicit
2108  * naps in the hope that some of these pages can be written.  But if the
2109  * allocating task holds filesystem locks which prevent writeout this might not
2110  * work, and the allocation attempt will fail.
2111  *
2112  * returns:     0, if no pages reclaimed
2113  *              else, the number of pages reclaimed
2114  */
2115 static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
2116                                         struct scan_control *sc,
2117                                         struct shrink_control *shrink)
2118 {
2119         int priority;
2120         unsigned long total_scanned = 0;
2121         struct reclaim_state *reclaim_state = current->reclaim_state;
2122         struct zoneref *z;
2123         struct zone *zone;
2124         unsigned long writeback_threshold;
2125         bool aborted_reclaim;
2126
2127         delayacct_freepages_start();
2128
2129         if (global_reclaim(sc))
2130                 count_vm_event(ALLOCSTALL);
2131
2132         for (priority = DEF_PRIORITY; priority >= 0; priority--) {
2133                 sc->nr_scanned = 0;
2134                 aborted_reclaim = shrink_zones(priority, zonelist, sc);
2135
2136                 /*
2137                  * Don't shrink slabs when reclaiming memory from
2138                  * over limit cgroups
2139                  */
2140                 if (global_reclaim(sc)) {
2141                         unsigned long lru_pages = 0;
2142                         for_each_zone_zonelist(zone, z, zonelist,
2143                                         gfp_zone(sc->gfp_mask)) {
2144                                 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
2145                                         continue;
2146
2147                                 lru_pages += zone_reclaimable_pages(zone);
2148                         }
2149
2150                         shrink_slab(shrink, sc->nr_scanned, lru_pages);
2151                         if (reclaim_state) {
2152                                 sc->nr_reclaimed += reclaim_state->reclaimed_slab;
2153                                 reclaim_state->reclaimed_slab = 0;
2154                         }
2155                 }
2156                 total_scanned += sc->nr_scanned;
2157                 if (sc->nr_reclaimed >= sc->nr_to_reclaim)
2158                         goto out;
2159
2160                 /*
2161                  * Try to write back as many pages as we just scanned.  This
2162                  * tends to cause slow streaming writers to write data to the
2163                  * disk smoothly, at the dirtying rate, which is nice.   But
2164                  * that's undesirable in laptop mode, where we *want* lumpy
2165                  * writeout.  So in laptop mode, write out the whole world.
2166                  */
2167                 writeback_threshold = sc->nr_to_reclaim + sc->nr_to_reclaim / 2;
2168                 if (total_scanned > writeback_threshold) {
2169                         wakeup_flusher_threads(laptop_mode ? 0 : total_scanned,
2170                                                 WB_REASON_TRY_TO_FREE_PAGES);
2171                         sc->may_writepage = 1;
2172                 }
2173
2174                 /* Take a nap, wait for some writeback to complete */
2175                 if (!sc->hibernation_mode && sc->nr_scanned &&
2176                     priority < DEF_PRIORITY - 2) {
2177                         struct zone *preferred_zone;
2178
2179                         first_zones_zonelist(zonelist, gfp_zone(sc->gfp_mask),
2180                                                 &cpuset_current_mems_allowed,
2181                                                 &preferred_zone);
2182                         wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/10);
2183                 }
2184         }
2185
2186 out:
2187         delayacct_freepages_end();
2188
2189         if (sc->nr_reclaimed)
2190                 return sc->nr_reclaimed;
2191
2192         /*
2193          * As hibernation is going on, kswapd is freezed so that it can't mark
2194          * the zone into all_unreclaimable. Thus bypassing all_unreclaimable
2195          * check.
2196          */
2197         if (oom_killer_disabled)
2198                 return 0;
2199
2200         /* Aborted reclaim to try compaction? don't OOM, then */
2201         if (aborted_reclaim)
2202                 return 1;
2203
2204         /* top priority shrink_zones still had more to do? don't OOM, then */
2205         if (global_reclaim(sc) && !all_unreclaimable(zonelist, sc))
2206                 return 1;
2207
2208         return 0;
2209 }
2210
2211 unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
2212                                 gfp_t gfp_mask, nodemask_t *nodemask)
2213 {
2214         unsigned long nr_reclaimed;
2215         struct scan_control sc = {
2216                 .gfp_mask = gfp_mask,
2217                 .may_writepage = !laptop_mode,
2218                 .nr_to_reclaim = SWAP_CLUSTER_MAX,
2219                 .may_unmap = 1,
2220                 .may_swap = 1,
2221                 .order = order,
2222                 .target_mem_cgroup = NULL,
2223                 .nodemask = nodemask,
2224         };
2225         struct shrink_control shrink = {
2226                 .gfp_mask = sc.gfp_mask,
2227         };
2228
2229         trace_mm_vmscan_direct_reclaim_begin(order,
2230                                 sc.may_writepage,
2231                                 gfp_mask);
2232
2233         nr_reclaimed = do_try_to_free_pages(zonelist, &sc, &shrink);
2234
2235         trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);
2236
2237         return nr_reclaimed;
2238 }
2239
2240 #ifdef CONFIG_CGROUP_MEM_RES_CTLR
2241
2242 unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *memcg,
2243                                                 gfp_t gfp_mask, bool noswap,
2244                                                 struct zone *zone,
2245                                                 unsigned long *nr_scanned)
2246 {
2247         struct scan_control sc = {
2248                 .nr_scanned = 0,
2249                 .nr_to_reclaim = SWAP_CLUSTER_MAX,
2250                 .may_writepage = !laptop_mode,
2251                 .may_unmap = 1,
2252                 .may_swap = !noswap,
2253                 .order = 0,
2254                 .target_mem_cgroup = memcg,
2255         };
2256         struct mem_cgroup_zone mz = {
2257                 .mem_cgroup = memcg,
2258                 .zone = zone,
2259         };
2260
2261         sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
2262                         (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
2263
2264         trace_mm_vmscan_memcg_softlimit_reclaim_begin(0,
2265                                                       sc.may_writepage,
2266                                                       sc.gfp_mask);
2267
2268         /*
2269          * NOTE: Although we can get the priority field, using it
2270          * here is not a good idea, since it limits the pages we can scan.
2271          * if we don't reclaim here, the shrink_zone from balance_pgdat
2272          * will pick up pages from other mem cgroup's as well. We hack
2273          * the priority and make it zero.
2274          */
2275         shrink_mem_cgroup_zone(0, &mz, &sc);
2276
2277         trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
2278
2279         *nr_scanned = sc.nr_scanned;
2280         return sc.nr_reclaimed;
2281 }
2282
2283 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
2284                                            gfp_t gfp_mask,
2285                                            bool noswap)
2286 {
2287         struct zonelist *zonelist;
2288         unsigned long nr_reclaimed;
2289         int nid;
2290         struct scan_control sc = {
2291                 .may_writepage = !laptop_mode,
2292                 .may_unmap = 1,
2293                 .may_swap = !noswap,
2294                 .nr_to_reclaim = SWAP_CLUSTER_MAX,
2295                 .order = 0,
2296                 .target_mem_cgroup = memcg,
2297                 .nodemask = NULL, /* we don't care the placement */
2298                 .gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
2299                                 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK),
2300         };
2301         struct shrink_control shrink = {
2302                 .gfp_mask = sc.gfp_mask,
2303         };
2304
2305         /*
2306          * Unlike direct reclaim via alloc_pages(), memcg's reclaim doesn't
2307          * take care of from where we get pages. So the node where we start the
2308          * scan does not need to be the current node.
2309          */
2310         nid = mem_cgroup_select_victim_node(memcg);
2311
2312         zonelist = NODE_DATA(nid)->node_zonelists;
2313
2314         trace_mm_vmscan_memcg_reclaim_begin(0,
2315                                             sc.may_writepage,
2316                                             sc.gfp_mask);
2317
2318         nr_reclaimed = do_try_to_free_pages(zonelist, &sc, &shrink);
2319
2320         trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
2321
2322         return nr_reclaimed;
2323 }
2324 #endif
2325
2326 static void age_active_anon(struct zone *zone, struct scan_control *sc,
2327                             int priority)
2328 {
2329         struct mem_cgroup *memcg;
2330
2331         if (!total_swap_pages)
2332                 return;
2333
2334         memcg = mem_cgroup_iter(NULL, NULL, NULL);
2335         do {
2336                 struct mem_cgroup_zone mz = {
2337                         .mem_cgroup = memcg,
2338                         .zone = zone,
2339                 };
2340
2341                 if (inactive_anon_is_low(&mz))
2342                         shrink_active_list(SWAP_CLUSTER_MAX, &mz,
2343                                            sc, priority, 0);
2344
2345                 memcg = mem_cgroup_iter(NULL, memcg, NULL);
2346         } while (memcg);
2347 }
2348
2349 /*
2350  * pgdat_balanced is used when checking if a node is balanced for high-order
2351  * allocations. Only zones that meet watermarks and are in a zone allowed
2352  * by the callers classzone_idx are added to balanced_pages. The total of
2353  * balanced pages must be at least 25% of the zones allowed by classzone_idx
2354  * for the node to be considered balanced. Forcing all zones to be balanced
2355  * for high orders can cause excessive reclaim when there are imbalanced zones.
2356  * The choice of 25% is due to
2357  *   o a 16M DMA zone that is balanced will not balance a zone on any
2358  *     reasonable sized machine
2359  *   o On all other machines, the top zone must be at least a reasonable
2360  *     percentage of the middle zones. For example, on 32-bit x86, highmem
2361  *     would need to be at least 256M for it to be balance a whole node.
2362  *     Similarly, on x86-64 the Normal zone would need to be at least 1G
2363  *     to balance a node on its own. These seemed like reasonable ratios.
2364  */
2365 static bool pgdat_balanced(pg_data_t *pgdat, unsigned long balanced_pages,
2366                                                 int classzone_idx)
2367 {
2368         unsigned long present_pages = 0;
2369         int i;
2370
2371         for (i = 0; i <= classzone_idx; i++)
2372                 present_pages += pgdat->node_zones[i].present_pages;
2373
2374         /* A special case here: if zone has no page, we think it's balanced */
2375         return balanced_pages >= (present_pages >> 2);
2376 }
2377
2378 /* is kswapd sleeping prematurely? */
2379 static bool sleeping_prematurely(pg_data_t *pgdat, int order, long remaining,
2380                                         int classzone_idx)
2381 {
2382         int i;
2383         unsigned long balanced = 0;
2384         bool all_zones_ok = true;
2385
2386         /* If a direct reclaimer woke kswapd within HZ/10, it's premature */
2387         if (remaining)
2388                 return true;
2389
2390         /* Check the watermark levels */
2391         for (i = 0; i <= classzone_idx; i++) {
2392                 struct zone *zone = pgdat->node_zones + i;
2393
2394                 if (!populated_zone(zone))
2395                         continue;
2396
2397                 /*
2398                  * balance_pgdat() skips over all_unreclaimable after
2399                  * DEF_PRIORITY. Effectively, it considers them balanced so
2400                  * they must be considered balanced here as well if kswapd
2401                  * is to sleep
2402                  */
2403                 if (zone->all_unreclaimable) {
2404                         balanced += zone->present_pages;
2405                         continue;
2406                 }
2407
2408                 if (!zone_watermark_ok_safe(zone, order, high_wmark_pages(zone),
2409                                                         i, 0))
2410                         all_zones_ok = false;
2411                 else
2412                         balanced += zone->present_pages;
2413         }
2414
2415         /*
2416          * For high-order requests, the balanced zones must contain at least
2417          * 25% of the nodes pages for kswapd to sleep. For order-0, all zones
2418          * must be balanced
2419          */
2420         if (order)
2421                 return !pgdat_balanced(pgdat, balanced, classzone_idx);
2422         else
2423                 return !all_zones_ok;
2424 }
2425
2426 /*
2427  * For kswapd, balance_pgdat() will work across all this node's zones until
2428  * they are all at high_wmark_pages(zone).
2429  *
2430  * Returns the final order kswapd was reclaiming at
2431  *
2432  * There is special handling here for zones which are full of pinned pages.
2433  * This can happen if the pages are all mlocked, or if they are all used by
2434  * device drivers (say, ZONE_DMA).  Or if they are all in use by hugetlb.
2435  * What we do is to detect the case where all pages in the zone have been
2436  * scanned twice and there has been zero successful reclaim.  Mark the zone as
2437  * dead and from now on, only perform a short scan.  Basically we're polling
2438  * the zone for when the problem goes away.
2439  *
2440  * kswapd scans the zones in the highmem->normal->dma direction.  It skips
2441  * zones which have free_pages > high_wmark_pages(zone), but once a zone is
2442  * found to have free_pages <= high_wmark_pages(zone), we scan that zone and the
2443  * lower zones regardless of the number of free pages in the lower zones. This
2444  * interoperates with the page allocator fallback scheme to ensure that aging
2445  * of pages is balanced across the zones.
2446  */
2447 static unsigned long balance_pgdat(pg_data_t *pgdat, int order,
2448                                                         int *classzone_idx)
2449 {
2450         int all_zones_ok;
2451         unsigned long balanced;
2452         int priority;
2453         int i;
2454         int end_zone = 0;       /* Inclusive.  0 = ZONE_DMA */
2455         unsigned long total_scanned;
2456         struct reclaim_state *reclaim_state = current->reclaim_state;
2457         unsigned long nr_soft_reclaimed;
2458         unsigned long nr_soft_scanned;
2459         struct scan_control sc = {
2460                 .gfp_mask = GFP_KERNEL,
2461                 .may_unmap = 1,
2462                 .may_swap = 1,
2463                 /*
2464                  * kswapd doesn't want to be bailed out while reclaim. because
2465                  * we want to put equal scanning pressure on each zone.
2466                  */
2467                 .nr_to_reclaim = ULONG_MAX,
2468                 .order = order,
2469                 .target_mem_cgroup = NULL,
2470         };
2471         struct shrink_control shrink = {
2472                 .gfp_mask = sc.gfp_mask,
2473         };
2474 loop_again:
2475         total_scanned = 0;
2476         sc.nr_reclaimed = 0;
2477         sc.may_writepage = !laptop_mode;
2478         count_vm_event(PAGEOUTRUN);
2479
2480         for (priority = DEF_PRIORITY; priority >= 0; priority--) {
2481                 unsigned long lru_pages = 0;
2482                 int has_under_min_watermark_zone = 0;
2483
2484                 all_zones_ok = 1;
2485                 balanced = 0;
2486
2487                 /*
2488                  * Scan in the highmem->dma direction for the highest
2489                  * zone which needs scanning
2490                  */
2491                 for (i = pgdat->nr_zones - 1; i >= 0; i--) {
2492                         struct zone *zone = pgdat->node_zones + i;
2493
2494                         if (!populated_zone(zone))
2495                                 continue;
2496
2497                         if (zone->all_unreclaimable && priority != DEF_PRIORITY)
2498                                 continue;
2499
2500                         /*
2501                          * Do some background aging of the anon list, to give
2502                          * pages a chance to be referenced before reclaiming.
2503                          */
2504                         age_active_anon(zone, &sc, priority);
2505
2506                         /*
2507                          * If the number of buffer_heads in the machine
2508                          * exceeds the maximum allowed level and this node
2509                          * has a highmem zone, force kswapd to reclaim from
2510                          * it to relieve lowmem pressure.
2511                          */
2512                         if (buffer_heads_over_limit && is_highmem_idx(i)) {
2513                                 end_zone = i;
2514                                 break;
2515                         }
2516
2517                         if (!zone_watermark_ok_safe(zone, order,
2518                                         high_wmark_pages(zone), 0, 0)) {
2519                                 end_zone = i;
2520                                 break;
2521                         } else {
2522                                 /* If balanced, clear the congested flag */
2523                                 zone_clear_flag(zone, ZONE_CONGESTED);
2524                         }
2525                 }
2526                 if (i < 0)
2527                         goto out;
2528
2529                 for (i = 0; i <= end_zone; i++) {
2530                         struct zone *zone = pgdat->node_zones + i;
2531
2532                         lru_pages += zone_reclaimable_pages(zone);
2533                 }
2534
2535                 /*
2536                  * Now scan the zone in the dma->highmem direction, stopping
2537                  * at the last zone which needs scanning.
2538                  *
2539                  * We do this because the page allocator works in the opposite
2540                  * direction.  This prevents the page allocator from allocating
2541                  * pages behind kswapd's direction of progress, which would
2542                  * cause too much scanning of the lower zones.
2543                  */
2544                 for (i = 0; i <= end_zone; i++) {
2545                         struct zone *zone = pgdat->node_zones + i;
2546                         int nr_slab, testorder;
2547                         unsigned long balance_gap;
2548
2549                         if (!populated_zone(zone))
2550                                 continue;
2551
2552                         if (zone->all_unreclaimable && priority != DEF_PRIORITY)
2553                                 continue;
2554
2555                         sc.nr_scanned = 0;
2556
2557                         nr_soft_scanned = 0;
2558                         /*
2559                          * Call soft limit reclaim before calling shrink_zone.
2560                          */
2561                         nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone,
2562                                                         order, sc.gfp_mask,
2563                                                         &nr_soft_scanned);
2564                         sc.nr_reclaimed += nr_soft_reclaimed;
2565                         total_scanned += nr_soft_scanned;
2566
2567                         /*
2568                          * We put equal pressure on every zone, unless
2569                          * one zone has way too many pages free
2570                          * already. The "too many pages" is defined
2571                          * as the high wmark plus a "gap" where the
2572                          * gap is either the low watermark or 1%
2573                          * of the zone, whichever is smaller.
2574                          */
2575                         balance_gap = min(low_wmark_pages(zone),
2576                                 (zone->present_pages +
2577                                         KSWAPD_ZONE_BALANCE_GAP_RATIO-1) /
2578                                 KSWAPD_ZONE_BALANCE_GAP_RATIO);
2579                         /*
2580                          * Kswapd reclaims only single pages with compaction
2581                          * enabled. Trying too hard to reclaim until contiguous
2582                          * free pages have become available can hurt performance
2583                          * by evicting too much useful data from memory.
2584                          * Do not reclaim more than needed for compaction.
2585                          */
2586                         testorder = order;
2587                         if (COMPACTION_BUILD && order &&
2588                                         compaction_suitable(zone, order) !=
2589                                                 COMPACT_SKIPPED)
2590                                 testorder = 0;
2591
2592                         if ((buffer_heads_over_limit && is_highmem_idx(i)) ||
2593                                     !zone_watermark_ok_safe(zone, testorder,
2594                                         high_wmark_pages(zone) + balance_gap,
2595                                         end_zone, 0)) {
2596                                 shrink_zone(priority, zone, &sc);
2597
2598                                 reclaim_state->reclaimed_slab = 0;
2599                                 nr_slab = shrink_slab(&shrink, sc.nr_scanned, lru_pages);
2600                                 sc.nr_reclaimed += reclaim_state->reclaimed_slab;
2601                                 total_scanned += sc.nr_scanned;
2602
2603                                 if (nr_slab == 0 && !zone_reclaimable(zone))
2604                                         zone->all_unreclaimable = 1;
2605                         }
2606
2607                         /*
2608                          * If we've done a decent amount of scanning and
2609                          * the reclaim ratio is low, start doing writepage
2610                          * even in laptop mode
2611                          */
2612                         if (total_scanned > SWAP_CLUSTER_MAX * 2 &&
2613                             total_scanned > sc.nr_reclaimed + sc.nr_reclaimed / 2)
2614                                 sc.may_writepage = 1;
2615
2616                         if (zone->all_unreclaimable) {
2617                                 if (end_zone && end_zone == i)
2618                                         end_zone--;
2619                                 continue;
2620                         }
2621
2622                         if (!zone_watermark_ok_safe(zone, testorder,
2623                                         high_wmark_pages(zone), end_zone, 0)) {
2624                                 all_zones_ok = 0;
2625                                 /*
2626                                  * We are still under min water mark.  This
2627                                  * means that we have a GFP_ATOMIC allocation
2628                                  * failure risk. Hurry up!
2629                                  */
2630                                 if (!zone_watermark_ok_safe(zone, order,
2631                                             min_wmark_pages(zone), end_zone, 0))
2632                                         has_under_min_watermark_zone = 1;
2633                         } else {
2634                                 /*
2635                                  * If a zone reaches its high watermark,
2636                                  * consider it to be no longer congested. It's
2637                                  * possible there are dirty pages backed by
2638                                  * congested BDIs but as pressure is relieved,
2639                                  * speculatively avoid congestion waits
2640                                  */
2641                                 zone_clear_flag(zone, ZONE_CONGESTED);
2642                                 if (i <= *classzone_idx)
2643                                         balanced += zone->present_pages;
2644                         }
2645
2646                 }
2647                 if (all_zones_ok || (order && pgdat_balanced(pgdat, balanced, *classzone_idx)))
2648                         break;          /* kswapd: all done */
2649                 /*
2650                  * OK, kswapd is getting into trouble.  Take a nap, then take
2651                  * another pass across the zones.
2652                  */
2653                 if (total_scanned && (priority < DEF_PRIORITY - 2)) {
2654                         if (has_under_min_watermark_zone)
2655                                 count_vm_event(KSWAPD_SKIP_CONGESTION_WAIT);
2656                         else
2657                                 congestion_wait(BLK_RW_ASYNC, HZ/10);
2658                 }
2659
2660                 /*
2661                  * We do this so kswapd doesn't build up large priorities for
2662                  * example when it is freeing in parallel with allocators. It
2663                  * matches the direct reclaim path behaviour in terms of impact
2664                  * on zone->*_priority.
2665                  */
2666                 if (sc.nr_reclaimed >= SWAP_CLUSTER_MAX)
2667                         break;
2668         }
2669 out:
2670
2671         /*
2672          * order-0: All zones must meet high watermark for a balanced node
2673          * high-order: Balanced zones must make up at least 25% of the node
2674          *             for the node to be balanced
2675          */
2676         if (!(all_zones_ok || (order && pgdat_balanced(pgdat, balanced, *classzone_idx)))) {
2677                 cond_resched();
2678
2679                 try_to_freeze();
2680
2681                 /*
2682                  * Fragmentation may mean that the system cannot be
2683                  * rebalanced for high-order allocations in all zones.
2684                  * At this point, if nr_reclaimed < SWAP_CLUSTER_MAX,
2685                  * it means the zones have been fully scanned and are still
2686                  * not balanced. For high-order allocations, there is
2687                  * little point trying all over again as kswapd may
2688                  * infinite loop.
2689                  *
2690                  * Instead, recheck all watermarks at order-0 as they
2691                  * are the most important. If watermarks are ok, kswapd will go
2692                  * back to sleep. High-order users can still perform direct
2693                  * reclaim if they wish.
2694                  */
2695                 if (sc.nr_reclaimed < SWAP_CLUSTER_MAX)
2696                         order = sc.order = 0;
2697
2698                 goto loop_again;
2699         }
2700
2701         /*
2702          * If kswapd was reclaiming at a higher order, it has the option of
2703          * sleeping without all zones being balanced. Before it does, it must
2704          * ensure that the watermarks for order-0 on *all* zones are met and
2705          * that the congestion flags are cleared. The congestion flag must
2706          * be cleared as kswapd is the only mechanism that clears the flag
2707          * and it is potentially going to sleep here.
2708          */
2709         if (order) {
2710                 int zones_need_compaction = 1;
2711
2712                 for (i = 0; i <= end_zone; i++) {
2713                         struct zone *zone = pgdat->node_zones + i;
2714
2715                         if (!populated_zone(zone))
2716                                 continue;
2717
2718                         if (zone->all_unreclaimable && priority != DEF_PRIORITY)
2719                                 continue;
2720
2721                         /* Would compaction fail due to lack of free memory? */
2722                         if (COMPACTION_BUILD &&
2723                             compaction_suitable(zone, order) == COMPACT_SKIPPED)
2724                                 goto loop_again;
2725
2726                         /* Confirm the zone is balanced for order-0 */
2727                         if (!zone_watermark_ok(zone, 0,
2728                                         high_wmark_pages(zone), 0, 0)) {
2729                                 order = sc.order = 0;
2730                                 goto loop_again;
2731                         }
2732
2733                         /* Check if the memory needs to be defragmented. */
2734                         if (zone_watermark_ok(zone, order,
2735                                     low_wmark_pages(zone), *classzone_idx, 0))
2736                                 zones_need_compaction = 0;
2737
2738                         /* If balanced, clear the congested flag */
2739                         zone_clear_flag(zone, ZONE_CONGESTED);
2740                 }
2741
2742                 if (zones_need_compaction)
2743                         compact_pgdat(pgdat, order);
2744         }
2745
2746         /*
2747          * Return the order we were reclaiming at so sleeping_prematurely()
2748          * makes a decision on the order we were last reclaiming at. However,
2749          * if another caller entered the allocator slow path while kswapd
2750          * was awake, order will remain at the higher level
2751          */
2752         *classzone_idx = end_zone;
2753         return order;
2754 }
2755
2756 static void kswapd_try_to_sleep(pg_data_t *pgdat, int order, int classzone_idx)
2757 {
2758         long remaining = 0;
2759         DEFINE_WAIT(wait);
2760
2761         if (freezing(current) || kthread_should_stop())
2762                 return;
2763
2764         prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
2765
2766         /* Try to sleep for a short interval */
2767         if (!sleeping_prematurely(pgdat, order, remaining, classzone_idx)) {
2768                 remaining = schedule_timeout(HZ/10);
2769                 finish_wait(&pgdat->kswapd_wait, &wait);
2770                 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
2771         }
2772
2773         /*
2774          * After a short sleep, check if it was a premature sleep. If not, then
2775          * go fully to sleep until explicitly woken up.
2776          */
2777         if (!sleeping_prematurely(pgdat, order, remaining, classzone_idx)) {
2778                 trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
2779
2780                 /*
2781                  * vmstat counters are not perfectly accurate and the estimated
2782                  * value for counters such as NR_FREE_PAGES can deviate from the
2783                  * true value by nr_online_cpus * threshold. To avoid the zone
2784                  * watermarks being breached while under pressure, we reduce the
2785                  * per-cpu vmstat threshold while kswapd is awake and restore
2786                  * them before going back to sleep.
2787                  */
2788                 set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);
2789                 schedule();
2790                 set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);
2791         } else {
2792                 if (remaining)
2793                         count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY);
2794                 else
2795                         count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY);
2796         }
2797         finish_wait(&pgdat->kswapd_wait, &wait);
2798 }
2799
2800 /*
2801  * The background pageout daemon, started as a kernel thread
2802  * from the init process.
2803  *
2804  * This basically trickles out pages so that we have _some_
2805  * free memory available even if there is no other activity
2806  * that frees anything up. This is needed for things like routing
2807  * etc, where we otherwise might have all activity going on in
2808  * asynchronous contexts that cannot page things out.
2809  *
2810  * If there are applications that are active memory-allocators
2811  * (most normal use), this basically shouldn't matter.
2812  */
2813 static int kswapd(void *p)
2814 {
2815         unsigned long order, new_order;
2816         unsigned balanced_order;
2817         int classzone_idx, new_classzone_idx;
2818         int balanced_classzone_idx;
2819         pg_data_t *pgdat = (pg_data_t*)p;
2820         struct task_struct *tsk = current;
2821
2822         struct reclaim_state reclaim_state = {
2823                 .reclaimed_slab = 0,
2824         };
2825         const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
2826
2827         lockdep_set_current_reclaim_state(GFP_KERNEL);
2828
2829         if (!cpumask_empty(cpumask))
2830                 set_cpus_allowed_ptr(tsk, cpumask);
2831         current->reclaim_state = &reclaim_state;
2832
2833         /*
2834          * Tell the memory management that we're a "memory allocator",
2835          * and that if we need more memory we should get access to it
2836          * regardless (see "__alloc_pages()"). "kswapd" should
2837          * never get caught in the normal page freeing logic.
2838          *
2839          * (Kswapd normally doesn't need memory anyway, but sometimes
2840          * you need a small amount of memory in order to be able to
2841          * page out something else, and this flag essentially protects
2842          * us from recursively trying to free more memory as we're
2843          * trying to free the first piece of memory in the first place).
2844          */
2845         tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD;
2846         set_freezable();
2847
2848         order = new_order = 0;
2849         balanced_order = 0;
2850         classzone_idx = new_classzone_idx = pgdat->nr_zones - 1;
2851         balanced_classzone_idx = classzone_idx;
2852         for ( ; ; ) {
2853                 int ret;
2854
2855                 /*
2856                  * If the last balance_pgdat was unsuccessful it's unlikely a
2857                  * new request of a similar or harder type will succeed soon
2858                  * so consider going to sleep on the basis we reclaimed at
2859                  */
2860                 if (balanced_classzone_idx >= new_classzone_idx &&
2861                                         balanced_order == new_order) {
2862                         new_order = pgdat->kswapd_max_order;
2863                         new_classzone_idx = pgdat->classzone_idx;
2864                         pgdat->kswapd_max_order =  0;
2865                         pgdat->classzone_idx = pgdat->nr_zones - 1;
2866                 }
2867
2868                 if (order < new_order || classzone_idx > new_classzone_idx) {
2869                         /*
2870                          * Don't sleep if someone wants a larger 'order'
2871                          * allocation or has tigher zone constraints
2872                          */
2873                         order = new_order;
2874                         classzone_idx = new_classzone_idx;
2875                 } else {
2876                         kswapd_try_to_sleep(pgdat, balanced_order,
2877                                                 balanced_classzone_idx);
2878                         order = pgdat->kswapd_max_order;
2879                         classzone_idx = pgdat->classzone_idx;
2880                         new_order = order;
2881                         new_classzone_idx = classzone_idx;
2882                         pgdat->kswapd_max_order = 0;
2883                         pgdat->classzone_idx = pgdat->nr_zones - 1;
2884                 }
2885
2886                 ret = try_to_freeze();
2887                 if (kthread_should_stop())
2888                         break;
2889
2890                 /*
2891                  * We can speed up thawing tasks if we don't call balance_pgdat
2892                  * after returning from the refrigerator
2893                  */
2894                 if (!ret) {
2895                         trace_mm_vmscan_kswapd_wake(pgdat->node_id, order);
2896                         balanced_classzone_idx = classzone_idx;
2897                         balanced_order = balance_pgdat(pgdat, order,
2898                                                 &balanced_classzone_idx);
2899                 }
2900         }
2901         return 0;
2902 }
2903
2904 /*
2905  * A zone is low on free memory, so wake its kswapd task to service it.
2906  */
2907 void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx)
2908 {
2909         pg_data_t *pgdat;
2910
2911         if (!populated_zone(zone))
2912                 return;
2913
2914         if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
2915                 return;
2916         pgdat = zone->zone_pgdat;
2917         if (pgdat->kswapd_max_order < order) {
2918                 pgdat->kswapd_max_order = order;
2919                 pgdat->classzone_idx = min(pgdat->classzone_idx, classzone_idx);
2920         }
2921         if (!waitqueue_active(&pgdat->kswapd_wait))
2922                 return;
2923         if (zone_watermark_ok_safe(zone, order, low_wmark_pages(zone), 0, 0))
2924                 return;
2925
2926         trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, zone_idx(zone), order);
2927         wake_up_interruptible(&pgdat->kswapd_wait);
2928 }
2929
2930 /*
2931  * The reclaimable count would be mostly accurate.
2932  * The less reclaimable pages may be
2933  * - mlocked pages, which will be moved to unevictable list when encountered
2934  * - mapped pages, which may require several travels to be reclaimed
2935  * - dirty pages, which is not "instantly" reclaimable
2936  */
2937 unsigned long global_reclaimable_pages(void)
2938 {
2939         int nr;
2940
2941         nr = global_page_state(NR_ACTIVE_FILE) +
2942              global_page_state(NR_INACTIVE_FILE);
2943
2944         if (nr_swap_pages > 0)
2945                 nr += global_page_state(NR_ACTIVE_ANON) +
2946                       global_page_state(NR_INACTIVE_ANON);
2947
2948         return nr;
2949 }
2950
2951 unsigned long zone_reclaimable_pages(struct zone *zone)
2952 {
2953         int nr;
2954
2955         nr = zone_page_state(zone, NR_ACTIVE_FILE) +
2956              zone_page_state(zone, NR_INACTIVE_FILE);
2957
2958         if (nr_swap_pages > 0)
2959                 nr += zone_page_state(zone, NR_ACTIVE_ANON) +
2960                       zone_page_state(zone, NR_INACTIVE_ANON);
2961
2962         return nr;
2963 }
2964
2965 #ifdef CONFIG_HIBERNATION
2966 /*
2967  * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
2968  * freed pages.
2969  *
2970  * Rather than trying to age LRUs the aim is to preserve the overall
2971  * LRU order by reclaiming preferentially
2972  * inactive > active > active referenced > active mapped
2973  */
2974 unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
2975 {
2976         struct reclaim_state reclaim_state;
2977         struct scan_control sc = {
2978                 .gfp_mask = GFP_HIGHUSER_MOVABLE,
2979                 .may_swap = 1,
2980                 .may_unmap = 1,
2981                 .may_writepage = 1,
2982                 .nr_to_reclaim = nr_to_reclaim,
2983                 .hibernation_mode = 1,
2984                 .order = 0,
2985         };
2986         struct shrink_control shrink = {
2987                 .gfp_mask = sc.gfp_mask,
2988         };
2989         struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
2990         struct task_struct *p = current;
2991         unsigned long nr_reclaimed;
2992
2993         p->flags |= PF_MEMALLOC;
2994         lockdep_set_current_reclaim_state(sc.gfp_mask);
2995         reclaim_state.reclaimed_slab = 0;
2996         p->reclaim_state = &reclaim_state;
2997
2998         nr_reclaimed = do_try_to_free_pages(zonelist, &sc, &shrink);
2999
3000         p->reclaim_state = NULL;
3001         lockdep_clear_current_reclaim_state();
3002         p->flags &= ~PF_MEMALLOC;
3003
3004         return nr_reclaimed;
3005 }
3006 #endif /* CONFIG_HIBERNATION */
3007
3008 /* It's optimal to keep kswapds on the same CPUs as their memory, but
3009    not required for correctness.  So if the last cpu in a node goes
3010    away, we get changed to run anywhere: as the first one comes back,
3011    restore their cpu bindings. */
3012 static int __devinit cpu_callback(struct notifier_block *nfb,
3013                                   unsigned long action, void *hcpu)
3014 {
3015         int nid;
3016
3017         if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) {
3018                 for_each_node_state(nid, N_HIGH_MEMORY) {
3019                         pg_data_t *pgdat = NODE_DATA(nid);
3020                         const struct cpumask *mask;
3021
3022                         mask = cpumask_of_node(pgdat->node_id);
3023
3024                         if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
3025                                 /* One of our CPUs online: restore mask */
3026                                 set_cpus_allowed_ptr(pgdat->kswapd, mask);
3027                 }
3028         }
3029         return NOTIFY_OK;
3030 }
3031
3032 /*
3033  * This kswapd start function will be called by init and node-hot-add.
3034  * On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added.
3035  */
3036 int kswapd_run(int nid)
3037 {
3038         pg_data_t *pgdat = NODE_DATA(nid);
3039         int ret = 0;
3040
3041         if (pgdat->kswapd)
3042                 return 0;
3043
3044         pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
3045         if (IS_ERR(pgdat->kswapd)) {
3046                 /* failure at boot is fatal */
3047                 BUG_ON(system_state == SYSTEM_BOOTING);
3048                 printk("Failed to start kswapd on node %d\n",nid);
3049                 ret = -1;
3050         }
3051         return ret;
3052 }
3053
3054 /*
3055  * Called by memory hotplug when all memory in a node is offlined.
3056  */
3057 void kswapd_stop(int nid)
3058 {
3059         struct task_struct *kswapd = NODE_DATA(nid)->kswapd;
3060
3061         if (kswapd)
3062                 kthread_stop(kswapd);
3063 }
3064
3065 static int __init kswapd_init(void)
3066 {
3067         int nid;
3068
3069         swap_setup();
3070         for_each_node_state(nid, N_HIGH_MEMORY)
3071                 kswapd_run(nid);
3072         hotcpu_notifier(cpu_callback, 0);
3073         return 0;
3074 }
3075
3076 module_init(kswapd_init)
3077
3078 #ifdef CONFIG_NUMA
3079 /*
3080  * Zone reclaim mode
3081  *
3082  * If non-zero call zone_reclaim when the number of free pages falls below
3083  * the watermarks.
3084  */
3085 int zone_reclaim_mode __read_mostly;
3086
3087 #define RECLAIM_OFF 0
3088 #define RECLAIM_ZONE (1<<0)     /* Run shrink_inactive_list on the zone */
3089 #define RECLAIM_WRITE (1<<1)    /* Writeout pages during reclaim */
3090 #define RECLAIM_SWAP (1<<2)     /* Swap pages out during reclaim */
3091
3092 /*
3093  * Priority for ZONE_RECLAIM. This determines the fraction of pages
3094  * of a node considered for each zone_reclaim. 4 scans 1/16th of
3095  * a zone.
3096  */
3097 #define ZONE_RECLAIM_PRIORITY 4
3098
3099 /*
3100  * Percentage of pages in a zone that must be unmapped for zone_reclaim to
3101  * occur.
3102  */
3103 int sysctl_min_unmapped_ratio = 1;
3104
3105 /*
3106  * If the number of slab pages in a zone grows beyond this percentage then
3107  * slab reclaim needs to occur.
3108  */
3109 int sysctl_min_slab_ratio = 5;
3110
3111 static inline unsigned long zone_unmapped_file_pages(struct zone *zone)
3112 {
3113         unsigned long file_mapped = zone_page_state(zone, NR_FILE_MAPPED);
3114         unsigned long file_lru = zone_page_state(zone, NR_INACTIVE_FILE) +
3115                 zone_page_state(zone, NR_ACTIVE_FILE);
3116
3117         /*
3118          * It's possible for there to be more file mapped pages than
3119          * accounted for by the pages on the file LRU lists because
3120          * tmpfs pages accounted for as ANON can also be FILE_MAPPED
3121          */
3122         return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0;
3123 }
3124
3125 /* Work out how many page cache pages we can reclaim in this reclaim_mode */
3126 static long zone_pagecache_reclaimable(struct zone *zone)
3127 {
3128         long nr_pagecache_reclaimable;
3129         long delta = 0;
3130
3131         /*
3132          * If RECLAIM_SWAP is set, then all file pages are considered
3133          * potentially reclaimable. Otherwise, we have to worry about
3134          * pages like swapcache and zone_unmapped_file_pages() provides
3135          * a better estimate
3136          */
3137         if (zone_reclaim_mode & RECLAIM_SWAP)
3138                 nr_pagecache_reclaimable = zone_page_state(zone, NR_FILE_PAGES);
3139         else
3140                 nr_pagecache_reclaimable = zone_unmapped_file_pages(zone);
3141
3142         /* If we can't clean pages, remove dirty pages from consideration */
3143         if (!(zone_reclaim_mode & RECLAIM_WRITE))
3144                 delta += zone_page_state(zone, NR_FILE_DIRTY);
3145
3146         /* Watch for any possible underflows due to delta */
3147         if (unlikely(delta > nr_pagecache_reclaimable))
3148                 delta = nr_pagecache_reclaimable;
3149
3150         return nr_pagecache_reclaimable - delta;
3151 }
3152
3153 /*
3154  * Try to free up some pages from this zone through reclaim.
3155  */
3156 static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
3157 {
3158         /* Minimum pages needed in order to stay on node */
3159         const unsigned long nr_pages = 1 << order;
3160         struct task_struct *p = current;
3161         struct reclaim_state reclaim_state;
3162         int priority;
3163         struct scan_control sc = {
3164                 .may_writepage = !!(zone_reclaim_mode & RECLAIM_WRITE),
3165                 .may_unmap = !!(zone_reclaim_mode & RECLAIM_SWAP),
3166                 .may_swap = 1,
3167                 .nr_to_reclaim = max_t(unsigned long, nr_pages,
3168                                        SWAP_CLUSTER_MAX),
3169                 .gfp_mask = gfp_mask,
3170                 .order = order,
3171         };
3172         struct shrink_control shrink = {
3173                 .gfp_mask = sc.gfp_mask,
3174         };
3175         unsigned long nr_slab_pages0, nr_slab_pages1;
3176
3177         cond_resched();
3178         /*
3179          * We need to be able to allocate from the reserves for RECLAIM_SWAP
3180          * and we also need to be able to write out pages for RECLAIM_WRITE
3181          * and RECLAIM_SWAP.
3182          */
3183         p->flags |= PF_MEMALLOC | PF_SWAPWRITE;
3184         lockdep_set_current_reclaim_state(gfp_mask);
3185         reclaim_state.reclaimed_slab = 0;
3186         p->reclaim_state = &reclaim_state;
3187
3188         if (zone_pagecache_reclaimable(zone) > zone->min_unmapped_pages) {
3189                 /*
3190                  * Free memory by calling shrink zone with increasing
3191                  * priorities until we have enough memory freed.
3192                  */
3193                 priority = ZONE_RECLAIM_PRIORITY;
3194                 do {
3195                         shrink_zone(priority, zone, &sc);
3196                         priority--;
3197                 } while (priority >= 0 && sc.nr_reclaimed < nr_pages);
3198         }
3199
3200         nr_slab_pages0 = zone_page_state(zone, NR_SLAB_RECLAIMABLE);
3201         if (nr_slab_pages0 > zone->min_slab_pages) {
3202                 /*
3203                  * shrink_slab() does not currently allow us to determine how
3204                  * many pages were freed in this zone. So we take the current
3205                  * number of slab pages and shake the slab until it is reduced
3206                  * by the same nr_pages that we used for reclaiming unmapped
3207                  * pages.
3208                  *
3209                  * Note that shrink_slab will free memory on all zones and may
3210                  * take a long time.
3211                  */
3212                 for (;;) {
3213                         unsigned long lru_pages = zone_reclaimable_pages(zone);
3214
3215                         /* No reclaimable slab or very low memory pressure */
3216                         if (!shrink_slab(&shrink, sc.nr_scanned, lru_pages))
3217                                 break;
3218
3219                         /* Freed enough memory */
3220                         nr_slab_pages1 = zone_page_state(zone,
3221                                                         NR_SLAB_RECLAIMABLE);
3222                         if (nr_slab_pages1 + nr_pages <= nr_slab_pages0)
3223                                 break;
3224                 }
3225
3226                 /*
3227                  * Update nr_reclaimed by the number of slab pages we
3228                  * reclaimed from this zone.
3229                  */
3230                 nr_slab_pages1 = zone_page_state(zone, NR_SLAB_RECLAIMABLE);
3231                 if (nr_slab_pages1 < nr_slab_pages0)
3232                         sc.nr_reclaimed += nr_slab_pages0 - nr_slab_pages1;
3233         }
3234
3235         p->reclaim_state = NULL;
3236         current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE);
3237         lockdep_clear_current_reclaim_state();
3238         return sc.nr_reclaimed >= nr_pages;
3239 }
3240
3241 int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
3242 {
3243         int node_id;
3244         int ret;
3245
3246         /*
3247          * Zone reclaim reclaims unmapped file backed pages and
3248          * slab pages if we are over the defined limits.
3249          *
3250          * A small portion of unmapped file backed pages is needed for
3251          * file I/O otherwise pages read by file I/O will be immediately
3252          * thrown out if the zone is overallocated. So we do not reclaim
3253          * if less than a specified percentage of the zone is used by
3254          * unmapped file backed pages.
3255          */
3256         if (zone_pagecache_reclaimable(zone) <= zone->min_unmapped_pages &&
3257             zone_page_state(zone, NR_SLAB_RECLAIMABLE) <= zone->min_slab_pages)
3258                 return ZONE_RECLAIM_FULL;
3259
3260         if (zone->all_unreclaimable)
3261                 return ZONE_RECLAIM_FULL;
3262
3263         /*
3264          * Do not scan if the allocation should not be delayed.
3265          */
3266         if (!(gfp_mask & __GFP_WAIT) || (current->flags & PF_MEMALLOC))
3267                 return ZONE_RECLAIM_NOSCAN;
3268
3269         /*
3270          * Only run zone reclaim on the local zone or on zones that do not
3271          * have associated processors. This will favor the local processor
3272          * over remote processors and spread off node memory allocations
3273          * as wide as possible.
3274          */
3275         node_id = zone_to_nid(zone);
3276         if (node_state(node_id, N_CPU) && node_id != numa_node_id())
3277                 return ZONE_RECLAIM_NOSCAN;
3278
3279         if (zone_test_and_set_flag(zone, ZONE_RECLAIM_LOCKED))
3280                 return ZONE_RECLAIM_NOSCAN;
3281
3282         ret = __zone_reclaim(zone, gfp_mask, order);
3283         zone_clear_flag(zone, ZONE_RECLAIM_LOCKED);
3284
3285         if (!ret)
3286                 count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);
3287
3288         return ret;
3289 }
3290 #endif
3291
3292 /*
3293  * page_evictable - test whether a page is evictable
3294  * @page: the page to test
3295  * @vma: the VMA in which the page is or will be mapped, may be NULL
3296  *
3297  * Test whether page is evictable--i.e., should be placed on active/inactive
3298  * lists vs unevictable list.  The vma argument is !NULL when called from the
3299  * fault path to determine how to instantate a new page.
3300  *
3301  * Reasons page might not be evictable:
3302  * (1) page's mapping marked unevictable
3303  * (2) page is part of an mlocked VMA
3304  *
3305  */
3306 int page_evictable(struct page *page, struct vm_area_struct *vma)
3307 {
3308
3309         if (mapping_unevictable(page_mapping(page)))
3310                 return 0;
3311
3312         if (PageMlocked(page) || (vma && mlocked_vma_newpage(vma, page)))
3313                 return 0;
3314
3315         return 1;
3316 }
3317
3318 #ifdef CONFIG_SHMEM
3319 /**
3320  * check_move_unevictable_pages - check pages for evictability and move to appropriate zone lru list
3321  * @pages:      array of pages to check
3322  * @nr_pages:   number of pages to check
3323  *
3324  * Checks pages for evictability and moves them to the appropriate lru list.
3325  *
3326  * This function is only used for SysV IPC SHM_UNLOCK.
3327  */
3328 void check_move_unevictable_pages(struct page **pages, int nr_pages)
3329 {
3330         struct lruvec *lruvec;
3331         struct zone *zone = NULL;
3332         int pgscanned = 0;
3333         int pgrescued = 0;
3334         int i;
3335
3336         for (i = 0; i < nr_pages; i++) {
3337                 struct page *page = pages[i];
3338                 struct zone *pagezone;
3339
3340                 pgscanned++;
3341                 pagezone = page_zone(page);
3342                 if (pagezone != zone) {
3343                         if (zone)
3344                                 spin_unlock_irq(&zone->lru_lock);
3345                         zone = pagezone;
3346                         spin_lock_irq(&zone->lru_lock);
3347                 }
3348
3349                 if (!PageLRU(page) || !PageUnevictable(page))
3350                         continue;
3351
3352                 if (page_evictable(page, NULL)) {
3353                         enum lru_list lru = page_lru_base_type(page);
3354
3355                         VM_BUG_ON(PageActive(page));
3356                         ClearPageUnevictable(page);
3357                         __dec_zone_state(zone, NR_UNEVICTABLE);
3358                         lruvec = mem_cgroup_lru_move_lists(zone, page,
3359                                                 LRU_UNEVICTABLE, lru);
3360                         list_move(&page->lru, &lruvec->lists[lru]);
3361                         __inc_zone_state(zone, NR_INACTIVE_ANON + lru);
3362                         pgrescued++;
3363                 }
3364         }
3365
3366         if (zone) {
3367                 __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
3368                 __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
3369                 spin_unlock_irq(&zone->lru_lock);
3370         }
3371 }
3372 #endif /* CONFIG_SHMEM */
3373
3374 static void warn_scan_unevictable_pages(void)
3375 {
3376         printk_once(KERN_WARNING
3377                     "%s: The scan_unevictable_pages sysctl/node-interface has been "
3378                     "disabled for lack of a legitimate use case.  If you have "
3379                     "one, please send an email to linux-mm@kvack.org.\n",
3380                     current->comm);
3381 }
3382
3383 /*
3384  * scan_unevictable_pages [vm] sysctl handler.  On demand re-scan of
3385  * all nodes' unevictable lists for evictable pages
3386  */
3387 unsigned long scan_unevictable_pages;
3388
3389 int scan_unevictable_handler(struct ctl_table *table, int write,
3390                            void __user *buffer,
3391                            size_t *length, loff_t *ppos)
3392 {
3393         warn_scan_unevictable_pages();
3394         proc_doulongvec_minmax(table, write, buffer, length, ppos);
3395         scan_unevictable_pages = 0;
3396         return 0;
3397 }
3398
3399 #ifdef CONFIG_NUMA
3400 /*
3401  * per node 'scan_unevictable_pages' attribute.  On demand re-scan of
3402  * a specified node's per zone unevictable lists for evictable pages.
3403  */
3404
3405 static ssize_t read_scan_unevictable_node(struct device *dev,
3406                                           struct device_attribute *attr,
3407                                           char *buf)
3408 {
3409         warn_scan_unevictable_pages();
3410         return sprintf(buf, "0\n");     /* always zero; should fit... */
3411 }
3412
3413 static ssize_t write_scan_unevictable_node(struct device *dev,
3414                                            struct device_attribute *attr,
3415                                         const char *buf, size_t count)
3416 {
3417         warn_scan_unevictable_pages();
3418         return 1;
3419 }
3420
3421
3422 static DEVICE_ATTR(scan_unevictable_pages, S_IRUGO | S_IWUSR,
3423                         read_scan_unevictable_node,
3424                         write_scan_unevictable_node);
3425
3426 int scan_unevictable_register_node(struct node *node)
3427 {
3428         return device_create_file(&node->dev, &dev_attr_scan_unevictable_pages);
3429 }
3430
3431 void scan_unevictable_unregister_node(struct node *node)
3432 {
3433         device_remove_file(&node->dev, &dev_attr_scan_unevictable_pages);
3434 }
3435 #endif