mm: Close races between THP migration and PMD numa clearing
[linux-3.10.git] / mm / bounce.c
index 13b6dad..5a7d58f 100644 (file)
@@ -4,7 +4,7 @@
  */
 
 #include <linux/mm.h>
-#include <linux/module.h>
+#include <linux/export.h>
 #include <linux/swap.h>
 #include <linux/gfp.h>
 #include <linux/bio.h>
@@ -14,6 +14,7 @@
 #include <linux/init.h>
 #include <linux/hash.h>
 #include <linux/highmem.h>
+#include <linux/bootmem.h>
 #include <asm/tlbflush.h>
 
 #include <trace/events/block.h>
 
 static mempool_t *page_pool, *isa_page_pool;
 
-#ifdef CONFIG_HIGHMEM
+#if defined(CONFIG_HIGHMEM) || defined(CONFIG_NEED_BOUNCE_POOL)
 static __init int init_emergency_pool(void)
 {
-       struct sysinfo i;
-       si_meminfo(&i);
-       si_swapinfo(&i);
-
-       if (!i.totalhigh)
+#if defined(CONFIG_HIGHMEM) && !defined(CONFIG_MEMORY_HOTPLUG)
+       if (max_pfn <= max_low_pfn)
                return 0;
+#endif
 
        page_pool = mempool_create_page_pool(POOL_SIZE, 0);
        BUG_ON(!page_pool);
-       printk("highmem bounce pool size: %d pages\n", POOL_SIZE);
+       printk("bounce pool size: %d pages\n", POOL_SIZE);
 
        return 0;
 }
 
 __initcall(init_emergency_pool);
+#endif
 
+#ifdef CONFIG_HIGHMEM
 /*
  * highmem version, map in to vec
  */
@@ -51,9 +52,9 @@ static void bounce_copy_vec(struct bio_vec *to, unsigned char *vfrom)
        unsigned char *vto;
 
        local_irq_save(flags);
-       vto = kmap_atomic(to->bv_page, KM_BOUNCE_READ);
+       vto = kmap_atomic(to->bv_page);
        memcpy(vto + to->bv_offset, vfrom, to->bv_len);
-       kunmap_atomic(vto, KM_BOUNCE_READ);
+       kunmap_atomic(vto);
        local_irq_restore(flags);
 }
 
@@ -100,7 +101,7 @@ static void copy_to_high_bio_irq(struct bio *to, struct bio *from)
        struct bio_vec *tovec, *fromvec;
        int i;
 
-       __bio_for_each_segment(tovec, to, i, 0) {
+       bio_for_each_segment(tovec, to, i) {
                fromvec = from->bi_io_vec + i;
 
                /*
@@ -116,8 +117,8 @@ static void copy_to_high_bio_irq(struct bio *to, struct bio *from)
                 */
                vfrom = page_address(fromvec->bv_page) + tovec->bv_offset;
 
-               flush_dcache_page(tovec->bv_page);
                bounce_copy_vec(tovec, vfrom);
+               flush_dcache_page(tovec->bv_page);
        }
 }
 
@@ -133,7 +134,7 @@ static void bounce_end_io(struct bio *bio, mempool_t *pool, int err)
        /*
         * free up bounce indirect pages used
         */
-       __bio_for_each_segment(bvec, bio, i, 0) {
+       bio_for_each_segment_all(bvec, bio, i) {
                org_vec = bio_orig->bi_io_vec + i;
                if (bvec->bv_page == org_vec->bv_page)
                        continue;
@@ -177,81 +178,66 @@ static void bounce_end_io_read_isa(struct bio *bio, int err)
        __bounce_end_io_read(bio, isa_page_pool, err);
 }
 
+#ifdef CONFIG_NEED_BOUNCE_POOL
+static int must_snapshot_stable_pages(struct request_queue *q, struct bio *bio)
+{
+       if (bio_data_dir(bio) != WRITE)
+               return 0;
+
+       if (!bdi_cap_stable_pages_required(&q->backing_dev_info))
+               return 0;
+
+       return test_bit(BIO_SNAP_STABLE, &bio->bi_flags);
+}
+#else
+static int must_snapshot_stable_pages(struct request_queue *q, struct bio *bio)
+{
+       return 0;
+}
+#endif /* CONFIG_NEED_BOUNCE_POOL */
+
 static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
-                              mempool_t *pool)
+                              mempool_t *pool, int force)
 {
-       struct page *page;
-       struct bio *bio = NULL;
-       int i, rw = bio_data_dir(*bio_orig);
+       struct bio *bio;
+       int rw = bio_data_dir(*bio_orig);
        struct bio_vec *to, *from;
+       unsigned i;
 
-       bio_for_each_segment(from, *bio_orig, i) {
-               page = from->bv_page;
+       if (force)
+               goto bounce;
+       bio_for_each_segment(from, *bio_orig, i)
+               if (page_to_pfn(from->bv_page) > queue_bounce_pfn(q))
+                       goto bounce;
 
-               /*
-                * is destination page below bounce pfn?
-                */
-               if (page_to_pfn(page) <= queue_bounce_pfn(q))
-                       continue;
+       return;
+bounce:
+       bio = bio_clone_bioset(*bio_orig, GFP_NOIO, fs_bio_set);
 
-               /*
-                * irk, bounce it
-                */
-               if (!bio) {
-                       unsigned int cnt = (*bio_orig)->bi_vcnt;
-
-                       bio = bio_alloc(GFP_NOIO, cnt);
-                       memset(bio->bi_io_vec, 0, cnt * sizeof(struct bio_vec));
-               }
-                       
+       bio_for_each_segment_all(to, bio, i) {
+               struct page *page = to->bv_page;
 
-               to = bio->bi_io_vec + i;
+               if (page_to_pfn(page) <= queue_bounce_pfn(q) && !force)
+                       continue;
 
-               to->bv_page = mempool_alloc(pool, q->bounce_gfp);
-               to->bv_len = from->bv_len;
-               to->bv_offset = from->bv_offset;
                inc_zone_page_state(to->bv_page, NR_BOUNCE);
+               to->bv_page = mempool_alloc(pool, q->bounce_gfp);
 
                if (rw == WRITE) {
                        char *vto, *vfrom;
 
-                       flush_dcache_page(from->bv_page);
+                       flush_dcache_page(page);
+
                        vto = page_address(to->bv_page) + to->bv_offset;
-                       vfrom = kmap(from->bv_page) + from->bv_offset;
+                       vfrom = kmap_atomic(page) + to->bv_offset;
                        memcpy(vto, vfrom, to->bv_len);
-                       kunmap(from->bv_page);
+                       kunmap_atomic(vfrom);
                }
        }
 
-       /*
-        * no pages bounced
-        */
-       if (!bio)
-               return;
-
        trace_block_bio_bounce(q, *bio_orig);
 
-       /*
-        * at least one page was bounced, fill in possible non-highmem
-        * pages
-        */
-       __bio_for_each_segment(from, *bio_orig, i, 0) {
-               to = bio_iovec_idx(bio, i);
-               if (!to->bv_page) {
-                       to->bv_page = from->bv_page;
-                       to->bv_len = from->bv_len;
-                       to->bv_offset = from->bv_offset;
-               }
-       }
-
-       bio->bi_bdev = (*bio_orig)->bi_bdev;
        bio->bi_flags |= (1 << BIO_BOUNCED);
-       bio->bi_sector = (*bio_orig)->bi_sector;
-       bio->bi_rw = (*bio_orig)->bi_rw;
-
-       bio->bi_vcnt = (*bio_orig)->bi_vcnt;
-       bio->bi_idx = (*bio_orig)->bi_idx;
-       bio->bi_size = (*bio_orig)->bi_size;
 
        if (pool == page_pool) {
                bio->bi_end_io = bounce_end_io_write;
@@ -269,6 +255,7 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
 
 void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)
 {
+       int must_bounce;
        mempool_t *pool;
 
        /*
@@ -277,13 +264,15 @@ void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)
        if (!bio_has_data(*bio_orig))
                return;
 
+       must_bounce = must_snapshot_stable_pages(q, *bio_orig);
+
        /*
         * for non-isa bounce case, just check if the bounce pfn is equal
         * to or bigger than the highest pfn in the system -- in that case,
         * don't waste time iterating over bio segments
         */
        if (!(q->bounce_gfp & GFP_DMA)) {
-               if (queue_bounce_pfn(q) >= blk_max_pfn)
+               if (queue_bounce_pfn(q) >= blk_max_pfn && !must_bounce)
                        return;
                pool = page_pool;
        } else {
@@ -294,7 +283,7 @@ void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)
        /*
         * slow path
         */
-       __blk_queue_bounce(q, bio_orig, pool);
+       __blk_queue_bounce(q, bio_orig, pool, must_bounce);
 }
 
 EXPORT_SYMBOL(blk_queue_bounce);