media: video: tegra: sh532u: fix out-of-bounds read
[linux-2.6.git] / mm / readahead.c
index 5b3c9b7..867f9dd 100644 (file)
@@ -3,39 +3,20 @@
  *
  * Copyright (C) 2002, Linus Torvalds
  *
- * 09Apr2002   akpm@zip.com.au
+ * 09Apr2002   Andrew Morton
  *             Initial version.
  */
 
 #include <linux/kernel.h>
 #include <linux/fs.h>
+#include <linux/gfp.h>
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/blkdev.h>
 #include <linux/backing-dev.h>
 #include <linux/task_io_accounting_ops.h>
 #include <linux/pagevec.h>
-
-void default_unplug_io_fn(struct backing_dev_info *bdi, struct page *page)
-{
-}
-EXPORT_SYMBOL(default_unplug_io_fn);
-
-/*
- * Convienent macros for min/max read-ahead pages.
- * Note that MAX_RA_PAGES is rounded down, while MIN_RA_PAGES is rounded up.
- * The latter is necessary for systems with large page size(i.e. 64k).
- */
-#define MAX_RA_PAGES   (VM_MAX_READAHEAD*1024 / PAGE_CACHE_SIZE)
-#define MIN_RA_PAGES   DIV_ROUND_UP(VM_MIN_READAHEAD*1024, PAGE_CACHE_SIZE)
-
-struct backing_dev_info default_backing_dev_info = {
-       .ra_pages       = MAX_RA_PAGES,
-       .state          = 0,
-       .capabilities   = BDI_CAP_MAP_COPY,
-       .unplug_io_fn   = default_unplug_io_fn,
-};
-EXPORT_SYMBOL_GPL(default_backing_dev_info);
+#include <linux/pagemap.h>
 
 /*
  * Initialise a struct file's readahead state.  Assumes that the caller has
@@ -45,12 +26,48 @@ void
 file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping)
 {
        ra->ra_pages = mapping->backing_dev_info->ra_pages;
-       ra->prev_index = -1;
+       ra->prev_pos = -1;
 }
 EXPORT_SYMBOL_GPL(file_ra_state_init);
 
 #define list_to_page(head) (list_entry((head)->prev, struct page, lru))
 
+/*
+ * see if a page needs releasing upon read_cache_pages() failure
+ * - the caller of read_cache_pages() may have set PG_private or PG_fscache
+ *   before calling, such as the NFS fs marking pages that are cached locally
+ *   on disk, thus we need to give the fs a chance to clean up in the event of
+ *   an error
+ */
+static void read_cache_pages_invalidate_page(struct address_space *mapping,
+                                            struct page *page)
+{
+       if (page_has_private(page)) {
+               if (!trylock_page(page))
+                       BUG();
+               page->mapping = mapping;
+               do_invalidatepage(page, 0);
+               page->mapping = NULL;
+               unlock_page(page);
+       }
+       page_cache_release(page);
+}
+
+/*
+ * release a list of pages, invalidating them first if need be
+ */
+static void read_cache_pages_invalidate_pages(struct address_space *mapping,
+                                             struct list_head *pages)
+{
+       struct page *victim;
+
+       while (!list_empty(pages)) {
+               victim = list_to_page(pages);
+               list_del(&victim->lru);
+               read_cache_pages_invalidate_page(mapping, victim);
+       }
+}
+
 /**
  * read_cache_pages - populate an address space with some pages & start reads against them
  * @mapping: the address_space
@@ -65,28 +82,25 @@ int read_cache_pages(struct address_space *mapping, struct list_head *pages,
                        int (*filler)(void *, struct page *), void *data)
 {
        struct page *page;
-       struct pagevec lru_pvec;
        int ret = 0;
 
-       pagevec_init(&lru_pvec, 0);
-
        while (!list_empty(pages)) {
                page = list_to_page(pages);
                list_del(&page->lru);
-               if (add_to_page_cache(page, mapping, page->index, GFP_KERNEL)) {
-                       page_cache_release(page);
+               if (add_to_page_cache_lru(page, mapping,
+                                       page->index, GFP_KERNEL)) {
+                       read_cache_pages_invalidate_page(mapping, page);
                        continue;
                }
+               page_cache_release(page);
+
                ret = filler(data, page);
-               if (!pagevec_add(&lru_pvec, page))
-                       __pagevec_lru_add(&lru_pvec);
-               if (ret) {
-                       put_pages_list(pages);
+               if (unlikely(ret)) {
+                       read_cache_pages_invalidate_pages(mapping, pages);
                        break;
                }
                task_io_account_read(PAGE_CACHE_SIZE);
        }
-       pagevec_lru_add(&lru_pvec);
        return ret;
 }
 
@@ -95,10 +109,12 @@ EXPORT_SYMBOL(read_cache_pages);
 static int read_pages(struct address_space *mapping, struct file *filp,
                struct list_head *pages, unsigned nr_pages)
 {
+       struct blk_plug plug;
        unsigned page_idx;
-       struct pagevec lru_pvec;
        int ret;
 
+       blk_start_plug(&plug);
+
        if (mapping->a_ops->readpages) {
                ret = mapping->a_ops->readpages(filp, mapping, pages, nr_pages);
                /* Clean up the remaining pages */
@@ -106,34 +122,30 @@ static int read_pages(struct address_space *mapping, struct file *filp,
                goto out;
        }
 
-       pagevec_init(&lru_pvec, 0);
        for (page_idx = 0; page_idx < nr_pages; page_idx++) {
                struct page *page = list_to_page(pages);
                list_del(&page->lru);
-               if (!add_to_page_cache(page, mapping,
+               if (!add_to_page_cache_lru(page, mapping,
                                        page->index, GFP_KERNEL)) {
                        mapping->a_ops->readpage(filp, page);
-                       if (!pagevec_add(&lru_pvec, page))
-                               __pagevec_lru_add(&lru_pvec);
-               } else
-                       page_cache_release(page);
+               }
+               page_cache_release(page);
        }
-       pagevec_lru_add(&lru_pvec);
        ret = 0;
+
 out:
+       blk_finish_plug(&plug);
+
        return ret;
 }
 
 /*
- * do_page_cache_readahead actually reads a chunk of disk.  It allocates all
+ * __do_page_cache_readahead() actually reads a chunk of disk.  It allocates all
  * the pages first, then submits them all for I/O. This avoids the very bad
  * behaviour which would occur if page allocations are causing VM writeback.
  * We really don't want to intermingle reads and writes like that.
  *
  * Returns the number of pages requested, or the maximum amount of I/O allowed.
- *
- * do_page_cache_readahead() returns -1 if it encountered request queue
- * congestion.
  */
 static int
 __do_page_cache_readahead(struct address_space *mapping, struct file *filp,
@@ -156,20 +168,19 @@ __do_page_cache_readahead(struct address_space *mapping, struct file *filp,
        /*
         * Preallocate as many pages as we will need.
         */
-       read_lock_irq(&mapping->tree_lock);
        for (page_idx = 0; page_idx < nr_to_read; page_idx++) {
                pgoff_t page_offset = offset + page_idx;
 
                if (page_offset > end_index)
                        break;
 
+               rcu_read_lock();
                page = radix_tree_lookup(&mapping->page_tree, page_offset);
+               rcu_read_unlock();
                if (page)
                        continue;
 
-               read_unlock_irq(&mapping->tree_lock);
-               page = page_cache_alloc_cold(mapping);
-               read_lock_irq(&mapping->tree_lock);
+               page = page_cache_alloc_readahead(mapping);
                if (!page)
                        break;
                page->index = page_offset;
@@ -178,7 +189,6 @@ __do_page_cache_readahead(struct address_space *mapping, struct file *filp,
                        SetPageReadahead(page);
                ret++;
        }
-       read_unlock_irq(&mapping->tree_lock);
 
        /*
         * Now start the IO.  We ignore I/O errors - if the page is not
@@ -204,6 +214,7 @@ int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
        if (unlikely(!mapping->a_ops->readpage && !mapping->a_ops->readpages))
                return -EINVAL;
 
+       nr_to_read = max_sane_readahead(nr_to_read);
        while (nr_to_read) {
                int err;
 
@@ -225,28 +236,12 @@ int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
 }
 
 /*
- * This version skips the IO if the queue is read-congested, and will tell the
- * block layer to abandon the readahead if request allocation would block.
- *
- * force_page_cache_readahead() will ignore queue congestion and will block on
- * request queues.
- */
-int do_page_cache_readahead(struct address_space *mapping, struct file *filp,
-                       pgoff_t offset, unsigned long nr_to_read)
-{
-       if (bdi_read_congested(mapping->backing_dev_info))
-               return -1;
-
-       return __do_page_cache_readahead(mapping, filp, offset, nr_to_read, 0);
-}
-
-/*
  * Given a desired number of PAGE_CACHE_SIZE readahead pages, return a
  * sensible upper limit.
  */
 unsigned long max_sane_readahead(unsigned long nr)
 {
-       return min(nr, (node_page_state(numa_node_id(), NR_INACTIVE)
+       return min(nr, (node_page_state(numa_node_id(), NR_INACTIVE_FILE)
                + node_page_state(numa_node_id(), NR_FREE_PAGES)) / 2);
 }
 
@@ -256,18 +251,13 @@ unsigned long max_sane_readahead(unsigned long nr)
 unsigned long ra_submit(struct file_ra_state *ra,
                       struct address_space *mapping, struct file *filp)
 {
-       unsigned long ra_size;
-       unsigned long la_size;
        int actual;
 
-       ra_size = ra_readahead_size(ra);
-       la_size = ra_lookahead_size(ra);
        actual = __do_page_cache_readahead(mapping, filp,
-                                       ra->ra_index, ra_size, la_size);
+                                       ra->start, ra->size, ra->async_size);
 
        return actual;
 }
-EXPORT_SYMBOL_GPL(ra_submit);
 
 /*
  * Set the initial window size, round to next power of 2 and square
@@ -296,7 +286,7 @@ static unsigned long get_init_ra_size(unsigned long size, unsigned long max)
 static unsigned long get_next_ra_size(struct file_ra_state *ra,
                                                unsigned long max)
 {
-       unsigned long cur = ra->readahead_index - ra->ra_index;
+       unsigned long cur = ra->size;
        unsigned long newsize;
 
        if (cur < max / 16)
@@ -313,32 +303,25 @@ static unsigned long get_next_ra_size(struct file_ra_state *ra,
  * The fields in struct file_ra_state represent the most-recently-executed
  * readahead attempt:
  *
- *                    |-------- last readahead window -------->|
- *       |-- application walking here -->|
- * ======#============|==================#=====================|
- *       ^la_index    ^ra_index          ^lookahead_index      ^readahead_index
- *
- * [ra_index, readahead_index) represents the last readahead window.
- *
- * [la_index, lookahead_index] is where the application would be walking(in
- * the common case of cache-cold sequential reads): the last window was
- * established when the application was at la_index, and the next window will
- * be bring in when the application reaches lookahead_index.
+ *                        |<----- async_size ---------|
+ *     |------------------- size -------------------->|
+ *     |==================#===========================|
+ *     ^start             ^page marked with PG_readahead
  *
  * To overlap application thinking time and disk I/O time, we do
  * `readahead pipelining': Do not wait until the application consumed all
  * readahead pages and stalled on the missing page at readahead_index;
- * Instead, submit an asynchronous readahead I/O as early as the application
- * reads on the page at lookahead_index. Normally lookahead_index will be
- * equal to ra_index, for maximum pipelining.
+ * Instead, submit an asynchronous readahead I/O as soon as there are
+ * only async_size pages left in the readahead window. Normally async_size
+ * will be equal to size, for maximum pipelining.
  *
  * In interleaved sequential reads, concurrent streams on the same fd can
  * be invalidating each other's readahead state. So we flag the new readahead
- * page at lookahead_index with PG_readahead, and use it as readahead
+ * page at (start+size-async_size) with PG_readahead, and use it as readahead
  * indicator. The flag won't be set on already cached pages, to avoid the
  * readahead-for-nothing fuss, saving pointless page cache lookups.
  *
- * prev_index tracks the last visited page in the _previous_ read request.
+ * prev_pos tracks the last visited byte in the _previous_ read request.
  * It should be maintained by the caller, and will be used for detecting
  * small random reads. Note that the readahead algorithm checks loosely
  * for sequential patterns. Hence interleaved reads might be served as
@@ -354,111 +337,228 @@ static unsigned long get_next_ra_size(struct file_ra_state *ra,
  */
 
 /*
+ * Count contiguously cached pages from @offset-1 to @offset-@max,
+ * this count is a conservative estimation of
+ *     - length of the sequential read sequence, or
+ *     - thrashing threshold in memory tight systems
+ */
+static pgoff_t count_history_pages(struct address_space *mapping,
+                                  struct file_ra_state *ra,
+                                  pgoff_t offset, unsigned long max)
+{
+       pgoff_t head;
+
+       rcu_read_lock();
+       head = radix_tree_prev_hole(&mapping->page_tree, offset - 1, max);
+       rcu_read_unlock();
+
+       return offset - 1 - head;
+}
+
+/*
+ * page cache context based read-ahead
+ */
+static int try_context_readahead(struct address_space *mapping,
+                                struct file_ra_state *ra,
+                                pgoff_t offset,
+                                unsigned long req_size,
+                                unsigned long max)
+{
+       pgoff_t size;
+
+       size = count_history_pages(mapping, ra, offset, max);
+
+       /*
+        * no history pages:
+        * it could be a random read
+        */
+       if (!size)
+               return 0;
+
+       /*
+        * starts from beginning of file:
+        * it is a strong indication of long-run stream (or whole-file-read)
+        */
+       if (size >= offset)
+               size *= 2;
+
+       ra->start = offset;
+       ra->size = get_init_ra_size(size + req_size, max);
+       ra->async_size = ra->size;
+
+       return 1;
+}
+
+/*
  * A minimal readahead algorithm for trivial sequential/random reads.
  */
 static unsigned long
 ondemand_readahead(struct address_space *mapping,
                   struct file_ra_state *ra, struct file *filp,
-                  struct page *page, pgoff_t offset,
+                  bool hit_readahead_marker, pgoff_t offset,
                   unsigned long req_size)
 {
-       unsigned long max;      /* max readahead pages */
-       pgoff_t ra_index;       /* readahead index */
-       unsigned long ra_size;  /* readahead size */
-       unsigned long la_size;  /* lookahead size */
-       int sequential;
+       unsigned long max = max_sane_readahead(ra->ra_pages);
 
-       max = ra->ra_pages;
-       sequential = (offset - ra->prev_index <= 1UL) || (req_size > max);
+       /*
+        * start of file
+        */
+       if (!offset)
+               goto initial_readahead;
 
        /*
-        * Lookahead/readahead hit, assume sequential access.
+        * It's the expected callback offset, assume sequential access.
         * Ramp up sizes, and push forward the readahead window.
         */
-       if (offset && (offset == ra->lookahead_index ||
-                       offset == ra->readahead_index)) {
-               ra_index = ra->readahead_index;
-               ra_size = get_next_ra_size(ra, max);
-               la_size = ra_size;
-               goto fill_ra;
+       if ((offset == (ra->start + ra->size - ra->async_size) ||
+            offset == (ra->start + ra->size))) {
+               ra->start += ra->size;
+               ra->size = get_next_ra_size(ra, max);
+               ra->async_size = ra->size;
+               goto readit;
        }
 
        /*
-        * Standalone, small read.
-        * Read as is, and do not pollute the readahead state.
+        * Hit a marked page without valid readahead state.
+        * E.g. interleaved reads.
+        * Query the pagecache for async_size, which normally equals to
+        * readahead size. Ramp it up and use it as the new readahead size.
         */
-       if (!page && !sequential) {
-               return __do_page_cache_readahead(mapping, filp,
-                                               offset, req_size, 0);
+       if (hit_readahead_marker) {
+               pgoff_t start;
+
+               rcu_read_lock();
+               start = radix_tree_next_hole(&mapping->page_tree, offset+1,max);
+               rcu_read_unlock();
+
+               if (!start || start - offset > max)
+                       return 0;
+
+               ra->start = start;
+               ra->size = start - offset;      /* old async_size */
+               ra->size += req_size;
+               ra->size = get_next_ra_size(ra, max);
+               ra->async_size = ra->size;
+               goto readit;
        }
 
        /*
-        * It may be one of
-        *      - first read on start of file
-        *      - sequential cache miss
-        *      - oversize random read
-        * Start readahead for it.
+        * oversize read
         */
-       ra_index = offset;
-       ra_size = get_init_ra_size(req_size, max);
-       la_size = ra_size > req_size ? ra_size - req_size : ra_size;
+       if (req_size > max)
+               goto initial_readahead;
 
        /*
-        * Hit on a lookahead page without valid readahead state.
-        * E.g. interleaved reads.
-        * Not knowing its readahead pos/size, bet on the minimal possible one.
+        * sequential cache miss
         */
-       if (page) {
-               ra_index++;
-               ra_size = min(4 * ra_size, max);
-       }
+       if (offset - (ra->prev_pos >> PAGE_CACHE_SHIFT) <= 1UL)
+               goto initial_readahead;
 
-fill_ra:
-       ra_set_index(ra, offset, ra_index);
-       ra_set_size(ra, ra_size, la_size);
+       /*
+        * Query the page cache and look for the traces(cached history pages)
+        * that a sequential stream would leave behind.
+        */
+       if (try_context_readahead(mapping, ra, offset, req_size, max))
+               goto readit;
+
+       /*
+        * standalone, small random read
+        * Read as is, and do not pollute the readahead state.
+        */
+       return __do_page_cache_readahead(mapping, filp, offset, req_size, 0);
+
+initial_readahead:
+       ra->start = offset;
+       ra->size = get_init_ra_size(req_size, max);
+       ra->async_size = ra->size > req_size ? ra->size - req_size : ra->size;
+
+readit:
+       /*
+        * Will this read hit the readahead marker made by itself?
+        * If so, trigger the readahead marker hit now, and merge
+        * the resulted next readahead window into the current one.
+        */
+       if (offset == ra->start && ra->size == ra->async_size) {
+               ra->async_size = get_next_ra_size(ra, max);
+               ra->size += ra->async_size;
+       }
 
        return ra_submit(ra, mapping, filp);
 }
 
 /**
- * page_cache_readahead_ondemand - generic file readahead
+ * page_cache_sync_readahead - generic file readahead
  * @mapping: address_space which holds the pagecache and I/O vectors
  * @ra: file_ra_state which holds the readahead state
  * @filp: passed on to ->readpage() and ->readpages()
- * @page: the page at @offset, or NULL if non-present
- * @offset: start offset into @mapping, in PAGE_CACHE_SIZE units
+ * @offset: start offset into @mapping, in pagecache page-sized units
  * @req_size: hint: total size of the read which the caller is performing in
- *            PAGE_CACHE_SIZE units
+ *            pagecache pages
  *
- * page_cache_readahead_ondemand() is the entry point of readahead logic.
- * This function should be called when it is time to perform readahead:
- * 1) @page == NULL
- *    A cache miss happened, time for synchronous readahead.
- * 2) @page != NULL && PageReadahead(@page)
- *    A look-ahead hit occured, time for asynchronous readahead.
+ * page_cache_sync_readahead() should be called when a cache miss happened:
+ * it will submit the read.  The readahead logic may decide to piggyback more
+ * pages onto the read request if access patterns suggest it will improve
+ * performance.
  */
-unsigned long
-page_cache_readahead_ondemand(struct address_space *mapping,
-                               struct file_ra_state *ra, struct file *filp,
-                               struct page *page, pgoff_t offset,
-                               unsigned long req_size)
+void page_cache_sync_readahead(struct address_space *mapping,
+                              struct file_ra_state *ra, struct file *filp,
+                              pgoff_t offset, unsigned long req_size)
 {
        /* no read-ahead */
        if (!ra->ra_pages)
-               return 0;
+               return;
 
-       if (page) {
-               ClearPageReadahead(page);
-
-               /*
-                * Defer asynchronous read-ahead on IO congestion.
-                */
-               if (bdi_read_congested(mapping->backing_dev_info))
-                       return 0;
+       /* be dumb */
+       if (filp && (filp->f_mode & FMODE_RANDOM)) {
+               force_page_cache_readahead(mapping, filp, offset, req_size);
+               return;
        }
 
        /* do read-ahead */
-       return ondemand_readahead(mapping, ra, filp, page,
-                                       offset, req_size);
+       ondemand_readahead(mapping, ra, filp, false, offset, req_size);
+}
+EXPORT_SYMBOL_GPL(page_cache_sync_readahead);
+
+/**
+ * page_cache_async_readahead - file readahead for marked pages
+ * @mapping: address_space which holds the pagecache and I/O vectors
+ * @ra: file_ra_state which holds the readahead state
+ * @filp: passed on to ->readpage() and ->readpages()
+ * @page: the page at @offset which has the PG_readahead flag set
+ * @offset: start offset into @mapping, in pagecache page-sized units
+ * @req_size: hint: total size of the read which the caller is performing in
+ *            pagecache pages
+ *
+ * page_cache_async_readahead() should be called when a page is used which
+ * has the PG_readahead flag; this is a marker to suggest that the application
+ * has used up enough of the readahead window that we should start pulling in
+ * more pages.
+ */
+void
+page_cache_async_readahead(struct address_space *mapping,
+                          struct file_ra_state *ra, struct file *filp,
+                          struct page *page, pgoff_t offset,
+                          unsigned long req_size)
+{
+       /* no read-ahead */
+       if (!ra->ra_pages)
+               return;
+
+       /*
+        * Same bit is used for PG_readahead and PG_reclaim.
+        */
+       if (PageWriteback(page))
+               return;
+
+       ClearPageReadahead(page);
+
+       /*
+        * Defer asynchronous read-ahead on IO congestion.
+        */
+       if (bdi_read_congested(mapping->backing_dev_info))
+               return;
+
+       /* do read-ahead */
+       ondemand_readahead(mapping, ra, filp, true, offset, req_size);
 }
-EXPORT_SYMBOL_GPL(page_cache_readahead_ondemand);
+EXPORT_SYMBOL_GPL(page_cache_async_readahead);