cris: fix a build error in kernel/fork.c
[linux-2.6.git] / mm / page-writeback.c
index da95995..d196074 100644 (file)
 #include <trace/events/writeback.h>
 
 /*
+ * Sleep at most 200ms at a time in balance_dirty_pages().
+ */
+#define MAX_PAUSE              max(HZ/5, 1)
+
+/*
  * Estimate write bandwidth at 200ms intervals.
  */
 #define BANDWIDTH_INTERVAL     max(HZ/5, 1)
@@ -276,12 +281,13 @@ static inline void task_dirties_fraction(struct task_struct *tsk,
  * effectively curb the growth of dirty pages. Light dirtiers with high enough
  * dirty threshold may never get throttled.
  */
+#define TASK_LIMIT_FRACTION 8
 static unsigned long task_dirty_limit(struct task_struct *tsk,
                                       unsigned long bdi_dirty)
 {
        long numerator, denominator;
        unsigned long dirty = bdi_dirty;
-       u64 inv = dirty >> 3;
+       u64 inv = dirty / TASK_LIMIT_FRACTION;
 
        task_dirties_fraction(tsk, &numerator, &denominator);
        inv *= numerator;
@@ -292,6 +298,12 @@ static unsigned long task_dirty_limit(struct task_struct *tsk,
        return max(dirty, bdi_dirty/2);
 }
 
+/* Minimum limit for any task */
+static unsigned long task_min_dirty_limit(unsigned long bdi_dirty)
+{
+       return bdi_dirty - bdi_dirty / TASK_LIMIT_FRACTION;
+}
+
 /*
  *
  */
@@ -399,6 +411,11 @@ unsigned long determine_dirtyable_memory(void)
        return x + 1;   /* Ensure that we never return 0 */
 }
 
+static unsigned long hard_dirty_limit(unsigned long thresh)
+{
+       return max(thresh, global_dirty_limit);
+}
+
 /*
  * global_dirty_limits - background-writeback and dirty-throttling thresholds
  *
@@ -437,6 +454,7 @@ void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty)
        }
        *pbackground = background;
        *pdirty = dirty;
+       trace_global_dirty_state(background, dirty);
 }
 
 /**
@@ -640,9 +658,12 @@ static void balance_dirty_pages(struct address_space *mapping,
        unsigned long background_thresh;
        unsigned long dirty_thresh;
        unsigned long bdi_thresh;
+       unsigned long task_bdi_thresh;
+       unsigned long min_task_bdi_thresh;
        unsigned long pages_written = 0;
        unsigned long pause = 1;
        bool dirty_exceeded = false;
+       bool clear_dirty_exceeded = true;
        struct backing_dev_info *bdi = mapping->backing_dev_info;
        unsigned long start_time = jiffies;
 
@@ -662,7 +683,8 @@ static void balance_dirty_pages(struct address_space *mapping,
                        break;
 
                bdi_thresh = bdi_dirty_limit(bdi, dirty_thresh);
-               bdi_thresh = task_dirty_limit(current, bdi_thresh);
+               min_task_bdi_thresh = task_min_dirty_limit(bdi_thresh);
+               task_bdi_thresh = task_dirty_limit(current, bdi_thresh);
 
                /*
                 * In order to avoid the stacked BDI deadlock we need
@@ -674,7 +696,7 @@ static void balance_dirty_pages(struct address_space *mapping,
                 * actually dirty; with m+n sitting in the percpu
                 * deltas.
                 */
-               if (bdi_thresh < 2*bdi_stat_error(bdi)) {
+               if (task_bdi_thresh < 2 * bdi_stat_error(bdi)) {
                        bdi_nr_reclaimable = bdi_stat_sum(bdi, BDI_RECLAIMABLE);
                        bdi_dirty = bdi_nr_reclaimable +
                                    bdi_stat_sum(bdi, BDI_WRITEBACK);
@@ -690,8 +712,10 @@ static void balance_dirty_pages(struct address_space *mapping,
                 * bdi or process from holding back light ones; The latter is
                 * the last resort safeguard.
                 */
-               dirty_exceeded = (bdi_dirty > bdi_thresh) ||
+               dirty_exceeded = (bdi_dirty > task_bdi_thresh) ||
                                  (nr_dirty > dirty_thresh);
+               clear_dirty_exceeded = (bdi_dirty <= min_task_bdi_thresh) &&
+                                       (nr_dirty <= dirty_thresh);
 
                if (!dirty_exceeded)
                        break;
@@ -712,7 +736,7 @@ static void balance_dirty_pages(struct address_space *mapping,
                 * up.
                 */
                trace_balance_dirty_start(bdi);
-               if (bdi_nr_reclaimable > bdi_thresh) {
+               if (bdi_nr_reclaimable > task_bdi_thresh) {
                        pages_written += writeback_inodes_wb(&bdi->wb,
                                                             write_chunk);
                        trace_balance_dirty_written(bdi, pages_written);
@@ -723,6 +747,29 @@ static void balance_dirty_pages(struct address_space *mapping,
                io_schedule_timeout(pause);
                trace_balance_dirty_wait(bdi);
 
+               dirty_thresh = hard_dirty_limit(dirty_thresh);
+               /*
+                * max-pause area. If dirty exceeded but still within this
+                * area, no need to sleep for more than 200ms: (a) 8 pages per
+                * 200ms is typically more than enough to curb heavy dirtiers;
+                * (b) the pause time limit makes the dirtiers more responsive.
+                */
+               if (nr_dirty < dirty_thresh +
+                              dirty_thresh / DIRTY_MAXPAUSE_AREA &&
+                   time_after(jiffies, start_time + MAX_PAUSE))
+                       break;
+               /*
+                * pass-good area. When some bdi gets blocked (eg. NFS server
+                * not responding), or write bandwidth dropped dramatically due
+                * to concurrent reads, or dirty threshold suddenly dropped and
+                * the dirty pages cannot be brought down anytime soon (eg. on
+                * slow USB stick), at least let go of the good bdi's.
+                */
+               if (nr_dirty < dirty_thresh +
+                              dirty_thresh / DIRTY_PASSGOOD_AREA &&
+                   bdi_dirty < bdi_thresh)
+                       break;
+
                /*
                 * Increase the delay for each loop, up to our previous
                 * default of taking a 100ms nap.
@@ -732,7 +779,8 @@ static void balance_dirty_pages(struct address_space *mapping,
                        pause = HZ / 10;
        }
 
-       if (!dirty_exceeded && bdi->dirty_exceeded)
+       /* Clear dirty_exceeded flag only when no task can exceed the limit */
+       if (clear_dirty_exceeded && bdi->dirty_exceeded)
                bdi->dirty_exceeded = 0;
 
        if (writeback_in_progress(bdi))
@@ -1299,7 +1347,6 @@ EXPORT_SYMBOL(account_page_dirtied);
 void account_page_writeback(struct page *page)
 {
        inc_zone_page_state(page, NR_WRITEBACK);
-       inc_zone_page_state(page, NR_WRITTEN);
 }
 EXPORT_SYMBOL(account_page_writeback);
 
@@ -1516,8 +1563,10 @@ int test_clear_page_writeback(struct page *page)
        } else {
                ret = TestClearPageWriteback(page);
        }
-       if (ret)
+       if (ret) {
                dec_zone_page_state(page, NR_WRITEBACK);
+               inc_zone_page_state(page, NR_WRITTEN);
+       }
        return ret;
 }
 
@@ -1563,10 +1612,6 @@ EXPORT_SYMBOL(test_set_page_writeback);
  */
 int mapping_tagged(struct address_space *mapping, int tag)
 {
-       int ret;
-       rcu_read_lock();
-       ret = radix_tree_tagged(&mapping->page_tree, tag);
-       rcu_read_unlock();
-       return ret;
+       return radix_tree_tagged(&mapping->page_tree, tag);
 }
 EXPORT_SYMBOL(mapping_tagged);