btrfs: move inline function code to header file
[linux-3.10.git] / fs / btrfs / extent_io.c
1 #include <linux/bitops.h>
2 #include <linux/slab.h>
3 #include <linux/bio.h>
4 #include <linux/mm.h>
5 #include <linux/pagemap.h>
6 #include <linux/page-flags.h>
7 #include <linux/module.h>
8 #include <linux/spinlock.h>
9 #include <linux/blkdev.h>
10 #include <linux/swap.h>
11 #include <linux/writeback.h>
12 #include <linux/pagevec.h>
13 #include <linux/prefetch.h>
14 #include <linux/cleancache.h>
15 #include "extent_io.h"
16 #include "extent_map.h"
17 #include "compat.h"
18 #include "ctree.h"
19 #include "btrfs_inode.h"
20 #include "volumes.h"
21 #include "check-integrity.h"
22 #include "locking.h"
23 #include "rcu-string.h"
24
25 static struct kmem_cache *extent_state_cache;
26 static struct kmem_cache *extent_buffer_cache;
27
28 static LIST_HEAD(buffers);
29 static LIST_HEAD(states);
30
31 #define LEAK_DEBUG 0
32 #if LEAK_DEBUG
33 static DEFINE_SPINLOCK(leak_lock);
34 #endif
35
36 #define BUFFER_LRU_MAX 64
37
38 struct tree_entry {
39         u64 start;
40         u64 end;
41         struct rb_node rb_node;
42 };
43
44 struct extent_page_data {
45         struct bio *bio;
46         struct extent_io_tree *tree;
47         get_extent_t *get_extent;
48         unsigned long bio_flags;
49
50         /* tells writepage not to lock the state bits for this range
51          * it still does the unlocking
52          */
53         unsigned int extent_locked:1;
54
55         /* tells the submit_bio code to use a WRITE_SYNC */
56         unsigned int sync_io:1;
57 };
58
59 static noinline void flush_write_bio(void *data);
60 static inline struct btrfs_fs_info *
61 tree_fs_info(struct extent_io_tree *tree)
62 {
63         return btrfs_sb(tree->mapping->host->i_sb);
64 }
65
66 int __init extent_io_init(void)
67 {
68         extent_state_cache = kmem_cache_create("btrfs_extent_state",
69                         sizeof(struct extent_state), 0,
70                         SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
71         if (!extent_state_cache)
72                 return -ENOMEM;
73
74         extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
75                         sizeof(struct extent_buffer), 0,
76                         SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
77         if (!extent_buffer_cache)
78                 goto free_state_cache;
79         return 0;
80
81 free_state_cache:
82         kmem_cache_destroy(extent_state_cache);
83         return -ENOMEM;
84 }
85
86 void extent_io_exit(void)
87 {
88         struct extent_state *state;
89         struct extent_buffer *eb;
90
91         while (!list_empty(&states)) {
92                 state = list_entry(states.next, struct extent_state, leak_list);
93                 printk(KERN_ERR "btrfs state leak: start %llu end %llu "
94                        "state %lu in tree %p refs %d\n",
95                        (unsigned long long)state->start,
96                        (unsigned long long)state->end,
97                        state->state, state->tree, atomic_read(&state->refs));
98                 list_del(&state->leak_list);
99                 kmem_cache_free(extent_state_cache, state);
100
101         }
102
103         while (!list_empty(&buffers)) {
104                 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
105                 printk(KERN_ERR "btrfs buffer leak start %llu len %lu "
106                        "refs %d\n", (unsigned long long)eb->start,
107                        eb->len, atomic_read(&eb->refs));
108                 list_del(&eb->leak_list);
109                 kmem_cache_free(extent_buffer_cache, eb);
110         }
111         if (extent_state_cache)
112                 kmem_cache_destroy(extent_state_cache);
113         if (extent_buffer_cache)
114                 kmem_cache_destroy(extent_buffer_cache);
115 }
116
117 void extent_io_tree_init(struct extent_io_tree *tree,
118                          struct address_space *mapping)
119 {
120         tree->state = RB_ROOT;
121         INIT_RADIX_TREE(&tree->buffer, GFP_ATOMIC);
122         tree->ops = NULL;
123         tree->dirty_bytes = 0;
124         spin_lock_init(&tree->lock);
125         spin_lock_init(&tree->buffer_lock);
126         tree->mapping = mapping;
127 }
128
129 static struct extent_state *alloc_extent_state(gfp_t mask)
130 {
131         struct extent_state *state;
132 #if LEAK_DEBUG
133         unsigned long flags;
134 #endif
135
136         state = kmem_cache_alloc(extent_state_cache, mask);
137         if (!state)
138                 return state;
139         state->state = 0;
140         state->private = 0;
141         state->tree = NULL;
142 #if LEAK_DEBUG
143         spin_lock_irqsave(&leak_lock, flags);
144         list_add(&state->leak_list, &states);
145         spin_unlock_irqrestore(&leak_lock, flags);
146 #endif
147         atomic_set(&state->refs, 1);
148         init_waitqueue_head(&state->wq);
149         trace_alloc_extent_state(state, mask, _RET_IP_);
150         return state;
151 }
152
153 void free_extent_state(struct extent_state *state)
154 {
155         if (!state)
156                 return;
157         if (atomic_dec_and_test(&state->refs)) {
158 #if LEAK_DEBUG
159                 unsigned long flags;
160 #endif
161                 WARN_ON(state->tree);
162 #if LEAK_DEBUG
163                 spin_lock_irqsave(&leak_lock, flags);
164                 list_del(&state->leak_list);
165                 spin_unlock_irqrestore(&leak_lock, flags);
166 #endif
167                 trace_free_extent_state(state, _RET_IP_);
168                 kmem_cache_free(extent_state_cache, state);
169         }
170 }
171
172 static struct rb_node *tree_insert(struct rb_root *root, u64 offset,
173                                    struct rb_node *node)
174 {
175         struct rb_node **p = &root->rb_node;
176         struct rb_node *parent = NULL;
177         struct tree_entry *entry;
178
179         while (*p) {
180                 parent = *p;
181                 entry = rb_entry(parent, struct tree_entry, rb_node);
182
183                 if (offset < entry->start)
184                         p = &(*p)->rb_left;
185                 else if (offset > entry->end)
186                         p = &(*p)->rb_right;
187                 else
188                         return parent;
189         }
190
191         rb_link_node(node, parent, p);
192         rb_insert_color(node, root);
193         return NULL;
194 }
195
196 static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
197                                      struct rb_node **prev_ret,
198                                      struct rb_node **next_ret)
199 {
200         struct rb_root *root = &tree->state;
201         struct rb_node *n = root->rb_node;
202         struct rb_node *prev = NULL;
203         struct rb_node *orig_prev = NULL;
204         struct tree_entry *entry;
205         struct tree_entry *prev_entry = NULL;
206
207         while (n) {
208                 entry = rb_entry(n, struct tree_entry, rb_node);
209                 prev = n;
210                 prev_entry = entry;
211
212                 if (offset < entry->start)
213                         n = n->rb_left;
214                 else if (offset > entry->end)
215                         n = n->rb_right;
216                 else
217                         return n;
218         }
219
220         if (prev_ret) {
221                 orig_prev = prev;
222                 while (prev && offset > prev_entry->end) {
223                         prev = rb_next(prev);
224                         prev_entry = rb_entry(prev, struct tree_entry, rb_node);
225                 }
226                 *prev_ret = prev;
227                 prev = orig_prev;
228         }
229
230         if (next_ret) {
231                 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
232                 while (prev && offset < prev_entry->start) {
233                         prev = rb_prev(prev);
234                         prev_entry = rb_entry(prev, struct tree_entry, rb_node);
235                 }
236                 *next_ret = prev;
237         }
238         return NULL;
239 }
240
241 static inline struct rb_node *tree_search(struct extent_io_tree *tree,
242                                           u64 offset)
243 {
244         struct rb_node *prev = NULL;
245         struct rb_node *ret;
246
247         ret = __etree_search(tree, offset, &prev, NULL);
248         if (!ret)
249                 return prev;
250         return ret;
251 }
252
253 static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
254                      struct extent_state *other)
255 {
256         if (tree->ops && tree->ops->merge_extent_hook)
257                 tree->ops->merge_extent_hook(tree->mapping->host, new,
258                                              other);
259 }
260
261 /*
262  * utility function to look for merge candidates inside a given range.
263  * Any extents with matching state are merged together into a single
264  * extent in the tree.  Extents with EXTENT_IO in their state field
265  * are not merged because the end_io handlers need to be able to do
266  * operations on them without sleeping (or doing allocations/splits).
267  *
268  * This should be called with the tree lock held.
269  */
270 static void merge_state(struct extent_io_tree *tree,
271                         struct extent_state *state)
272 {
273         struct extent_state *other;
274         struct rb_node *other_node;
275
276         if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
277                 return;
278
279         other_node = rb_prev(&state->rb_node);
280         if (other_node) {
281                 other = rb_entry(other_node, struct extent_state, rb_node);
282                 if (other->end == state->start - 1 &&
283                     other->state == state->state) {
284                         merge_cb(tree, state, other);
285                         state->start = other->start;
286                         other->tree = NULL;
287                         rb_erase(&other->rb_node, &tree->state);
288                         free_extent_state(other);
289                 }
290         }
291         other_node = rb_next(&state->rb_node);
292         if (other_node) {
293                 other = rb_entry(other_node, struct extent_state, rb_node);
294                 if (other->start == state->end + 1 &&
295                     other->state == state->state) {
296                         merge_cb(tree, state, other);
297                         state->end = other->end;
298                         other->tree = NULL;
299                         rb_erase(&other->rb_node, &tree->state);
300                         free_extent_state(other);
301                 }
302         }
303 }
304
305 static void set_state_cb(struct extent_io_tree *tree,
306                          struct extent_state *state, int *bits)
307 {
308         if (tree->ops && tree->ops->set_bit_hook)
309                 tree->ops->set_bit_hook(tree->mapping->host, state, bits);
310 }
311
312 static void clear_state_cb(struct extent_io_tree *tree,
313                            struct extent_state *state, int *bits)
314 {
315         if (tree->ops && tree->ops->clear_bit_hook)
316                 tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
317 }
318
319 static void set_state_bits(struct extent_io_tree *tree,
320                            struct extent_state *state, int *bits);
321
322 /*
323  * insert an extent_state struct into the tree.  'bits' are set on the
324  * struct before it is inserted.
325  *
326  * This may return -EEXIST if the extent is already there, in which case the
327  * state struct is freed.
328  *
329  * The tree lock is not taken internally.  This is a utility function and
330  * probably isn't what you want to call (see set/clear_extent_bit).
331  */
332 static int insert_state(struct extent_io_tree *tree,
333                         struct extent_state *state, u64 start, u64 end,
334                         int *bits)
335 {
336         struct rb_node *node;
337
338         if (end < start) {
339                 printk(KERN_ERR "btrfs end < start %llu %llu\n",
340                        (unsigned long long)end,
341                        (unsigned long long)start);
342                 WARN_ON(1);
343         }
344         state->start = start;
345         state->end = end;
346
347         set_state_bits(tree, state, bits);
348
349         node = tree_insert(&tree->state, end, &state->rb_node);
350         if (node) {
351                 struct extent_state *found;
352                 found = rb_entry(node, struct extent_state, rb_node);
353                 printk(KERN_ERR "btrfs found node %llu %llu on insert of "
354                        "%llu %llu\n", (unsigned long long)found->start,
355                        (unsigned long long)found->end,
356                        (unsigned long long)start, (unsigned long long)end);
357                 return -EEXIST;
358         }
359         state->tree = tree;
360         merge_state(tree, state);
361         return 0;
362 }
363
364 static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
365                      u64 split)
366 {
367         if (tree->ops && tree->ops->split_extent_hook)
368                 tree->ops->split_extent_hook(tree->mapping->host, orig, split);
369 }
370
371 /*
372  * split a given extent state struct in two, inserting the preallocated
373  * struct 'prealloc' as the newly created second half.  'split' indicates an
374  * offset inside 'orig' where it should be split.
375  *
376  * Before calling,
377  * the tree has 'orig' at [orig->start, orig->end].  After calling, there
378  * are two extent state structs in the tree:
379  * prealloc: [orig->start, split - 1]
380  * orig: [ split, orig->end ]
381  *
382  * The tree locks are not taken by this function. They need to be held
383  * by the caller.
384  */
385 static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
386                        struct extent_state *prealloc, u64 split)
387 {
388         struct rb_node *node;
389
390         split_cb(tree, orig, split);
391
392         prealloc->start = orig->start;
393         prealloc->end = split - 1;
394         prealloc->state = orig->state;
395         orig->start = split;
396
397         node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node);
398         if (node) {
399                 free_extent_state(prealloc);
400                 return -EEXIST;
401         }
402         prealloc->tree = tree;
403         return 0;
404 }
405
406 static struct extent_state *next_state(struct extent_state *state)
407 {
408         struct rb_node *next = rb_next(&state->rb_node);
409         if (next)
410                 return rb_entry(next, struct extent_state, rb_node);
411         else
412                 return NULL;
413 }
414
415 /*
416  * utility function to clear some bits in an extent state struct.
417  * it will optionally wake up any one waiting on this state (wake == 1).
418  *
419  * If no bits are set on the state struct after clearing things, the
420  * struct is freed and removed from the tree
421  */
422 static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
423                                             struct extent_state *state,
424                                             int *bits, int wake)
425 {
426         struct extent_state *next;
427         int bits_to_clear = *bits & ~EXTENT_CTLBITS;
428
429         if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
430                 u64 range = state->end - state->start + 1;
431                 WARN_ON(range > tree->dirty_bytes);
432                 tree->dirty_bytes -= range;
433         }
434         clear_state_cb(tree, state, bits);
435         state->state &= ~bits_to_clear;
436         if (wake)
437                 wake_up(&state->wq);
438         if (state->state == 0) {
439                 next = next_state(state);
440                 if (state->tree) {
441                         rb_erase(&state->rb_node, &tree->state);
442                         state->tree = NULL;
443                         free_extent_state(state);
444                 } else {
445                         WARN_ON(1);
446                 }
447         } else {
448                 merge_state(tree, state);
449                 next = next_state(state);
450         }
451         return next;
452 }
453
454 static struct extent_state *
455 alloc_extent_state_atomic(struct extent_state *prealloc)
456 {
457         if (!prealloc)
458                 prealloc = alloc_extent_state(GFP_ATOMIC);
459
460         return prealloc;
461 }
462
463 void extent_io_tree_panic(struct extent_io_tree *tree, int err)
464 {
465         btrfs_panic(tree_fs_info(tree), err, "Locking error: "
466                     "Extent tree was modified by another "
467                     "thread while locked.");
468 }
469
470 /*
471  * clear some bits on a range in the tree.  This may require splitting
472  * or inserting elements in the tree, so the gfp mask is used to
473  * indicate which allocations or sleeping are allowed.
474  *
475  * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
476  * the given range from the tree regardless of state (ie for truncate).
477  *
478  * the range [start, end] is inclusive.
479  *
480  * This takes the tree lock, and returns 0 on success and < 0 on error.
481  */
482 int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
483                      int bits, int wake, int delete,
484                      struct extent_state **cached_state,
485                      gfp_t mask)
486 {
487         struct extent_state *state;
488         struct extent_state *cached;
489         struct extent_state *prealloc = NULL;
490         struct rb_node *node;
491         u64 last_end;
492         int err;
493         int clear = 0;
494
495         if (delete)
496                 bits |= ~EXTENT_CTLBITS;
497         bits |= EXTENT_FIRST_DELALLOC;
498
499         if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
500                 clear = 1;
501 again:
502         if (!prealloc && (mask & __GFP_WAIT)) {
503                 prealloc = alloc_extent_state(mask);
504                 if (!prealloc)
505                         return -ENOMEM;
506         }
507
508         spin_lock(&tree->lock);
509         if (cached_state) {
510                 cached = *cached_state;
511
512                 if (clear) {
513                         *cached_state = NULL;
514                         cached_state = NULL;
515                 }
516
517                 if (cached && cached->tree && cached->start <= start &&
518                     cached->end > start) {
519                         if (clear)
520                                 atomic_dec(&cached->refs);
521                         state = cached;
522                         goto hit_next;
523                 }
524                 if (clear)
525                         free_extent_state(cached);
526         }
527         /*
528          * this search will find the extents that end after
529          * our range starts
530          */
531         node = tree_search(tree, start);
532         if (!node)
533                 goto out;
534         state = rb_entry(node, struct extent_state, rb_node);
535 hit_next:
536         if (state->start > end)
537                 goto out;
538         WARN_ON(state->end < start);
539         last_end = state->end;
540
541         /* the state doesn't have the wanted bits, go ahead */
542         if (!(state->state & bits)) {
543                 state = next_state(state);
544                 goto next;
545         }
546
547         /*
548          *     | ---- desired range ---- |
549          *  | state | or
550          *  | ------------- state -------------- |
551          *
552          * We need to split the extent we found, and may flip
553          * bits on second half.
554          *
555          * If the extent we found extends past our range, we
556          * just split and search again.  It'll get split again
557          * the next time though.
558          *
559          * If the extent we found is inside our range, we clear
560          * the desired bit on it.
561          */
562
563         if (state->start < start) {
564                 prealloc = alloc_extent_state_atomic(prealloc);
565                 BUG_ON(!prealloc);
566                 err = split_state(tree, state, prealloc, start);
567                 if (err)
568                         extent_io_tree_panic(tree, err);
569
570                 prealloc = NULL;
571                 if (err)
572                         goto out;
573                 if (state->end <= end) {
574                         state = clear_state_bit(tree, state, &bits, wake);
575                         goto next;
576                 }
577                 goto search_again;
578         }
579         /*
580          * | ---- desired range ---- |
581          *                        | state |
582          * We need to split the extent, and clear the bit
583          * on the first half
584          */
585         if (state->start <= end && state->end > end) {
586                 prealloc = alloc_extent_state_atomic(prealloc);
587                 BUG_ON(!prealloc);
588                 err = split_state(tree, state, prealloc, end + 1);
589                 if (err)
590                         extent_io_tree_panic(tree, err);
591
592                 if (wake)
593                         wake_up(&state->wq);
594
595                 clear_state_bit(tree, prealloc, &bits, wake);
596
597                 prealloc = NULL;
598                 goto out;
599         }
600
601         state = clear_state_bit(tree, state, &bits, wake);
602 next:
603         if (last_end == (u64)-1)
604                 goto out;
605         start = last_end + 1;
606         if (start <= end && state && !need_resched())
607                 goto hit_next;
608         goto search_again;
609
610 out:
611         spin_unlock(&tree->lock);
612         if (prealloc)
613                 free_extent_state(prealloc);
614
615         return 0;
616
617 search_again:
618         if (start > end)
619                 goto out;
620         spin_unlock(&tree->lock);
621         if (mask & __GFP_WAIT)
622                 cond_resched();
623         goto again;
624 }
625
626 static void wait_on_state(struct extent_io_tree *tree,
627                           struct extent_state *state)
628                 __releases(tree->lock)
629                 __acquires(tree->lock)
630 {
631         DEFINE_WAIT(wait);
632         prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
633         spin_unlock(&tree->lock);
634         schedule();
635         spin_lock(&tree->lock);
636         finish_wait(&state->wq, &wait);
637 }
638
639 /*
640  * waits for one or more bits to clear on a range in the state tree.
641  * The range [start, end] is inclusive.
642  * The tree lock is taken by this function
643  */
644 void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits)
645 {
646         struct extent_state *state;
647         struct rb_node *node;
648
649         spin_lock(&tree->lock);
650 again:
651         while (1) {
652                 /*
653                  * this search will find all the extents that end after
654                  * our range starts
655                  */
656                 node = tree_search(tree, start);
657                 if (!node)
658                         break;
659
660                 state = rb_entry(node, struct extent_state, rb_node);
661
662                 if (state->start > end)
663                         goto out;
664
665                 if (state->state & bits) {
666                         start = state->start;
667                         atomic_inc(&state->refs);
668                         wait_on_state(tree, state);
669                         free_extent_state(state);
670                         goto again;
671                 }
672                 start = state->end + 1;
673
674                 if (start > end)
675                         break;
676
677                 cond_resched_lock(&tree->lock);
678         }
679 out:
680         spin_unlock(&tree->lock);
681 }
682
683 static void set_state_bits(struct extent_io_tree *tree,
684                            struct extent_state *state,
685                            int *bits)
686 {
687         int bits_to_set = *bits & ~EXTENT_CTLBITS;
688
689         set_state_cb(tree, state, bits);
690         if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
691                 u64 range = state->end - state->start + 1;
692                 tree->dirty_bytes += range;
693         }
694         state->state |= bits_to_set;
695 }
696
697 static void cache_state(struct extent_state *state,
698                         struct extent_state **cached_ptr)
699 {
700         if (cached_ptr && !(*cached_ptr)) {
701                 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY)) {
702                         *cached_ptr = state;
703                         atomic_inc(&state->refs);
704                 }
705         }
706 }
707
708 static void uncache_state(struct extent_state **cached_ptr)
709 {
710         if (cached_ptr && (*cached_ptr)) {
711                 struct extent_state *state = *cached_ptr;
712                 *cached_ptr = NULL;
713                 free_extent_state(state);
714         }
715 }
716
717 /*
718  * set some bits on a range in the tree.  This may require allocations or
719  * sleeping, so the gfp mask is used to indicate what is allowed.
720  *
721  * If any of the exclusive bits are set, this will fail with -EEXIST if some
722  * part of the range already has the desired bits set.  The start of the
723  * existing range is returned in failed_start in this case.
724  *
725  * [start, end] is inclusive This takes the tree lock.
726  */
727
728 static int __must_check
729 __set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
730                  int bits, int exclusive_bits, u64 *failed_start,
731                  struct extent_state **cached_state, gfp_t mask)
732 {
733         struct extent_state *state;
734         struct extent_state *prealloc = NULL;
735         struct rb_node *node;
736         int err = 0;
737         u64 last_start;
738         u64 last_end;
739
740         bits |= EXTENT_FIRST_DELALLOC;
741 again:
742         if (!prealloc && (mask & __GFP_WAIT)) {
743                 prealloc = alloc_extent_state(mask);
744                 BUG_ON(!prealloc);
745         }
746
747         spin_lock(&tree->lock);
748         if (cached_state && *cached_state) {
749                 state = *cached_state;
750                 if (state->start <= start && state->end > start &&
751                     state->tree) {
752                         node = &state->rb_node;
753                         goto hit_next;
754                 }
755         }
756         /*
757          * this search will find all the extents that end after
758          * our range starts.
759          */
760         node = tree_search(tree, start);
761         if (!node) {
762                 prealloc = alloc_extent_state_atomic(prealloc);
763                 BUG_ON(!prealloc);
764                 err = insert_state(tree, prealloc, start, end, &bits);
765                 if (err)
766                         extent_io_tree_panic(tree, err);
767
768                 prealloc = NULL;
769                 goto out;
770         }
771         state = rb_entry(node, struct extent_state, rb_node);
772 hit_next:
773         last_start = state->start;
774         last_end = state->end;
775
776         /*
777          * | ---- desired range ---- |
778          * | state |
779          *
780          * Just lock what we found and keep going
781          */
782         if (state->start == start && state->end <= end) {
783                 if (state->state & exclusive_bits) {
784                         *failed_start = state->start;
785                         err = -EEXIST;
786                         goto out;
787                 }
788
789                 set_state_bits(tree, state, &bits);
790                 cache_state(state, cached_state);
791                 merge_state(tree, state);
792                 if (last_end == (u64)-1)
793                         goto out;
794                 start = last_end + 1;
795                 state = next_state(state);
796                 if (start < end && state && state->start == start &&
797                     !need_resched())
798                         goto hit_next;
799                 goto search_again;
800         }
801
802         /*
803          *     | ---- desired range ---- |
804          * | state |
805          *   or
806          * | ------------- state -------------- |
807          *
808          * We need to split the extent we found, and may flip bits on
809          * second half.
810          *
811          * If the extent we found extends past our
812          * range, we just split and search again.  It'll get split
813          * again the next time though.
814          *
815          * If the extent we found is inside our range, we set the
816          * desired bit on it.
817          */
818         if (state->start < start) {
819                 if (state->state & exclusive_bits) {
820                         *failed_start = start;
821                         err = -EEXIST;
822                         goto out;
823                 }
824
825                 prealloc = alloc_extent_state_atomic(prealloc);
826                 BUG_ON(!prealloc);
827                 err = split_state(tree, state, prealloc, start);
828                 if (err)
829                         extent_io_tree_panic(tree, err);
830
831                 prealloc = NULL;
832                 if (err)
833                         goto out;
834                 if (state->end <= end) {
835                         set_state_bits(tree, state, &bits);
836                         cache_state(state, cached_state);
837                         merge_state(tree, state);
838                         if (last_end == (u64)-1)
839                                 goto out;
840                         start = last_end + 1;
841                         state = next_state(state);
842                         if (start < end && state && state->start == start &&
843                             !need_resched())
844                                 goto hit_next;
845                 }
846                 goto search_again;
847         }
848         /*
849          * | ---- desired range ---- |
850          *     | state | or               | state |
851          *
852          * There's a hole, we need to insert something in it and
853          * ignore the extent we found.
854          */
855         if (state->start > start) {
856                 u64 this_end;
857                 if (end < last_start)
858                         this_end = end;
859                 else
860                         this_end = last_start - 1;
861
862                 prealloc = alloc_extent_state_atomic(prealloc);
863                 BUG_ON(!prealloc);
864
865                 /*
866                  * Avoid to free 'prealloc' if it can be merged with
867                  * the later extent.
868                  */
869                 err = insert_state(tree, prealloc, start, this_end,
870                                    &bits);
871                 if (err)
872                         extent_io_tree_panic(tree, err);
873
874                 cache_state(prealloc, cached_state);
875                 prealloc = NULL;
876                 start = this_end + 1;
877                 goto search_again;
878         }
879         /*
880          * | ---- desired range ---- |
881          *                        | state |
882          * We need to split the extent, and set the bit
883          * on the first half
884          */
885         if (state->start <= end && state->end > end) {
886                 if (state->state & exclusive_bits) {
887                         *failed_start = start;
888                         err = -EEXIST;
889                         goto out;
890                 }
891
892                 prealloc = alloc_extent_state_atomic(prealloc);
893                 BUG_ON(!prealloc);
894                 err = split_state(tree, state, prealloc, end + 1);
895                 if (err)
896                         extent_io_tree_panic(tree, err);
897
898                 set_state_bits(tree, prealloc, &bits);
899                 cache_state(prealloc, cached_state);
900                 merge_state(tree, prealloc);
901                 prealloc = NULL;
902                 goto out;
903         }
904
905         goto search_again;
906
907 out:
908         spin_unlock(&tree->lock);
909         if (prealloc)
910                 free_extent_state(prealloc);
911
912         return err;
913
914 search_again:
915         if (start > end)
916                 goto out;
917         spin_unlock(&tree->lock);
918         if (mask & __GFP_WAIT)
919                 cond_resched();
920         goto again;
921 }
922
923 int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits,
924                    u64 *failed_start, struct extent_state **cached_state,
925                    gfp_t mask)
926 {
927         return __set_extent_bit(tree, start, end, bits, 0, failed_start,
928                                 cached_state, mask);
929 }
930
931
932 /**
933  * convert_extent_bit - convert all bits in a given range from one bit to
934  *                      another
935  * @tree:       the io tree to search
936  * @start:      the start offset in bytes
937  * @end:        the end offset in bytes (inclusive)
938  * @bits:       the bits to set in this range
939  * @clear_bits: the bits to clear in this range
940  * @cached_state:       state that we're going to cache
941  * @mask:       the allocation mask
942  *
943  * This will go through and set bits for the given range.  If any states exist
944  * already in this range they are set with the given bit and cleared of the
945  * clear_bits.  This is only meant to be used by things that are mergeable, ie
946  * converting from say DELALLOC to DIRTY.  This is not meant to be used with
947  * boundary bits like LOCK.
948  */
949 int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
950                        int bits, int clear_bits,
951                        struct extent_state **cached_state, gfp_t mask)
952 {
953         struct extent_state *state;
954         struct extent_state *prealloc = NULL;
955         struct rb_node *node;
956         int err = 0;
957         u64 last_start;
958         u64 last_end;
959
960 again:
961         if (!prealloc && (mask & __GFP_WAIT)) {
962                 prealloc = alloc_extent_state(mask);
963                 if (!prealloc)
964                         return -ENOMEM;
965         }
966
967         spin_lock(&tree->lock);
968         if (cached_state && *cached_state) {
969                 state = *cached_state;
970                 if (state->start <= start && state->end > start &&
971                     state->tree) {
972                         node = &state->rb_node;
973                         goto hit_next;
974                 }
975         }
976
977         /*
978          * this search will find all the extents that end after
979          * our range starts.
980          */
981         node = tree_search(tree, start);
982         if (!node) {
983                 prealloc = alloc_extent_state_atomic(prealloc);
984                 if (!prealloc) {
985                         err = -ENOMEM;
986                         goto out;
987                 }
988                 err = insert_state(tree, prealloc, start, end, &bits);
989                 prealloc = NULL;
990                 if (err)
991                         extent_io_tree_panic(tree, err);
992                 goto out;
993         }
994         state = rb_entry(node, struct extent_state, rb_node);
995 hit_next:
996         last_start = state->start;
997         last_end = state->end;
998
999         /*
1000          * | ---- desired range ---- |
1001          * | state |
1002          *
1003          * Just lock what we found and keep going
1004          */
1005         if (state->start == start && state->end <= end) {
1006                 set_state_bits(tree, state, &bits);
1007                 cache_state(state, cached_state);
1008                 state = clear_state_bit(tree, state, &clear_bits, 0);
1009                 if (last_end == (u64)-1)
1010                         goto out;
1011                 start = last_end + 1;
1012                 if (start < end && state && state->start == start &&
1013                     !need_resched())
1014                         goto hit_next;
1015                 goto search_again;
1016         }
1017
1018         /*
1019          *     | ---- desired range ---- |
1020          * | state |
1021          *   or
1022          * | ------------- state -------------- |
1023          *
1024          * We need to split the extent we found, and may flip bits on
1025          * second half.
1026          *
1027          * If the extent we found extends past our
1028          * range, we just split and search again.  It'll get split
1029          * again the next time though.
1030          *
1031          * If the extent we found is inside our range, we set the
1032          * desired bit on it.
1033          */
1034         if (state->start < start) {
1035                 prealloc = alloc_extent_state_atomic(prealloc);
1036                 if (!prealloc) {
1037                         err = -ENOMEM;
1038                         goto out;
1039                 }
1040                 err = split_state(tree, state, prealloc, start);
1041                 if (err)
1042                         extent_io_tree_panic(tree, err);
1043                 prealloc = NULL;
1044                 if (err)
1045                         goto out;
1046                 if (state->end <= end) {
1047                         set_state_bits(tree, state, &bits);
1048                         cache_state(state, cached_state);
1049                         state = clear_state_bit(tree, state, &clear_bits, 0);
1050                         if (last_end == (u64)-1)
1051                                 goto out;
1052                         start = last_end + 1;
1053                         if (start < end && state && state->start == start &&
1054                             !need_resched())
1055                                 goto hit_next;
1056                 }
1057                 goto search_again;
1058         }
1059         /*
1060          * | ---- desired range ---- |
1061          *     | state | or               | state |
1062          *
1063          * There's a hole, we need to insert something in it and
1064          * ignore the extent we found.
1065          */
1066         if (state->start > start) {
1067                 u64 this_end;
1068                 if (end < last_start)
1069                         this_end = end;
1070                 else
1071                         this_end = last_start - 1;
1072
1073                 prealloc = alloc_extent_state_atomic(prealloc);
1074                 if (!prealloc) {
1075                         err = -ENOMEM;
1076                         goto out;
1077                 }
1078
1079                 /*
1080                  * Avoid to free 'prealloc' if it can be merged with
1081                  * the later extent.
1082                  */
1083                 err = insert_state(tree, prealloc, start, this_end,
1084                                    &bits);
1085                 if (err)
1086                         extent_io_tree_panic(tree, err);
1087                 cache_state(prealloc, cached_state);
1088                 prealloc = NULL;
1089                 start = this_end + 1;
1090                 goto search_again;
1091         }
1092         /*
1093          * | ---- desired range ---- |
1094          *                        | state |
1095          * We need to split the extent, and set the bit
1096          * on the first half
1097          */
1098         if (state->start <= end && state->end > end) {
1099                 prealloc = alloc_extent_state_atomic(prealloc);
1100                 if (!prealloc) {
1101                         err = -ENOMEM;
1102                         goto out;
1103                 }
1104
1105                 err = split_state(tree, state, prealloc, end + 1);
1106                 if (err)
1107                         extent_io_tree_panic(tree, err);
1108
1109                 set_state_bits(tree, prealloc, &bits);
1110                 cache_state(prealloc, cached_state);
1111                 clear_state_bit(tree, prealloc, &clear_bits, 0);
1112                 prealloc = NULL;
1113                 goto out;
1114         }
1115
1116         goto search_again;
1117
1118 out:
1119         spin_unlock(&tree->lock);
1120         if (prealloc)
1121                 free_extent_state(prealloc);
1122
1123         return err;
1124
1125 search_again:
1126         if (start > end)
1127                 goto out;
1128         spin_unlock(&tree->lock);
1129         if (mask & __GFP_WAIT)
1130                 cond_resched();
1131         goto again;
1132 }
1133
1134 /* wrappers around set/clear extent bit */
1135 int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1136                      gfp_t mask)
1137 {
1138         return set_extent_bit(tree, start, end, EXTENT_DIRTY, NULL,
1139                               NULL, mask);
1140 }
1141
1142 int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1143                     int bits, gfp_t mask)
1144 {
1145         return set_extent_bit(tree, start, end, bits, NULL,
1146                               NULL, mask);
1147 }
1148
1149 int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1150                       int bits, gfp_t mask)
1151 {
1152         return clear_extent_bit(tree, start, end, bits, 0, 0, NULL, mask);
1153 }
1154
1155 int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
1156                         struct extent_state **cached_state, gfp_t mask)
1157 {
1158         return set_extent_bit(tree, start, end,
1159                               EXTENT_DELALLOC | EXTENT_UPTODATE,
1160                               NULL, cached_state, mask);
1161 }
1162
1163 int set_extent_defrag(struct extent_io_tree *tree, u64 start, u64 end,
1164                       struct extent_state **cached_state, gfp_t mask)
1165 {
1166         return set_extent_bit(tree, start, end,
1167                               EXTENT_DELALLOC | EXTENT_UPTODATE | EXTENT_DEFRAG,
1168                               NULL, cached_state, mask);
1169 }
1170
1171 int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1172                        gfp_t mask)
1173 {
1174         return clear_extent_bit(tree, start, end,
1175                                 EXTENT_DIRTY | EXTENT_DELALLOC |
1176                                 EXTENT_DO_ACCOUNTING, 0, 0, NULL, mask);
1177 }
1178
1179 int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
1180                      gfp_t mask)
1181 {
1182         return set_extent_bit(tree, start, end, EXTENT_NEW, NULL,
1183                               NULL, mask);
1184 }
1185
1186 int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
1187                         struct extent_state **cached_state, gfp_t mask)
1188 {
1189         return set_extent_bit(tree, start, end, EXTENT_UPTODATE, 0,
1190                               cached_state, mask);
1191 }
1192
1193 int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
1194                           struct extent_state **cached_state, gfp_t mask)
1195 {
1196         return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0,
1197                                 cached_state, mask);
1198 }
1199
1200 /*
1201  * either insert or lock state struct between start and end use mask to tell
1202  * us if waiting is desired.
1203  */
1204 int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1205                      int bits, struct extent_state **cached_state)
1206 {
1207         int err;
1208         u64 failed_start;
1209         while (1) {
1210                 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED | bits,
1211                                        EXTENT_LOCKED, &failed_start,
1212                                        cached_state, GFP_NOFS);
1213                 if (err == -EEXIST) {
1214                         wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1215                         start = failed_start;
1216                 } else
1217                         break;
1218                 WARN_ON(start > end);
1219         }
1220         return err;
1221 }
1222
1223 int lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1224 {
1225         return lock_extent_bits(tree, start, end, 0, NULL);
1226 }
1227
1228 int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1229 {
1230         int err;
1231         u64 failed_start;
1232
1233         err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1234                                &failed_start, NULL, GFP_NOFS);
1235         if (err == -EEXIST) {
1236                 if (failed_start > start)
1237                         clear_extent_bit(tree, start, failed_start - 1,
1238                                          EXTENT_LOCKED, 1, 0, NULL, GFP_NOFS);
1239                 return 0;
1240         }
1241         return 1;
1242 }
1243
1244 int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end,
1245                          struct extent_state **cached, gfp_t mask)
1246 {
1247         return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached,
1248                                 mask);
1249 }
1250
1251 int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1252 {
1253         return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL,
1254                                 GFP_NOFS);
1255 }
1256
1257 /*
1258  * helper function to set both pages and extents in the tree writeback
1259  */
1260 static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
1261 {
1262         unsigned long index = start >> PAGE_CACHE_SHIFT;
1263         unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1264         struct page *page;
1265
1266         while (index <= end_index) {
1267                 page = find_get_page(tree->mapping, index);
1268                 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1269                 set_page_writeback(page);
1270                 page_cache_release(page);
1271                 index++;
1272         }
1273         return 0;
1274 }
1275
1276 /* find the first state struct with 'bits' set after 'start', and
1277  * return it.  tree->lock must be held.  NULL will returned if
1278  * nothing was found after 'start'
1279  */
1280 struct extent_state *find_first_extent_bit_state(struct extent_io_tree *tree,
1281                                                  u64 start, int bits)
1282 {
1283         struct rb_node *node;
1284         struct extent_state *state;
1285
1286         /*
1287          * this search will find all the extents that end after
1288          * our range starts.
1289          */
1290         node = tree_search(tree, start);
1291         if (!node)
1292                 goto out;
1293
1294         while (1) {
1295                 state = rb_entry(node, struct extent_state, rb_node);
1296                 if (state->end >= start && (state->state & bits))
1297                         return state;
1298
1299                 node = rb_next(node);
1300                 if (!node)
1301                         break;
1302         }
1303 out:
1304         return NULL;
1305 }
1306
1307 /*
1308  * find the first offset in the io tree with 'bits' set. zero is
1309  * returned if we find something, and *start_ret and *end_ret are
1310  * set to reflect the state struct that was found.
1311  *
1312  * If nothing was found, 1 is returned. If found something, return 0.
1313  */
1314 int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
1315                           u64 *start_ret, u64 *end_ret, int bits,
1316                           struct extent_state **cached_state)
1317 {
1318         struct extent_state *state;
1319         struct rb_node *n;
1320         int ret = 1;
1321
1322         spin_lock(&tree->lock);
1323         if (cached_state && *cached_state) {
1324                 state = *cached_state;
1325                 if (state->end == start - 1 && state->tree) {
1326                         n = rb_next(&state->rb_node);
1327                         while (n) {
1328                                 state = rb_entry(n, struct extent_state,
1329                                                  rb_node);
1330                                 if (state->state & bits)
1331                                         goto got_it;
1332                                 n = rb_next(n);
1333                         }
1334                         free_extent_state(*cached_state);
1335                         *cached_state = NULL;
1336                         goto out;
1337                 }
1338                 free_extent_state(*cached_state);
1339                 *cached_state = NULL;
1340         }
1341
1342         state = find_first_extent_bit_state(tree, start, bits);
1343 got_it:
1344         if (state) {
1345                 cache_state(state, cached_state);
1346                 *start_ret = state->start;
1347                 *end_ret = state->end;
1348                 ret = 0;
1349         }
1350 out:
1351         spin_unlock(&tree->lock);
1352         return ret;
1353 }
1354
1355 /*
1356  * find a contiguous range of bytes in the file marked as delalloc, not
1357  * more than 'max_bytes'.  start and end are used to return the range,
1358  *
1359  * 1 is returned if we find something, 0 if nothing was in the tree
1360  */
1361 static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
1362                                         u64 *start, u64 *end, u64 max_bytes,
1363                                         struct extent_state **cached_state)
1364 {
1365         struct rb_node *node;
1366         struct extent_state *state;
1367         u64 cur_start = *start;
1368         u64 found = 0;
1369         u64 total_bytes = 0;
1370
1371         spin_lock(&tree->lock);
1372
1373         /*
1374          * this search will find all the extents that end after
1375          * our range starts.
1376          */
1377         node = tree_search(tree, cur_start);
1378         if (!node) {
1379                 if (!found)
1380                         *end = (u64)-1;
1381                 goto out;
1382         }
1383
1384         while (1) {
1385                 state = rb_entry(node, struct extent_state, rb_node);
1386                 if (found && (state->start != cur_start ||
1387                               (state->state & EXTENT_BOUNDARY))) {
1388                         goto out;
1389                 }
1390                 if (!(state->state & EXTENT_DELALLOC)) {
1391                         if (!found)
1392                                 *end = state->end;
1393                         goto out;
1394                 }
1395                 if (!found) {
1396                         *start = state->start;
1397                         *cached_state = state;
1398                         atomic_inc(&state->refs);
1399                 }
1400                 found++;
1401                 *end = state->end;
1402                 cur_start = state->end + 1;
1403                 node = rb_next(node);
1404                 if (!node)
1405                         break;
1406                 total_bytes += state->end - state->start + 1;
1407                 if (total_bytes >= max_bytes)
1408                         break;
1409         }
1410 out:
1411         spin_unlock(&tree->lock);
1412         return found;
1413 }
1414
1415 static noinline void __unlock_for_delalloc(struct inode *inode,
1416                                            struct page *locked_page,
1417                                            u64 start, u64 end)
1418 {
1419         int ret;
1420         struct page *pages[16];
1421         unsigned long index = start >> PAGE_CACHE_SHIFT;
1422         unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1423         unsigned long nr_pages = end_index - index + 1;
1424         int i;
1425
1426         if (index == locked_page->index && end_index == index)
1427                 return;
1428
1429         while (nr_pages > 0) {
1430                 ret = find_get_pages_contig(inode->i_mapping, index,
1431                                      min_t(unsigned long, nr_pages,
1432                                      ARRAY_SIZE(pages)), pages);
1433                 for (i = 0; i < ret; i++) {
1434                         if (pages[i] != locked_page)
1435                                 unlock_page(pages[i]);
1436                         page_cache_release(pages[i]);
1437                 }
1438                 nr_pages -= ret;
1439                 index += ret;
1440                 cond_resched();
1441         }
1442 }
1443
1444 static noinline int lock_delalloc_pages(struct inode *inode,
1445                                         struct page *locked_page,
1446                                         u64 delalloc_start,
1447                                         u64 delalloc_end)
1448 {
1449         unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT;
1450         unsigned long start_index = index;
1451         unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT;
1452         unsigned long pages_locked = 0;
1453         struct page *pages[16];
1454         unsigned long nrpages;
1455         int ret;
1456         int i;
1457
1458         /* the caller is responsible for locking the start index */
1459         if (index == locked_page->index && index == end_index)
1460                 return 0;
1461
1462         /* skip the page at the start index */
1463         nrpages = end_index - index + 1;
1464         while (nrpages > 0) {
1465                 ret = find_get_pages_contig(inode->i_mapping, index,
1466                                      min_t(unsigned long,
1467                                      nrpages, ARRAY_SIZE(pages)), pages);
1468                 if (ret == 0) {
1469                         ret = -EAGAIN;
1470                         goto done;
1471                 }
1472                 /* now we have an array of pages, lock them all */
1473                 for (i = 0; i < ret; i++) {
1474                         /*
1475                          * the caller is taking responsibility for
1476                          * locked_page
1477                          */
1478                         if (pages[i] != locked_page) {
1479                                 lock_page(pages[i]);
1480                                 if (!PageDirty(pages[i]) ||
1481                                     pages[i]->mapping != inode->i_mapping) {
1482                                         ret = -EAGAIN;
1483                                         unlock_page(pages[i]);
1484                                         page_cache_release(pages[i]);
1485                                         goto done;
1486                                 }
1487                         }
1488                         page_cache_release(pages[i]);
1489                         pages_locked++;
1490                 }
1491                 nrpages -= ret;
1492                 index += ret;
1493                 cond_resched();
1494         }
1495         ret = 0;
1496 done:
1497         if (ret && pages_locked) {
1498                 __unlock_for_delalloc(inode, locked_page,
1499                               delalloc_start,
1500                               ((u64)(start_index + pages_locked - 1)) <<
1501                               PAGE_CACHE_SHIFT);
1502         }
1503         return ret;
1504 }
1505
1506 /*
1507  * find a contiguous range of bytes in the file marked as delalloc, not
1508  * more than 'max_bytes'.  start and end are used to return the range,
1509  *
1510  * 1 is returned if we find something, 0 if nothing was in the tree
1511  */
1512 static noinline u64 find_lock_delalloc_range(struct inode *inode,
1513                                              struct extent_io_tree *tree,
1514                                              struct page *locked_page,
1515                                              u64 *start, u64 *end,
1516                                              u64 max_bytes)
1517 {
1518         u64 delalloc_start;
1519         u64 delalloc_end;
1520         u64 found;
1521         struct extent_state *cached_state = NULL;
1522         int ret;
1523         int loops = 0;
1524
1525 again:
1526         /* step one, find a bunch of delalloc bytes starting at start */
1527         delalloc_start = *start;
1528         delalloc_end = 0;
1529         found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
1530                                     max_bytes, &cached_state);
1531         if (!found || delalloc_end <= *start) {
1532                 *start = delalloc_start;
1533                 *end = delalloc_end;
1534                 free_extent_state(cached_state);
1535                 return found;
1536         }
1537
1538         /*
1539          * start comes from the offset of locked_page.  We have to lock
1540          * pages in order, so we can't process delalloc bytes before
1541          * locked_page
1542          */
1543         if (delalloc_start < *start)
1544                 delalloc_start = *start;
1545
1546         /*
1547          * make sure to limit the number of pages we try to lock down
1548          * if we're looping.
1549          */
1550         if (delalloc_end + 1 - delalloc_start > max_bytes && loops)
1551                 delalloc_end = delalloc_start + PAGE_CACHE_SIZE - 1;
1552
1553         /* step two, lock all the pages after the page that has start */
1554         ret = lock_delalloc_pages(inode, locked_page,
1555                                   delalloc_start, delalloc_end);
1556         if (ret == -EAGAIN) {
1557                 /* some of the pages are gone, lets avoid looping by
1558                  * shortening the size of the delalloc range we're searching
1559                  */
1560                 free_extent_state(cached_state);
1561                 if (!loops) {
1562                         unsigned long offset = (*start) & (PAGE_CACHE_SIZE - 1);
1563                         max_bytes = PAGE_CACHE_SIZE - offset;
1564                         loops = 1;
1565                         goto again;
1566                 } else {
1567                         found = 0;
1568                         goto out_failed;
1569                 }
1570         }
1571         BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */
1572
1573         /* step three, lock the state bits for the whole range */
1574         lock_extent_bits(tree, delalloc_start, delalloc_end, 0, &cached_state);
1575
1576         /* then test to make sure it is all still delalloc */
1577         ret = test_range_bit(tree, delalloc_start, delalloc_end,
1578                              EXTENT_DELALLOC, 1, cached_state);
1579         if (!ret) {
1580                 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1581                                      &cached_state, GFP_NOFS);
1582                 __unlock_for_delalloc(inode, locked_page,
1583                               delalloc_start, delalloc_end);
1584                 cond_resched();
1585                 goto again;
1586         }
1587         free_extent_state(cached_state);
1588         *start = delalloc_start;
1589         *end = delalloc_end;
1590 out_failed:
1591         return found;
1592 }
1593
1594 int extent_clear_unlock_delalloc(struct inode *inode,
1595                                 struct extent_io_tree *tree,
1596                                 u64 start, u64 end, struct page *locked_page,
1597                                 unsigned long op)
1598 {
1599         int ret;
1600         struct page *pages[16];
1601         unsigned long index = start >> PAGE_CACHE_SHIFT;
1602         unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1603         unsigned long nr_pages = end_index - index + 1;
1604         int i;
1605         int clear_bits = 0;
1606
1607         if (op & EXTENT_CLEAR_UNLOCK)
1608                 clear_bits |= EXTENT_LOCKED;
1609         if (op & EXTENT_CLEAR_DIRTY)
1610                 clear_bits |= EXTENT_DIRTY;
1611
1612         if (op & EXTENT_CLEAR_DELALLOC)
1613                 clear_bits |= EXTENT_DELALLOC;
1614
1615         clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
1616         if (!(op & (EXTENT_CLEAR_UNLOCK_PAGE | EXTENT_CLEAR_DIRTY |
1617                     EXTENT_SET_WRITEBACK | EXTENT_END_WRITEBACK |
1618                     EXTENT_SET_PRIVATE2)))
1619                 return 0;
1620
1621         while (nr_pages > 0) {
1622                 ret = find_get_pages_contig(inode->i_mapping, index,
1623                                      min_t(unsigned long,
1624                                      nr_pages, ARRAY_SIZE(pages)), pages);
1625                 for (i = 0; i < ret; i++) {
1626
1627                         if (op & EXTENT_SET_PRIVATE2)
1628                                 SetPagePrivate2(pages[i]);
1629
1630                         if (pages[i] == locked_page) {
1631                                 page_cache_release(pages[i]);
1632                                 continue;
1633                         }
1634                         if (op & EXTENT_CLEAR_DIRTY)
1635                                 clear_page_dirty_for_io(pages[i]);
1636                         if (op & EXTENT_SET_WRITEBACK)
1637                                 set_page_writeback(pages[i]);
1638                         if (op & EXTENT_END_WRITEBACK)
1639                                 end_page_writeback(pages[i]);
1640                         if (op & EXTENT_CLEAR_UNLOCK_PAGE)
1641                                 unlock_page(pages[i]);
1642                         page_cache_release(pages[i]);
1643                 }
1644                 nr_pages -= ret;
1645                 index += ret;
1646                 cond_resched();
1647         }
1648         return 0;
1649 }
1650
1651 /*
1652  * count the number of bytes in the tree that have a given bit(s)
1653  * set.  This can be fairly slow, except for EXTENT_DIRTY which is
1654  * cached.  The total number found is returned.
1655  */
1656 u64 count_range_bits(struct extent_io_tree *tree,
1657                      u64 *start, u64 search_end, u64 max_bytes,
1658                      unsigned long bits, int contig)
1659 {
1660         struct rb_node *node;
1661         struct extent_state *state;
1662         u64 cur_start = *start;
1663         u64 total_bytes = 0;
1664         u64 last = 0;
1665         int found = 0;
1666
1667         if (search_end <= cur_start) {
1668                 WARN_ON(1);
1669                 return 0;
1670         }
1671
1672         spin_lock(&tree->lock);
1673         if (cur_start == 0 && bits == EXTENT_DIRTY) {
1674                 total_bytes = tree->dirty_bytes;
1675                 goto out;
1676         }
1677         /*
1678          * this search will find all the extents that end after
1679          * our range starts.
1680          */
1681         node = tree_search(tree, cur_start);
1682         if (!node)
1683                 goto out;
1684
1685         while (1) {
1686                 state = rb_entry(node, struct extent_state, rb_node);
1687                 if (state->start > search_end)
1688                         break;
1689                 if (contig && found && state->start > last + 1)
1690                         break;
1691                 if (state->end >= cur_start && (state->state & bits) == bits) {
1692                         total_bytes += min(search_end, state->end) + 1 -
1693                                        max(cur_start, state->start);
1694                         if (total_bytes >= max_bytes)
1695                                 break;
1696                         if (!found) {
1697                                 *start = max(cur_start, state->start);
1698                                 found = 1;
1699                         }
1700                         last = state->end;
1701                 } else if (contig && found) {
1702                         break;
1703                 }
1704                 node = rb_next(node);
1705                 if (!node)
1706                         break;
1707         }
1708 out:
1709         spin_unlock(&tree->lock);
1710         return total_bytes;
1711 }
1712
1713 /*
1714  * set the private field for a given byte offset in the tree.  If there isn't
1715  * an extent_state there already, this does nothing.
1716  */
1717 int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
1718 {
1719         struct rb_node *node;
1720         struct extent_state *state;
1721         int ret = 0;
1722
1723         spin_lock(&tree->lock);
1724         /*
1725          * this search will find all the extents that end after
1726          * our range starts.
1727          */
1728         node = tree_search(tree, start);
1729         if (!node) {
1730                 ret = -ENOENT;
1731                 goto out;
1732         }
1733         state = rb_entry(node, struct extent_state, rb_node);
1734         if (state->start != start) {
1735                 ret = -ENOENT;
1736                 goto out;
1737         }
1738         state->private = private;
1739 out:
1740         spin_unlock(&tree->lock);
1741         return ret;
1742 }
1743
1744 int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
1745 {
1746         struct rb_node *node;
1747         struct extent_state *state;
1748         int ret = 0;
1749
1750         spin_lock(&tree->lock);
1751         /*
1752          * this search will find all the extents that end after
1753          * our range starts.
1754          */
1755         node = tree_search(tree, start);
1756         if (!node) {
1757                 ret = -ENOENT;
1758                 goto out;
1759         }
1760         state = rb_entry(node, struct extent_state, rb_node);
1761         if (state->start != start) {
1762                 ret = -ENOENT;
1763                 goto out;
1764         }
1765         *private = state->private;
1766 out:
1767         spin_unlock(&tree->lock);
1768         return ret;
1769 }
1770
1771 /*
1772  * searches a range in the state tree for a given mask.
1773  * If 'filled' == 1, this returns 1 only if every extent in the tree
1774  * has the bits set.  Otherwise, 1 is returned if any bit in the
1775  * range is found set.
1776  */
1777 int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
1778                    int bits, int filled, struct extent_state *cached)
1779 {
1780         struct extent_state *state = NULL;
1781         struct rb_node *node;
1782         int bitset = 0;
1783
1784         spin_lock(&tree->lock);
1785         if (cached && cached->tree && cached->start <= start &&
1786             cached->end > start)
1787                 node = &cached->rb_node;
1788         else
1789                 node = tree_search(tree, start);
1790         while (node && start <= end) {
1791                 state = rb_entry(node, struct extent_state, rb_node);
1792
1793                 if (filled && state->start > start) {
1794                         bitset = 0;
1795                         break;
1796                 }
1797
1798                 if (state->start > end)
1799                         break;
1800
1801                 if (state->state & bits) {
1802                         bitset = 1;
1803                         if (!filled)
1804                                 break;
1805                 } else if (filled) {
1806                         bitset = 0;
1807                         break;
1808                 }
1809
1810                 if (state->end == (u64)-1)
1811                         break;
1812
1813                 start = state->end + 1;
1814                 if (start > end)
1815                         break;
1816                 node = rb_next(node);
1817                 if (!node) {
1818                         if (filled)
1819                                 bitset = 0;
1820                         break;
1821                 }
1822         }
1823         spin_unlock(&tree->lock);
1824         return bitset;
1825 }
1826
1827 /*
1828  * helper function to set a given page up to date if all the
1829  * extents in the tree for that page are up to date
1830  */
1831 static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
1832 {
1833         u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1834         u64 end = start + PAGE_CACHE_SIZE - 1;
1835         if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
1836                 SetPageUptodate(page);
1837 }
1838
1839 /*
1840  * helper function to unlock a page if all the extents in the tree
1841  * for that page are unlocked
1842  */
1843 static void check_page_locked(struct extent_io_tree *tree, struct page *page)
1844 {
1845         u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1846         u64 end = start + PAGE_CACHE_SIZE - 1;
1847         if (!test_range_bit(tree, start, end, EXTENT_LOCKED, 0, NULL))
1848                 unlock_page(page);
1849 }
1850
1851 /*
1852  * helper function to end page writeback if all the extents
1853  * in the tree for that page are done with writeback
1854  */
1855 static void check_page_writeback(struct extent_io_tree *tree,
1856                                  struct page *page)
1857 {
1858         end_page_writeback(page);
1859 }
1860
1861 /*
1862  * When IO fails, either with EIO or csum verification fails, we
1863  * try other mirrors that might have a good copy of the data.  This
1864  * io_failure_record is used to record state as we go through all the
1865  * mirrors.  If another mirror has good data, the page is set up to date
1866  * and things continue.  If a good mirror can't be found, the original
1867  * bio end_io callback is called to indicate things have failed.
1868  */
1869 struct io_failure_record {
1870         struct page *page;
1871         u64 start;
1872         u64 len;
1873         u64 logical;
1874         unsigned long bio_flags;
1875         int this_mirror;
1876         int failed_mirror;
1877         int in_validation;
1878 };
1879
1880 static int free_io_failure(struct inode *inode, struct io_failure_record *rec,
1881                                 int did_repair)
1882 {
1883         int ret;
1884         int err = 0;
1885         struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
1886
1887         set_state_private(failure_tree, rec->start, 0);
1888         ret = clear_extent_bits(failure_tree, rec->start,
1889                                 rec->start + rec->len - 1,
1890                                 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
1891         if (ret)
1892                 err = ret;
1893
1894         if (did_repair) {
1895                 ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start,
1896                                         rec->start + rec->len - 1,
1897                                         EXTENT_DAMAGED, GFP_NOFS);
1898                 if (ret && !err)
1899                         err = ret;
1900         }
1901
1902         kfree(rec);
1903         return err;
1904 }
1905
1906 static void repair_io_failure_callback(struct bio *bio, int err)
1907 {
1908         complete(bio->bi_private);
1909 }
1910
1911 /*
1912  * this bypasses the standard btrfs submit functions deliberately, as
1913  * the standard behavior is to write all copies in a raid setup. here we only
1914  * want to write the one bad copy. so we do the mapping for ourselves and issue
1915  * submit_bio directly.
1916  * to avoid any synchonization issues, wait for the data after writing, which
1917  * actually prevents the read that triggered the error from finishing.
1918  * currently, there can be no more than two copies of every data bit. thus,
1919  * exactly one rewrite is required.
1920  */
1921 int repair_io_failure(struct btrfs_mapping_tree *map_tree, u64 start,
1922                         u64 length, u64 logical, struct page *page,
1923                         int mirror_num)
1924 {
1925         struct bio *bio;
1926         struct btrfs_device *dev;
1927         DECLARE_COMPLETION_ONSTACK(compl);
1928         u64 map_length = 0;
1929         u64 sector;
1930         struct btrfs_bio *bbio = NULL;
1931         int ret;
1932
1933         BUG_ON(!mirror_num);
1934
1935         bio = bio_alloc(GFP_NOFS, 1);
1936         if (!bio)
1937                 return -EIO;
1938         bio->bi_private = &compl;
1939         bio->bi_end_io = repair_io_failure_callback;
1940         bio->bi_size = 0;
1941         map_length = length;
1942
1943         ret = btrfs_map_block(map_tree, WRITE, logical,
1944                               &map_length, &bbio, mirror_num);
1945         if (ret) {
1946                 bio_put(bio);
1947                 return -EIO;
1948         }
1949         BUG_ON(mirror_num != bbio->mirror_num);
1950         sector = bbio->stripes[mirror_num-1].physical >> 9;
1951         bio->bi_sector = sector;
1952         dev = bbio->stripes[mirror_num-1].dev;
1953         kfree(bbio);
1954         if (!dev || !dev->bdev || !dev->writeable) {
1955                 bio_put(bio);
1956                 return -EIO;
1957         }
1958         bio->bi_bdev = dev->bdev;
1959         bio_add_page(bio, page, length, start-page_offset(page));
1960         btrfsic_submit_bio(WRITE_SYNC, bio);
1961         wait_for_completion(&compl);
1962
1963         if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) {
1964                 /* try to remap that extent elsewhere? */
1965                 bio_put(bio);
1966                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
1967                 return -EIO;
1968         }
1969
1970         printk_ratelimited_in_rcu(KERN_INFO "btrfs read error corrected: ino %lu off %llu "
1971                       "(dev %s sector %llu)\n", page->mapping->host->i_ino,
1972                       start, rcu_str_deref(dev->name), sector);
1973
1974         bio_put(bio);
1975         return 0;
1976 }
1977
1978 int repair_eb_io_failure(struct btrfs_root *root, struct extent_buffer *eb,
1979                          int mirror_num)
1980 {
1981         struct btrfs_mapping_tree *map_tree = &root->fs_info->mapping_tree;
1982         u64 start = eb->start;
1983         unsigned long i, num_pages = num_extent_pages(eb->start, eb->len);
1984         int ret = 0;
1985
1986         for (i = 0; i < num_pages; i++) {
1987                 struct page *p = extent_buffer_page(eb, i);
1988                 ret = repair_io_failure(map_tree, start, PAGE_CACHE_SIZE,
1989                                         start, p, mirror_num);
1990                 if (ret)
1991                         break;
1992                 start += PAGE_CACHE_SIZE;
1993         }
1994
1995         return ret;
1996 }
1997
1998 /*
1999  * each time an IO finishes, we do a fast check in the IO failure tree
2000  * to see if we need to process or clean up an io_failure_record
2001  */
2002 static int clean_io_failure(u64 start, struct page *page)
2003 {
2004         u64 private;
2005         u64 private_failure;
2006         struct io_failure_record *failrec;
2007         struct btrfs_mapping_tree *map_tree;
2008         struct extent_state *state;
2009         int num_copies;
2010         int did_repair = 0;
2011         int ret;
2012         struct inode *inode = page->mapping->host;
2013
2014         private = 0;
2015         ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private,
2016                                 (u64)-1, 1, EXTENT_DIRTY, 0);
2017         if (!ret)
2018                 return 0;
2019
2020         ret = get_state_private(&BTRFS_I(inode)->io_failure_tree, start,
2021                                 &private_failure);
2022         if (ret)
2023                 return 0;
2024
2025         failrec = (struct io_failure_record *)(unsigned long) private_failure;
2026         BUG_ON(!failrec->this_mirror);
2027
2028         if (failrec->in_validation) {
2029                 /* there was no real error, just free the record */
2030                 pr_debug("clean_io_failure: freeing dummy error at %llu\n",
2031                          failrec->start);
2032                 did_repair = 1;
2033                 goto out;
2034         }
2035
2036         spin_lock(&BTRFS_I(inode)->io_tree.lock);
2037         state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree,
2038                                             failrec->start,
2039                                             EXTENT_LOCKED);
2040         spin_unlock(&BTRFS_I(inode)->io_tree.lock);
2041
2042         if (state && state->start == failrec->start) {
2043                 map_tree = &BTRFS_I(inode)->root->fs_info->mapping_tree;
2044                 num_copies = btrfs_num_copies(map_tree, failrec->logical,
2045                                                 failrec->len);
2046                 if (num_copies > 1)  {
2047                         ret = repair_io_failure(map_tree, start, failrec->len,
2048                                                 failrec->logical, page,
2049                                                 failrec->failed_mirror);
2050                         did_repair = !ret;
2051                 }
2052         }
2053
2054 out:
2055         if (!ret)
2056                 ret = free_io_failure(inode, failrec, did_repair);
2057
2058         return ret;
2059 }
2060
2061 /*
2062  * this is a generic handler for readpage errors (default
2063  * readpage_io_failed_hook). if other copies exist, read those and write back
2064  * good data to the failed position. does not investigate in remapping the
2065  * failed extent elsewhere, hoping the device will be smart enough to do this as
2066  * needed
2067  */
2068
2069 static int bio_readpage_error(struct bio *failed_bio, struct page *page,
2070                                 u64 start, u64 end, int failed_mirror,
2071                                 struct extent_state *state)
2072 {
2073         struct io_failure_record *failrec = NULL;
2074         u64 private;
2075         struct extent_map *em;
2076         struct inode *inode = page->mapping->host;
2077         struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2078         struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2079         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2080         struct bio *bio;
2081         int num_copies;
2082         int ret;
2083         int read_mode;
2084         u64 logical;
2085
2086         BUG_ON(failed_bio->bi_rw & REQ_WRITE);
2087
2088         ret = get_state_private(failure_tree, start, &private);
2089         if (ret) {
2090                 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2091                 if (!failrec)
2092                         return -ENOMEM;
2093                 failrec->start = start;
2094                 failrec->len = end - start + 1;
2095                 failrec->this_mirror = 0;
2096                 failrec->bio_flags = 0;
2097                 failrec->in_validation = 0;
2098
2099                 read_lock(&em_tree->lock);
2100                 em = lookup_extent_mapping(em_tree, start, failrec->len);
2101                 if (!em) {
2102                         read_unlock(&em_tree->lock);
2103                         kfree(failrec);
2104                         return -EIO;
2105                 }
2106
2107                 if (em->start > start || em->start + em->len < start) {
2108                         free_extent_map(em);
2109                         em = NULL;
2110                 }
2111                 read_unlock(&em_tree->lock);
2112
2113                 if (!em) {
2114                         kfree(failrec);
2115                         return -EIO;
2116                 }
2117                 logical = start - em->start;
2118                 logical = em->block_start + logical;
2119                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2120                         logical = em->block_start;
2121                         failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2122                         extent_set_compress_type(&failrec->bio_flags,
2123                                                  em->compress_type);
2124                 }
2125                 pr_debug("bio_readpage_error: (new) logical=%llu, start=%llu, "
2126                          "len=%llu\n", logical, start, failrec->len);
2127                 failrec->logical = logical;
2128                 free_extent_map(em);
2129
2130                 /* set the bits in the private failure tree */
2131                 ret = set_extent_bits(failure_tree, start, end,
2132                                         EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2133                 if (ret >= 0)
2134                         ret = set_state_private(failure_tree, start,
2135                                                 (u64)(unsigned long)failrec);
2136                 /* set the bits in the inode's tree */
2137                 if (ret >= 0)
2138                         ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED,
2139                                                 GFP_NOFS);
2140                 if (ret < 0) {
2141                         kfree(failrec);
2142                         return ret;
2143                 }
2144         } else {
2145                 failrec = (struct io_failure_record *)(unsigned long)private;
2146                 pr_debug("bio_readpage_error: (found) logical=%llu, "
2147                          "start=%llu, len=%llu, validation=%d\n",
2148                          failrec->logical, failrec->start, failrec->len,
2149                          failrec->in_validation);
2150                 /*
2151                  * when data can be on disk more than twice, add to failrec here
2152                  * (e.g. with a list for failed_mirror) to make
2153                  * clean_io_failure() clean all those errors at once.
2154                  */
2155         }
2156         num_copies = btrfs_num_copies(
2157                               &BTRFS_I(inode)->root->fs_info->mapping_tree,
2158                               failrec->logical, failrec->len);
2159         if (num_copies == 1) {
2160                 /*
2161                  * we only have a single copy of the data, so don't bother with
2162                  * all the retry and error correction code that follows. no
2163                  * matter what the error is, it is very likely to persist.
2164                  */
2165                 pr_debug("bio_readpage_error: cannot repair, num_copies == 1. "
2166                          "state=%p, num_copies=%d, next_mirror %d, "
2167                          "failed_mirror %d\n", state, num_copies,
2168                          failrec->this_mirror, failed_mirror);
2169                 free_io_failure(inode, failrec, 0);
2170                 return -EIO;
2171         }
2172
2173         if (!state) {
2174                 spin_lock(&tree->lock);
2175                 state = find_first_extent_bit_state(tree, failrec->start,
2176                                                     EXTENT_LOCKED);
2177                 if (state && state->start != failrec->start)
2178                         state = NULL;
2179                 spin_unlock(&tree->lock);
2180         }
2181
2182         /*
2183          * there are two premises:
2184          *      a) deliver good data to the caller
2185          *      b) correct the bad sectors on disk
2186          */
2187         if (failed_bio->bi_vcnt > 1) {
2188                 /*
2189                  * to fulfill b), we need to know the exact failing sectors, as
2190                  * we don't want to rewrite any more than the failed ones. thus,
2191                  * we need separate read requests for the failed bio
2192                  *
2193                  * if the following BUG_ON triggers, our validation request got
2194                  * merged. we need separate requests for our algorithm to work.
2195                  */
2196                 BUG_ON(failrec->in_validation);
2197                 failrec->in_validation = 1;
2198                 failrec->this_mirror = failed_mirror;
2199                 read_mode = READ_SYNC | REQ_FAILFAST_DEV;
2200         } else {
2201                 /*
2202                  * we're ready to fulfill a) and b) alongside. get a good copy
2203                  * of the failed sector and if we succeed, we have setup
2204                  * everything for repair_io_failure to do the rest for us.
2205                  */
2206                 if (failrec->in_validation) {
2207                         BUG_ON(failrec->this_mirror != failed_mirror);
2208                         failrec->in_validation = 0;
2209                         failrec->this_mirror = 0;
2210                 }
2211                 failrec->failed_mirror = failed_mirror;
2212                 failrec->this_mirror++;
2213                 if (failrec->this_mirror == failed_mirror)
2214                         failrec->this_mirror++;
2215                 read_mode = READ_SYNC;
2216         }
2217
2218         if (!state || failrec->this_mirror > num_copies) {
2219                 pr_debug("bio_readpage_error: (fail) state=%p, num_copies=%d, "
2220                          "next_mirror %d, failed_mirror %d\n", state,
2221                          num_copies, failrec->this_mirror, failed_mirror);
2222                 free_io_failure(inode, failrec, 0);
2223                 return -EIO;
2224         }
2225
2226         bio = bio_alloc(GFP_NOFS, 1);
2227         if (!bio) {
2228                 free_io_failure(inode, failrec, 0);
2229                 return -EIO;
2230         }
2231         bio->bi_private = state;
2232         bio->bi_end_io = failed_bio->bi_end_io;
2233         bio->bi_sector = failrec->logical >> 9;
2234         bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
2235         bio->bi_size = 0;
2236
2237         bio_add_page(bio, page, failrec->len, start - page_offset(page));
2238
2239         pr_debug("bio_readpage_error: submitting new read[%#x] to "
2240                  "this_mirror=%d, num_copies=%d, in_validation=%d\n", read_mode,
2241                  failrec->this_mirror, num_copies, failrec->in_validation);
2242
2243         ret = tree->ops->submit_bio_hook(inode, read_mode, bio,
2244                                          failrec->this_mirror,
2245                                          failrec->bio_flags, 0);
2246         return ret;
2247 }
2248
2249 /* lots and lots of room for performance fixes in the end_bio funcs */
2250
2251 int end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2252 {
2253         int uptodate = (err == 0);
2254         struct extent_io_tree *tree;
2255         int ret;
2256
2257         tree = &BTRFS_I(page->mapping->host)->io_tree;
2258
2259         if (tree->ops && tree->ops->writepage_end_io_hook) {
2260                 ret = tree->ops->writepage_end_io_hook(page, start,
2261                                                end, NULL, uptodate);
2262                 if (ret)
2263                         uptodate = 0;
2264         }
2265
2266         if (!uptodate) {
2267                 ClearPageUptodate(page);
2268                 SetPageError(page);
2269         }
2270         return 0;
2271 }
2272
2273 /*
2274  * after a writepage IO is done, we need to:
2275  * clear the uptodate bits on error
2276  * clear the writeback bits in the extent tree for this IO
2277  * end_page_writeback if the page has no more pending IO
2278  *
2279  * Scheduling is not allowed, so the extent state tree is expected
2280  * to have one and only one object corresponding to this IO.
2281  */
2282 static void end_bio_extent_writepage(struct bio *bio, int err)
2283 {
2284         struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2285         struct extent_io_tree *tree;
2286         u64 start;
2287         u64 end;
2288         int whole_page;
2289
2290         do {
2291                 struct page *page = bvec->bv_page;
2292                 tree = &BTRFS_I(page->mapping->host)->io_tree;
2293
2294                 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
2295                          bvec->bv_offset;
2296                 end = start + bvec->bv_len - 1;
2297
2298                 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
2299                         whole_page = 1;
2300                 else
2301                         whole_page = 0;
2302
2303                 if (--bvec >= bio->bi_io_vec)
2304                         prefetchw(&bvec->bv_page->flags);
2305
2306                 if (end_extent_writepage(page, err, start, end))
2307                         continue;
2308
2309                 if (whole_page)
2310                         end_page_writeback(page);
2311                 else
2312                         check_page_writeback(tree, page);
2313         } while (bvec >= bio->bi_io_vec);
2314
2315         bio_put(bio);
2316 }
2317
2318 /*
2319  * after a readpage IO is done, we need to:
2320  * clear the uptodate bits on error
2321  * set the uptodate bits if things worked
2322  * set the page up to date if all extents in the tree are uptodate
2323  * clear the lock bit in the extent tree
2324  * unlock the page if there are no other extents locked for it
2325  *
2326  * Scheduling is not allowed, so the extent state tree is expected
2327  * to have one and only one object corresponding to this IO.
2328  */
2329 static void end_bio_extent_readpage(struct bio *bio, int err)
2330 {
2331         int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
2332         struct bio_vec *bvec_end = bio->bi_io_vec + bio->bi_vcnt - 1;
2333         struct bio_vec *bvec = bio->bi_io_vec;
2334         struct extent_io_tree *tree;
2335         u64 start;
2336         u64 end;
2337         int whole_page;
2338         int mirror;
2339         int ret;
2340
2341         if (err)
2342                 uptodate = 0;
2343
2344         do {
2345                 struct page *page = bvec->bv_page;
2346                 struct extent_state *cached = NULL;
2347                 struct extent_state *state;
2348
2349                 pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, "
2350                          "mirror=%ld\n", (u64)bio->bi_sector, err,
2351                          (long int)bio->bi_bdev);
2352                 tree = &BTRFS_I(page->mapping->host)->io_tree;
2353
2354                 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
2355                         bvec->bv_offset;
2356                 end = start + bvec->bv_len - 1;
2357
2358                 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
2359                         whole_page = 1;
2360                 else
2361                         whole_page = 0;
2362
2363                 if (++bvec <= bvec_end)
2364                         prefetchw(&bvec->bv_page->flags);
2365
2366                 spin_lock(&tree->lock);
2367                 state = find_first_extent_bit_state(tree, start, EXTENT_LOCKED);
2368                 if (state && state->start == start) {
2369                         /*
2370                          * take a reference on the state, unlock will drop
2371                          * the ref
2372                          */
2373                         cache_state(state, &cached);
2374                 }
2375                 spin_unlock(&tree->lock);
2376
2377                 mirror = (int)(unsigned long)bio->bi_bdev;
2378                 if (uptodate && tree->ops && tree->ops->readpage_end_io_hook) {
2379                         ret = tree->ops->readpage_end_io_hook(page, start, end,
2380                                                               state, mirror);
2381                         if (ret)
2382                                 uptodate = 0;
2383                         else
2384                                 clean_io_failure(start, page);
2385                 }
2386
2387                 if (!uptodate && tree->ops && tree->ops->readpage_io_failed_hook) {
2388                         ret = tree->ops->readpage_io_failed_hook(page, mirror);
2389                         if (!ret && !err &&
2390                             test_bit(BIO_UPTODATE, &bio->bi_flags))
2391                                 uptodate = 1;
2392                 } else if (!uptodate) {
2393                         /*
2394                          * The generic bio_readpage_error handles errors the
2395                          * following way: If possible, new read requests are
2396                          * created and submitted and will end up in
2397                          * end_bio_extent_readpage as well (if we're lucky, not
2398                          * in the !uptodate case). In that case it returns 0 and
2399                          * we just go on with the next page in our bio. If it
2400                          * can't handle the error it will return -EIO and we
2401                          * remain responsible for that page.
2402                          */
2403                         ret = bio_readpage_error(bio, page, start, end, mirror, NULL);
2404                         if (ret == 0) {
2405                                 uptodate =
2406                                         test_bit(BIO_UPTODATE, &bio->bi_flags);
2407                                 if (err)
2408                                         uptodate = 0;
2409                                 uncache_state(&cached);
2410                                 continue;
2411                         }
2412                 }
2413
2414                 if (uptodate && tree->track_uptodate) {
2415                         set_extent_uptodate(tree, start, end, &cached,
2416                                             GFP_ATOMIC);
2417                 }
2418                 unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC);
2419
2420                 if (whole_page) {
2421                         if (uptodate) {
2422                                 SetPageUptodate(page);
2423                         } else {
2424                                 ClearPageUptodate(page);
2425                                 SetPageError(page);
2426                         }
2427                         unlock_page(page);
2428                 } else {
2429                         if (uptodate) {
2430                                 check_page_uptodate(tree, page);
2431                         } else {
2432                                 ClearPageUptodate(page);
2433                                 SetPageError(page);
2434                         }
2435                         check_page_locked(tree, page);
2436                 }
2437         } while (bvec <= bvec_end);
2438
2439         bio_put(bio);
2440 }
2441
2442 struct bio *
2443 btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
2444                 gfp_t gfp_flags)
2445 {
2446         struct bio *bio;
2447
2448         bio = bio_alloc(gfp_flags, nr_vecs);
2449
2450         if (bio == NULL && (current->flags & PF_MEMALLOC)) {
2451                 while (!bio && (nr_vecs /= 2))
2452                         bio = bio_alloc(gfp_flags, nr_vecs);
2453         }
2454
2455         if (bio) {
2456                 bio->bi_size = 0;
2457                 bio->bi_bdev = bdev;
2458                 bio->bi_sector = first_sector;
2459         }
2460         return bio;
2461 }
2462
2463 /*
2464  * Since writes are async, they will only return -ENOMEM.
2465  * Reads can return the full range of I/O error conditions.
2466  */
2467 static int __must_check submit_one_bio(int rw, struct bio *bio,
2468                                        int mirror_num, unsigned long bio_flags)
2469 {
2470         int ret = 0;
2471         struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2472         struct page *page = bvec->bv_page;
2473         struct extent_io_tree *tree = bio->bi_private;
2474         u64 start;
2475
2476         start = ((u64)page->index << PAGE_CACHE_SHIFT) + bvec->bv_offset;
2477
2478         bio->bi_private = NULL;
2479
2480         bio_get(bio);
2481
2482         if (tree->ops && tree->ops->submit_bio_hook)
2483                 ret = tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
2484                                            mirror_num, bio_flags, start);
2485         else
2486                 btrfsic_submit_bio(rw, bio);
2487
2488         if (bio_flagged(bio, BIO_EOPNOTSUPP))
2489                 ret = -EOPNOTSUPP;
2490         bio_put(bio);
2491         return ret;
2492 }
2493
2494 static int merge_bio(struct extent_io_tree *tree, struct page *page,
2495                      unsigned long offset, size_t size, struct bio *bio,
2496                      unsigned long bio_flags)
2497 {
2498         int ret = 0;
2499         if (tree->ops && tree->ops->merge_bio_hook)
2500                 ret = tree->ops->merge_bio_hook(page, offset, size, bio,
2501                                                 bio_flags);
2502         BUG_ON(ret < 0);
2503         return ret;
2504
2505 }
2506
2507 static int submit_extent_page(int rw, struct extent_io_tree *tree,
2508                               struct page *page, sector_t sector,
2509                               size_t size, unsigned long offset,
2510                               struct block_device *bdev,
2511                               struct bio **bio_ret,
2512                               unsigned long max_pages,
2513                               bio_end_io_t end_io_func,
2514                               int mirror_num,
2515                               unsigned long prev_bio_flags,
2516                               unsigned long bio_flags)
2517 {
2518         int ret = 0;
2519         struct bio *bio;
2520         int nr;
2521         int contig = 0;
2522         int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED;
2523         int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
2524         size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE);
2525
2526         if (bio_ret && *bio_ret) {
2527                 bio = *bio_ret;
2528                 if (old_compressed)
2529                         contig = bio->bi_sector == sector;
2530                 else
2531                         contig = bio->bi_sector + (bio->bi_size >> 9) ==
2532                                 sector;
2533
2534                 if (prev_bio_flags != bio_flags || !contig ||
2535                     merge_bio(tree, page, offset, page_size, bio, bio_flags) ||
2536                     bio_add_page(bio, page, page_size, offset) < page_size) {
2537                         ret = submit_one_bio(rw, bio, mirror_num,
2538                                              prev_bio_flags);
2539                         if (ret < 0)
2540                                 return ret;
2541                         bio = NULL;
2542                 } else {
2543                         return 0;
2544                 }
2545         }
2546         if (this_compressed)
2547                 nr = BIO_MAX_PAGES;
2548         else
2549                 nr = bio_get_nr_vecs(bdev);
2550
2551         bio = btrfs_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
2552         if (!bio)
2553                 return -ENOMEM;
2554
2555         bio_add_page(bio, page, page_size, offset);
2556         bio->bi_end_io = end_io_func;
2557         bio->bi_private = tree;
2558
2559         if (bio_ret)
2560                 *bio_ret = bio;
2561         else
2562                 ret = submit_one_bio(rw, bio, mirror_num, bio_flags);
2563
2564         return ret;
2565 }
2566
2567 void attach_extent_buffer_page(struct extent_buffer *eb, struct page *page)
2568 {
2569         if (!PagePrivate(page)) {
2570                 SetPagePrivate(page);
2571                 page_cache_get(page);
2572                 set_page_private(page, (unsigned long)eb);
2573         } else {
2574                 WARN_ON(page->private != (unsigned long)eb);
2575         }
2576 }
2577
2578 void set_page_extent_mapped(struct page *page)
2579 {
2580         if (!PagePrivate(page)) {
2581                 SetPagePrivate(page);
2582                 page_cache_get(page);
2583                 set_page_private(page, EXTENT_PAGE_PRIVATE);
2584         }
2585 }
2586
2587 /*
2588  * basic readpage implementation.  Locked extent state structs are inserted
2589  * into the tree that are removed when the IO is done (by the end_io
2590  * handlers)
2591  * XXX JDM: This needs looking at to ensure proper page locking
2592  */
2593 static int __extent_read_full_page(struct extent_io_tree *tree,
2594                                    struct page *page,
2595                                    get_extent_t *get_extent,
2596                                    struct bio **bio, int mirror_num,
2597                                    unsigned long *bio_flags)
2598 {
2599         struct inode *inode = page->mapping->host;
2600         u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
2601         u64 page_end = start + PAGE_CACHE_SIZE - 1;
2602         u64 end;
2603         u64 cur = start;
2604         u64 extent_offset;
2605         u64 last_byte = i_size_read(inode);
2606         u64 block_start;
2607         u64 cur_end;
2608         sector_t sector;
2609         struct extent_map *em;
2610         struct block_device *bdev;
2611         struct btrfs_ordered_extent *ordered;
2612         int ret;
2613         int nr = 0;
2614         size_t pg_offset = 0;
2615         size_t iosize;
2616         size_t disk_io_size;
2617         size_t blocksize = inode->i_sb->s_blocksize;
2618         unsigned long this_bio_flag = 0;
2619
2620         set_page_extent_mapped(page);
2621
2622         if (!PageUptodate(page)) {
2623                 if (cleancache_get_page(page) == 0) {
2624                         BUG_ON(blocksize != PAGE_SIZE);
2625                         goto out;
2626                 }
2627         }
2628
2629         end = page_end;
2630         while (1) {
2631                 lock_extent(tree, start, end);
2632                 ordered = btrfs_lookup_ordered_extent(inode, start);
2633                 if (!ordered)
2634                         break;
2635                 unlock_extent(tree, start, end);
2636                 btrfs_start_ordered_extent(inode, ordered, 1);
2637                 btrfs_put_ordered_extent(ordered);
2638         }
2639
2640         if (page->index == last_byte >> PAGE_CACHE_SHIFT) {
2641                 char *userpage;
2642                 size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1);
2643
2644                 if (zero_offset) {
2645                         iosize = PAGE_CACHE_SIZE - zero_offset;
2646                         userpage = kmap_atomic(page);
2647                         memset(userpage + zero_offset, 0, iosize);
2648                         flush_dcache_page(page);
2649                         kunmap_atomic(userpage);
2650                 }
2651         }
2652         while (cur <= end) {
2653                 if (cur >= last_byte) {
2654                         char *userpage;
2655                         struct extent_state *cached = NULL;
2656
2657                         iosize = PAGE_CACHE_SIZE - pg_offset;
2658                         userpage = kmap_atomic(page);
2659                         memset(userpage + pg_offset, 0, iosize);
2660                         flush_dcache_page(page);
2661                         kunmap_atomic(userpage);
2662                         set_extent_uptodate(tree, cur, cur + iosize - 1,
2663                                             &cached, GFP_NOFS);
2664                         unlock_extent_cached(tree, cur, cur + iosize - 1,
2665                                              &cached, GFP_NOFS);
2666                         break;
2667                 }
2668                 em = get_extent(inode, page, pg_offset, cur,
2669                                 end - cur + 1, 0);
2670                 if (IS_ERR_OR_NULL(em)) {
2671                         SetPageError(page);
2672                         unlock_extent(tree, cur, end);
2673                         break;
2674                 }
2675                 extent_offset = cur - em->start;
2676                 BUG_ON(extent_map_end(em) <= cur);
2677                 BUG_ON(end < cur);
2678
2679                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2680                         this_bio_flag = EXTENT_BIO_COMPRESSED;
2681                         extent_set_compress_type(&this_bio_flag,
2682                                                  em->compress_type);
2683                 }
2684
2685                 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2686                 cur_end = min(extent_map_end(em) - 1, end);
2687                 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
2688                 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2689                         disk_io_size = em->block_len;
2690                         sector = em->block_start >> 9;
2691                 } else {
2692                         sector = (em->block_start + extent_offset) >> 9;
2693                         disk_io_size = iosize;
2694                 }
2695                 bdev = em->bdev;
2696                 block_start = em->block_start;
2697                 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2698                         block_start = EXTENT_MAP_HOLE;
2699                 free_extent_map(em);
2700                 em = NULL;
2701
2702                 /* we've found a hole, just zero and go on */
2703                 if (block_start == EXTENT_MAP_HOLE) {
2704                         char *userpage;
2705                         struct extent_state *cached = NULL;
2706
2707                         userpage = kmap_atomic(page);
2708                         memset(userpage + pg_offset, 0, iosize);
2709                         flush_dcache_page(page);
2710                         kunmap_atomic(userpage);
2711
2712                         set_extent_uptodate(tree, cur, cur + iosize - 1,
2713                                             &cached, GFP_NOFS);
2714                         unlock_extent_cached(tree, cur, cur + iosize - 1,
2715                                              &cached, GFP_NOFS);
2716                         cur = cur + iosize;
2717                         pg_offset += iosize;
2718                         continue;
2719                 }
2720                 /* the get_extent function already copied into the page */
2721                 if (test_range_bit(tree, cur, cur_end,
2722                                    EXTENT_UPTODATE, 1, NULL)) {
2723                         check_page_uptodate(tree, page);
2724                         unlock_extent(tree, cur, cur + iosize - 1);
2725                         cur = cur + iosize;
2726                         pg_offset += iosize;
2727                         continue;
2728                 }
2729                 /* we have an inline extent but it didn't get marked up
2730                  * to date.  Error out
2731                  */
2732                 if (block_start == EXTENT_MAP_INLINE) {
2733                         SetPageError(page);
2734                         unlock_extent(tree, cur, cur + iosize - 1);
2735                         cur = cur + iosize;
2736                         pg_offset += iosize;
2737                         continue;
2738                 }
2739
2740                 ret = 0;
2741                 if (tree->ops && tree->ops->readpage_io_hook) {
2742                         ret = tree->ops->readpage_io_hook(page, cur,
2743                                                           cur + iosize - 1);
2744                 }
2745                 if (!ret) {
2746                         unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
2747                         pnr -= page->index;
2748                         ret = submit_extent_page(READ, tree, page,
2749                                          sector, disk_io_size, pg_offset,
2750                                          bdev, bio, pnr,
2751                                          end_bio_extent_readpage, mirror_num,
2752                                          *bio_flags,
2753                                          this_bio_flag);
2754                         BUG_ON(ret == -ENOMEM);
2755                         nr++;
2756                         *bio_flags = this_bio_flag;
2757                 }
2758                 if (ret)
2759                         SetPageError(page);
2760                 cur = cur + iosize;
2761                 pg_offset += iosize;
2762         }
2763 out:
2764         if (!nr) {
2765                 if (!PageError(page))
2766                         SetPageUptodate(page);
2767                 unlock_page(page);
2768         }
2769         return 0;
2770 }
2771
2772 int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
2773                             get_extent_t *get_extent, int mirror_num)
2774 {
2775         struct bio *bio = NULL;
2776         unsigned long bio_flags = 0;
2777         int ret;
2778
2779         ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
2780                                       &bio_flags);
2781         if (bio)
2782                 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
2783         return ret;
2784 }
2785
2786 static noinline void update_nr_written(struct page *page,
2787                                       struct writeback_control *wbc,
2788                                       unsigned long nr_written)
2789 {
2790         wbc->nr_to_write -= nr_written;
2791         if (wbc->range_cyclic || (wbc->nr_to_write > 0 &&
2792             wbc->range_start == 0 && wbc->range_end == LLONG_MAX))
2793                 page->mapping->writeback_index = page->index + nr_written;
2794 }
2795
2796 /*
2797  * the writepage semantics are similar to regular writepage.  extent
2798  * records are inserted to lock ranges in the tree, and as dirty areas
2799  * are found, they are marked writeback.  Then the lock bits are removed
2800  * and the end_io handler clears the writeback ranges
2801  */
2802 static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2803                               void *data)
2804 {
2805         struct inode *inode = page->mapping->host;
2806         struct extent_page_data *epd = data;
2807         struct extent_io_tree *tree = epd->tree;
2808         u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
2809         u64 delalloc_start;
2810         u64 page_end = start + PAGE_CACHE_SIZE - 1;
2811         u64 end;
2812         u64 cur = start;
2813         u64 extent_offset;
2814         u64 last_byte = i_size_read(inode);
2815         u64 block_start;
2816         u64 iosize;
2817         sector_t sector;
2818         struct extent_state *cached_state = NULL;
2819         struct extent_map *em;
2820         struct block_device *bdev;
2821         int ret;
2822         int nr = 0;
2823         size_t pg_offset = 0;
2824         size_t blocksize;
2825         loff_t i_size = i_size_read(inode);
2826         unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
2827         u64 nr_delalloc;
2828         u64 delalloc_end;
2829         int page_started;
2830         int compressed;
2831         int write_flags;
2832         unsigned long nr_written = 0;
2833         bool fill_delalloc = true;
2834
2835         if (wbc->sync_mode == WB_SYNC_ALL)
2836                 write_flags = WRITE_SYNC;
2837         else
2838                 write_flags = WRITE;
2839
2840         trace___extent_writepage(page, inode, wbc);
2841
2842         WARN_ON(!PageLocked(page));
2843
2844         ClearPageError(page);
2845
2846         pg_offset = i_size & (PAGE_CACHE_SIZE - 1);
2847         if (page->index > end_index ||
2848            (page->index == end_index && !pg_offset)) {
2849                 page->mapping->a_ops->invalidatepage(page, 0);
2850                 unlock_page(page);
2851                 return 0;
2852         }
2853
2854         if (page->index == end_index) {
2855                 char *userpage;
2856
2857                 userpage = kmap_atomic(page);
2858                 memset(userpage + pg_offset, 0,
2859                        PAGE_CACHE_SIZE - pg_offset);
2860                 kunmap_atomic(userpage);
2861                 flush_dcache_page(page);
2862         }
2863         pg_offset = 0;
2864
2865         set_page_extent_mapped(page);
2866
2867         if (!tree->ops || !tree->ops->fill_delalloc)
2868                 fill_delalloc = false;
2869
2870         delalloc_start = start;
2871         delalloc_end = 0;
2872         page_started = 0;
2873         if (!epd->extent_locked && fill_delalloc) {
2874                 u64 delalloc_to_write = 0;
2875                 /*
2876                  * make sure the wbc mapping index is at least updated
2877                  * to this page.
2878                  */
2879                 update_nr_written(page, wbc, 0);
2880
2881                 while (delalloc_end < page_end) {
2882                         nr_delalloc = find_lock_delalloc_range(inode, tree,
2883                                                        page,
2884                                                        &delalloc_start,
2885                                                        &delalloc_end,
2886                                                        128 * 1024 * 1024);
2887                         if (nr_delalloc == 0) {
2888                                 delalloc_start = delalloc_end + 1;
2889                                 continue;
2890                         }
2891                         ret = tree->ops->fill_delalloc(inode, page,
2892                                                        delalloc_start,
2893                                                        delalloc_end,
2894                                                        &page_started,
2895                                                        &nr_written);
2896                         /* File system has been set read-only */
2897                         if (ret) {
2898                                 SetPageError(page);
2899                                 goto done;
2900                         }
2901                         /*
2902                          * delalloc_end is already one less than the total
2903                          * length, so we don't subtract one from
2904                          * PAGE_CACHE_SIZE
2905                          */
2906                         delalloc_to_write += (delalloc_end - delalloc_start +
2907                                               PAGE_CACHE_SIZE) >>
2908                                               PAGE_CACHE_SHIFT;
2909                         delalloc_start = delalloc_end + 1;
2910                 }
2911                 if (wbc->nr_to_write < delalloc_to_write) {
2912                         int thresh = 8192;
2913
2914                         if (delalloc_to_write < thresh * 2)
2915                                 thresh = delalloc_to_write;
2916                         wbc->nr_to_write = min_t(u64, delalloc_to_write,
2917                                                  thresh);
2918                 }
2919
2920                 /* did the fill delalloc function already unlock and start
2921                  * the IO?
2922                  */
2923                 if (page_started) {
2924                         ret = 0;
2925                         /*
2926                          * we've unlocked the page, so we can't update
2927                          * the mapping's writeback index, just update
2928                          * nr_to_write.
2929                          */
2930                         wbc->nr_to_write -= nr_written;
2931                         goto done_unlocked;
2932                 }
2933         }
2934         if (tree->ops && tree->ops->writepage_start_hook) {
2935                 ret = tree->ops->writepage_start_hook(page, start,
2936                                                       page_end);
2937                 if (ret) {
2938                         /* Fixup worker will requeue */
2939                         if (ret == -EBUSY)
2940                                 wbc->pages_skipped++;
2941                         else
2942                                 redirty_page_for_writepage(wbc, page);
2943                         update_nr_written(page, wbc, nr_written);
2944                         unlock_page(page);
2945                         ret = 0;
2946                         goto done_unlocked;
2947                 }
2948         }
2949
2950         /*
2951          * we don't want to touch the inode after unlocking the page,
2952          * so we update the mapping writeback index now
2953          */
2954         update_nr_written(page, wbc, nr_written + 1);
2955
2956         end = page_end;
2957         if (last_byte <= start) {
2958                 if (tree->ops && tree->ops->writepage_end_io_hook)
2959                         tree->ops->writepage_end_io_hook(page, start,
2960                                                          page_end, NULL, 1);
2961                 goto done;
2962         }
2963
2964         blocksize = inode->i_sb->s_blocksize;
2965
2966         while (cur <= end) {
2967                 if (cur >= last_byte) {
2968                         if (tree->ops && tree->ops->writepage_end_io_hook)
2969                                 tree->ops->writepage_end_io_hook(page, cur,
2970                                                          page_end, NULL, 1);
2971                         break;
2972                 }
2973                 em = epd->get_extent(inode, page, pg_offset, cur,
2974                                      end - cur + 1, 1);
2975                 if (IS_ERR_OR_NULL(em)) {
2976                         SetPageError(page);
2977                         break;
2978                 }
2979
2980                 extent_offset = cur - em->start;
2981                 BUG_ON(extent_map_end(em) <= cur);
2982                 BUG_ON(end < cur);
2983                 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2984                 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
2985                 sector = (em->block_start + extent_offset) >> 9;
2986                 bdev = em->bdev;
2987                 block_start = em->block_start;
2988                 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
2989                 free_extent_map(em);
2990                 em = NULL;
2991
2992                 /*
2993                  * compressed and inline extents are written through other
2994                  * paths in the FS
2995                  */
2996                 if (compressed || block_start == EXTENT_MAP_HOLE ||
2997                     block_start == EXTENT_MAP_INLINE) {
2998                         /*
2999                          * end_io notification does not happen here for
3000                          * compressed extents
3001                          */
3002                         if (!compressed && tree->ops &&
3003                             tree->ops->writepage_end_io_hook)
3004                                 tree->ops->writepage_end_io_hook(page, cur,
3005                                                          cur + iosize - 1,
3006                                                          NULL, 1);
3007                         else if (compressed) {
3008                                 /* we don't want to end_page_writeback on
3009                                  * a compressed extent.  this happens
3010                                  * elsewhere
3011                                  */
3012                                 nr++;
3013                         }
3014
3015                         cur += iosize;
3016                         pg_offset += iosize;
3017                         continue;
3018                 }
3019                 /* leave this out until we have a page_mkwrite call */
3020                 if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
3021                                    EXTENT_DIRTY, 0, NULL)) {
3022                         cur = cur + iosize;
3023                         pg_offset += iosize;
3024                         continue;
3025                 }
3026
3027                 if (tree->ops && tree->ops->writepage_io_hook) {
3028                         ret = tree->ops->writepage_io_hook(page, cur,
3029                                                 cur + iosize - 1);
3030                 } else {
3031                         ret = 0;
3032                 }
3033                 if (ret) {
3034                         SetPageError(page);
3035                 } else {
3036                         unsigned long max_nr = end_index + 1;
3037
3038                         set_range_writeback(tree, cur, cur + iosize - 1);
3039                         if (!PageWriteback(page)) {
3040                                 printk(KERN_ERR "btrfs warning page %lu not "
3041                                        "writeback, cur %llu end %llu\n",
3042                                        page->index, (unsigned long long)cur,
3043                                        (unsigned long long)end);
3044                         }
3045
3046                         ret = submit_extent_page(write_flags, tree, page,
3047                                                  sector, iosize, pg_offset,
3048                                                  bdev, &epd->bio, max_nr,
3049                                                  end_bio_extent_writepage,
3050                                                  0, 0, 0);
3051                         if (ret)
3052                                 SetPageError(page);
3053                 }
3054                 cur = cur + iosize;
3055                 pg_offset += iosize;
3056                 nr++;
3057         }
3058 done:
3059         if (nr == 0) {
3060                 /* make sure the mapping tag for page dirty gets cleared */
3061                 set_page_writeback(page);
3062                 end_page_writeback(page);
3063         }
3064         unlock_page(page);
3065
3066 done_unlocked:
3067
3068         /* drop our reference on any cached states */
3069         free_extent_state(cached_state);
3070         return 0;
3071 }
3072
3073 static int eb_wait(void *word)
3074 {
3075         io_schedule();
3076         return 0;
3077 }
3078
3079 static void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
3080 {
3081         wait_on_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK, eb_wait,
3082                     TASK_UNINTERRUPTIBLE);
3083 }
3084
3085 static int lock_extent_buffer_for_io(struct extent_buffer *eb,
3086                                      struct btrfs_fs_info *fs_info,
3087                                      struct extent_page_data *epd)
3088 {
3089         unsigned long i, num_pages;
3090         int flush = 0;
3091         int ret = 0;
3092
3093         if (!btrfs_try_tree_write_lock(eb)) {
3094                 flush = 1;
3095                 flush_write_bio(epd);
3096                 btrfs_tree_lock(eb);
3097         }
3098
3099         if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3100                 btrfs_tree_unlock(eb);
3101                 if (!epd->sync_io)
3102                         return 0;
3103                 if (!flush) {
3104                         flush_write_bio(epd);
3105                         flush = 1;
3106                 }
3107                 while (1) {
3108                         wait_on_extent_buffer_writeback(eb);
3109                         btrfs_tree_lock(eb);
3110                         if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3111                                 break;
3112                         btrfs_tree_unlock(eb);
3113                 }
3114         }
3115
3116         /*
3117          * We need to do this to prevent races in people who check if the eb is
3118          * under IO since we can end up having no IO bits set for a short period
3119          * of time.
3120          */
3121         spin_lock(&eb->refs_lock);
3122         if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3123                 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3124                 spin_unlock(&eb->refs_lock);
3125                 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
3126                 spin_lock(&fs_info->delalloc_lock);
3127                 if (fs_info->dirty_metadata_bytes >= eb->len)
3128                         fs_info->dirty_metadata_bytes -= eb->len;
3129                 else
3130                         WARN_ON(1);
3131                 spin_unlock(&fs_info->delalloc_lock);
3132                 ret = 1;
3133         } else {
3134                 spin_unlock(&eb->refs_lock);
3135         }
3136
3137         btrfs_tree_unlock(eb);
3138
3139         if (!ret)
3140                 return ret;
3141
3142         num_pages = num_extent_pages(eb->start, eb->len);
3143         for (i = 0; i < num_pages; i++) {
3144                 struct page *p = extent_buffer_page(eb, i);
3145
3146                 if (!trylock_page(p)) {
3147                         if (!flush) {
3148                                 flush_write_bio(epd);
3149                                 flush = 1;
3150                         }
3151                         lock_page(p);
3152                 }
3153         }
3154
3155         return ret;
3156 }
3157
3158 static void end_extent_buffer_writeback(struct extent_buffer *eb)
3159 {
3160         clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3161         smp_mb__after_clear_bit();
3162         wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3163 }
3164
3165 static void end_bio_extent_buffer_writepage(struct bio *bio, int err)
3166 {
3167         int uptodate = err == 0;
3168         struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
3169         struct extent_buffer *eb;
3170         int done;
3171
3172         do {
3173                 struct page *page = bvec->bv_page;
3174
3175                 bvec--;
3176                 eb = (struct extent_buffer *)page->private;
3177                 BUG_ON(!eb);
3178                 done = atomic_dec_and_test(&eb->io_pages);
3179
3180                 if (!uptodate || test_bit(EXTENT_BUFFER_IOERR, &eb->bflags)) {
3181                         set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3182                         ClearPageUptodate(page);
3183                         SetPageError(page);
3184                 }
3185
3186                 end_page_writeback(page);
3187
3188                 if (!done)
3189                         continue;
3190
3191                 end_extent_buffer_writeback(eb);
3192         } while (bvec >= bio->bi_io_vec);
3193
3194         bio_put(bio);
3195
3196 }
3197
3198 static int write_one_eb(struct extent_buffer *eb,
3199                         struct btrfs_fs_info *fs_info,
3200                         struct writeback_control *wbc,
3201                         struct extent_page_data *epd)
3202 {
3203         struct block_device *bdev = fs_info->fs_devices->latest_bdev;
3204         u64 offset = eb->start;
3205         unsigned long i, num_pages;
3206         unsigned long bio_flags = 0;
3207         int rw = (epd->sync_io ? WRITE_SYNC : WRITE);
3208         int ret = 0;
3209
3210         clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3211         num_pages = num_extent_pages(eb->start, eb->len);
3212         atomic_set(&eb->io_pages, num_pages);
3213         if (btrfs_header_owner(eb) == BTRFS_TREE_LOG_OBJECTID)
3214                 bio_flags = EXTENT_BIO_TREE_LOG;
3215
3216         for (i = 0; i < num_pages; i++) {
3217                 struct page *p = extent_buffer_page(eb, i);
3218
3219                 clear_page_dirty_for_io(p);
3220                 set_page_writeback(p);
3221                 ret = submit_extent_page(rw, eb->tree, p, offset >> 9,
3222                                          PAGE_CACHE_SIZE, 0, bdev, &epd->bio,
3223                                          -1, end_bio_extent_buffer_writepage,
3224                                          0, epd->bio_flags, bio_flags);
3225                 epd->bio_flags = bio_flags;
3226                 if (ret) {
3227                         set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3228                         SetPageError(p);
3229                         if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3230                                 end_extent_buffer_writeback(eb);
3231                         ret = -EIO;
3232                         break;
3233                 }
3234                 offset += PAGE_CACHE_SIZE;
3235                 update_nr_written(p, wbc, 1);
3236                 unlock_page(p);
3237         }
3238
3239         if (unlikely(ret)) {
3240                 for (; i < num_pages; i++) {
3241                         struct page *p = extent_buffer_page(eb, i);
3242                         unlock_page(p);
3243                 }
3244         }
3245
3246         return ret;
3247 }
3248
3249 int btree_write_cache_pages(struct address_space *mapping,
3250                                    struct writeback_control *wbc)
3251 {
3252         struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
3253         struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
3254         struct extent_buffer *eb, *prev_eb = NULL;
3255         struct extent_page_data epd = {
3256                 .bio = NULL,
3257                 .tree = tree,
3258                 .extent_locked = 0,
3259                 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3260                 .bio_flags = 0,
3261         };
3262         int ret = 0;
3263         int done = 0;
3264         int nr_to_write_done = 0;
3265         struct pagevec pvec;
3266         int nr_pages;
3267         pgoff_t index;
3268         pgoff_t end;            /* Inclusive */
3269         int scanned = 0;
3270         int tag;
3271
3272         pagevec_init(&pvec, 0);
3273         if (wbc->range_cyclic) {
3274                 index = mapping->writeback_index; /* Start from prev offset */
3275                 end = -1;
3276         } else {
3277                 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3278                 end = wbc->range_end >> PAGE_CACHE_SHIFT;
3279                 scanned = 1;
3280         }
3281         if (wbc->sync_mode == WB_SYNC_ALL)
3282                 tag = PAGECACHE_TAG_TOWRITE;
3283         else
3284                 tag = PAGECACHE_TAG_DIRTY;
3285 retry:
3286         if (wbc->sync_mode == WB_SYNC_ALL)
3287                 tag_pages_for_writeback(mapping, index, end);
3288         while (!done && !nr_to_write_done && (index <= end) &&
3289                (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3290                         min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3291                 unsigned i;
3292
3293                 scanned = 1;
3294                 for (i = 0; i < nr_pages; i++) {
3295                         struct page *page = pvec.pages[i];
3296
3297                         if (!PagePrivate(page))
3298                                 continue;
3299
3300                         if (!wbc->range_cyclic && page->index > end) {
3301                                 done = 1;
3302                                 break;
3303                         }
3304
3305                         spin_lock(&mapping->private_lock);
3306                         if (!PagePrivate(page)) {
3307                                 spin_unlock(&mapping->private_lock);
3308                                 continue;
3309                         }
3310
3311                         eb = (struct extent_buffer *)page->private;
3312
3313                         /*
3314                          * Shouldn't happen and normally this would be a BUG_ON
3315                          * but no sense in crashing the users box for something
3316                          * we can survive anyway.
3317                          */
3318                         if (!eb) {
3319                                 spin_unlock(&mapping->private_lock);
3320                                 WARN_ON(1);
3321                                 continue;
3322                         }
3323
3324                         if (eb == prev_eb) {
3325                                 spin_unlock(&mapping->private_lock);
3326                                 continue;
3327                         }
3328
3329                         ret = atomic_inc_not_zero(&eb->refs);
3330                         spin_unlock(&mapping->private_lock);
3331                         if (!ret)
3332                                 continue;
3333
3334                         prev_eb = eb;
3335                         ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
3336                         if (!ret) {
3337                                 free_extent_buffer(eb);
3338                                 continue;
3339                         }
3340
3341                         ret = write_one_eb(eb, fs_info, wbc, &epd);
3342                         if (ret) {
3343                                 done = 1;
3344                                 free_extent_buffer(eb);
3345                                 break;
3346                         }
3347                         free_extent_buffer(eb);
3348
3349                         /*
3350                          * the filesystem may choose to bump up nr_to_write.
3351                          * We have to make sure to honor the new nr_to_write
3352                          * at any time
3353                          */
3354                         nr_to_write_done = wbc->nr_to_write <= 0;
3355                 }
3356                 pagevec_release(&pvec);
3357                 cond_resched();
3358         }
3359         if (!scanned && !done) {
3360                 /*
3361                  * We hit the last page and there is more work to be done: wrap
3362                  * back to the start of the file
3363                  */
3364                 scanned = 1;
3365                 index = 0;
3366                 goto retry;
3367         }
3368         flush_write_bio(&epd);
3369         return ret;
3370 }
3371
3372 /**
3373  * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
3374  * @mapping: address space structure to write
3375  * @wbc: subtract the number of written pages from *@wbc->nr_to_write
3376  * @writepage: function called for each page
3377  * @data: data passed to writepage function
3378  *
3379  * If a page is already under I/O, write_cache_pages() skips it, even
3380  * if it's dirty.  This is desirable behaviour for memory-cleaning writeback,
3381  * but it is INCORRECT for data-integrity system calls such as fsync().  fsync()
3382  * and msync() need to guarantee that all the data which was dirty at the time
3383  * the call was made get new I/O started against them.  If wbc->sync_mode is
3384  * WB_SYNC_ALL then we were called for data integrity and we must wait for
3385  * existing IO to complete.
3386  */
3387 static int extent_write_cache_pages(struct extent_io_tree *tree,
3388                              struct address_space *mapping,
3389                              struct writeback_control *wbc,
3390                              writepage_t writepage, void *data,
3391                              void (*flush_fn)(void *))
3392 {
3393         struct inode *inode = mapping->host;
3394         int ret = 0;
3395         int done = 0;
3396         int nr_to_write_done = 0;
3397         struct pagevec pvec;
3398         int nr_pages;
3399         pgoff_t index;
3400         pgoff_t end;            /* Inclusive */
3401         int scanned = 0;
3402         int tag;
3403
3404         /*
3405          * We have to hold onto the inode so that ordered extents can do their
3406          * work when the IO finishes.  The alternative to this is failing to add
3407          * an ordered extent if the igrab() fails there and that is a huge pain
3408          * to deal with, so instead just hold onto the inode throughout the
3409          * writepages operation.  If it fails here we are freeing up the inode
3410          * anyway and we'd rather not waste our time writing out stuff that is
3411          * going to be truncated anyway.
3412          */
3413         if (!igrab(inode))
3414                 return 0;
3415
3416         pagevec_init(&pvec, 0);
3417         if (wbc->range_cyclic) {
3418                 index = mapping->writeback_index; /* Start from prev offset */
3419                 end = -1;
3420         } else {
3421                 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3422                 end = wbc->range_end >> PAGE_CACHE_SHIFT;
3423                 scanned = 1;
3424         }
3425         if (wbc->sync_mode == WB_SYNC_ALL)
3426                 tag = PAGECACHE_TAG_TOWRITE;
3427         else
3428                 tag = PAGECACHE_TAG_DIRTY;
3429 retry:
3430         if (wbc->sync_mode == WB_SYNC_ALL)
3431                 tag_pages_for_writeback(mapping, index, end);
3432         while (!done && !nr_to_write_done && (index <= end) &&
3433                (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3434                         min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3435                 unsigned i;
3436
3437                 scanned = 1;
3438                 for (i = 0; i < nr_pages; i++) {
3439                         struct page *page = pvec.pages[i];
3440
3441                         /*
3442                          * At this point we hold neither mapping->tree_lock nor
3443                          * lock on the page itself: the page may be truncated or
3444                          * invalidated (changing page->mapping to NULL), or even
3445                          * swizzled back from swapper_space to tmpfs file
3446                          * mapping
3447                          */
3448                         if (tree->ops &&
3449                             tree->ops->write_cache_pages_lock_hook) {
3450                                 tree->ops->write_cache_pages_lock_hook(page,
3451                                                                data, flush_fn);
3452                         } else {
3453                                 if (!trylock_page(page)) {
3454                                         flush_fn(data);
3455                                         lock_page(page);
3456                                 }
3457                         }
3458
3459                         if (unlikely(page->mapping != mapping)) {
3460                                 unlock_page(page);
3461                                 continue;
3462                         }
3463
3464                         if (!wbc->range_cyclic && page->index > end) {
3465                                 done = 1;
3466                                 unlock_page(page);
3467                                 continue;
3468                         }
3469
3470                         if (wbc->sync_mode != WB_SYNC_NONE) {
3471                                 if (PageWriteback(page))
3472                                         flush_fn(data);
3473                                 wait_on_page_writeback(page);
3474                         }
3475
3476                         if (PageWriteback(page) ||
3477                             !clear_page_dirty_for_io(page)) {
3478                                 unlock_page(page);
3479                                 continue;
3480                         }
3481
3482                         ret = (*writepage)(page, wbc, data);
3483
3484                         if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
3485                                 unlock_page(page);
3486                                 ret = 0;
3487                         }
3488                         if (ret)
3489                                 done = 1;
3490
3491                         /*
3492                          * the filesystem may choose to bump up nr_to_write.
3493                          * We have to make sure to honor the new nr_to_write
3494                          * at any time
3495                          */
3496                         nr_to_write_done = wbc->nr_to_write <= 0;
3497                 }
3498                 pagevec_release(&pvec);
3499                 cond_resched();
3500         }
3501         if (!scanned && !done) {
3502                 /*
3503                  * We hit the last page and there is more work to be done: wrap
3504                  * back to the start of the file
3505                  */
3506                 scanned = 1;
3507                 index = 0;
3508                 goto retry;
3509         }
3510         btrfs_add_delayed_iput(inode);
3511         return ret;
3512 }
3513
3514 static void flush_epd_write_bio(struct extent_page_data *epd)
3515 {
3516         if (epd->bio) {
3517                 int rw = WRITE;
3518                 int ret;
3519
3520                 if (epd->sync_io)
3521                         rw = WRITE_SYNC;
3522
3523                 ret = submit_one_bio(rw, epd->bio, 0, epd->bio_flags);
3524                 BUG_ON(ret < 0); /* -ENOMEM */
3525                 epd->bio = NULL;
3526         }
3527 }
3528
3529 static noinline void flush_write_bio(void *data)
3530 {
3531         struct extent_page_data *epd = data;
3532         flush_epd_write_bio(epd);
3533 }
3534
3535 int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
3536                           get_extent_t *get_extent,
3537                           struct writeback_control *wbc)
3538 {
3539         int ret;
3540         struct extent_page_data epd = {
3541                 .bio = NULL,
3542                 .tree = tree,
3543                 .get_extent = get_extent,
3544                 .extent_locked = 0,
3545                 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3546                 .bio_flags = 0,
3547         };
3548
3549         ret = __extent_writepage(page, wbc, &epd);
3550
3551         flush_epd_write_bio(&epd);
3552         return ret;
3553 }
3554
3555 int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
3556                               u64 start, u64 end, get_extent_t *get_extent,
3557                               int mode)
3558 {
3559         int ret = 0;
3560         struct address_space *mapping = inode->i_mapping;
3561         struct page *page;
3562         unsigned long nr_pages = (end - start + PAGE_CACHE_SIZE) >>
3563                 PAGE_CACHE_SHIFT;
3564
3565         struct extent_page_data epd = {
3566                 .bio = NULL,
3567                 .tree = tree,
3568                 .get_extent = get_extent,
3569                 .extent_locked = 1,
3570                 .sync_io = mode == WB_SYNC_ALL,
3571                 .bio_flags = 0,
3572         };
3573         struct writeback_control wbc_writepages = {
3574                 .sync_mode      = mode,
3575                 .nr_to_write    = nr_pages * 2,
3576                 .range_start    = start,
3577                 .range_end      = end + 1,
3578         };
3579
3580         while (start <= end) {
3581                 page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT);
3582                 if (clear_page_dirty_for_io(page))
3583                         ret = __extent_writepage(page, &wbc_writepages, &epd);
3584                 else {
3585                         if (tree->ops && tree->ops->writepage_end_io_hook)
3586                                 tree->ops->writepage_end_io_hook(page, start,
3587                                                  start + PAGE_CACHE_SIZE - 1,
3588                                                  NULL, 1);
3589                         unlock_page(page);
3590                 }
3591                 page_cache_release(page);
3592                 start += PAGE_CACHE_SIZE;
3593         }
3594
3595         flush_epd_write_bio(&epd);
3596         return ret;
3597 }
3598
3599 int extent_writepages(struct extent_io_tree *tree,
3600                       struct address_space *mapping,
3601                       get_extent_t *get_extent,
3602                       struct writeback_control *wbc)
3603 {
3604         int ret = 0;
3605         struct extent_page_data epd = {
3606                 .bio = NULL,
3607                 .tree = tree,
3608                 .get_extent = get_extent,
3609                 .extent_locked = 0,
3610                 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3611                 .bio_flags = 0,
3612         };
3613
3614         ret = extent_write_cache_pages(tree, mapping, wbc,
3615                                        __extent_writepage, &epd,
3616                                        flush_write_bio);
3617         flush_epd_write_bio(&epd);
3618         return ret;
3619 }
3620
3621 int extent_readpages(struct extent_io_tree *tree,
3622                      struct address_space *mapping,
3623                      struct list_head *pages, unsigned nr_pages,
3624                      get_extent_t get_extent)
3625 {
3626         struct bio *bio = NULL;
3627         unsigned page_idx;
3628         unsigned long bio_flags = 0;
3629         struct page *pagepool[16];
3630         struct page *page;
3631         int i = 0;
3632         int nr = 0;
3633
3634         for (page_idx = 0; page_idx < nr_pages; page_idx++) {
3635                 page = list_entry(pages->prev, struct page, lru);
3636
3637                 prefetchw(&page->flags);
3638                 list_del(&page->lru);
3639                 if (add_to_page_cache_lru(page, mapping,
3640                                         page->index, GFP_NOFS)) {
3641                         page_cache_release(page);
3642                         continue;
3643                 }
3644
3645                 pagepool[nr++] = page;
3646                 if (nr < ARRAY_SIZE(pagepool))
3647                         continue;
3648                 for (i = 0; i < nr; i++) {
3649                         __extent_read_full_page(tree, pagepool[i], get_extent,
3650                                         &bio, 0, &bio_flags);
3651                         page_cache_release(pagepool[i]);
3652                 }
3653                 nr = 0;
3654         }
3655         for (i = 0; i < nr; i++) {
3656                 __extent_read_full_page(tree, pagepool[i], get_extent,
3657                                         &bio, 0, &bio_flags);
3658                 page_cache_release(pagepool[i]);
3659         }
3660
3661         BUG_ON(!list_empty(pages));
3662         if (bio)
3663                 return submit_one_bio(READ, bio, 0, bio_flags);
3664         return 0;
3665 }
3666
3667 /*
3668  * basic invalidatepage code, this waits on any locked or writeback
3669  * ranges corresponding to the page, and then deletes any extent state
3670  * records from the tree
3671  */
3672 int extent_invalidatepage(struct extent_io_tree *tree,
3673                           struct page *page, unsigned long offset)
3674 {
3675         struct extent_state *cached_state = NULL;
3676         u64 start = ((u64)page->index << PAGE_CACHE_SHIFT);
3677         u64 end = start + PAGE_CACHE_SIZE - 1;
3678         size_t blocksize = page->mapping->host->i_sb->s_blocksize;
3679
3680         start += (offset + blocksize - 1) & ~(blocksize - 1);
3681         if (start > end)
3682                 return 0;
3683
3684         lock_extent_bits(tree, start, end, 0, &cached_state);
3685         wait_on_page_writeback(page);
3686         clear_extent_bit(tree, start, end,
3687                          EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
3688                          EXTENT_DO_ACCOUNTING,
3689                          1, 1, &cached_state, GFP_NOFS);
3690         return 0;
3691 }
3692
3693 /*
3694  * a helper for releasepage, this tests for areas of the page that
3695  * are locked or under IO and drops the related state bits if it is safe
3696  * to drop the page.
3697  */
3698 int try_release_extent_state(struct extent_map_tree *map,
3699                              struct extent_io_tree *tree, struct page *page,
3700                              gfp_t mask)
3701 {
3702         u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
3703         u64 end = start + PAGE_CACHE_SIZE - 1;
3704         int ret = 1;
3705
3706         if (test_range_bit(tree, start, end,
3707                            EXTENT_IOBITS, 0, NULL))
3708                 ret = 0;
3709         else {
3710                 if ((mask & GFP_NOFS) == GFP_NOFS)
3711                         mask = GFP_NOFS;
3712                 /*
3713                  * at this point we can safely clear everything except the
3714                  * locked bit and the nodatasum bit
3715                  */
3716                 ret = clear_extent_bit(tree, start, end,
3717                                  ~(EXTENT_LOCKED | EXTENT_NODATASUM),
3718                                  0, 0, NULL, mask);
3719
3720                 /* if clear_extent_bit failed for enomem reasons,
3721                  * we can't allow the release to continue.
3722                  */
3723                 if (ret < 0)
3724                         ret = 0;
3725                 else
3726                         ret = 1;
3727         }
3728         return ret;
3729 }
3730
3731 /*
3732  * a helper for releasepage.  As long as there are no locked extents
3733  * in the range corresponding to the page, both state records and extent
3734  * map records are removed
3735  */
3736 int try_release_extent_mapping(struct extent_map_tree *map,
3737                                struct extent_io_tree *tree, struct page *page,
3738                                gfp_t mask)
3739 {
3740         struct extent_map *em;
3741         u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
3742         u64 end = start + PAGE_CACHE_SIZE - 1;
3743
3744         if ((mask & __GFP_WAIT) &&
3745             page->mapping->host->i_size > 16 * 1024 * 1024) {
3746                 u64 len;
3747                 while (start <= end) {
3748                         len = end - start + 1;
3749                         write_lock(&map->lock);
3750                         em = lookup_extent_mapping(map, start, len);
3751                         if (!em) {
3752                                 write_unlock(&map->lock);
3753                                 break;
3754                         }
3755                         if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
3756                             em->start != start) {
3757                                 write_unlock(&map->lock);
3758                                 free_extent_map(em);
3759                                 break;
3760                         }
3761                         if (!test_range_bit(tree, em->start,
3762                                             extent_map_end(em) - 1,
3763                                             EXTENT_LOCKED | EXTENT_WRITEBACK,
3764                                             0, NULL)) {
3765                                 remove_extent_mapping(map, em);
3766                                 /* once for the rb tree */
3767                                 free_extent_map(em);
3768                         }
3769                         start = extent_map_end(em);
3770                         write_unlock(&map->lock);
3771
3772                         /* once for us */
3773                         free_extent_map(em);
3774                 }
3775         }
3776         return try_release_extent_state(map, tree, page, mask);
3777 }
3778
3779 /*
3780  * helper function for fiemap, which doesn't want to see any holes.
3781  * This maps until we find something past 'last'
3782  */
3783 static struct extent_map *get_extent_skip_holes(struct inode *inode,
3784                                                 u64 offset,
3785                                                 u64 last,
3786                                                 get_extent_t *get_extent)
3787 {
3788         u64 sectorsize = BTRFS_I(inode)->root->sectorsize;
3789         struct extent_map *em;
3790         u64 len;
3791
3792         if (offset >= last)
3793                 return NULL;
3794
3795         while(1) {
3796                 len = last - offset;
3797                 if (len == 0)
3798                         break;
3799                 len = (len + sectorsize - 1) & ~(sectorsize - 1);
3800                 em = get_extent(inode, NULL, 0, offset, len, 0);
3801                 if (IS_ERR_OR_NULL(em))
3802                         return em;
3803
3804                 /* if this isn't a hole return it */
3805                 if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) &&
3806                     em->block_start != EXTENT_MAP_HOLE) {
3807                         return em;
3808                 }
3809
3810                 /* this is a hole, advance to the next extent */
3811                 offset = extent_map_end(em);
3812                 free_extent_map(em);
3813                 if (offset >= last)
3814                         break;
3815         }
3816         return NULL;
3817 }
3818
3819 int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
3820                 __u64 start, __u64 len, get_extent_t *get_extent)
3821 {
3822         int ret = 0;
3823         u64 off = start;
3824         u64 max = start + len;
3825         u32 flags = 0;
3826         u32 found_type;
3827         u64 last;
3828         u64 last_for_get_extent = 0;
3829         u64 disko = 0;
3830         u64 isize = i_size_read(inode);
3831         struct btrfs_key found_key;
3832         struct extent_map *em = NULL;
3833         struct extent_state *cached_state = NULL;
3834         struct btrfs_path *path;
3835         struct btrfs_file_extent_item *item;
3836         int end = 0;
3837         u64 em_start = 0;
3838         u64 em_len = 0;
3839         u64 em_end = 0;
3840         unsigned long emflags;
3841
3842         if (len == 0)
3843                 return -EINVAL;
3844
3845         path = btrfs_alloc_path();
3846         if (!path)
3847                 return -ENOMEM;
3848         path->leave_spinning = 1;
3849
3850         start = ALIGN(start, BTRFS_I(inode)->root->sectorsize);
3851         len = ALIGN(len, BTRFS_I(inode)->root->sectorsize);
3852
3853         /*
3854          * lookup the last file extent.  We're not using i_size here
3855          * because there might be preallocation past i_size
3856          */
3857         ret = btrfs_lookup_file_extent(NULL, BTRFS_I(inode)->root,
3858                                        path, btrfs_ino(inode), -1, 0);
3859         if (ret < 0) {
3860                 btrfs_free_path(path);
3861                 return ret;
3862         }
3863         WARN_ON(!ret);
3864         path->slots[0]--;
3865         item = btrfs_item_ptr(path->nodes[0], path->slots[0],
3866                               struct btrfs_file_extent_item);
3867         btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
3868         found_type = btrfs_key_type(&found_key);
3869
3870         /* No extents, but there might be delalloc bits */
3871         if (found_key.objectid != btrfs_ino(inode) ||
3872             found_type != BTRFS_EXTENT_DATA_KEY) {
3873                 /* have to trust i_size as the end */
3874                 last = (u64)-1;
3875                 last_for_get_extent = isize;
3876         } else {
3877                 /*
3878                  * remember the start of the last extent.  There are a
3879                  * bunch of different factors that go into the length of the
3880                  * extent, so its much less complex to remember where it started
3881                  */
3882                 last = found_key.offset;
3883                 last_for_get_extent = last + 1;
3884         }
3885         btrfs_free_path(path);
3886
3887         /*
3888          * we might have some extents allocated but more delalloc past those
3889          * extents.  so, we trust isize unless the start of the last extent is
3890          * beyond isize
3891          */
3892         if (last < isize) {
3893                 last = (u64)-1;
3894                 last_for_get_extent = isize;
3895         }
3896
3897         lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len, 0,
3898                          &cached_state);
3899
3900         em = get_extent_skip_holes(inode, start, last_for_get_extent,
3901                                    get_extent);
3902         if (!em)
3903                 goto out;
3904         if (IS_ERR(em)) {
3905                 ret = PTR_ERR(em);
3906                 goto out;
3907         }
3908
3909         while (!end) {
3910                 u64 offset_in_extent;
3911
3912                 /* break if the extent we found is outside the range */
3913                 if (em->start >= max || extent_map_end(em) < off)
3914                         break;
3915
3916                 /*
3917                  * get_extent may return an extent that starts before our
3918                  * requested range.  We have to make sure the ranges
3919                  * we return to fiemap always move forward and don't
3920                  * overlap, so adjust the offsets here
3921                  */
3922                 em_start = max(em->start, off);
3923
3924                 /*
3925                  * record the offset from the start of the extent
3926                  * for adjusting the disk offset below
3927                  */
3928                 offset_in_extent = em_start - em->start;
3929                 em_end = extent_map_end(em);
3930                 em_len = em_end - em_start;
3931                 emflags = em->flags;
3932                 disko = 0;
3933                 flags = 0;
3934
3935                 /*
3936                  * bump off for our next call to get_extent
3937                  */
3938                 off = extent_map_end(em);
3939                 if (off >= max)
3940                         end = 1;
3941
3942                 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
3943                         end = 1;
3944                         flags |= FIEMAP_EXTENT_LAST;
3945                 } else if (em->block_start == EXTENT_MAP_INLINE) {
3946                         flags |= (FIEMAP_EXTENT_DATA_INLINE |
3947                                   FIEMAP_EXTENT_NOT_ALIGNED);
3948                 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
3949                         flags |= (FIEMAP_EXTENT_DELALLOC |
3950                                   FIEMAP_EXTENT_UNKNOWN);
3951                 } else {
3952                         disko = em->block_start + offset_in_extent;
3953                 }
3954                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
3955                         flags |= FIEMAP_EXTENT_ENCODED;
3956
3957                 free_extent_map(em);
3958                 em = NULL;
3959                 if ((em_start >= last) || em_len == (u64)-1 ||
3960                    (last == (u64)-1 && isize <= em_end)) {
3961                         flags |= FIEMAP_EXTENT_LAST;
3962                         end = 1;
3963                 }
3964
3965                 /* now scan forward to see if this is really the last extent. */
3966                 em = get_extent_skip_holes(inode, off, last_for_get_extent,
3967                                            get_extent);
3968                 if (IS_ERR(em)) {
3969                         ret = PTR_ERR(em);
3970                         goto out;
3971                 }
3972                 if (!em) {
3973                         flags |= FIEMAP_EXTENT_LAST;
3974                         end = 1;
3975                 }
3976                 ret = fiemap_fill_next_extent(fieinfo, em_start, disko,
3977                                               em_len, flags);
3978                 if (ret)
3979                         goto out_free;
3980         }
3981 out_free:
3982         free_extent_map(em);
3983 out:
3984         unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len,
3985                              &cached_state, GFP_NOFS);
3986         return ret;
3987 }
3988
3989 static void __free_extent_buffer(struct extent_buffer *eb)
3990 {
3991 #if LEAK_DEBUG
3992         unsigned long flags;
3993         spin_lock_irqsave(&leak_lock, flags);
3994         list_del(&eb->leak_list);
3995         spin_unlock_irqrestore(&leak_lock, flags);
3996 #endif
3997         if (eb->pages && eb->pages != eb->inline_pages)
3998                 kfree(eb->pages);
3999         kmem_cache_free(extent_buffer_cache, eb);
4000 }
4001
4002 static struct extent_buffer *__alloc_extent_buffer(struct extent_io_tree *tree,
4003                                                    u64 start,
4004                                                    unsigned long len,
4005                                                    gfp_t mask)
4006 {
4007         struct extent_buffer *eb = NULL;
4008 #if LEAK_DEBUG
4009         unsigned long flags;
4010 #endif
4011
4012         eb = kmem_cache_zalloc(extent_buffer_cache, mask);
4013         if (eb == NULL)
4014                 return NULL;
4015         eb->start = start;
4016         eb->len = len;
4017         eb->tree = tree;
4018         eb->bflags = 0;
4019         rwlock_init(&eb->lock);
4020         atomic_set(&eb->write_locks, 0);
4021         atomic_set(&eb->read_locks, 0);
4022         atomic_set(&eb->blocking_readers, 0);
4023         atomic_set(&eb->blocking_writers, 0);
4024         atomic_set(&eb->spinning_readers, 0);
4025         atomic_set(&eb->spinning_writers, 0);
4026         eb->lock_nested = 0;
4027         init_waitqueue_head(&eb->write_lock_wq);
4028         init_waitqueue_head(&eb->read_lock_wq);
4029
4030 #if LEAK_DEBUG
4031         spin_lock_irqsave(&leak_lock, flags);
4032         list_add(&eb->leak_list, &buffers);
4033         spin_unlock_irqrestore(&leak_lock, flags);
4034 #endif
4035         spin_lock_init(&eb->refs_lock);
4036         atomic_set(&eb->refs, 1);
4037         atomic_set(&eb->io_pages, 0);
4038
4039         if (len > MAX_INLINE_EXTENT_BUFFER_SIZE) {
4040                 struct page **pages;
4041                 int num_pages = (len + PAGE_CACHE_SIZE - 1) >>
4042                         PAGE_CACHE_SHIFT;
4043                 pages = kzalloc(num_pages, mask);
4044                 if (!pages) {
4045                         __free_extent_buffer(eb);
4046                         return NULL;
4047                 }
4048                 eb->pages = pages;
4049         } else {
4050                 eb->pages = eb->inline_pages;
4051         }
4052
4053         return eb;
4054 }
4055
4056 struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
4057 {
4058         unsigned long i;
4059         struct page *p;
4060         struct extent_buffer *new;
4061         unsigned long num_pages = num_extent_pages(src->start, src->len);
4062
4063         new = __alloc_extent_buffer(NULL, src->start, src->len, GFP_ATOMIC);
4064         if (new == NULL)
4065                 return NULL;
4066
4067         for (i = 0; i < num_pages; i++) {
4068                 p = alloc_page(GFP_ATOMIC);
4069                 BUG_ON(!p);
4070                 attach_extent_buffer_page(new, p);
4071                 WARN_ON(PageDirty(p));
4072                 SetPageUptodate(p);
4073                 new->pages[i] = p;
4074         }
4075
4076         copy_extent_buffer(new, src, 0, 0, src->len);
4077         set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags);
4078         set_bit(EXTENT_BUFFER_DUMMY, &new->bflags);
4079
4080         return new;
4081 }
4082
4083 struct extent_buffer *alloc_dummy_extent_buffer(u64 start, unsigned long len)
4084 {
4085         struct extent_buffer *eb;
4086         unsigned long num_pages = num_extent_pages(0, len);
4087         unsigned long i;
4088
4089         eb = __alloc_extent_buffer(NULL, start, len, GFP_ATOMIC);
4090         if (!eb)
4091                 return NULL;
4092
4093         for (i = 0; i < num_pages; i++) {
4094                 eb->pages[i] = alloc_page(GFP_ATOMIC);
4095                 if (!eb->pages[i])
4096                         goto err;
4097         }
4098         set_extent_buffer_uptodate(eb);
4099         btrfs_set_header_nritems(eb, 0);
4100         set_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4101
4102         return eb;
4103 err:
4104         for (i--; i > 0; i--)
4105                 __free_page(eb->pages[i]);
4106         __free_extent_buffer(eb);
4107         return NULL;
4108 }
4109
4110 static int extent_buffer_under_io(struct extent_buffer *eb)
4111 {
4112         return (atomic_read(&eb->io_pages) ||
4113                 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
4114                 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4115 }
4116
4117 /*
4118  * Helper for releasing extent buffer page.
4119  */
4120 static void btrfs_release_extent_buffer_page(struct extent_buffer *eb,
4121                                                 unsigned long start_idx)
4122 {
4123         unsigned long index;
4124         unsigned long num_pages;
4125         struct page *page;
4126         int mapped = !test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4127
4128         BUG_ON(extent_buffer_under_io(eb));
4129
4130         num_pages = num_extent_pages(eb->start, eb->len);
4131         index = start_idx + num_pages;
4132         if (start_idx >= index)
4133                 return;
4134
4135         do {
4136                 index--;
4137                 page = extent_buffer_page(eb, index);
4138                 if (page && mapped) {
4139                         spin_lock(&page->mapping->private_lock);
4140                         /*
4141                          * We do this since we'll remove the pages after we've
4142                          * removed the eb from the radix tree, so we could race
4143                          * and have this page now attached to the new eb.  So
4144                          * only clear page_private if it's still connected to
4145                          * this eb.
4146                          */
4147                         if (PagePrivate(page) &&
4148                             page->private == (unsigned long)eb) {
4149                                 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4150                                 BUG_ON(PageDirty(page));
4151                                 BUG_ON(PageWriteback(page));
4152                                 /*
4153                                  * We need to make sure we haven't be attached
4154                                  * to a new eb.
4155                                  */
4156                                 ClearPagePrivate(page);
4157                                 set_page_private(page, 0);
4158                                 /* One for the page private */
4159                                 page_cache_release(page);
4160                         }
4161                         spin_unlock(&page->mapping->private_lock);
4162
4163                 }
4164                 if (page) {
4165                         /* One for when we alloced the page */
4166                         page_cache_release(page);
4167                 }
4168         } while (index != start_idx);
4169 }
4170
4171 /*
4172  * Helper for releasing the extent buffer.
4173  */
4174 static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4175 {
4176         btrfs_release_extent_buffer_page(eb, 0);
4177         __free_extent_buffer(eb);
4178 }
4179
4180 static void check_buffer_tree_ref(struct extent_buffer *eb)
4181 {
4182         /* the ref bit is tricky.  We have to make sure it is set
4183          * if we have the buffer dirty.   Otherwise the
4184          * code to free a buffer can end up dropping a dirty
4185          * page
4186          *
4187          * Once the ref bit is set, it won't go away while the
4188          * buffer is dirty or in writeback, and it also won't
4189          * go away while we have the reference count on the
4190          * eb bumped.
4191          *
4192          * We can't just set the ref bit without bumping the
4193          * ref on the eb because free_extent_buffer might
4194          * see the ref bit and try to clear it.  If this happens
4195          * free_extent_buffer might end up dropping our original
4196          * ref by mistake and freeing the page before we are able
4197          * to add one more ref.
4198          *
4199          * So bump the ref count first, then set the bit.  If someone
4200          * beat us to it, drop the ref we added.
4201          */
4202         spin_lock(&eb->refs_lock);
4203         if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4204                 atomic_inc(&eb->refs);
4205         spin_unlock(&eb->refs_lock);
4206 }
4207
4208 static void mark_extent_buffer_accessed(struct extent_buffer *eb)
4209 {
4210         unsigned long num_pages, i;
4211
4212         check_buffer_tree_ref(eb);
4213
4214         num_pages = num_extent_pages(eb->start, eb->len);
4215         for (i = 0; i < num_pages; i++) {
4216                 struct page *p = extent_buffer_page(eb, i);
4217                 mark_page_accessed(p);
4218         }
4219 }
4220
4221 struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree,
4222                                           u64 start, unsigned long len)
4223 {
4224         unsigned long num_pages = num_extent_pages(start, len);
4225         unsigned long i;
4226         unsigned long index = start >> PAGE_CACHE_SHIFT;
4227         struct extent_buffer *eb;
4228         struct extent_buffer *exists = NULL;
4229         struct page *p;
4230         struct address_space *mapping = tree->mapping;
4231         int uptodate = 1;
4232         int ret;
4233
4234         rcu_read_lock();
4235         eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
4236         if (eb && atomic_inc_not_zero(&eb->refs)) {
4237                 rcu_read_unlock();
4238                 mark_extent_buffer_accessed(eb);
4239                 return eb;
4240         }
4241         rcu_read_unlock();
4242
4243         eb = __alloc_extent_buffer(tree, start, len, GFP_NOFS);
4244         if (!eb)
4245                 return NULL;
4246
4247         for (i = 0; i < num_pages; i++, index++) {
4248                 p = find_or_create_page(mapping, index, GFP_NOFS);
4249                 if (!p) {
4250                         WARN_ON(1);
4251                         goto free_eb;
4252                 }
4253
4254                 spin_lock(&mapping->private_lock);
4255                 if (PagePrivate(p)) {
4256                         /*
4257                          * We could have already allocated an eb for this page
4258                          * and attached one so lets see if we can get a ref on
4259                          * the existing eb, and if we can we know it's good and
4260                          * we can just return that one, else we know we can just
4261                          * overwrite page->private.
4262                          */
4263                         exists = (struct extent_buffer *)p->private;
4264                         if (atomic_inc_not_zero(&exists->refs)) {
4265                                 spin_unlock(&mapping->private_lock);
4266                                 unlock_page(p);
4267                                 page_cache_release(p);
4268                                 mark_extent_buffer_accessed(exists);
4269                                 goto free_eb;
4270                         }
4271
4272                         /*
4273                          * Do this so attach doesn't complain and we need to
4274                          * drop the ref the old guy had.
4275                          */
4276                         ClearPagePrivate(p);
4277                         WARN_ON(PageDirty(p));
4278                         page_cache_release(p);
4279                 }
4280                 attach_extent_buffer_page(eb, p);
4281                 spin_unlock(&mapping->private_lock);
4282                 WARN_ON(PageDirty(p));
4283                 mark_page_accessed(p);
4284                 eb->pages[i] = p;
4285                 if (!PageUptodate(p))
4286                         uptodate = 0;
4287
4288                 /*
4289                  * see below about how we avoid a nasty race with release page
4290                  * and why we unlock later
4291                  */
4292         }
4293         if (uptodate)
4294                 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4295 again:
4296         ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4297         if (ret)
4298                 goto free_eb;
4299
4300         spin_lock(&tree->buffer_lock);
4301         ret = radix_tree_insert(&tree->buffer, start >> PAGE_CACHE_SHIFT, eb);
4302         if (ret == -EEXIST) {
4303                 exists = radix_tree_lookup(&tree->buffer,
4304                                                 start >> PAGE_CACHE_SHIFT);
4305                 if (!atomic_inc_not_zero(&exists->refs)) {
4306                         spin_unlock(&tree->buffer_lock);
4307                         radix_tree_preload_end();
4308                         exists = NULL;
4309                         goto again;
4310                 }
4311                 spin_unlock(&tree->buffer_lock);
4312                 radix_tree_preload_end();
4313                 mark_extent_buffer_accessed(exists);
4314                 goto free_eb;
4315         }
4316         /* add one reference for the tree */
4317         check_buffer_tree_ref(eb);
4318         spin_unlock(&tree->buffer_lock);
4319         radix_tree_preload_end();
4320
4321         /*
4322          * there is a race where release page may have
4323          * tried to find this extent buffer in the radix
4324          * but failed.  It will tell the VM it is safe to
4325          * reclaim the, and it will clear the page private bit.
4326          * We must make sure to set the page private bit properly
4327          * after the extent buffer is in the radix tree so
4328          * it doesn't get lost
4329          */
4330         SetPageChecked(eb->pages[0]);
4331         for (i = 1; i < num_pages; i++) {
4332                 p = extent_buffer_page(eb, i);
4333                 ClearPageChecked(p);
4334                 unlock_page(p);
4335         }
4336         unlock_page(eb->pages[0]);
4337         return eb;
4338
4339 free_eb:
4340         for (i = 0; i < num_pages; i++) {
4341                 if (eb->pages[i])
4342                         unlock_page(eb->pages[i]);
4343         }
4344
4345         WARN_ON(!atomic_dec_and_test(&eb->refs));
4346         btrfs_release_extent_buffer(eb);
4347         return exists;
4348 }
4349
4350 struct extent_buffer *find_extent_buffer(struct extent_io_tree *tree,
4351                                          u64 start, unsigned long len)
4352 {
4353         struct extent_buffer *eb;
4354
4355         rcu_read_lock();
4356         eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
4357         if (eb && atomic_inc_not_zero(&eb->refs)) {
4358                 rcu_read_unlock();
4359                 mark_extent_buffer_accessed(eb);
4360                 return eb;
4361         }
4362         rcu_read_unlock();
4363
4364         return NULL;
4365 }
4366
4367 static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
4368 {
4369         struct extent_buffer *eb =
4370                         container_of(head, struct extent_buffer, rcu_head);
4371
4372         __free_extent_buffer(eb);
4373 }
4374
4375 /* Expects to have eb->eb_lock already held */
4376 static int release_extent_buffer(struct extent_buffer *eb, gfp_t mask)
4377 {
4378         WARN_ON(atomic_read(&eb->refs) == 0);
4379         if (atomic_dec_and_test(&eb->refs)) {
4380                 if (test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags)) {
4381                         spin_unlock(&eb->refs_lock);
4382                 } else {
4383                         struct extent_io_tree *tree = eb->tree;
4384
4385                         spin_unlock(&eb->refs_lock);
4386
4387                         spin_lock(&tree->buffer_lock);
4388                         radix_tree_delete(&tree->buffer,
4389                                           eb->start >> PAGE_CACHE_SHIFT);
4390                         spin_unlock(&tree->buffer_lock);
4391                 }
4392
4393                 /* Should be safe to release our pages at this point */
4394                 btrfs_release_extent_buffer_page(eb, 0);
4395                 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
4396                 return 1;
4397         }
4398         spin_unlock(&eb->refs_lock);
4399
4400         return 0;
4401 }
4402
4403 void free_extent_buffer(struct extent_buffer *eb)
4404 {
4405         if (!eb)
4406                 return;
4407
4408         spin_lock(&eb->refs_lock);
4409         if (atomic_read(&eb->refs) == 2 &&
4410             test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))
4411                 atomic_dec(&eb->refs);
4412
4413         if (atomic_read(&eb->refs) == 2 &&
4414             test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
4415             !extent_buffer_under_io(eb) &&
4416             test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4417                 atomic_dec(&eb->refs);
4418
4419         /*
4420          * I know this is terrible, but it's temporary until we stop tracking
4421          * the uptodate bits and such for the extent buffers.
4422          */
4423         release_extent_buffer(eb, GFP_ATOMIC);
4424 }
4425
4426 void free_extent_buffer_stale(struct extent_buffer *eb)
4427 {
4428         if (!eb)
4429                 return;
4430
4431         spin_lock(&eb->refs_lock);
4432         set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
4433
4434         if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
4435             test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4436                 atomic_dec(&eb->refs);
4437         release_extent_buffer(eb, GFP_NOFS);
4438 }
4439
4440 void clear_extent_buffer_dirty(struct extent_buffer *eb)
4441 {
4442         unsigned long i;
4443         unsigned long num_pages;
4444         struct page *page;
4445
4446         num_pages = num_extent_pages(eb->start, eb->len);
4447
4448         for (i = 0; i < num_pages; i++) {
4449                 page = extent_buffer_page(eb, i);
4450                 if (!PageDirty(page))
4451                         continue;
4452
4453                 lock_page(page);
4454                 WARN_ON(!PagePrivate(page));
4455
4456                 clear_page_dirty_for_io(page);
4457                 spin_lock_irq(&page->mapping->tree_lock);
4458                 if (!PageDirty(page)) {
4459                         radix_tree_tag_clear(&page->mapping->page_tree,
4460                                                 page_index(page),
4461                                                 PAGECACHE_TAG_DIRTY);
4462                 }
4463                 spin_unlock_irq(&page->mapping->tree_lock);
4464                 ClearPageError(page);
4465                 unlock_page(page);
4466         }
4467         WARN_ON(atomic_read(&eb->refs) == 0);
4468 }
4469
4470 int set_extent_buffer_dirty(struct extent_buffer *eb)
4471 {
4472         unsigned long i;
4473         unsigned long num_pages;
4474         int was_dirty = 0;
4475
4476         check_buffer_tree_ref(eb);
4477
4478         was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
4479
4480         num_pages = num_extent_pages(eb->start, eb->len);
4481         WARN_ON(atomic_read(&eb->refs) == 0);
4482         WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
4483
4484         for (i = 0; i < num_pages; i++)
4485                 set_page_dirty(extent_buffer_page(eb, i));
4486         return was_dirty;
4487 }
4488
4489 static int range_straddles_pages(u64 start, u64 len)
4490 {
4491         if (len < PAGE_CACHE_SIZE)
4492                 return 1;
4493         if (start & (PAGE_CACHE_SIZE - 1))
4494                 return 1;
4495         if ((start + len) & (PAGE_CACHE_SIZE - 1))
4496                 return 1;
4497         return 0;
4498 }
4499
4500 int clear_extent_buffer_uptodate(struct extent_buffer *eb)
4501 {
4502         unsigned long i;
4503         struct page *page;
4504         unsigned long num_pages;
4505
4506         clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4507         num_pages = num_extent_pages(eb->start, eb->len);
4508         for (i = 0; i < num_pages; i++) {
4509                 page = extent_buffer_page(eb, i);
4510                 if (page)
4511                         ClearPageUptodate(page);
4512         }
4513         return 0;
4514 }
4515
4516 int set_extent_buffer_uptodate(struct extent_buffer *eb)
4517 {
4518         unsigned long i;
4519         struct page *page;
4520         unsigned long num_pages;
4521
4522         set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4523         num_pages = num_extent_pages(eb->start, eb->len);
4524         for (i = 0; i < num_pages; i++) {
4525                 page = extent_buffer_page(eb, i);
4526                 SetPageUptodate(page);
4527         }
4528         return 0;
4529 }
4530
4531 int extent_range_uptodate(struct extent_io_tree *tree,
4532                           u64 start, u64 end)
4533 {
4534         struct page *page;
4535         int ret;
4536         int pg_uptodate = 1;
4537         int uptodate;
4538         unsigned long index;
4539
4540         if (range_straddles_pages(start, end - start + 1)) {
4541                 ret = test_range_bit(tree, start, end,
4542                                      EXTENT_UPTODATE, 1, NULL);
4543                 if (ret)
4544                         return 1;
4545         }
4546         while (start <= end) {
4547                 index = start >> PAGE_CACHE_SHIFT;
4548                 page = find_get_page(tree->mapping, index);
4549                 if (!page)
4550                         return 1;
4551                 uptodate = PageUptodate(page);
4552                 page_cache_release(page);
4553                 if (!uptodate) {
4554                         pg_uptodate = 0;
4555                         break;
4556                 }
4557                 start += PAGE_CACHE_SIZE;
4558         }
4559         return pg_uptodate;
4560 }
4561
4562 int extent_buffer_uptodate(struct extent_buffer *eb)
4563 {
4564         return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4565 }
4566
4567 int read_extent_buffer_pages(struct extent_io_tree *tree,
4568                              struct extent_buffer *eb, u64 start, int wait,
4569                              get_extent_t *get_extent, int mirror_num)
4570 {
4571         unsigned long i;
4572         unsigned long start_i;
4573         struct page *page;
4574         int err;
4575         int ret = 0;
4576         int locked_pages = 0;
4577         int all_uptodate = 1;
4578         unsigned long num_pages;
4579         unsigned long num_reads = 0;
4580         struct bio *bio = NULL;
4581         unsigned long bio_flags = 0;
4582
4583         if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
4584                 return 0;
4585
4586         if (start) {
4587                 WARN_ON(start < eb->start);
4588                 start_i = (start >> PAGE_CACHE_SHIFT) -
4589                         (eb->start >> PAGE_CACHE_SHIFT);
4590         } else {
4591                 start_i = 0;
4592         }
4593
4594         num_pages = num_extent_pages(eb->start, eb->len);
4595         for (i = start_i; i < num_pages; i++) {
4596                 page = extent_buffer_page(eb, i);
4597                 if (wait == WAIT_NONE) {
4598                         if (!trylock_page(page))
4599                                 goto unlock_exit;
4600                 } else {
4601                         lock_page(page);
4602                 }
4603                 locked_pages++;
4604                 if (!PageUptodate(page)) {
4605                         num_reads++;
4606                         all_uptodate = 0;
4607                 }
4608         }
4609         if (all_uptodate) {
4610                 if (start_i == 0)
4611                         set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4612                 goto unlock_exit;
4613         }
4614
4615         clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
4616         eb->read_mirror = 0;
4617         atomic_set(&eb->io_pages, num_reads);
4618         for (i = start_i; i < num_pages; i++) {
4619                 page = extent_buffer_page(eb, i);
4620                 if (!PageUptodate(page)) {
4621                         ClearPageError(page);
4622                         err = __extent_read_full_page(tree, page,
4623                                                       get_extent, &bio,
4624                                                       mirror_num, &bio_flags);
4625                         if (err)
4626                                 ret = err;
4627                 } else {
4628                         unlock_page(page);
4629                 }
4630         }
4631
4632         if (bio) {
4633                 err = submit_one_bio(READ, bio, mirror_num, bio_flags);
4634                 if (err)
4635                         return err;
4636         }
4637
4638         if (ret || wait != WAIT_COMPLETE)
4639                 return ret;
4640
4641         for (i = start_i; i < num_pages; i++) {
4642                 page = extent_buffer_page(eb, i);
4643                 wait_on_page_locked(page);
4644                 if (!PageUptodate(page))
4645                         ret = -EIO;
4646         }
4647
4648         return ret;
4649
4650 unlock_exit:
4651         i = start_i;
4652         while (locked_pages > 0) {
4653                 page = extent_buffer_page(eb, i);
4654                 i++;
4655                 unlock_page(page);
4656                 locked_pages--;
4657         }
4658         return ret;
4659 }
4660
4661 void read_extent_buffer(struct extent_buffer *eb, void *dstv,
4662                         unsigned long start,
4663                         unsigned long len)
4664 {
4665         size_t cur;
4666         size_t offset;
4667         struct page *page;
4668         char *kaddr;
4669         char *dst = (char *)dstv;
4670         size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4671         unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4672
4673         WARN_ON(start > eb->len);
4674         WARN_ON(start + len > eb->start + eb->len);
4675
4676         offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4677
4678         while (len > 0) {
4679                 page = extent_buffer_page(eb, i);
4680
4681                 cur = min(len, (PAGE_CACHE_SIZE - offset));
4682                 kaddr = page_address(page);
4683                 memcpy(dst, kaddr + offset, cur);
4684
4685                 dst += cur;
4686                 len -= cur;
4687                 offset = 0;
4688                 i++;
4689         }
4690 }
4691
4692 int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
4693                                unsigned long min_len, char **map,
4694                                unsigned long *map_start,
4695                                unsigned long *map_len)
4696 {
4697         size_t offset = start & (PAGE_CACHE_SIZE - 1);
4698         char *kaddr;
4699         struct page *p;
4700         size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4701         unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4702         unsigned long end_i = (start_offset + start + min_len - 1) >>
4703                 PAGE_CACHE_SHIFT;
4704
4705         if (i != end_i)
4706                 return -EINVAL;
4707
4708         if (i == 0) {
4709                 offset = start_offset;
4710                 *map_start = 0;
4711         } else {
4712                 offset = 0;
4713                 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
4714         }
4715
4716         if (start + min_len > eb->len) {
4717                 printk(KERN_ERR "btrfs bad mapping eb start %llu len %lu, "
4718                        "wanted %lu %lu\n", (unsigned long long)eb->start,
4719                        eb->len, start, min_len);
4720                 WARN_ON(1);
4721                 return -EINVAL;
4722         }
4723
4724         p = extent_buffer_page(eb, i);
4725         kaddr = page_address(p);
4726         *map = kaddr + offset;
4727         *map_len = PAGE_CACHE_SIZE - offset;
4728         return 0;
4729 }
4730
4731 int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
4732                           unsigned long start,
4733                           unsigned long len)
4734 {
4735         size_t cur;
4736         size_t offset;
4737         struct page *page;
4738         char *kaddr;
4739         char *ptr = (char *)ptrv;
4740         size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4741         unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4742         int ret = 0;
4743
4744         WARN_ON(start > eb->len);
4745         WARN_ON(start + len > eb->start + eb->len);
4746
4747         offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4748
4749         while (len > 0) {
4750                 page = extent_buffer_page(eb, i);
4751
4752                 cur = min(len, (PAGE_CACHE_SIZE - offset));
4753
4754                 kaddr = page_address(page);
4755                 ret = memcmp(ptr, kaddr + offset, cur);
4756                 if (ret)
4757                         break;
4758
4759                 ptr += cur;
4760                 len -= cur;
4761                 offset = 0;
4762                 i++;
4763         }
4764         return ret;
4765 }
4766
4767 void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
4768                          unsigned long start, unsigned long len)
4769 {
4770         size_t cur;
4771         size_t offset;
4772         struct page *page;
4773         char *kaddr;
4774         char *src = (char *)srcv;
4775         size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4776         unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4777
4778         WARN_ON(start > eb->len);
4779         WARN_ON(start + len > eb->start + eb->len);
4780
4781         offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4782
4783         while (len > 0) {
4784                 page = extent_buffer_page(eb, i);
4785                 WARN_ON(!PageUptodate(page));
4786
4787                 cur = min(len, PAGE_CACHE_SIZE - offset);
4788                 kaddr = page_address(page);
4789                 memcpy(kaddr + offset, src, cur);
4790
4791                 src += cur;
4792                 len -= cur;
4793                 offset = 0;
4794                 i++;
4795         }
4796 }
4797
4798 void memset_extent_buffer(struct extent_buffer *eb, char c,
4799                           unsigned long start, unsigned long len)
4800 {
4801         size_t cur;
4802         size_t offset;
4803         struct page *page;
4804         char *kaddr;
4805         size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4806         unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4807
4808         WARN_ON(start > eb->len);
4809         WARN_ON(start + len > eb->start + eb->len);
4810
4811         offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4812
4813         while (len > 0) {
4814                 page = extent_buffer_page(eb, i);
4815                 WARN_ON(!PageUptodate(page));
4816
4817                 cur = min(len, PAGE_CACHE_SIZE - offset);
4818                 kaddr = page_address(page);
4819                 memset(kaddr + offset, c, cur);
4820
4821                 len -= cur;
4822                 offset = 0;
4823                 i++;
4824         }
4825 }
4826
4827 void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
4828                         unsigned long dst_offset, unsigned long src_offset,
4829                         unsigned long len)
4830 {
4831         u64 dst_len = dst->len;
4832         size_t cur;
4833         size_t offset;
4834         struct page *page;
4835         char *kaddr;
4836         size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
4837         unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
4838
4839         WARN_ON(src->len != dst_len);
4840
4841         offset = (start_offset + dst_offset) &
4842                 ((unsigned long)PAGE_CACHE_SIZE - 1);
4843
4844         while (len > 0) {
4845                 page = extent_buffer_page(dst, i);
4846                 WARN_ON(!PageUptodate(page));
4847
4848                 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
4849
4850                 kaddr = page_address(page);
4851                 read_extent_buffer(src, kaddr + offset, src_offset, cur);
4852
4853                 src_offset += cur;
4854                 len -= cur;
4855                 offset = 0;
4856                 i++;
4857         }
4858 }
4859
4860 static void move_pages(struct page *dst_page, struct page *src_page,
4861                        unsigned long dst_off, unsigned long src_off,
4862                        unsigned long len)
4863 {
4864         char *dst_kaddr = page_address(dst_page);
4865         if (dst_page == src_page) {
4866                 memmove(dst_kaddr + dst_off, dst_kaddr + src_off, len);
4867         } else {
4868                 char *src_kaddr = page_address(src_page);
4869                 char *p = dst_kaddr + dst_off + len;
4870                 char *s = src_kaddr + src_off + len;
4871
4872                 while (len--)
4873                         *--p = *--s;
4874         }
4875 }
4876
4877 static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
4878 {
4879         unsigned long distance = (src > dst) ? src - dst : dst - src;
4880         return distance < len;
4881 }
4882
4883 static void copy_pages(struct page *dst_page, struct page *src_page,
4884                        unsigned long dst_off, unsigned long src_off,
4885                        unsigned long len)
4886 {
4887         char *dst_kaddr = page_address(dst_page);
4888         char *src_kaddr;
4889         int must_memmove = 0;
4890
4891         if (dst_page != src_page) {
4892                 src_kaddr = page_address(src_page);
4893         } else {
4894                 src_kaddr = dst_kaddr;
4895                 if (areas_overlap(src_off, dst_off, len))
4896                         must_memmove = 1;
4897         }
4898
4899         if (must_memmove)
4900                 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
4901         else
4902                 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
4903 }
4904
4905 void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
4906                            unsigned long src_offset, unsigned long len)
4907 {
4908         size_t cur;
4909         size_t dst_off_in_page;
4910         size_t src_off_in_page;
4911         size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
4912         unsigned long dst_i;
4913         unsigned long src_i;
4914
4915         if (src_offset + len > dst->len) {
4916                 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
4917                        "len %lu dst len %lu\n", src_offset, len, dst->len);
4918                 BUG_ON(1);
4919         }
4920         if (dst_offset + len > dst->len) {
4921                 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
4922                        "len %lu dst len %lu\n", dst_offset, len, dst->len);
4923                 BUG_ON(1);
4924         }
4925
4926         while (len > 0) {
4927                 dst_off_in_page = (start_offset + dst_offset) &
4928                         ((unsigned long)PAGE_CACHE_SIZE - 1);
4929                 src_off_in_page = (start_offset + src_offset) &
4930                         ((unsigned long)PAGE_CACHE_SIZE - 1);
4931
4932                 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
4933                 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
4934
4935                 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
4936                                                src_off_in_page));
4937                 cur = min_t(unsigned long, cur,
4938                         (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
4939
4940                 copy_pages(extent_buffer_page(dst, dst_i),
4941                            extent_buffer_page(dst, src_i),
4942                            dst_off_in_page, src_off_in_page, cur);
4943
4944                 src_offset += cur;
4945                 dst_offset += cur;
4946                 len -= cur;
4947         }
4948 }
4949
4950 void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
4951                            unsigned long src_offset, unsigned long len)
4952 {
4953         size_t cur;
4954         size_t dst_off_in_page;
4955         size_t src_off_in_page;
4956         unsigned long dst_end = dst_offset + len - 1;
4957         unsigned long src_end = src_offset + len - 1;
4958         size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
4959         unsigned long dst_i;
4960         unsigned long src_i;
4961
4962         if (src_offset + len > dst->len) {
4963                 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
4964                        "len %lu len %lu\n", src_offset, len, dst->len);
4965                 BUG_ON(1);
4966         }
4967         if (dst_offset + len > dst->len) {
4968                 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
4969                        "len %lu len %lu\n", dst_offset, len, dst->len);
4970                 BUG_ON(1);
4971         }
4972         if (dst_offset < src_offset) {
4973                 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
4974                 return;
4975         }
4976         while (len > 0) {
4977                 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
4978                 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
4979
4980                 dst_off_in_page = (start_offset + dst_end) &
4981                         ((unsigned long)PAGE_CACHE_SIZE - 1);
4982                 src_off_in_page = (start_offset + src_end) &
4983                         ((unsigned long)PAGE_CACHE_SIZE - 1);
4984
4985                 cur = min_t(unsigned long, len, src_off_in_page + 1);
4986                 cur = min(cur, dst_off_in_page + 1);
4987                 move_pages(extent_buffer_page(dst, dst_i),
4988                            extent_buffer_page(dst, src_i),
4989                            dst_off_in_page - cur + 1,
4990                            src_off_in_page - cur + 1, cur);
4991
4992                 dst_end -= cur;
4993                 src_end -= cur;
4994                 len -= cur;
4995         }
4996 }
4997
4998 int try_release_extent_buffer(struct page *page, gfp_t mask)
4999 {
5000         struct extent_buffer *eb;
5001
5002         /*
5003          * We need to make sure noboody is attaching this page to an eb right
5004          * now.
5005          */
5006         spin_lock(&page->mapping->private_lock);
5007         if (!PagePrivate(page)) {
5008                 spin_unlock(&page->mapping->private_lock);
5009                 return 1;
5010         }
5011
5012         eb = (struct extent_buffer *)page->private;
5013         BUG_ON(!eb);
5014
5015         /*
5016          * This is a little awful but should be ok, we need to make sure that
5017          * the eb doesn't disappear out from under us while we're looking at
5018          * this page.
5019          */
5020         spin_lock(&eb->refs_lock);
5021         if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
5022                 spin_unlock(&eb->refs_lock);
5023                 spin_unlock(&page->mapping->private_lock);
5024                 return 0;
5025         }
5026         spin_unlock(&page->mapping->private_lock);
5027
5028         if ((mask & GFP_NOFS) == GFP_NOFS)
5029                 mask = GFP_NOFS;
5030
5031         /*
5032          * If tree ref isn't set then we know the ref on this eb is a real ref,
5033          * so just return, this page will likely be freed soon anyway.
5034          */
5035         if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
5036                 spin_unlock(&eb->refs_lock);
5037                 return 0;
5038         }
5039
5040         return release_extent_buffer(eb, mask);
5041 }