block: remove per-queue plugging
[linux-2.6.git] / fs / xfs / linux-2.6 / xfs_buf.c
1 /*
2  * Copyright (c) 2000-2006 Silicon Graphics, Inc.
3  * All Rights Reserved.
4  *
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU General Public License as
7  * published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope that it would be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write the Free Software Foundation,
16  * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17  */
18 #include "xfs.h"
19 #include <linux/stddef.h>
20 #include <linux/errno.h>
21 #include <linux/gfp.h>
22 #include <linux/pagemap.h>
23 #include <linux/init.h>
24 #include <linux/vmalloc.h>
25 #include <linux/bio.h>
26 #include <linux/sysctl.h>
27 #include <linux/proc_fs.h>
28 #include <linux/workqueue.h>
29 #include <linux/percpu.h>
30 #include <linux/blkdev.h>
31 #include <linux/hash.h>
32 #include <linux/kthread.h>
33 #include <linux/migrate.h>
34 #include <linux/backing-dev.h>
35 #include <linux/freezer.h>
36 #include <linux/list_sort.h>
37
38 #include "xfs_sb.h"
39 #include "xfs_inum.h"
40 #include "xfs_log.h"
41 #include "xfs_ag.h"
42 #include "xfs_mount.h"
43 #include "xfs_trace.h"
44
45 static kmem_zone_t *xfs_buf_zone;
46 STATIC int xfsbufd(void *);
47 STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
48
49 static struct workqueue_struct *xfslogd_workqueue;
50 struct workqueue_struct *xfsdatad_workqueue;
51 struct workqueue_struct *xfsconvertd_workqueue;
52
53 #ifdef XFS_BUF_LOCK_TRACKING
54 # define XB_SET_OWNER(bp)       ((bp)->b_last_holder = current->pid)
55 # define XB_CLEAR_OWNER(bp)     ((bp)->b_last_holder = -1)
56 # define XB_GET_OWNER(bp)       ((bp)->b_last_holder)
57 #else
58 # define XB_SET_OWNER(bp)       do { } while (0)
59 # define XB_CLEAR_OWNER(bp)     do { } while (0)
60 # define XB_GET_OWNER(bp)       do { } while (0)
61 #endif
62
63 #define xb_to_gfp(flags) \
64         ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
65           ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
66
67 #define xb_to_km(flags) \
68          (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
69
70 #define xfs_buf_allocate(flags) \
71         kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
72 #define xfs_buf_deallocate(bp) \
73         kmem_zone_free(xfs_buf_zone, (bp));
74
75 static inline int
76 xfs_buf_is_vmapped(
77         struct xfs_buf  *bp)
78 {
79         /*
80          * Return true if the buffer is vmapped.
81          *
82          * The XBF_MAPPED flag is set if the buffer should be mapped, but the
83          * code is clever enough to know it doesn't have to map a single page,
84          * so the check has to be both for XBF_MAPPED and bp->b_page_count > 1.
85          */
86         return (bp->b_flags & XBF_MAPPED) && bp->b_page_count > 1;
87 }
88
89 static inline int
90 xfs_buf_vmap_len(
91         struct xfs_buf  *bp)
92 {
93         return (bp->b_page_count * PAGE_SIZE) - bp->b_offset;
94 }
95
96 /*
97  *      Page Region interfaces.
98  *
99  *      For pages in filesystems where the blocksize is smaller than the
100  *      pagesize, we use the page->private field (long) to hold a bitmap
101  *      of uptodate regions within the page.
102  *
103  *      Each such region is "bytes per page / bits per long" bytes long.
104  *
105  *      NBPPR == number-of-bytes-per-page-region
106  *      BTOPR == bytes-to-page-region (rounded up)
107  *      BTOPRT == bytes-to-page-region-truncated (rounded down)
108  */
109 #if (BITS_PER_LONG == 32)
110 #define PRSHIFT         (PAGE_CACHE_SHIFT - 5)  /* (32 == 1<<5) */
111 #elif (BITS_PER_LONG == 64)
112 #define PRSHIFT         (PAGE_CACHE_SHIFT - 6)  /* (64 == 1<<6) */
113 #else
114 #error BITS_PER_LONG must be 32 or 64
115 #endif
116 #define NBPPR           (PAGE_CACHE_SIZE/BITS_PER_LONG)
117 #define BTOPR(b)        (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
118 #define BTOPRT(b)       (((unsigned int)(b) >> PRSHIFT))
119
120 STATIC unsigned long
121 page_region_mask(
122         size_t          offset,
123         size_t          length)
124 {
125         unsigned long   mask;
126         int             first, final;
127
128         first = BTOPR(offset);
129         final = BTOPRT(offset + length - 1);
130         first = min(first, final);
131
132         mask = ~0UL;
133         mask <<= BITS_PER_LONG - (final - first);
134         mask >>= BITS_PER_LONG - (final);
135
136         ASSERT(offset + length <= PAGE_CACHE_SIZE);
137         ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
138
139         return mask;
140 }
141
142 STATIC void
143 set_page_region(
144         struct page     *page,
145         size_t          offset,
146         size_t          length)
147 {
148         set_page_private(page,
149                 page_private(page) | page_region_mask(offset, length));
150         if (page_private(page) == ~0UL)
151                 SetPageUptodate(page);
152 }
153
154 STATIC int
155 test_page_region(
156         struct page     *page,
157         size_t          offset,
158         size_t          length)
159 {
160         unsigned long   mask = page_region_mask(offset, length);
161
162         return (mask && (page_private(page) & mask) == mask);
163 }
164
165 /*
166  * xfs_buf_lru_add - add a buffer to the LRU.
167  *
168  * The LRU takes a new reference to the buffer so that it will only be freed
169  * once the shrinker takes the buffer off the LRU.
170  */
171 STATIC void
172 xfs_buf_lru_add(
173         struct xfs_buf  *bp)
174 {
175         struct xfs_buftarg *btp = bp->b_target;
176
177         spin_lock(&btp->bt_lru_lock);
178         if (list_empty(&bp->b_lru)) {
179                 atomic_inc(&bp->b_hold);
180                 list_add_tail(&bp->b_lru, &btp->bt_lru);
181                 btp->bt_lru_nr++;
182         }
183         spin_unlock(&btp->bt_lru_lock);
184 }
185
186 /*
187  * xfs_buf_lru_del - remove a buffer from the LRU
188  *
189  * The unlocked check is safe here because it only occurs when there are not
190  * b_lru_ref counts left on the inode under the pag->pag_buf_lock. it is there
191  * to optimise the shrinker removing the buffer from the LRU and calling
192  * xfs_buf_free(). i.e. it removes an unneccessary round trip on the
193  * bt_lru_lock.
194  */
195 STATIC void
196 xfs_buf_lru_del(
197         struct xfs_buf  *bp)
198 {
199         struct xfs_buftarg *btp = bp->b_target;
200
201         if (list_empty(&bp->b_lru))
202                 return;
203
204         spin_lock(&btp->bt_lru_lock);
205         if (!list_empty(&bp->b_lru)) {
206                 list_del_init(&bp->b_lru);
207                 btp->bt_lru_nr--;
208         }
209         spin_unlock(&btp->bt_lru_lock);
210 }
211
212 /*
213  * When we mark a buffer stale, we remove the buffer from the LRU and clear the
214  * b_lru_ref count so that the buffer is freed immediately when the buffer
215  * reference count falls to zero. If the buffer is already on the LRU, we need
216  * to remove the reference that LRU holds on the buffer.
217  *
218  * This prevents build-up of stale buffers on the LRU.
219  */
220 void
221 xfs_buf_stale(
222         struct xfs_buf  *bp)
223 {
224         bp->b_flags |= XBF_STALE;
225         atomic_set(&(bp)->b_lru_ref, 0);
226         if (!list_empty(&bp->b_lru)) {
227                 struct xfs_buftarg *btp = bp->b_target;
228
229                 spin_lock(&btp->bt_lru_lock);
230                 if (!list_empty(&bp->b_lru)) {
231                         list_del_init(&bp->b_lru);
232                         btp->bt_lru_nr--;
233                         atomic_dec(&bp->b_hold);
234                 }
235                 spin_unlock(&btp->bt_lru_lock);
236         }
237         ASSERT(atomic_read(&bp->b_hold) >= 1);
238 }
239
240 STATIC void
241 _xfs_buf_initialize(
242         xfs_buf_t               *bp,
243         xfs_buftarg_t           *target,
244         xfs_off_t               range_base,
245         size_t                  range_length,
246         xfs_buf_flags_t         flags)
247 {
248         /*
249          * We don't want certain flags to appear in b_flags.
250          */
251         flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
252
253         memset(bp, 0, sizeof(xfs_buf_t));
254         atomic_set(&bp->b_hold, 1);
255         atomic_set(&bp->b_lru_ref, 1);
256         init_completion(&bp->b_iowait);
257         INIT_LIST_HEAD(&bp->b_lru);
258         INIT_LIST_HEAD(&bp->b_list);
259         RB_CLEAR_NODE(&bp->b_rbnode);
260         sema_init(&bp->b_sema, 0); /* held, no waiters */
261         XB_SET_OWNER(bp);
262         bp->b_target = target;
263         bp->b_file_offset = range_base;
264         /*
265          * Set buffer_length and count_desired to the same value initially.
266          * I/O routines should use count_desired, which will be the same in
267          * most cases but may be reset (e.g. XFS recovery).
268          */
269         bp->b_buffer_length = bp->b_count_desired = range_length;
270         bp->b_flags = flags;
271         bp->b_bn = XFS_BUF_DADDR_NULL;
272         atomic_set(&bp->b_pin_count, 0);
273         init_waitqueue_head(&bp->b_waiters);
274
275         XFS_STATS_INC(xb_create);
276
277         trace_xfs_buf_init(bp, _RET_IP_);
278 }
279
280 /*
281  *      Allocate a page array capable of holding a specified number
282  *      of pages, and point the page buf at it.
283  */
284 STATIC int
285 _xfs_buf_get_pages(
286         xfs_buf_t               *bp,
287         int                     page_count,
288         xfs_buf_flags_t         flags)
289 {
290         /* Make sure that we have a page list */
291         if (bp->b_pages == NULL) {
292                 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
293                 bp->b_page_count = page_count;
294                 if (page_count <= XB_PAGES) {
295                         bp->b_pages = bp->b_page_array;
296                 } else {
297                         bp->b_pages = kmem_alloc(sizeof(struct page *) *
298                                         page_count, xb_to_km(flags));
299                         if (bp->b_pages == NULL)
300                                 return -ENOMEM;
301                 }
302                 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
303         }
304         return 0;
305 }
306
307 /*
308  *      Frees b_pages if it was allocated.
309  */
310 STATIC void
311 _xfs_buf_free_pages(
312         xfs_buf_t       *bp)
313 {
314         if (bp->b_pages != bp->b_page_array) {
315                 kmem_free(bp->b_pages);
316                 bp->b_pages = NULL;
317         }
318 }
319
320 /*
321  *      Releases the specified buffer.
322  *
323  *      The modification state of any associated pages is left unchanged.
324  *      The buffer most not be on any hash - use xfs_buf_rele instead for
325  *      hashed and refcounted buffers
326  */
327 void
328 xfs_buf_free(
329         xfs_buf_t               *bp)
330 {
331         trace_xfs_buf_free(bp, _RET_IP_);
332
333         ASSERT(list_empty(&bp->b_lru));
334
335         if (bp->b_flags & (_XBF_PAGE_CACHE|_XBF_PAGES)) {
336                 uint            i;
337
338                 if (xfs_buf_is_vmapped(bp))
339                         vm_unmap_ram(bp->b_addr - bp->b_offset,
340                                         bp->b_page_count);
341
342                 for (i = 0; i < bp->b_page_count; i++) {
343                         struct page     *page = bp->b_pages[i];
344
345                         if (bp->b_flags & _XBF_PAGE_CACHE)
346                                 ASSERT(!PagePrivate(page));
347                         page_cache_release(page);
348                 }
349         }
350         _xfs_buf_free_pages(bp);
351         xfs_buf_deallocate(bp);
352 }
353
354 /*
355  *      Finds all pages for buffer in question and builds it's page list.
356  */
357 STATIC int
358 _xfs_buf_lookup_pages(
359         xfs_buf_t               *bp,
360         uint                    flags)
361 {
362         struct address_space    *mapping = bp->b_target->bt_mapping;
363         size_t                  blocksize = bp->b_target->bt_bsize;
364         size_t                  size = bp->b_count_desired;
365         size_t                  nbytes, offset;
366         gfp_t                   gfp_mask = xb_to_gfp(flags);
367         unsigned short          page_count, i;
368         pgoff_t                 first;
369         xfs_off_t               end;
370         int                     error;
371
372         end = bp->b_file_offset + bp->b_buffer_length;
373         page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
374
375         error = _xfs_buf_get_pages(bp, page_count, flags);
376         if (unlikely(error))
377                 return error;
378         bp->b_flags |= _XBF_PAGE_CACHE;
379
380         offset = bp->b_offset;
381         first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
382
383         for (i = 0; i < bp->b_page_count; i++) {
384                 struct page     *page;
385                 uint            retries = 0;
386
387               retry:
388                 page = find_or_create_page(mapping, first + i, gfp_mask);
389                 if (unlikely(page == NULL)) {
390                         if (flags & XBF_READ_AHEAD) {
391                                 bp->b_page_count = i;
392                                 for (i = 0; i < bp->b_page_count; i++)
393                                         unlock_page(bp->b_pages[i]);
394                                 return -ENOMEM;
395                         }
396
397                         /*
398                          * This could deadlock.
399                          *
400                          * But until all the XFS lowlevel code is revamped to
401                          * handle buffer allocation failures we can't do much.
402                          */
403                         if (!(++retries % 100))
404                                 printk(KERN_ERR
405                                         "XFS: possible memory allocation "
406                                         "deadlock in %s (mode:0x%x)\n",
407                                         __func__, gfp_mask);
408
409                         XFS_STATS_INC(xb_page_retries);
410                         congestion_wait(BLK_RW_ASYNC, HZ/50);
411                         goto retry;
412                 }
413
414                 XFS_STATS_INC(xb_page_found);
415
416                 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
417                 size -= nbytes;
418
419                 ASSERT(!PagePrivate(page));
420                 if (!PageUptodate(page)) {
421                         page_count--;
422                         if (blocksize >= PAGE_CACHE_SIZE) {
423                                 if (flags & XBF_READ)
424                                         bp->b_flags |= _XBF_PAGE_LOCKED;
425                         } else if (!PagePrivate(page)) {
426                                 if (test_page_region(page, offset, nbytes))
427                                         page_count++;
428                         }
429                 }
430
431                 bp->b_pages[i] = page;
432                 offset = 0;
433         }
434
435         if (!(bp->b_flags & _XBF_PAGE_LOCKED)) {
436                 for (i = 0; i < bp->b_page_count; i++)
437                         unlock_page(bp->b_pages[i]);
438         }
439
440         if (page_count == bp->b_page_count)
441                 bp->b_flags |= XBF_DONE;
442
443         return error;
444 }
445
446 /*
447  *      Map buffer into kernel address-space if nessecary.
448  */
449 STATIC int
450 _xfs_buf_map_pages(
451         xfs_buf_t               *bp,
452         uint                    flags)
453 {
454         /* A single page buffer is always mappable */
455         if (bp->b_page_count == 1) {
456                 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
457                 bp->b_flags |= XBF_MAPPED;
458         } else if (flags & XBF_MAPPED) {
459                 bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
460                                         -1, PAGE_KERNEL);
461                 if (unlikely(bp->b_addr == NULL))
462                         return -ENOMEM;
463                 bp->b_addr += bp->b_offset;
464                 bp->b_flags |= XBF_MAPPED;
465         }
466
467         return 0;
468 }
469
470 /*
471  *      Finding and Reading Buffers
472  */
473
474 /*
475  *      Look up, and creates if absent, a lockable buffer for
476  *      a given range of an inode.  The buffer is returned
477  *      locked.  If other overlapping buffers exist, they are
478  *      released before the new buffer is created and locked,
479  *      which may imply that this call will block until those buffers
480  *      are unlocked.  No I/O is implied by this call.
481  */
482 xfs_buf_t *
483 _xfs_buf_find(
484         xfs_buftarg_t           *btp,   /* block device target          */
485         xfs_off_t               ioff,   /* starting offset of range     */
486         size_t                  isize,  /* length of range              */
487         xfs_buf_flags_t         flags,
488         xfs_buf_t               *new_bp)
489 {
490         xfs_off_t               range_base;
491         size_t                  range_length;
492         struct xfs_perag        *pag;
493         struct rb_node          **rbp;
494         struct rb_node          *parent;
495         xfs_buf_t               *bp;
496
497         range_base = (ioff << BBSHIFT);
498         range_length = (isize << BBSHIFT);
499
500         /* Check for IOs smaller than the sector size / not sector aligned */
501         ASSERT(!(range_length < (1 << btp->bt_sshift)));
502         ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
503
504         /* get tree root */
505         pag = xfs_perag_get(btp->bt_mount,
506                                 xfs_daddr_to_agno(btp->bt_mount, ioff));
507
508         /* walk tree */
509         spin_lock(&pag->pag_buf_lock);
510         rbp = &pag->pag_buf_tree.rb_node;
511         parent = NULL;
512         bp = NULL;
513         while (*rbp) {
514                 parent = *rbp;
515                 bp = rb_entry(parent, struct xfs_buf, b_rbnode);
516
517                 if (range_base < bp->b_file_offset)
518                         rbp = &(*rbp)->rb_left;
519                 else if (range_base > bp->b_file_offset)
520                         rbp = &(*rbp)->rb_right;
521                 else {
522                         /*
523                          * found a block offset match. If the range doesn't
524                          * match, the only way this is allowed is if the buffer
525                          * in the cache is stale and the transaction that made
526                          * it stale has not yet committed. i.e. we are
527                          * reallocating a busy extent. Skip this buffer and
528                          * continue searching to the right for an exact match.
529                          */
530                         if (bp->b_buffer_length != range_length) {
531                                 ASSERT(bp->b_flags & XBF_STALE);
532                                 rbp = &(*rbp)->rb_right;
533                                 continue;
534                         }
535                         atomic_inc(&bp->b_hold);
536                         goto found;
537                 }
538         }
539
540         /* No match found */
541         if (new_bp) {
542                 _xfs_buf_initialize(new_bp, btp, range_base,
543                                 range_length, flags);
544                 rb_link_node(&new_bp->b_rbnode, parent, rbp);
545                 rb_insert_color(&new_bp->b_rbnode, &pag->pag_buf_tree);
546                 /* the buffer keeps the perag reference until it is freed */
547                 new_bp->b_pag = pag;
548                 spin_unlock(&pag->pag_buf_lock);
549         } else {
550                 XFS_STATS_INC(xb_miss_locked);
551                 spin_unlock(&pag->pag_buf_lock);
552                 xfs_perag_put(pag);
553         }
554         return new_bp;
555
556 found:
557         spin_unlock(&pag->pag_buf_lock);
558         xfs_perag_put(pag);
559
560         if (xfs_buf_cond_lock(bp)) {
561                 /* failed, so wait for the lock if requested. */
562                 if (!(flags & XBF_TRYLOCK)) {
563                         xfs_buf_lock(bp);
564                         XFS_STATS_INC(xb_get_locked_waited);
565                 } else {
566                         xfs_buf_rele(bp);
567                         XFS_STATS_INC(xb_busy_locked);
568                         return NULL;
569                 }
570         }
571
572         if (bp->b_flags & XBF_STALE) {
573                 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
574                 bp->b_flags &= XBF_MAPPED;
575         }
576
577         trace_xfs_buf_find(bp, flags, _RET_IP_);
578         XFS_STATS_INC(xb_get_locked);
579         return bp;
580 }
581
582 /*
583  *      Assembles a buffer covering the specified range.
584  *      Storage in memory for all portions of the buffer will be allocated,
585  *      although backing storage may not be.
586  */
587 xfs_buf_t *
588 xfs_buf_get(
589         xfs_buftarg_t           *target,/* target for buffer            */
590         xfs_off_t               ioff,   /* starting offset of range     */
591         size_t                  isize,  /* length of range              */
592         xfs_buf_flags_t         flags)
593 {
594         xfs_buf_t               *bp, *new_bp;
595         int                     error = 0, i;
596
597         new_bp = xfs_buf_allocate(flags);
598         if (unlikely(!new_bp))
599                 return NULL;
600
601         bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
602         if (bp == new_bp) {
603                 error = _xfs_buf_lookup_pages(bp, flags);
604                 if (error)
605                         goto no_buffer;
606         } else {
607                 xfs_buf_deallocate(new_bp);
608                 if (unlikely(bp == NULL))
609                         return NULL;
610         }
611
612         for (i = 0; i < bp->b_page_count; i++)
613                 mark_page_accessed(bp->b_pages[i]);
614
615         if (!(bp->b_flags & XBF_MAPPED)) {
616                 error = _xfs_buf_map_pages(bp, flags);
617                 if (unlikely(error)) {
618                         printk(KERN_WARNING "%s: failed to map pages\n",
619                                         __func__);
620                         goto no_buffer;
621                 }
622         }
623
624         XFS_STATS_INC(xb_get);
625
626         /*
627          * Always fill in the block number now, the mapped cases can do
628          * their own overlay of this later.
629          */
630         bp->b_bn = ioff;
631         bp->b_count_desired = bp->b_buffer_length;
632
633         trace_xfs_buf_get(bp, flags, _RET_IP_);
634         return bp;
635
636  no_buffer:
637         if (flags & (XBF_LOCK | XBF_TRYLOCK))
638                 xfs_buf_unlock(bp);
639         xfs_buf_rele(bp);
640         return NULL;
641 }
642
643 STATIC int
644 _xfs_buf_read(
645         xfs_buf_t               *bp,
646         xfs_buf_flags_t         flags)
647 {
648         int                     status;
649
650         ASSERT(!(flags & (XBF_DELWRI|XBF_WRITE)));
651         ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
652
653         bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_DELWRI | \
654                         XBF_READ_AHEAD | _XBF_RUN_QUEUES);
655         bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | \
656                         XBF_READ_AHEAD | _XBF_RUN_QUEUES);
657
658         status = xfs_buf_iorequest(bp);
659         if (status || XFS_BUF_ISERROR(bp) || (flags & XBF_ASYNC))
660                 return status;
661         return xfs_buf_iowait(bp);
662 }
663
664 xfs_buf_t *
665 xfs_buf_read(
666         xfs_buftarg_t           *target,
667         xfs_off_t               ioff,
668         size_t                  isize,
669         xfs_buf_flags_t         flags)
670 {
671         xfs_buf_t               *bp;
672
673         flags |= XBF_READ;
674
675         bp = xfs_buf_get(target, ioff, isize, flags);
676         if (bp) {
677                 trace_xfs_buf_read(bp, flags, _RET_IP_);
678
679                 if (!XFS_BUF_ISDONE(bp)) {
680                         XFS_STATS_INC(xb_get_read);
681                         _xfs_buf_read(bp, flags);
682                 } else if (flags & XBF_ASYNC) {
683                         /*
684                          * Read ahead call which is already satisfied,
685                          * drop the buffer
686                          */
687                         goto no_buffer;
688                 } else {
689                         /* We do not want read in the flags */
690                         bp->b_flags &= ~XBF_READ;
691                 }
692         }
693
694         return bp;
695
696  no_buffer:
697         if (flags & (XBF_LOCK | XBF_TRYLOCK))
698                 xfs_buf_unlock(bp);
699         xfs_buf_rele(bp);
700         return NULL;
701 }
702
703 /*
704  *      If we are not low on memory then do the readahead in a deadlock
705  *      safe manner.
706  */
707 void
708 xfs_buf_readahead(
709         xfs_buftarg_t           *target,
710         xfs_off_t               ioff,
711         size_t                  isize)
712 {
713         struct backing_dev_info *bdi;
714
715         bdi = target->bt_mapping->backing_dev_info;
716         if (bdi_read_congested(bdi))
717                 return;
718
719         xfs_buf_read(target, ioff, isize,
720                      XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD|XBF_DONT_BLOCK);
721 }
722
723 /*
724  * Read an uncached buffer from disk. Allocates and returns a locked
725  * buffer containing the disk contents or nothing.
726  */
727 struct xfs_buf *
728 xfs_buf_read_uncached(
729         struct xfs_mount        *mp,
730         struct xfs_buftarg      *target,
731         xfs_daddr_t             daddr,
732         size_t                  length,
733         int                     flags)
734 {
735         xfs_buf_t               *bp;
736         int                     error;
737
738         bp = xfs_buf_get_uncached(target, length, flags);
739         if (!bp)
740                 return NULL;
741
742         /* set up the buffer for a read IO */
743         xfs_buf_lock(bp);
744         XFS_BUF_SET_ADDR(bp, daddr);
745         XFS_BUF_READ(bp);
746         XFS_BUF_BUSY(bp);
747
748         xfsbdstrat(mp, bp);
749         error = xfs_buf_iowait(bp);
750         if (error || bp->b_error) {
751                 xfs_buf_relse(bp);
752                 return NULL;
753         }
754         return bp;
755 }
756
757 xfs_buf_t *
758 xfs_buf_get_empty(
759         size_t                  len,
760         xfs_buftarg_t           *target)
761 {
762         xfs_buf_t               *bp;
763
764         bp = xfs_buf_allocate(0);
765         if (bp)
766                 _xfs_buf_initialize(bp, target, 0, len, 0);
767         return bp;
768 }
769
770 static inline struct page *
771 mem_to_page(
772         void                    *addr)
773 {
774         if ((!is_vmalloc_addr(addr))) {
775                 return virt_to_page(addr);
776         } else {
777                 return vmalloc_to_page(addr);
778         }
779 }
780
781 int
782 xfs_buf_associate_memory(
783         xfs_buf_t               *bp,
784         void                    *mem,
785         size_t                  len)
786 {
787         int                     rval;
788         int                     i = 0;
789         unsigned long           pageaddr;
790         unsigned long           offset;
791         size_t                  buflen;
792         int                     page_count;
793
794         pageaddr = (unsigned long)mem & PAGE_CACHE_MASK;
795         offset = (unsigned long)mem - pageaddr;
796         buflen = PAGE_CACHE_ALIGN(len + offset);
797         page_count = buflen >> PAGE_CACHE_SHIFT;
798
799         /* Free any previous set of page pointers */
800         if (bp->b_pages)
801                 _xfs_buf_free_pages(bp);
802
803         bp->b_pages = NULL;
804         bp->b_addr = mem;
805
806         rval = _xfs_buf_get_pages(bp, page_count, XBF_DONT_BLOCK);
807         if (rval)
808                 return rval;
809
810         bp->b_offset = offset;
811
812         for (i = 0; i < bp->b_page_count; i++) {
813                 bp->b_pages[i] = mem_to_page((void *)pageaddr);
814                 pageaddr += PAGE_CACHE_SIZE;
815         }
816
817         bp->b_count_desired = len;
818         bp->b_buffer_length = buflen;
819         bp->b_flags |= XBF_MAPPED;
820         bp->b_flags &= ~_XBF_PAGE_LOCKED;
821
822         return 0;
823 }
824
825 xfs_buf_t *
826 xfs_buf_get_uncached(
827         struct xfs_buftarg      *target,
828         size_t                  len,
829         int                     flags)
830 {
831         unsigned long           page_count = PAGE_ALIGN(len) >> PAGE_SHIFT;
832         int                     error, i;
833         xfs_buf_t               *bp;
834
835         bp = xfs_buf_allocate(0);
836         if (unlikely(bp == NULL))
837                 goto fail;
838         _xfs_buf_initialize(bp, target, 0, len, 0);
839
840         error = _xfs_buf_get_pages(bp, page_count, 0);
841         if (error)
842                 goto fail_free_buf;
843
844         for (i = 0; i < page_count; i++) {
845                 bp->b_pages[i] = alloc_page(xb_to_gfp(flags));
846                 if (!bp->b_pages[i])
847                         goto fail_free_mem;
848         }
849         bp->b_flags |= _XBF_PAGES;
850
851         error = _xfs_buf_map_pages(bp, XBF_MAPPED);
852         if (unlikely(error)) {
853                 printk(KERN_WARNING "%s: failed to map pages\n",
854                                 __func__);
855                 goto fail_free_mem;
856         }
857
858         xfs_buf_unlock(bp);
859
860         trace_xfs_buf_get_uncached(bp, _RET_IP_);
861         return bp;
862
863  fail_free_mem:
864         while (--i >= 0)
865                 __free_page(bp->b_pages[i]);
866         _xfs_buf_free_pages(bp);
867  fail_free_buf:
868         xfs_buf_deallocate(bp);
869  fail:
870         return NULL;
871 }
872
873 /*
874  *      Increment reference count on buffer, to hold the buffer concurrently
875  *      with another thread which may release (free) the buffer asynchronously.
876  *      Must hold the buffer already to call this function.
877  */
878 void
879 xfs_buf_hold(
880         xfs_buf_t               *bp)
881 {
882         trace_xfs_buf_hold(bp, _RET_IP_);
883         atomic_inc(&bp->b_hold);
884 }
885
886 /*
887  *      Releases a hold on the specified buffer.  If the
888  *      the hold count is 1, calls xfs_buf_free.
889  */
890 void
891 xfs_buf_rele(
892         xfs_buf_t               *bp)
893 {
894         struct xfs_perag        *pag = bp->b_pag;
895
896         trace_xfs_buf_rele(bp, _RET_IP_);
897
898         if (!pag) {
899                 ASSERT(list_empty(&bp->b_lru));
900                 ASSERT(RB_EMPTY_NODE(&bp->b_rbnode));
901                 if (atomic_dec_and_test(&bp->b_hold))
902                         xfs_buf_free(bp);
903                 return;
904         }
905
906         ASSERT(!RB_EMPTY_NODE(&bp->b_rbnode));
907
908         ASSERT(atomic_read(&bp->b_hold) > 0);
909         if (atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock)) {
910                 if (!(bp->b_flags & XBF_STALE) &&
911                            atomic_read(&bp->b_lru_ref)) {
912                         xfs_buf_lru_add(bp);
913                         spin_unlock(&pag->pag_buf_lock);
914                 } else {
915                         xfs_buf_lru_del(bp);
916                         ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
917                         rb_erase(&bp->b_rbnode, &pag->pag_buf_tree);
918                         spin_unlock(&pag->pag_buf_lock);
919                         xfs_perag_put(pag);
920                         xfs_buf_free(bp);
921                 }
922         }
923 }
924
925
926 /*
927  *      Mutual exclusion on buffers.  Locking model:
928  *
929  *      Buffers associated with inodes for which buffer locking
930  *      is not enabled are not protected by semaphores, and are
931  *      assumed to be exclusively owned by the caller.  There is a
932  *      spinlock in the buffer, used by the caller when concurrent
933  *      access is possible.
934  */
935
936 /*
937  *      Locks a buffer object, if it is not already locked.  Note that this in
938  *      no way locks the underlying pages, so it is only useful for
939  *      synchronizing concurrent use of buffer objects, not for synchronizing
940  *      independent access to the underlying pages.
941  *
942  *      If we come across a stale, pinned, locked buffer, we know that we are
943  *      being asked to lock a buffer that has been reallocated. Because it is
944  *      pinned, we know that the log has not been pushed to disk and hence it
945  *      will still be locked.  Rather than continuing to have trylock attempts
946  *      fail until someone else pushes the log, push it ourselves before
947  *      returning.  This means that the xfsaild will not get stuck trying
948  *      to push on stale inode buffers.
949  */
950 int
951 xfs_buf_cond_lock(
952         xfs_buf_t               *bp)
953 {
954         int                     locked;
955
956         locked = down_trylock(&bp->b_sema) == 0;
957         if (locked)
958                 XB_SET_OWNER(bp);
959         else if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
960                 xfs_log_force(bp->b_target->bt_mount, 0);
961
962         trace_xfs_buf_cond_lock(bp, _RET_IP_);
963         return locked ? 0 : -EBUSY;
964 }
965
966 int
967 xfs_buf_lock_value(
968         xfs_buf_t               *bp)
969 {
970         return bp->b_sema.count;
971 }
972
973 /*
974  *      Locks a buffer object.
975  *      Note that this in no way locks the underlying pages, so it is only
976  *      useful for synchronizing concurrent use of buffer objects, not for
977  *      synchronizing independent access to the underlying pages.
978  *
979  *      If we come across a stale, pinned, locked buffer, we know that we
980  *      are being asked to lock a buffer that has been reallocated. Because
981  *      it is pinned, we know that the log has not been pushed to disk and
982  *      hence it will still be locked. Rather than sleeping until someone
983  *      else pushes the log, push it ourselves before trying to get the lock.
984  */
985 void
986 xfs_buf_lock(
987         xfs_buf_t               *bp)
988 {
989         trace_xfs_buf_lock(bp, _RET_IP_);
990
991         if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
992                 xfs_log_force(bp->b_target->bt_mount, 0);
993         if (atomic_read(&bp->b_io_remaining))
994                 blk_flush_plug(current);
995         down(&bp->b_sema);
996         XB_SET_OWNER(bp);
997
998         trace_xfs_buf_lock_done(bp, _RET_IP_);
999 }
1000
1001 /*
1002  *      Releases the lock on the buffer object.
1003  *      If the buffer is marked delwri but is not queued, do so before we
1004  *      unlock the buffer as we need to set flags correctly.  We also need to
1005  *      take a reference for the delwri queue because the unlocker is going to
1006  *      drop their's and they don't know we just queued it.
1007  */
1008 void
1009 xfs_buf_unlock(
1010         xfs_buf_t               *bp)
1011 {
1012         if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
1013                 atomic_inc(&bp->b_hold);
1014                 bp->b_flags |= XBF_ASYNC;
1015                 xfs_buf_delwri_queue(bp, 0);
1016         }
1017
1018         XB_CLEAR_OWNER(bp);
1019         up(&bp->b_sema);
1020
1021         trace_xfs_buf_unlock(bp, _RET_IP_);
1022 }
1023
1024 STATIC void
1025 xfs_buf_wait_unpin(
1026         xfs_buf_t               *bp)
1027 {
1028         DECLARE_WAITQUEUE       (wait, current);
1029
1030         if (atomic_read(&bp->b_pin_count) == 0)
1031                 return;
1032
1033         add_wait_queue(&bp->b_waiters, &wait);
1034         for (;;) {
1035                 set_current_state(TASK_UNINTERRUPTIBLE);
1036                 if (atomic_read(&bp->b_pin_count) == 0)
1037                         break;
1038                 io_schedule();
1039         }
1040         remove_wait_queue(&bp->b_waiters, &wait);
1041         set_current_state(TASK_RUNNING);
1042 }
1043
1044 /*
1045  *      Buffer Utility Routines
1046  */
1047
1048 STATIC void
1049 xfs_buf_iodone_work(
1050         struct work_struct      *work)
1051 {
1052         xfs_buf_t               *bp =
1053                 container_of(work, xfs_buf_t, b_iodone_work);
1054
1055         if (bp->b_iodone)
1056                 (*(bp->b_iodone))(bp);
1057         else if (bp->b_flags & XBF_ASYNC)
1058                 xfs_buf_relse(bp);
1059 }
1060
1061 void
1062 xfs_buf_ioend(
1063         xfs_buf_t               *bp,
1064         int                     schedule)
1065 {
1066         trace_xfs_buf_iodone(bp, _RET_IP_);
1067
1068         bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
1069         if (bp->b_error == 0)
1070                 bp->b_flags |= XBF_DONE;
1071
1072         if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
1073                 if (schedule) {
1074                         INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
1075                         queue_work(xfslogd_workqueue, &bp->b_iodone_work);
1076                 } else {
1077                         xfs_buf_iodone_work(&bp->b_iodone_work);
1078                 }
1079         } else {
1080                 complete(&bp->b_iowait);
1081         }
1082 }
1083
1084 void
1085 xfs_buf_ioerror(
1086         xfs_buf_t               *bp,
1087         int                     error)
1088 {
1089         ASSERT(error >= 0 && error <= 0xffff);
1090         bp->b_error = (unsigned short)error;
1091         trace_xfs_buf_ioerror(bp, error, _RET_IP_);
1092 }
1093
1094 int
1095 xfs_bwrite(
1096         struct xfs_mount        *mp,
1097         struct xfs_buf          *bp)
1098 {
1099         int                     error;
1100
1101         bp->b_flags |= XBF_WRITE;
1102         bp->b_flags &= ~(XBF_ASYNC | XBF_READ);
1103
1104         xfs_buf_delwri_dequeue(bp);
1105         xfs_bdstrat_cb(bp);
1106
1107         error = xfs_buf_iowait(bp);
1108         if (error)
1109                 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
1110         xfs_buf_relse(bp);
1111         return error;
1112 }
1113
1114 void
1115 xfs_bdwrite(
1116         void                    *mp,
1117         struct xfs_buf          *bp)
1118 {
1119         trace_xfs_buf_bdwrite(bp, _RET_IP_);
1120
1121         bp->b_flags &= ~XBF_READ;
1122         bp->b_flags |= (XBF_DELWRI | XBF_ASYNC);
1123
1124         xfs_buf_delwri_queue(bp, 1);
1125 }
1126
1127 /*
1128  * Called when we want to stop a buffer from getting written or read.
1129  * We attach the EIO error, muck with its flags, and call xfs_buf_ioend
1130  * so that the proper iodone callbacks get called.
1131  */
1132 STATIC int
1133 xfs_bioerror(
1134         xfs_buf_t *bp)
1135 {
1136 #ifdef XFSERRORDEBUG
1137         ASSERT(XFS_BUF_ISREAD(bp) || bp->b_iodone);
1138 #endif
1139
1140         /*
1141          * No need to wait until the buffer is unpinned, we aren't flushing it.
1142          */
1143         XFS_BUF_ERROR(bp, EIO);
1144
1145         /*
1146          * We're calling xfs_buf_ioend, so delete XBF_DONE flag.
1147          */
1148         XFS_BUF_UNREAD(bp);
1149         XFS_BUF_UNDELAYWRITE(bp);
1150         XFS_BUF_UNDONE(bp);
1151         XFS_BUF_STALE(bp);
1152
1153         xfs_buf_ioend(bp, 0);
1154
1155         return EIO;
1156 }
1157
1158 /*
1159  * Same as xfs_bioerror, except that we are releasing the buffer
1160  * here ourselves, and avoiding the xfs_buf_ioend call.
1161  * This is meant for userdata errors; metadata bufs come with
1162  * iodone functions attached, so that we can track down errors.
1163  */
1164 STATIC int
1165 xfs_bioerror_relse(
1166         struct xfs_buf  *bp)
1167 {
1168         int64_t         fl = XFS_BUF_BFLAGS(bp);
1169         /*
1170          * No need to wait until the buffer is unpinned.
1171          * We aren't flushing it.
1172          *
1173          * chunkhold expects B_DONE to be set, whether
1174          * we actually finish the I/O or not. We don't want to
1175          * change that interface.
1176          */
1177         XFS_BUF_UNREAD(bp);
1178         XFS_BUF_UNDELAYWRITE(bp);
1179         XFS_BUF_DONE(bp);
1180         XFS_BUF_STALE(bp);
1181         XFS_BUF_CLR_IODONE_FUNC(bp);
1182         if (!(fl & XBF_ASYNC)) {
1183                 /*
1184                  * Mark b_error and B_ERROR _both_.
1185                  * Lot's of chunkcache code assumes that.
1186                  * There's no reason to mark error for
1187                  * ASYNC buffers.
1188                  */
1189                 XFS_BUF_ERROR(bp, EIO);
1190                 XFS_BUF_FINISH_IOWAIT(bp);
1191         } else {
1192                 xfs_buf_relse(bp);
1193         }
1194
1195         return EIO;
1196 }
1197
1198
1199 /*
1200  * All xfs metadata buffers except log state machine buffers
1201  * get this attached as their b_bdstrat callback function.
1202  * This is so that we can catch a buffer
1203  * after prematurely unpinning it to forcibly shutdown the filesystem.
1204  */
1205 int
1206 xfs_bdstrat_cb(
1207         struct xfs_buf  *bp)
1208 {
1209         if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) {
1210                 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1211                 /*
1212                  * Metadata write that didn't get logged but
1213                  * written delayed anyway. These aren't associated
1214                  * with a transaction, and can be ignored.
1215                  */
1216                 if (!bp->b_iodone && !XFS_BUF_ISREAD(bp))
1217                         return xfs_bioerror_relse(bp);
1218                 else
1219                         return xfs_bioerror(bp);
1220         }
1221
1222         xfs_buf_iorequest(bp);
1223         return 0;
1224 }
1225
1226 /*
1227  * Wrapper around bdstrat so that we can stop data from going to disk in case
1228  * we are shutting down the filesystem.  Typically user data goes thru this
1229  * path; one of the exceptions is the superblock.
1230  */
1231 void
1232 xfsbdstrat(
1233         struct xfs_mount        *mp,
1234         struct xfs_buf          *bp)
1235 {
1236         if (XFS_FORCED_SHUTDOWN(mp)) {
1237                 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1238                 xfs_bioerror_relse(bp);
1239                 return;
1240         }
1241
1242         xfs_buf_iorequest(bp);
1243 }
1244
1245 STATIC void
1246 _xfs_buf_ioend(
1247         xfs_buf_t               *bp,
1248         int                     schedule)
1249 {
1250         if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1251                 bp->b_flags &= ~_XBF_PAGE_LOCKED;
1252                 xfs_buf_ioend(bp, schedule);
1253         }
1254 }
1255
1256 STATIC void
1257 xfs_buf_bio_end_io(
1258         struct bio              *bio,
1259         int                     error)
1260 {
1261         xfs_buf_t               *bp = (xfs_buf_t *)bio->bi_private;
1262         unsigned int            blocksize = bp->b_target->bt_bsize;
1263         struct bio_vec          *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1264
1265         xfs_buf_ioerror(bp, -error);
1266
1267         if (!error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
1268                 invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
1269
1270         do {
1271                 struct page     *page = bvec->bv_page;
1272
1273                 ASSERT(!PagePrivate(page));
1274                 if (unlikely(bp->b_error)) {
1275                         if (bp->b_flags & XBF_READ)
1276                                 ClearPageUptodate(page);
1277                 } else if (blocksize >= PAGE_CACHE_SIZE) {
1278                         SetPageUptodate(page);
1279                 } else if (!PagePrivate(page) &&
1280                                 (bp->b_flags & _XBF_PAGE_CACHE)) {
1281                         set_page_region(page, bvec->bv_offset, bvec->bv_len);
1282                 }
1283
1284                 if (--bvec >= bio->bi_io_vec)
1285                         prefetchw(&bvec->bv_page->flags);
1286
1287                 if (bp->b_flags & _XBF_PAGE_LOCKED)
1288                         unlock_page(page);
1289         } while (bvec >= bio->bi_io_vec);
1290
1291         _xfs_buf_ioend(bp, 1);
1292         bio_put(bio);
1293 }
1294
1295 STATIC void
1296 _xfs_buf_ioapply(
1297         xfs_buf_t               *bp)
1298 {
1299         int                     rw, map_i, total_nr_pages, nr_pages;
1300         struct bio              *bio;
1301         int                     offset = bp->b_offset;
1302         int                     size = bp->b_count_desired;
1303         sector_t                sector = bp->b_bn;
1304         unsigned int            blocksize = bp->b_target->bt_bsize;
1305
1306         total_nr_pages = bp->b_page_count;
1307         map_i = 0;
1308
1309         if (bp->b_flags & XBF_ORDERED) {
1310                 ASSERT(!(bp->b_flags & XBF_READ));
1311                 rw = WRITE_FLUSH_FUA;
1312         } else if (bp->b_flags & XBF_LOG_BUFFER) {
1313                 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1314                 bp->b_flags &= ~_XBF_RUN_QUEUES;
1315                 rw = (bp->b_flags & XBF_WRITE) ? WRITE_SYNC : READ_SYNC;
1316         } else if (bp->b_flags & _XBF_RUN_QUEUES) {
1317                 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1318                 bp->b_flags &= ~_XBF_RUN_QUEUES;
1319                 rw = (bp->b_flags & XBF_WRITE) ? WRITE_META : READ_META;
1320         } else {
1321                 rw = (bp->b_flags & XBF_WRITE) ? WRITE :
1322                      (bp->b_flags & XBF_READ_AHEAD) ? READA : READ;
1323         }
1324
1325         /* Special code path for reading a sub page size buffer in --
1326          * we populate up the whole page, and hence the other metadata
1327          * in the same page.  This optimization is only valid when the
1328          * filesystem block size is not smaller than the page size.
1329          */
1330         if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
1331             ((bp->b_flags & (XBF_READ|_XBF_PAGE_LOCKED)) ==
1332               (XBF_READ|_XBF_PAGE_LOCKED)) &&
1333             (blocksize >= PAGE_CACHE_SIZE)) {
1334                 bio = bio_alloc(GFP_NOIO, 1);
1335
1336                 bio->bi_bdev = bp->b_target->bt_bdev;
1337                 bio->bi_sector = sector - (offset >> BBSHIFT);
1338                 bio->bi_end_io = xfs_buf_bio_end_io;
1339                 bio->bi_private = bp;
1340
1341                 bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
1342                 size = 0;
1343
1344                 atomic_inc(&bp->b_io_remaining);
1345
1346                 goto submit_io;
1347         }
1348
1349 next_chunk:
1350         atomic_inc(&bp->b_io_remaining);
1351         nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1352         if (nr_pages > total_nr_pages)
1353                 nr_pages = total_nr_pages;
1354
1355         bio = bio_alloc(GFP_NOIO, nr_pages);
1356         bio->bi_bdev = bp->b_target->bt_bdev;
1357         bio->bi_sector = sector;
1358         bio->bi_end_io = xfs_buf_bio_end_io;
1359         bio->bi_private = bp;
1360
1361         for (; size && nr_pages; nr_pages--, map_i++) {
1362                 int     rbytes, nbytes = PAGE_CACHE_SIZE - offset;
1363
1364                 if (nbytes > size)
1365                         nbytes = size;
1366
1367                 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1368                 if (rbytes < nbytes)
1369                         break;
1370
1371                 offset = 0;
1372                 sector += nbytes >> BBSHIFT;
1373                 size -= nbytes;
1374                 total_nr_pages--;
1375         }
1376
1377 submit_io:
1378         if (likely(bio->bi_size)) {
1379                 if (xfs_buf_is_vmapped(bp)) {
1380                         flush_kernel_vmap_range(bp->b_addr,
1381                                                 xfs_buf_vmap_len(bp));
1382                 }
1383                 submit_bio(rw, bio);
1384                 if (size)
1385                         goto next_chunk;
1386         } else {
1387                 /*
1388                  * if we get here, no pages were added to the bio. However,
1389                  * we can't just error out here - if the pages are locked then
1390                  * we have to unlock them otherwise we can hang on a later
1391                  * access to the page.
1392                  */
1393                 xfs_buf_ioerror(bp, EIO);
1394                 if (bp->b_flags & _XBF_PAGE_LOCKED) {
1395                         int i;
1396                         for (i = 0; i < bp->b_page_count; i++)
1397                                 unlock_page(bp->b_pages[i]);
1398                 }
1399                 bio_put(bio);
1400         }
1401 }
1402
1403 int
1404 xfs_buf_iorequest(
1405         xfs_buf_t               *bp)
1406 {
1407         trace_xfs_buf_iorequest(bp, _RET_IP_);
1408
1409         if (bp->b_flags & XBF_DELWRI) {
1410                 xfs_buf_delwri_queue(bp, 1);
1411                 return 0;
1412         }
1413
1414         if (bp->b_flags & XBF_WRITE) {
1415                 xfs_buf_wait_unpin(bp);
1416         }
1417
1418         xfs_buf_hold(bp);
1419
1420         /* Set the count to 1 initially, this will stop an I/O
1421          * completion callout which happens before we have started
1422          * all the I/O from calling xfs_buf_ioend too early.
1423          */
1424         atomic_set(&bp->b_io_remaining, 1);
1425         _xfs_buf_ioapply(bp);
1426         _xfs_buf_ioend(bp, 0);
1427
1428         xfs_buf_rele(bp);
1429         return 0;
1430 }
1431
1432 /*
1433  *      Waits for I/O to complete on the buffer supplied.
1434  *      It returns immediately if no I/O is pending.
1435  *      It returns the I/O error code, if any, or 0 if there was no error.
1436  */
1437 int
1438 xfs_buf_iowait(
1439         xfs_buf_t               *bp)
1440 {
1441         trace_xfs_buf_iowait(bp, _RET_IP_);
1442
1443         if (atomic_read(&bp->b_io_remaining))
1444                 blk_flush_plug(current);
1445         wait_for_completion(&bp->b_iowait);
1446
1447         trace_xfs_buf_iowait_done(bp, _RET_IP_);
1448         return bp->b_error;
1449 }
1450
1451 xfs_caddr_t
1452 xfs_buf_offset(
1453         xfs_buf_t               *bp,
1454         size_t                  offset)
1455 {
1456         struct page             *page;
1457
1458         if (bp->b_flags & XBF_MAPPED)
1459                 return XFS_BUF_PTR(bp) + offset;
1460
1461         offset += bp->b_offset;
1462         page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
1463         return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
1464 }
1465
1466 /*
1467  *      Move data into or out of a buffer.
1468  */
1469 void
1470 xfs_buf_iomove(
1471         xfs_buf_t               *bp,    /* buffer to process            */
1472         size_t                  boff,   /* starting buffer offset       */
1473         size_t                  bsize,  /* length to copy               */
1474         void                    *data,  /* data address                 */
1475         xfs_buf_rw_t            mode)   /* read/write/zero flag         */
1476 {
1477         size_t                  bend, cpoff, csize;
1478         struct page             *page;
1479
1480         bend = boff + bsize;
1481         while (boff < bend) {
1482                 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1483                 cpoff = xfs_buf_poff(boff + bp->b_offset);
1484                 csize = min_t(size_t,
1485                               PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
1486
1487                 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1488
1489                 switch (mode) {
1490                 case XBRW_ZERO:
1491                         memset(page_address(page) + cpoff, 0, csize);
1492                         break;
1493                 case XBRW_READ:
1494                         memcpy(data, page_address(page) + cpoff, csize);
1495                         break;
1496                 case XBRW_WRITE:
1497                         memcpy(page_address(page) + cpoff, data, csize);
1498                 }
1499
1500                 boff += csize;
1501                 data += csize;
1502         }
1503 }
1504
1505 /*
1506  *      Handling of buffer targets (buftargs).
1507  */
1508
1509 /*
1510  * Wait for any bufs with callbacks that have been submitted but have not yet
1511  * returned. These buffers will have an elevated hold count, so wait on those
1512  * while freeing all the buffers only held by the LRU.
1513  */
1514 void
1515 xfs_wait_buftarg(
1516         struct xfs_buftarg      *btp)
1517 {
1518         struct xfs_buf          *bp;
1519
1520 restart:
1521         spin_lock(&btp->bt_lru_lock);
1522         while (!list_empty(&btp->bt_lru)) {
1523                 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1524                 if (atomic_read(&bp->b_hold) > 1) {
1525                         spin_unlock(&btp->bt_lru_lock);
1526                         delay(100);
1527                         goto restart;
1528                 }
1529                 /*
1530                  * clear the LRU reference count so the bufer doesn't get
1531                  * ignored in xfs_buf_rele().
1532                  */
1533                 atomic_set(&bp->b_lru_ref, 0);
1534                 spin_unlock(&btp->bt_lru_lock);
1535                 xfs_buf_rele(bp);
1536                 spin_lock(&btp->bt_lru_lock);
1537         }
1538         spin_unlock(&btp->bt_lru_lock);
1539 }
1540
1541 int
1542 xfs_buftarg_shrink(
1543         struct shrinker         *shrink,
1544         int                     nr_to_scan,
1545         gfp_t                   mask)
1546 {
1547         struct xfs_buftarg      *btp = container_of(shrink,
1548                                         struct xfs_buftarg, bt_shrinker);
1549         struct xfs_buf          *bp;
1550         LIST_HEAD(dispose);
1551
1552         if (!nr_to_scan)
1553                 return btp->bt_lru_nr;
1554
1555         spin_lock(&btp->bt_lru_lock);
1556         while (!list_empty(&btp->bt_lru)) {
1557                 if (nr_to_scan-- <= 0)
1558                         break;
1559
1560                 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1561
1562                 /*
1563                  * Decrement the b_lru_ref count unless the value is already
1564                  * zero. If the value is already zero, we need to reclaim the
1565                  * buffer, otherwise it gets another trip through the LRU.
1566                  */
1567                 if (!atomic_add_unless(&bp->b_lru_ref, -1, 0)) {
1568                         list_move_tail(&bp->b_lru, &btp->bt_lru);
1569                         continue;
1570                 }
1571
1572                 /*
1573                  * remove the buffer from the LRU now to avoid needing another
1574                  * lock round trip inside xfs_buf_rele().
1575                  */
1576                 list_move(&bp->b_lru, &dispose);
1577                 btp->bt_lru_nr--;
1578         }
1579         spin_unlock(&btp->bt_lru_lock);
1580
1581         while (!list_empty(&dispose)) {
1582                 bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
1583                 list_del_init(&bp->b_lru);
1584                 xfs_buf_rele(bp);
1585         }
1586
1587         return btp->bt_lru_nr;
1588 }
1589
1590 void
1591 xfs_free_buftarg(
1592         struct xfs_mount        *mp,
1593         struct xfs_buftarg      *btp)
1594 {
1595         unregister_shrinker(&btp->bt_shrinker);
1596
1597         xfs_flush_buftarg(btp, 1);
1598         if (mp->m_flags & XFS_MOUNT_BARRIER)
1599                 xfs_blkdev_issue_flush(btp);
1600         iput(btp->bt_mapping->host);
1601
1602         kthread_stop(btp->bt_task);
1603         kmem_free(btp);
1604 }
1605
1606 STATIC int
1607 xfs_setsize_buftarg_flags(
1608         xfs_buftarg_t           *btp,
1609         unsigned int            blocksize,
1610         unsigned int            sectorsize,
1611         int                     verbose)
1612 {
1613         btp->bt_bsize = blocksize;
1614         btp->bt_sshift = ffs(sectorsize) - 1;
1615         btp->bt_smask = sectorsize - 1;
1616
1617         if (set_blocksize(btp->bt_bdev, sectorsize)) {
1618                 printk(KERN_WARNING
1619                         "XFS: Cannot set_blocksize to %u on device %s\n",
1620                         sectorsize, XFS_BUFTARG_NAME(btp));
1621                 return EINVAL;
1622         }
1623
1624         if (verbose &&
1625             (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1626                 printk(KERN_WARNING
1627                         "XFS: %u byte sectors in use on device %s.  "
1628                         "This is suboptimal; %u or greater is ideal.\n",
1629                         sectorsize, XFS_BUFTARG_NAME(btp),
1630                         (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1631         }
1632
1633         return 0;
1634 }
1635
1636 /*
1637  *      When allocating the initial buffer target we have not yet
1638  *      read in the superblock, so don't know what sized sectors
1639  *      are being used is at this early stage.  Play safe.
1640  */
1641 STATIC int
1642 xfs_setsize_buftarg_early(
1643         xfs_buftarg_t           *btp,
1644         struct block_device     *bdev)
1645 {
1646         return xfs_setsize_buftarg_flags(btp,
1647                         PAGE_CACHE_SIZE, bdev_logical_block_size(bdev), 0);
1648 }
1649
1650 int
1651 xfs_setsize_buftarg(
1652         xfs_buftarg_t           *btp,
1653         unsigned int            blocksize,
1654         unsigned int            sectorsize)
1655 {
1656         return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1657 }
1658
1659 STATIC int
1660 xfs_mapping_buftarg(
1661         xfs_buftarg_t           *btp,
1662         struct block_device     *bdev)
1663 {
1664         struct backing_dev_info *bdi;
1665         struct inode            *inode;
1666         struct address_space    *mapping;
1667         static const struct address_space_operations mapping_aops = {
1668                 .migratepage = fail_migrate_page,
1669         };
1670
1671         inode = new_inode(bdev->bd_inode->i_sb);
1672         if (!inode) {
1673                 printk(KERN_WARNING
1674                         "XFS: Cannot allocate mapping inode for device %s\n",
1675                         XFS_BUFTARG_NAME(btp));
1676                 return ENOMEM;
1677         }
1678         inode->i_ino = get_next_ino();
1679         inode->i_mode = S_IFBLK;
1680         inode->i_bdev = bdev;
1681         inode->i_rdev = bdev->bd_dev;
1682         bdi = blk_get_backing_dev_info(bdev);
1683         if (!bdi)
1684                 bdi = &default_backing_dev_info;
1685         mapping = &inode->i_data;
1686         mapping->a_ops = &mapping_aops;
1687         mapping->backing_dev_info = bdi;
1688         mapping_set_gfp_mask(mapping, GFP_NOFS);
1689         btp->bt_mapping = mapping;
1690         return 0;
1691 }
1692
1693 STATIC int
1694 xfs_alloc_delwrite_queue(
1695         xfs_buftarg_t           *btp,
1696         const char              *fsname)
1697 {
1698         INIT_LIST_HEAD(&btp->bt_delwrite_queue);
1699         spin_lock_init(&btp->bt_delwrite_lock);
1700         btp->bt_flags = 0;
1701         btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd/%s", fsname);
1702         if (IS_ERR(btp->bt_task))
1703                 return PTR_ERR(btp->bt_task);
1704         return 0;
1705 }
1706
1707 xfs_buftarg_t *
1708 xfs_alloc_buftarg(
1709         struct xfs_mount        *mp,
1710         struct block_device     *bdev,
1711         int                     external,
1712         const char              *fsname)
1713 {
1714         xfs_buftarg_t           *btp;
1715
1716         btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1717
1718         btp->bt_mount = mp;
1719         btp->bt_dev =  bdev->bd_dev;
1720         btp->bt_bdev = bdev;
1721         INIT_LIST_HEAD(&btp->bt_lru);
1722         spin_lock_init(&btp->bt_lru_lock);
1723         if (xfs_setsize_buftarg_early(btp, bdev))
1724                 goto error;
1725         if (xfs_mapping_buftarg(btp, bdev))
1726                 goto error;
1727         if (xfs_alloc_delwrite_queue(btp, fsname))
1728                 goto error;
1729         btp->bt_shrinker.shrink = xfs_buftarg_shrink;
1730         btp->bt_shrinker.seeks = DEFAULT_SEEKS;
1731         register_shrinker(&btp->bt_shrinker);
1732         return btp;
1733
1734 error:
1735         kmem_free(btp);
1736         return NULL;
1737 }
1738
1739
1740 /*
1741  *      Delayed write buffer handling
1742  */
1743 STATIC void
1744 xfs_buf_delwri_queue(
1745         xfs_buf_t               *bp,
1746         int                     unlock)
1747 {
1748         struct list_head        *dwq = &bp->b_target->bt_delwrite_queue;
1749         spinlock_t              *dwlk = &bp->b_target->bt_delwrite_lock;
1750
1751         trace_xfs_buf_delwri_queue(bp, _RET_IP_);
1752
1753         ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
1754
1755         spin_lock(dwlk);
1756         /* If already in the queue, dequeue and place at tail */
1757         if (!list_empty(&bp->b_list)) {
1758                 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1759                 if (unlock)
1760                         atomic_dec(&bp->b_hold);
1761                 list_del(&bp->b_list);
1762         }
1763
1764         if (list_empty(dwq)) {
1765                 /* start xfsbufd as it is about to have something to do */
1766                 wake_up_process(bp->b_target->bt_task);
1767         }
1768
1769         bp->b_flags |= _XBF_DELWRI_Q;
1770         list_add_tail(&bp->b_list, dwq);
1771         bp->b_queuetime = jiffies;
1772         spin_unlock(dwlk);
1773
1774         if (unlock)
1775                 xfs_buf_unlock(bp);
1776 }
1777
1778 void
1779 xfs_buf_delwri_dequeue(
1780         xfs_buf_t               *bp)
1781 {
1782         spinlock_t              *dwlk = &bp->b_target->bt_delwrite_lock;
1783         int                     dequeued = 0;
1784
1785         spin_lock(dwlk);
1786         if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1787                 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1788                 list_del_init(&bp->b_list);
1789                 dequeued = 1;
1790         }
1791         bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
1792         spin_unlock(dwlk);
1793
1794         if (dequeued)
1795                 xfs_buf_rele(bp);
1796
1797         trace_xfs_buf_delwri_dequeue(bp, _RET_IP_);
1798 }
1799
1800 /*
1801  * If a delwri buffer needs to be pushed before it has aged out, then promote
1802  * it to the head of the delwri queue so that it will be flushed on the next
1803  * xfsbufd run. We do this by resetting the queuetime of the buffer to be older
1804  * than the age currently needed to flush the buffer. Hence the next time the
1805  * xfsbufd sees it is guaranteed to be considered old enough to flush.
1806  */
1807 void
1808 xfs_buf_delwri_promote(
1809         struct xfs_buf  *bp)
1810 {
1811         struct xfs_buftarg *btp = bp->b_target;
1812         long            age = xfs_buf_age_centisecs * msecs_to_jiffies(10) + 1;
1813
1814         ASSERT(bp->b_flags & XBF_DELWRI);
1815         ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1816
1817         /*
1818          * Check the buffer age before locking the delayed write queue as we
1819          * don't need to promote buffers that are already past the flush age.
1820          */
1821         if (bp->b_queuetime < jiffies - age)
1822                 return;
1823         bp->b_queuetime = jiffies - age;
1824         spin_lock(&btp->bt_delwrite_lock);
1825         list_move(&bp->b_list, &btp->bt_delwrite_queue);
1826         spin_unlock(&btp->bt_delwrite_lock);
1827 }
1828
1829 STATIC void
1830 xfs_buf_runall_queues(
1831         struct workqueue_struct *queue)
1832 {
1833         flush_workqueue(queue);
1834 }
1835
1836 /*
1837  * Move as many buffers as specified to the supplied list
1838  * idicating if we skipped any buffers to prevent deadlocks.
1839  */
1840 STATIC int
1841 xfs_buf_delwri_split(
1842         xfs_buftarg_t   *target,
1843         struct list_head *list,
1844         unsigned long   age)
1845 {
1846         xfs_buf_t       *bp, *n;
1847         struct list_head *dwq = &target->bt_delwrite_queue;
1848         spinlock_t      *dwlk = &target->bt_delwrite_lock;
1849         int             skipped = 0;
1850         int             force;
1851
1852         force = test_and_clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1853         INIT_LIST_HEAD(list);
1854         spin_lock(dwlk);
1855         list_for_each_entry_safe(bp, n, dwq, b_list) {
1856                 ASSERT(bp->b_flags & XBF_DELWRI);
1857
1858                 if (!XFS_BUF_ISPINNED(bp) && !xfs_buf_cond_lock(bp)) {
1859                         if (!force &&
1860                             time_before(jiffies, bp->b_queuetime + age)) {
1861                                 xfs_buf_unlock(bp);
1862                                 break;
1863                         }
1864
1865                         bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q|
1866                                          _XBF_RUN_QUEUES);
1867                         bp->b_flags |= XBF_WRITE;
1868                         list_move_tail(&bp->b_list, list);
1869                         trace_xfs_buf_delwri_split(bp, _RET_IP_);
1870                 } else
1871                         skipped++;
1872         }
1873         spin_unlock(dwlk);
1874
1875         return skipped;
1876
1877 }
1878
1879 /*
1880  * Compare function is more complex than it needs to be because
1881  * the return value is only 32 bits and we are doing comparisons
1882  * on 64 bit values
1883  */
1884 static int
1885 xfs_buf_cmp(
1886         void            *priv,
1887         struct list_head *a,
1888         struct list_head *b)
1889 {
1890         struct xfs_buf  *ap = container_of(a, struct xfs_buf, b_list);
1891         struct xfs_buf  *bp = container_of(b, struct xfs_buf, b_list);
1892         xfs_daddr_t             diff;
1893
1894         diff = ap->b_bn - bp->b_bn;
1895         if (diff < 0)
1896                 return -1;
1897         if (diff > 0)
1898                 return 1;
1899         return 0;
1900 }
1901
1902 void
1903 xfs_buf_delwri_sort(
1904         xfs_buftarg_t   *target,
1905         struct list_head *list)
1906 {
1907         list_sort(NULL, list, xfs_buf_cmp);
1908 }
1909
1910 STATIC int
1911 xfsbufd(
1912         void            *data)
1913 {
1914         xfs_buftarg_t   *target = (xfs_buftarg_t *)data;
1915
1916         current->flags |= PF_MEMALLOC;
1917
1918         set_freezable();
1919
1920         do {
1921                 long    age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
1922                 long    tout = xfs_buf_timer_centisecs * msecs_to_jiffies(10);
1923                 int     count = 0;
1924                 struct list_head tmp;
1925
1926                 if (unlikely(freezing(current))) {
1927                         set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1928                         refrigerator();
1929                 } else {
1930                         clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1931                 }
1932
1933                 /* sleep for a long time if there is nothing to do. */
1934                 if (list_empty(&target->bt_delwrite_queue))
1935                         tout = MAX_SCHEDULE_TIMEOUT;
1936                 schedule_timeout_interruptible(tout);
1937
1938                 xfs_buf_delwri_split(target, &tmp, age);
1939                 list_sort(NULL, &tmp, xfs_buf_cmp);
1940                 while (!list_empty(&tmp)) {
1941                         struct xfs_buf *bp;
1942                         bp = list_first_entry(&tmp, struct xfs_buf, b_list);
1943                         list_del_init(&bp->b_list);
1944                         xfs_bdstrat_cb(bp);
1945                         count++;
1946                 }
1947                 if (count)
1948                         blk_flush_plug(current);
1949
1950         } while (!kthread_should_stop());
1951
1952         return 0;
1953 }
1954
1955 /*
1956  *      Go through all incore buffers, and release buffers if they belong to
1957  *      the given device. This is used in filesystem error handling to
1958  *      preserve the consistency of its metadata.
1959  */
1960 int
1961 xfs_flush_buftarg(
1962         xfs_buftarg_t   *target,
1963         int             wait)
1964 {
1965         xfs_buf_t       *bp;
1966         int             pincount = 0;
1967         LIST_HEAD(tmp_list);
1968         LIST_HEAD(wait_list);
1969
1970         xfs_buf_runall_queues(xfsconvertd_workqueue);
1971         xfs_buf_runall_queues(xfsdatad_workqueue);
1972         xfs_buf_runall_queues(xfslogd_workqueue);
1973
1974         set_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1975         pincount = xfs_buf_delwri_split(target, &tmp_list, 0);
1976
1977         /*
1978          * Dropped the delayed write list lock, now walk the temporary list.
1979          * All I/O is issued async and then if we need to wait for completion
1980          * we do that after issuing all the IO.
1981          */
1982         list_sort(NULL, &tmp_list, xfs_buf_cmp);
1983         while (!list_empty(&tmp_list)) {
1984                 bp = list_first_entry(&tmp_list, struct xfs_buf, b_list);
1985                 ASSERT(target == bp->b_target);
1986                 list_del_init(&bp->b_list);
1987                 if (wait) {
1988                         bp->b_flags &= ~XBF_ASYNC;
1989                         list_add(&bp->b_list, &wait_list);
1990                 }
1991                 xfs_bdstrat_cb(bp);
1992         }
1993
1994         if (wait) {
1995                 /* Expedite and wait for IO to complete. */
1996                 blk_flush_plug(current);
1997                 while (!list_empty(&wait_list)) {
1998                         bp = list_first_entry(&wait_list, struct xfs_buf, b_list);
1999
2000                         list_del_init(&bp->b_list);
2001                         xfs_buf_iowait(bp);
2002                         xfs_buf_relse(bp);
2003                 }
2004         }
2005
2006         return pincount;
2007 }
2008
2009 int __init
2010 xfs_buf_init(void)
2011 {
2012         xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
2013                                                 KM_ZONE_HWALIGN, NULL);
2014         if (!xfs_buf_zone)
2015                 goto out;
2016
2017         xfslogd_workqueue = alloc_workqueue("xfslogd",
2018                                         WQ_MEM_RECLAIM | WQ_HIGHPRI, 1);
2019         if (!xfslogd_workqueue)
2020                 goto out_free_buf_zone;
2021
2022         xfsdatad_workqueue = create_workqueue("xfsdatad");
2023         if (!xfsdatad_workqueue)
2024                 goto out_destroy_xfslogd_workqueue;
2025
2026         xfsconvertd_workqueue = create_workqueue("xfsconvertd");
2027         if (!xfsconvertd_workqueue)
2028                 goto out_destroy_xfsdatad_workqueue;
2029
2030         return 0;
2031
2032  out_destroy_xfsdatad_workqueue:
2033         destroy_workqueue(xfsdatad_workqueue);
2034  out_destroy_xfslogd_workqueue:
2035         destroy_workqueue(xfslogd_workqueue);
2036  out_free_buf_zone:
2037         kmem_zone_destroy(xfs_buf_zone);
2038  out:
2039         return -ENOMEM;
2040 }
2041
2042 void
2043 xfs_buf_terminate(void)
2044 {
2045         destroy_workqueue(xfsconvertd_workqueue);
2046         destroy_workqueue(xfsdatad_workqueue);
2047         destroy_workqueue(xfslogd_workqueue);
2048         kmem_zone_destroy(xfs_buf_zone);
2049 }
2050
2051 #ifdef CONFIG_KDB_MODULES
2052 struct list_head *
2053 xfs_get_buftarg_list(void)
2054 {
2055         return &xfs_buftarg_list;
2056 }
2057 #endif