xfs: Convert linux-2.6/ files to new logging interface
[linux-2.6.git] / fs / xfs / linux-2.6 / xfs_buf.c
1 /*
2  * Copyright (c) 2000-2006 Silicon Graphics, Inc.
3  * All Rights Reserved.
4  *
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU General Public License as
7  * published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope that it would be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write the Free Software Foundation,
16  * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17  */
18 #include "xfs.h"
19 #include <linux/stddef.h>
20 #include <linux/errno.h>
21 #include <linux/gfp.h>
22 #include <linux/pagemap.h>
23 #include <linux/init.h>
24 #include <linux/vmalloc.h>
25 #include <linux/bio.h>
26 #include <linux/sysctl.h>
27 #include <linux/proc_fs.h>
28 #include <linux/workqueue.h>
29 #include <linux/percpu.h>
30 #include <linux/blkdev.h>
31 #include <linux/hash.h>
32 #include <linux/kthread.h>
33 #include <linux/migrate.h>
34 #include <linux/backing-dev.h>
35 #include <linux/freezer.h>
36 #include <linux/list_sort.h>
37
38 #include "xfs_sb.h"
39 #include "xfs_inum.h"
40 #include "xfs_log.h"
41 #include "xfs_ag.h"
42 #include "xfs_mount.h"
43 #include "xfs_trace.h"
44
45 static kmem_zone_t *xfs_buf_zone;
46 STATIC int xfsbufd(void *);
47 STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
48
49 static struct workqueue_struct *xfslogd_workqueue;
50 struct workqueue_struct *xfsdatad_workqueue;
51 struct workqueue_struct *xfsconvertd_workqueue;
52
53 #ifdef XFS_BUF_LOCK_TRACKING
54 # define XB_SET_OWNER(bp)       ((bp)->b_last_holder = current->pid)
55 # define XB_CLEAR_OWNER(bp)     ((bp)->b_last_holder = -1)
56 # define XB_GET_OWNER(bp)       ((bp)->b_last_holder)
57 #else
58 # define XB_SET_OWNER(bp)       do { } while (0)
59 # define XB_CLEAR_OWNER(bp)     do { } while (0)
60 # define XB_GET_OWNER(bp)       do { } while (0)
61 #endif
62
63 #define xb_to_gfp(flags) \
64         ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
65           ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
66
67 #define xb_to_km(flags) \
68          (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
69
70 #define xfs_buf_allocate(flags) \
71         kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
72 #define xfs_buf_deallocate(bp) \
73         kmem_zone_free(xfs_buf_zone, (bp));
74
75 static inline int
76 xfs_buf_is_vmapped(
77         struct xfs_buf  *bp)
78 {
79         /*
80          * Return true if the buffer is vmapped.
81          *
82          * The XBF_MAPPED flag is set if the buffer should be mapped, but the
83          * code is clever enough to know it doesn't have to map a single page,
84          * so the check has to be both for XBF_MAPPED and bp->b_page_count > 1.
85          */
86         return (bp->b_flags & XBF_MAPPED) && bp->b_page_count > 1;
87 }
88
89 static inline int
90 xfs_buf_vmap_len(
91         struct xfs_buf  *bp)
92 {
93         return (bp->b_page_count * PAGE_SIZE) - bp->b_offset;
94 }
95
96 /*
97  *      Page Region interfaces.
98  *
99  *      For pages in filesystems where the blocksize is smaller than the
100  *      pagesize, we use the page->private field (long) to hold a bitmap
101  *      of uptodate regions within the page.
102  *
103  *      Each such region is "bytes per page / bits per long" bytes long.
104  *
105  *      NBPPR == number-of-bytes-per-page-region
106  *      BTOPR == bytes-to-page-region (rounded up)
107  *      BTOPRT == bytes-to-page-region-truncated (rounded down)
108  */
109 #if (BITS_PER_LONG == 32)
110 #define PRSHIFT         (PAGE_CACHE_SHIFT - 5)  /* (32 == 1<<5) */
111 #elif (BITS_PER_LONG == 64)
112 #define PRSHIFT         (PAGE_CACHE_SHIFT - 6)  /* (64 == 1<<6) */
113 #else
114 #error BITS_PER_LONG must be 32 or 64
115 #endif
116 #define NBPPR           (PAGE_CACHE_SIZE/BITS_PER_LONG)
117 #define BTOPR(b)        (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
118 #define BTOPRT(b)       (((unsigned int)(b) >> PRSHIFT))
119
120 STATIC unsigned long
121 page_region_mask(
122         size_t          offset,
123         size_t          length)
124 {
125         unsigned long   mask;
126         int             first, final;
127
128         first = BTOPR(offset);
129         final = BTOPRT(offset + length - 1);
130         first = min(first, final);
131
132         mask = ~0UL;
133         mask <<= BITS_PER_LONG - (final - first);
134         mask >>= BITS_PER_LONG - (final);
135
136         ASSERT(offset + length <= PAGE_CACHE_SIZE);
137         ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
138
139         return mask;
140 }
141
142 STATIC void
143 set_page_region(
144         struct page     *page,
145         size_t          offset,
146         size_t          length)
147 {
148         set_page_private(page,
149                 page_private(page) | page_region_mask(offset, length));
150         if (page_private(page) == ~0UL)
151                 SetPageUptodate(page);
152 }
153
154 STATIC int
155 test_page_region(
156         struct page     *page,
157         size_t          offset,
158         size_t          length)
159 {
160         unsigned long   mask = page_region_mask(offset, length);
161
162         return (mask && (page_private(page) & mask) == mask);
163 }
164
165 /*
166  * xfs_buf_lru_add - add a buffer to the LRU.
167  *
168  * The LRU takes a new reference to the buffer so that it will only be freed
169  * once the shrinker takes the buffer off the LRU.
170  */
171 STATIC void
172 xfs_buf_lru_add(
173         struct xfs_buf  *bp)
174 {
175         struct xfs_buftarg *btp = bp->b_target;
176
177         spin_lock(&btp->bt_lru_lock);
178         if (list_empty(&bp->b_lru)) {
179                 atomic_inc(&bp->b_hold);
180                 list_add_tail(&bp->b_lru, &btp->bt_lru);
181                 btp->bt_lru_nr++;
182         }
183         spin_unlock(&btp->bt_lru_lock);
184 }
185
186 /*
187  * xfs_buf_lru_del - remove a buffer from the LRU
188  *
189  * The unlocked check is safe here because it only occurs when there are not
190  * b_lru_ref counts left on the inode under the pag->pag_buf_lock. it is there
191  * to optimise the shrinker removing the buffer from the LRU and calling
192  * xfs_buf_free(). i.e. it removes an unneccessary round trip on the
193  * bt_lru_lock.
194  */
195 STATIC void
196 xfs_buf_lru_del(
197         struct xfs_buf  *bp)
198 {
199         struct xfs_buftarg *btp = bp->b_target;
200
201         if (list_empty(&bp->b_lru))
202                 return;
203
204         spin_lock(&btp->bt_lru_lock);
205         if (!list_empty(&bp->b_lru)) {
206                 list_del_init(&bp->b_lru);
207                 btp->bt_lru_nr--;
208         }
209         spin_unlock(&btp->bt_lru_lock);
210 }
211
212 /*
213  * When we mark a buffer stale, we remove the buffer from the LRU and clear the
214  * b_lru_ref count so that the buffer is freed immediately when the buffer
215  * reference count falls to zero. If the buffer is already on the LRU, we need
216  * to remove the reference that LRU holds on the buffer.
217  *
218  * This prevents build-up of stale buffers on the LRU.
219  */
220 void
221 xfs_buf_stale(
222         struct xfs_buf  *bp)
223 {
224         bp->b_flags |= XBF_STALE;
225         atomic_set(&(bp)->b_lru_ref, 0);
226         if (!list_empty(&bp->b_lru)) {
227                 struct xfs_buftarg *btp = bp->b_target;
228
229                 spin_lock(&btp->bt_lru_lock);
230                 if (!list_empty(&bp->b_lru)) {
231                         list_del_init(&bp->b_lru);
232                         btp->bt_lru_nr--;
233                         atomic_dec(&bp->b_hold);
234                 }
235                 spin_unlock(&btp->bt_lru_lock);
236         }
237         ASSERT(atomic_read(&bp->b_hold) >= 1);
238 }
239
240 STATIC void
241 _xfs_buf_initialize(
242         xfs_buf_t               *bp,
243         xfs_buftarg_t           *target,
244         xfs_off_t               range_base,
245         size_t                  range_length,
246         xfs_buf_flags_t         flags)
247 {
248         /*
249          * We don't want certain flags to appear in b_flags.
250          */
251         flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
252
253         memset(bp, 0, sizeof(xfs_buf_t));
254         atomic_set(&bp->b_hold, 1);
255         atomic_set(&bp->b_lru_ref, 1);
256         init_completion(&bp->b_iowait);
257         INIT_LIST_HEAD(&bp->b_lru);
258         INIT_LIST_HEAD(&bp->b_list);
259         RB_CLEAR_NODE(&bp->b_rbnode);
260         sema_init(&bp->b_sema, 0); /* held, no waiters */
261         XB_SET_OWNER(bp);
262         bp->b_target = target;
263         bp->b_file_offset = range_base;
264         /*
265          * Set buffer_length and count_desired to the same value initially.
266          * I/O routines should use count_desired, which will be the same in
267          * most cases but may be reset (e.g. XFS recovery).
268          */
269         bp->b_buffer_length = bp->b_count_desired = range_length;
270         bp->b_flags = flags;
271         bp->b_bn = XFS_BUF_DADDR_NULL;
272         atomic_set(&bp->b_pin_count, 0);
273         init_waitqueue_head(&bp->b_waiters);
274
275         XFS_STATS_INC(xb_create);
276
277         trace_xfs_buf_init(bp, _RET_IP_);
278 }
279
280 /*
281  *      Allocate a page array capable of holding a specified number
282  *      of pages, and point the page buf at it.
283  */
284 STATIC int
285 _xfs_buf_get_pages(
286         xfs_buf_t               *bp,
287         int                     page_count,
288         xfs_buf_flags_t         flags)
289 {
290         /* Make sure that we have a page list */
291         if (bp->b_pages == NULL) {
292                 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
293                 bp->b_page_count = page_count;
294                 if (page_count <= XB_PAGES) {
295                         bp->b_pages = bp->b_page_array;
296                 } else {
297                         bp->b_pages = kmem_alloc(sizeof(struct page *) *
298                                         page_count, xb_to_km(flags));
299                         if (bp->b_pages == NULL)
300                                 return -ENOMEM;
301                 }
302                 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
303         }
304         return 0;
305 }
306
307 /*
308  *      Frees b_pages if it was allocated.
309  */
310 STATIC void
311 _xfs_buf_free_pages(
312         xfs_buf_t       *bp)
313 {
314         if (bp->b_pages != bp->b_page_array) {
315                 kmem_free(bp->b_pages);
316                 bp->b_pages = NULL;
317         }
318 }
319
320 /*
321  *      Releases the specified buffer.
322  *
323  *      The modification state of any associated pages is left unchanged.
324  *      The buffer most not be on any hash - use xfs_buf_rele instead for
325  *      hashed and refcounted buffers
326  */
327 void
328 xfs_buf_free(
329         xfs_buf_t               *bp)
330 {
331         trace_xfs_buf_free(bp, _RET_IP_);
332
333         ASSERT(list_empty(&bp->b_lru));
334
335         if (bp->b_flags & (_XBF_PAGE_CACHE|_XBF_PAGES)) {
336                 uint            i;
337
338                 if (xfs_buf_is_vmapped(bp))
339                         vm_unmap_ram(bp->b_addr - bp->b_offset,
340                                         bp->b_page_count);
341
342                 for (i = 0; i < bp->b_page_count; i++) {
343                         struct page     *page = bp->b_pages[i];
344
345                         if (bp->b_flags & _XBF_PAGE_CACHE)
346                                 ASSERT(!PagePrivate(page));
347                         page_cache_release(page);
348                 }
349         }
350         _xfs_buf_free_pages(bp);
351         xfs_buf_deallocate(bp);
352 }
353
354 /*
355  *      Finds all pages for buffer in question and builds it's page list.
356  */
357 STATIC int
358 _xfs_buf_lookup_pages(
359         xfs_buf_t               *bp,
360         uint                    flags)
361 {
362         struct address_space    *mapping = bp->b_target->bt_mapping;
363         size_t                  blocksize = bp->b_target->bt_bsize;
364         size_t                  size = bp->b_count_desired;
365         size_t                  nbytes, offset;
366         gfp_t                   gfp_mask = xb_to_gfp(flags);
367         unsigned short          page_count, i;
368         pgoff_t                 first;
369         xfs_off_t               end;
370         int                     error;
371
372         end = bp->b_file_offset + bp->b_buffer_length;
373         page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
374
375         error = _xfs_buf_get_pages(bp, page_count, flags);
376         if (unlikely(error))
377                 return error;
378         bp->b_flags |= _XBF_PAGE_CACHE;
379
380         offset = bp->b_offset;
381         first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
382
383         for (i = 0; i < bp->b_page_count; i++) {
384                 struct page     *page;
385                 uint            retries = 0;
386
387               retry:
388                 page = find_or_create_page(mapping, first + i, gfp_mask);
389                 if (unlikely(page == NULL)) {
390                         if (flags & XBF_READ_AHEAD) {
391                                 bp->b_page_count = i;
392                                 for (i = 0; i < bp->b_page_count; i++)
393                                         unlock_page(bp->b_pages[i]);
394                                 return -ENOMEM;
395                         }
396
397                         /*
398                          * This could deadlock.
399                          *
400                          * But until all the XFS lowlevel code is revamped to
401                          * handle buffer allocation failures we can't do much.
402                          */
403                         if (!(++retries % 100))
404                                 xfs_err(NULL,
405                 "possible memory allocation deadlock in %s (mode:0x%x)",
406                                         __func__, gfp_mask);
407
408                         XFS_STATS_INC(xb_page_retries);
409                         congestion_wait(BLK_RW_ASYNC, HZ/50);
410                         goto retry;
411                 }
412
413                 XFS_STATS_INC(xb_page_found);
414
415                 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
416                 size -= nbytes;
417
418                 ASSERT(!PagePrivate(page));
419                 if (!PageUptodate(page)) {
420                         page_count--;
421                         if (blocksize >= PAGE_CACHE_SIZE) {
422                                 if (flags & XBF_READ)
423                                         bp->b_flags |= _XBF_PAGE_LOCKED;
424                         } else if (!PagePrivate(page)) {
425                                 if (test_page_region(page, offset, nbytes))
426                                         page_count++;
427                         }
428                 }
429
430                 bp->b_pages[i] = page;
431                 offset = 0;
432         }
433
434         if (!(bp->b_flags & _XBF_PAGE_LOCKED)) {
435                 for (i = 0; i < bp->b_page_count; i++)
436                         unlock_page(bp->b_pages[i]);
437         }
438
439         if (page_count == bp->b_page_count)
440                 bp->b_flags |= XBF_DONE;
441
442         return error;
443 }
444
445 /*
446  *      Map buffer into kernel address-space if nessecary.
447  */
448 STATIC int
449 _xfs_buf_map_pages(
450         xfs_buf_t               *bp,
451         uint                    flags)
452 {
453         /* A single page buffer is always mappable */
454         if (bp->b_page_count == 1) {
455                 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
456                 bp->b_flags |= XBF_MAPPED;
457         } else if (flags & XBF_MAPPED) {
458                 bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
459                                         -1, PAGE_KERNEL);
460                 if (unlikely(bp->b_addr == NULL))
461                         return -ENOMEM;
462                 bp->b_addr += bp->b_offset;
463                 bp->b_flags |= XBF_MAPPED;
464         }
465
466         return 0;
467 }
468
469 /*
470  *      Finding and Reading Buffers
471  */
472
473 /*
474  *      Look up, and creates if absent, a lockable buffer for
475  *      a given range of an inode.  The buffer is returned
476  *      locked.  If other overlapping buffers exist, they are
477  *      released before the new buffer is created and locked,
478  *      which may imply that this call will block until those buffers
479  *      are unlocked.  No I/O is implied by this call.
480  */
481 xfs_buf_t *
482 _xfs_buf_find(
483         xfs_buftarg_t           *btp,   /* block device target          */
484         xfs_off_t               ioff,   /* starting offset of range     */
485         size_t                  isize,  /* length of range              */
486         xfs_buf_flags_t         flags,
487         xfs_buf_t               *new_bp)
488 {
489         xfs_off_t               range_base;
490         size_t                  range_length;
491         struct xfs_perag        *pag;
492         struct rb_node          **rbp;
493         struct rb_node          *parent;
494         xfs_buf_t               *bp;
495
496         range_base = (ioff << BBSHIFT);
497         range_length = (isize << BBSHIFT);
498
499         /* Check for IOs smaller than the sector size / not sector aligned */
500         ASSERT(!(range_length < (1 << btp->bt_sshift)));
501         ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
502
503         /* get tree root */
504         pag = xfs_perag_get(btp->bt_mount,
505                                 xfs_daddr_to_agno(btp->bt_mount, ioff));
506
507         /* walk tree */
508         spin_lock(&pag->pag_buf_lock);
509         rbp = &pag->pag_buf_tree.rb_node;
510         parent = NULL;
511         bp = NULL;
512         while (*rbp) {
513                 parent = *rbp;
514                 bp = rb_entry(parent, struct xfs_buf, b_rbnode);
515
516                 if (range_base < bp->b_file_offset)
517                         rbp = &(*rbp)->rb_left;
518                 else if (range_base > bp->b_file_offset)
519                         rbp = &(*rbp)->rb_right;
520                 else {
521                         /*
522                          * found a block offset match. If the range doesn't
523                          * match, the only way this is allowed is if the buffer
524                          * in the cache is stale and the transaction that made
525                          * it stale has not yet committed. i.e. we are
526                          * reallocating a busy extent. Skip this buffer and
527                          * continue searching to the right for an exact match.
528                          */
529                         if (bp->b_buffer_length != range_length) {
530                                 ASSERT(bp->b_flags & XBF_STALE);
531                                 rbp = &(*rbp)->rb_right;
532                                 continue;
533                         }
534                         atomic_inc(&bp->b_hold);
535                         goto found;
536                 }
537         }
538
539         /* No match found */
540         if (new_bp) {
541                 _xfs_buf_initialize(new_bp, btp, range_base,
542                                 range_length, flags);
543                 rb_link_node(&new_bp->b_rbnode, parent, rbp);
544                 rb_insert_color(&new_bp->b_rbnode, &pag->pag_buf_tree);
545                 /* the buffer keeps the perag reference until it is freed */
546                 new_bp->b_pag = pag;
547                 spin_unlock(&pag->pag_buf_lock);
548         } else {
549                 XFS_STATS_INC(xb_miss_locked);
550                 spin_unlock(&pag->pag_buf_lock);
551                 xfs_perag_put(pag);
552         }
553         return new_bp;
554
555 found:
556         spin_unlock(&pag->pag_buf_lock);
557         xfs_perag_put(pag);
558
559         if (xfs_buf_cond_lock(bp)) {
560                 /* failed, so wait for the lock if requested. */
561                 if (!(flags & XBF_TRYLOCK)) {
562                         xfs_buf_lock(bp);
563                         XFS_STATS_INC(xb_get_locked_waited);
564                 } else {
565                         xfs_buf_rele(bp);
566                         XFS_STATS_INC(xb_busy_locked);
567                         return NULL;
568                 }
569         }
570
571         if (bp->b_flags & XBF_STALE) {
572                 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
573                 bp->b_flags &= XBF_MAPPED;
574         }
575
576         trace_xfs_buf_find(bp, flags, _RET_IP_);
577         XFS_STATS_INC(xb_get_locked);
578         return bp;
579 }
580
581 /*
582  *      Assembles a buffer covering the specified range.
583  *      Storage in memory for all portions of the buffer will be allocated,
584  *      although backing storage may not be.
585  */
586 xfs_buf_t *
587 xfs_buf_get(
588         xfs_buftarg_t           *target,/* target for buffer            */
589         xfs_off_t               ioff,   /* starting offset of range     */
590         size_t                  isize,  /* length of range              */
591         xfs_buf_flags_t         flags)
592 {
593         xfs_buf_t               *bp, *new_bp;
594         int                     error = 0, i;
595
596         new_bp = xfs_buf_allocate(flags);
597         if (unlikely(!new_bp))
598                 return NULL;
599
600         bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
601         if (bp == new_bp) {
602                 error = _xfs_buf_lookup_pages(bp, flags);
603                 if (error)
604                         goto no_buffer;
605         } else {
606                 xfs_buf_deallocate(new_bp);
607                 if (unlikely(bp == NULL))
608                         return NULL;
609         }
610
611         for (i = 0; i < bp->b_page_count; i++)
612                 mark_page_accessed(bp->b_pages[i]);
613
614         if (!(bp->b_flags & XBF_MAPPED)) {
615                 error = _xfs_buf_map_pages(bp, flags);
616                 if (unlikely(error)) {
617                         xfs_warn(target->bt_mount,
618                                 "%s: failed to map pages\n", __func__);
619                         goto no_buffer;
620                 }
621         }
622
623         XFS_STATS_INC(xb_get);
624
625         /*
626          * Always fill in the block number now, the mapped cases can do
627          * their own overlay of this later.
628          */
629         bp->b_bn = ioff;
630         bp->b_count_desired = bp->b_buffer_length;
631
632         trace_xfs_buf_get(bp, flags, _RET_IP_);
633         return bp;
634
635  no_buffer:
636         if (flags & (XBF_LOCK | XBF_TRYLOCK))
637                 xfs_buf_unlock(bp);
638         xfs_buf_rele(bp);
639         return NULL;
640 }
641
642 STATIC int
643 _xfs_buf_read(
644         xfs_buf_t               *bp,
645         xfs_buf_flags_t         flags)
646 {
647         int                     status;
648
649         ASSERT(!(flags & (XBF_DELWRI|XBF_WRITE)));
650         ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
651
652         bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_DELWRI | \
653                         XBF_READ_AHEAD | _XBF_RUN_QUEUES);
654         bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | \
655                         XBF_READ_AHEAD | _XBF_RUN_QUEUES);
656
657         status = xfs_buf_iorequest(bp);
658         if (status || XFS_BUF_ISERROR(bp) || (flags & XBF_ASYNC))
659                 return status;
660         return xfs_buf_iowait(bp);
661 }
662
663 xfs_buf_t *
664 xfs_buf_read(
665         xfs_buftarg_t           *target,
666         xfs_off_t               ioff,
667         size_t                  isize,
668         xfs_buf_flags_t         flags)
669 {
670         xfs_buf_t               *bp;
671
672         flags |= XBF_READ;
673
674         bp = xfs_buf_get(target, ioff, isize, flags);
675         if (bp) {
676                 trace_xfs_buf_read(bp, flags, _RET_IP_);
677
678                 if (!XFS_BUF_ISDONE(bp)) {
679                         XFS_STATS_INC(xb_get_read);
680                         _xfs_buf_read(bp, flags);
681                 } else if (flags & XBF_ASYNC) {
682                         /*
683                          * Read ahead call which is already satisfied,
684                          * drop the buffer
685                          */
686                         goto no_buffer;
687                 } else {
688                         /* We do not want read in the flags */
689                         bp->b_flags &= ~XBF_READ;
690                 }
691         }
692
693         return bp;
694
695  no_buffer:
696         if (flags & (XBF_LOCK | XBF_TRYLOCK))
697                 xfs_buf_unlock(bp);
698         xfs_buf_rele(bp);
699         return NULL;
700 }
701
702 /*
703  *      If we are not low on memory then do the readahead in a deadlock
704  *      safe manner.
705  */
706 void
707 xfs_buf_readahead(
708         xfs_buftarg_t           *target,
709         xfs_off_t               ioff,
710         size_t                  isize)
711 {
712         struct backing_dev_info *bdi;
713
714         bdi = target->bt_mapping->backing_dev_info;
715         if (bdi_read_congested(bdi))
716                 return;
717
718         xfs_buf_read(target, ioff, isize,
719                      XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD|XBF_DONT_BLOCK);
720 }
721
722 /*
723  * Read an uncached buffer from disk. Allocates and returns a locked
724  * buffer containing the disk contents or nothing.
725  */
726 struct xfs_buf *
727 xfs_buf_read_uncached(
728         struct xfs_mount        *mp,
729         struct xfs_buftarg      *target,
730         xfs_daddr_t             daddr,
731         size_t                  length,
732         int                     flags)
733 {
734         xfs_buf_t               *bp;
735         int                     error;
736
737         bp = xfs_buf_get_uncached(target, length, flags);
738         if (!bp)
739                 return NULL;
740
741         /* set up the buffer for a read IO */
742         xfs_buf_lock(bp);
743         XFS_BUF_SET_ADDR(bp, daddr);
744         XFS_BUF_READ(bp);
745         XFS_BUF_BUSY(bp);
746
747         xfsbdstrat(mp, bp);
748         error = xfs_buf_iowait(bp);
749         if (error || bp->b_error) {
750                 xfs_buf_relse(bp);
751                 return NULL;
752         }
753         return bp;
754 }
755
756 xfs_buf_t *
757 xfs_buf_get_empty(
758         size_t                  len,
759         xfs_buftarg_t           *target)
760 {
761         xfs_buf_t               *bp;
762
763         bp = xfs_buf_allocate(0);
764         if (bp)
765                 _xfs_buf_initialize(bp, target, 0, len, 0);
766         return bp;
767 }
768
769 static inline struct page *
770 mem_to_page(
771         void                    *addr)
772 {
773         if ((!is_vmalloc_addr(addr))) {
774                 return virt_to_page(addr);
775         } else {
776                 return vmalloc_to_page(addr);
777         }
778 }
779
780 int
781 xfs_buf_associate_memory(
782         xfs_buf_t               *bp,
783         void                    *mem,
784         size_t                  len)
785 {
786         int                     rval;
787         int                     i = 0;
788         unsigned long           pageaddr;
789         unsigned long           offset;
790         size_t                  buflen;
791         int                     page_count;
792
793         pageaddr = (unsigned long)mem & PAGE_CACHE_MASK;
794         offset = (unsigned long)mem - pageaddr;
795         buflen = PAGE_CACHE_ALIGN(len + offset);
796         page_count = buflen >> PAGE_CACHE_SHIFT;
797
798         /* Free any previous set of page pointers */
799         if (bp->b_pages)
800                 _xfs_buf_free_pages(bp);
801
802         bp->b_pages = NULL;
803         bp->b_addr = mem;
804
805         rval = _xfs_buf_get_pages(bp, page_count, XBF_DONT_BLOCK);
806         if (rval)
807                 return rval;
808
809         bp->b_offset = offset;
810
811         for (i = 0; i < bp->b_page_count; i++) {
812                 bp->b_pages[i] = mem_to_page((void *)pageaddr);
813                 pageaddr += PAGE_CACHE_SIZE;
814         }
815
816         bp->b_count_desired = len;
817         bp->b_buffer_length = buflen;
818         bp->b_flags |= XBF_MAPPED;
819         bp->b_flags &= ~_XBF_PAGE_LOCKED;
820
821         return 0;
822 }
823
824 xfs_buf_t *
825 xfs_buf_get_uncached(
826         struct xfs_buftarg      *target,
827         size_t                  len,
828         int                     flags)
829 {
830         unsigned long           page_count = PAGE_ALIGN(len) >> PAGE_SHIFT;
831         int                     error, i;
832         xfs_buf_t               *bp;
833
834         bp = xfs_buf_allocate(0);
835         if (unlikely(bp == NULL))
836                 goto fail;
837         _xfs_buf_initialize(bp, target, 0, len, 0);
838
839         error = _xfs_buf_get_pages(bp, page_count, 0);
840         if (error)
841                 goto fail_free_buf;
842
843         for (i = 0; i < page_count; i++) {
844                 bp->b_pages[i] = alloc_page(xb_to_gfp(flags));
845                 if (!bp->b_pages[i])
846                         goto fail_free_mem;
847         }
848         bp->b_flags |= _XBF_PAGES;
849
850         error = _xfs_buf_map_pages(bp, XBF_MAPPED);
851         if (unlikely(error)) {
852                 xfs_warn(target->bt_mount,
853                         "%s: failed to map pages\n", __func__);
854                 goto fail_free_mem;
855         }
856
857         xfs_buf_unlock(bp);
858
859         trace_xfs_buf_get_uncached(bp, _RET_IP_);
860         return bp;
861
862  fail_free_mem:
863         while (--i >= 0)
864                 __free_page(bp->b_pages[i]);
865         _xfs_buf_free_pages(bp);
866  fail_free_buf:
867         xfs_buf_deallocate(bp);
868  fail:
869         return NULL;
870 }
871
872 /*
873  *      Increment reference count on buffer, to hold the buffer concurrently
874  *      with another thread which may release (free) the buffer asynchronously.
875  *      Must hold the buffer already to call this function.
876  */
877 void
878 xfs_buf_hold(
879         xfs_buf_t               *bp)
880 {
881         trace_xfs_buf_hold(bp, _RET_IP_);
882         atomic_inc(&bp->b_hold);
883 }
884
885 /*
886  *      Releases a hold on the specified buffer.  If the
887  *      the hold count is 1, calls xfs_buf_free.
888  */
889 void
890 xfs_buf_rele(
891         xfs_buf_t               *bp)
892 {
893         struct xfs_perag        *pag = bp->b_pag;
894
895         trace_xfs_buf_rele(bp, _RET_IP_);
896
897         if (!pag) {
898                 ASSERT(list_empty(&bp->b_lru));
899                 ASSERT(RB_EMPTY_NODE(&bp->b_rbnode));
900                 if (atomic_dec_and_test(&bp->b_hold))
901                         xfs_buf_free(bp);
902                 return;
903         }
904
905         ASSERT(!RB_EMPTY_NODE(&bp->b_rbnode));
906
907         ASSERT(atomic_read(&bp->b_hold) > 0);
908         if (atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock)) {
909                 if (!(bp->b_flags & XBF_STALE) &&
910                            atomic_read(&bp->b_lru_ref)) {
911                         xfs_buf_lru_add(bp);
912                         spin_unlock(&pag->pag_buf_lock);
913                 } else {
914                         xfs_buf_lru_del(bp);
915                         ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
916                         rb_erase(&bp->b_rbnode, &pag->pag_buf_tree);
917                         spin_unlock(&pag->pag_buf_lock);
918                         xfs_perag_put(pag);
919                         xfs_buf_free(bp);
920                 }
921         }
922 }
923
924
925 /*
926  *      Mutual exclusion on buffers.  Locking model:
927  *
928  *      Buffers associated with inodes for which buffer locking
929  *      is not enabled are not protected by semaphores, and are
930  *      assumed to be exclusively owned by the caller.  There is a
931  *      spinlock in the buffer, used by the caller when concurrent
932  *      access is possible.
933  */
934
935 /*
936  *      Locks a buffer object, if it is not already locked.  Note that this in
937  *      no way locks the underlying pages, so it is only useful for
938  *      synchronizing concurrent use of buffer objects, not for synchronizing
939  *      independent access to the underlying pages.
940  *
941  *      If we come across a stale, pinned, locked buffer, we know that we are
942  *      being asked to lock a buffer that has been reallocated. Because it is
943  *      pinned, we know that the log has not been pushed to disk and hence it
944  *      will still be locked.  Rather than continuing to have trylock attempts
945  *      fail until someone else pushes the log, push it ourselves before
946  *      returning.  This means that the xfsaild will not get stuck trying
947  *      to push on stale inode buffers.
948  */
949 int
950 xfs_buf_cond_lock(
951         xfs_buf_t               *bp)
952 {
953         int                     locked;
954
955         locked = down_trylock(&bp->b_sema) == 0;
956         if (locked)
957                 XB_SET_OWNER(bp);
958         else if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
959                 xfs_log_force(bp->b_target->bt_mount, 0);
960
961         trace_xfs_buf_cond_lock(bp, _RET_IP_);
962         return locked ? 0 : -EBUSY;
963 }
964
965 int
966 xfs_buf_lock_value(
967         xfs_buf_t               *bp)
968 {
969         return bp->b_sema.count;
970 }
971
972 /*
973  *      Locks a buffer object.
974  *      Note that this in no way locks the underlying pages, so it is only
975  *      useful for synchronizing concurrent use of buffer objects, not for
976  *      synchronizing independent access to the underlying pages.
977  *
978  *      If we come across a stale, pinned, locked buffer, we know that we
979  *      are being asked to lock a buffer that has been reallocated. Because
980  *      it is pinned, we know that the log has not been pushed to disk and
981  *      hence it will still be locked. Rather than sleeping until someone
982  *      else pushes the log, push it ourselves before trying to get the lock.
983  */
984 void
985 xfs_buf_lock(
986         xfs_buf_t               *bp)
987 {
988         trace_xfs_buf_lock(bp, _RET_IP_);
989
990         if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
991                 xfs_log_force(bp->b_target->bt_mount, 0);
992         if (atomic_read(&bp->b_io_remaining))
993                 blk_run_address_space(bp->b_target->bt_mapping);
994         down(&bp->b_sema);
995         XB_SET_OWNER(bp);
996
997         trace_xfs_buf_lock_done(bp, _RET_IP_);
998 }
999
1000 /*
1001  *      Releases the lock on the buffer object.
1002  *      If the buffer is marked delwri but is not queued, do so before we
1003  *      unlock the buffer as we need to set flags correctly.  We also need to
1004  *      take a reference for the delwri queue because the unlocker is going to
1005  *      drop their's and they don't know we just queued it.
1006  */
1007 void
1008 xfs_buf_unlock(
1009         xfs_buf_t               *bp)
1010 {
1011         if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
1012                 atomic_inc(&bp->b_hold);
1013                 bp->b_flags |= XBF_ASYNC;
1014                 xfs_buf_delwri_queue(bp, 0);
1015         }
1016
1017         XB_CLEAR_OWNER(bp);
1018         up(&bp->b_sema);
1019
1020         trace_xfs_buf_unlock(bp, _RET_IP_);
1021 }
1022
1023 STATIC void
1024 xfs_buf_wait_unpin(
1025         xfs_buf_t               *bp)
1026 {
1027         DECLARE_WAITQUEUE       (wait, current);
1028
1029         if (atomic_read(&bp->b_pin_count) == 0)
1030                 return;
1031
1032         add_wait_queue(&bp->b_waiters, &wait);
1033         for (;;) {
1034                 set_current_state(TASK_UNINTERRUPTIBLE);
1035                 if (atomic_read(&bp->b_pin_count) == 0)
1036                         break;
1037                 if (atomic_read(&bp->b_io_remaining))
1038                         blk_run_address_space(bp->b_target->bt_mapping);
1039                 schedule();
1040         }
1041         remove_wait_queue(&bp->b_waiters, &wait);
1042         set_current_state(TASK_RUNNING);
1043 }
1044
1045 /*
1046  *      Buffer Utility Routines
1047  */
1048
1049 STATIC void
1050 xfs_buf_iodone_work(
1051         struct work_struct      *work)
1052 {
1053         xfs_buf_t               *bp =
1054                 container_of(work, xfs_buf_t, b_iodone_work);
1055
1056         if (bp->b_iodone)
1057                 (*(bp->b_iodone))(bp);
1058         else if (bp->b_flags & XBF_ASYNC)
1059                 xfs_buf_relse(bp);
1060 }
1061
1062 void
1063 xfs_buf_ioend(
1064         xfs_buf_t               *bp,
1065         int                     schedule)
1066 {
1067         trace_xfs_buf_iodone(bp, _RET_IP_);
1068
1069         bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
1070         if (bp->b_error == 0)
1071                 bp->b_flags |= XBF_DONE;
1072
1073         if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
1074                 if (schedule) {
1075                         INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
1076                         queue_work(xfslogd_workqueue, &bp->b_iodone_work);
1077                 } else {
1078                         xfs_buf_iodone_work(&bp->b_iodone_work);
1079                 }
1080         } else {
1081                 complete(&bp->b_iowait);
1082         }
1083 }
1084
1085 void
1086 xfs_buf_ioerror(
1087         xfs_buf_t               *bp,
1088         int                     error)
1089 {
1090         ASSERT(error >= 0 && error <= 0xffff);
1091         bp->b_error = (unsigned short)error;
1092         trace_xfs_buf_ioerror(bp, error, _RET_IP_);
1093 }
1094
1095 int
1096 xfs_bwrite(
1097         struct xfs_mount        *mp,
1098         struct xfs_buf          *bp)
1099 {
1100         int                     error;
1101
1102         bp->b_flags |= XBF_WRITE;
1103         bp->b_flags &= ~(XBF_ASYNC | XBF_READ);
1104
1105         xfs_buf_delwri_dequeue(bp);
1106         xfs_bdstrat_cb(bp);
1107
1108         error = xfs_buf_iowait(bp);
1109         if (error)
1110                 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
1111         xfs_buf_relse(bp);
1112         return error;
1113 }
1114
1115 void
1116 xfs_bdwrite(
1117         void                    *mp,
1118         struct xfs_buf          *bp)
1119 {
1120         trace_xfs_buf_bdwrite(bp, _RET_IP_);
1121
1122         bp->b_flags &= ~XBF_READ;
1123         bp->b_flags |= (XBF_DELWRI | XBF_ASYNC);
1124
1125         xfs_buf_delwri_queue(bp, 1);
1126 }
1127
1128 /*
1129  * Called when we want to stop a buffer from getting written or read.
1130  * We attach the EIO error, muck with its flags, and call xfs_buf_ioend
1131  * so that the proper iodone callbacks get called.
1132  */
1133 STATIC int
1134 xfs_bioerror(
1135         xfs_buf_t *bp)
1136 {
1137 #ifdef XFSERRORDEBUG
1138         ASSERT(XFS_BUF_ISREAD(bp) || bp->b_iodone);
1139 #endif
1140
1141         /*
1142          * No need to wait until the buffer is unpinned, we aren't flushing it.
1143          */
1144         XFS_BUF_ERROR(bp, EIO);
1145
1146         /*
1147          * We're calling xfs_buf_ioend, so delete XBF_DONE flag.
1148          */
1149         XFS_BUF_UNREAD(bp);
1150         XFS_BUF_UNDELAYWRITE(bp);
1151         XFS_BUF_UNDONE(bp);
1152         XFS_BUF_STALE(bp);
1153
1154         xfs_buf_ioend(bp, 0);
1155
1156         return EIO;
1157 }
1158
1159 /*
1160  * Same as xfs_bioerror, except that we are releasing the buffer
1161  * here ourselves, and avoiding the xfs_buf_ioend call.
1162  * This is meant for userdata errors; metadata bufs come with
1163  * iodone functions attached, so that we can track down errors.
1164  */
1165 STATIC int
1166 xfs_bioerror_relse(
1167         struct xfs_buf  *bp)
1168 {
1169         int64_t         fl = XFS_BUF_BFLAGS(bp);
1170         /*
1171          * No need to wait until the buffer is unpinned.
1172          * We aren't flushing it.
1173          *
1174          * chunkhold expects B_DONE to be set, whether
1175          * we actually finish the I/O or not. We don't want to
1176          * change that interface.
1177          */
1178         XFS_BUF_UNREAD(bp);
1179         XFS_BUF_UNDELAYWRITE(bp);
1180         XFS_BUF_DONE(bp);
1181         XFS_BUF_STALE(bp);
1182         XFS_BUF_CLR_IODONE_FUNC(bp);
1183         if (!(fl & XBF_ASYNC)) {
1184                 /*
1185                  * Mark b_error and B_ERROR _both_.
1186                  * Lot's of chunkcache code assumes that.
1187                  * There's no reason to mark error for
1188                  * ASYNC buffers.
1189                  */
1190                 XFS_BUF_ERROR(bp, EIO);
1191                 XFS_BUF_FINISH_IOWAIT(bp);
1192         } else {
1193                 xfs_buf_relse(bp);
1194         }
1195
1196         return EIO;
1197 }
1198
1199
1200 /*
1201  * All xfs metadata buffers except log state machine buffers
1202  * get this attached as their b_bdstrat callback function.
1203  * This is so that we can catch a buffer
1204  * after prematurely unpinning it to forcibly shutdown the filesystem.
1205  */
1206 int
1207 xfs_bdstrat_cb(
1208         struct xfs_buf  *bp)
1209 {
1210         if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) {
1211                 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1212                 /*
1213                  * Metadata write that didn't get logged but
1214                  * written delayed anyway. These aren't associated
1215                  * with a transaction, and can be ignored.
1216                  */
1217                 if (!bp->b_iodone && !XFS_BUF_ISREAD(bp))
1218                         return xfs_bioerror_relse(bp);
1219                 else
1220                         return xfs_bioerror(bp);
1221         }
1222
1223         xfs_buf_iorequest(bp);
1224         return 0;
1225 }
1226
1227 /*
1228  * Wrapper around bdstrat so that we can stop data from going to disk in case
1229  * we are shutting down the filesystem.  Typically user data goes thru this
1230  * path; one of the exceptions is the superblock.
1231  */
1232 void
1233 xfsbdstrat(
1234         struct xfs_mount        *mp,
1235         struct xfs_buf          *bp)
1236 {
1237         if (XFS_FORCED_SHUTDOWN(mp)) {
1238                 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1239                 xfs_bioerror_relse(bp);
1240                 return;
1241         }
1242
1243         xfs_buf_iorequest(bp);
1244 }
1245
1246 STATIC void
1247 _xfs_buf_ioend(
1248         xfs_buf_t               *bp,
1249         int                     schedule)
1250 {
1251         if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1252                 bp->b_flags &= ~_XBF_PAGE_LOCKED;
1253                 xfs_buf_ioend(bp, schedule);
1254         }
1255 }
1256
1257 STATIC void
1258 xfs_buf_bio_end_io(
1259         struct bio              *bio,
1260         int                     error)
1261 {
1262         xfs_buf_t               *bp = (xfs_buf_t *)bio->bi_private;
1263         unsigned int            blocksize = bp->b_target->bt_bsize;
1264         struct bio_vec          *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1265
1266         xfs_buf_ioerror(bp, -error);
1267
1268         if (!error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
1269                 invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
1270
1271         do {
1272                 struct page     *page = bvec->bv_page;
1273
1274                 ASSERT(!PagePrivate(page));
1275                 if (unlikely(bp->b_error)) {
1276                         if (bp->b_flags & XBF_READ)
1277                                 ClearPageUptodate(page);
1278                 } else if (blocksize >= PAGE_CACHE_SIZE) {
1279                         SetPageUptodate(page);
1280                 } else if (!PagePrivate(page) &&
1281                                 (bp->b_flags & _XBF_PAGE_CACHE)) {
1282                         set_page_region(page, bvec->bv_offset, bvec->bv_len);
1283                 }
1284
1285                 if (--bvec >= bio->bi_io_vec)
1286                         prefetchw(&bvec->bv_page->flags);
1287
1288                 if (bp->b_flags & _XBF_PAGE_LOCKED)
1289                         unlock_page(page);
1290         } while (bvec >= bio->bi_io_vec);
1291
1292         _xfs_buf_ioend(bp, 1);
1293         bio_put(bio);
1294 }
1295
1296 STATIC void
1297 _xfs_buf_ioapply(
1298         xfs_buf_t               *bp)
1299 {
1300         int                     rw, map_i, total_nr_pages, nr_pages;
1301         struct bio              *bio;
1302         int                     offset = bp->b_offset;
1303         int                     size = bp->b_count_desired;
1304         sector_t                sector = bp->b_bn;
1305         unsigned int            blocksize = bp->b_target->bt_bsize;
1306
1307         total_nr_pages = bp->b_page_count;
1308         map_i = 0;
1309
1310         if (bp->b_flags & XBF_ORDERED) {
1311                 ASSERT(!(bp->b_flags & XBF_READ));
1312                 rw = WRITE_FLUSH_FUA;
1313         } else if (bp->b_flags & XBF_LOG_BUFFER) {
1314                 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1315                 bp->b_flags &= ~_XBF_RUN_QUEUES;
1316                 rw = (bp->b_flags & XBF_WRITE) ? WRITE_SYNC : READ_SYNC;
1317         } else if (bp->b_flags & _XBF_RUN_QUEUES) {
1318                 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1319                 bp->b_flags &= ~_XBF_RUN_QUEUES;
1320                 rw = (bp->b_flags & XBF_WRITE) ? WRITE_META : READ_META;
1321         } else {
1322                 rw = (bp->b_flags & XBF_WRITE) ? WRITE :
1323                      (bp->b_flags & XBF_READ_AHEAD) ? READA : READ;
1324         }
1325
1326         /* Special code path for reading a sub page size buffer in --
1327          * we populate up the whole page, and hence the other metadata
1328          * in the same page.  This optimization is only valid when the
1329          * filesystem block size is not smaller than the page size.
1330          */
1331         if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
1332             ((bp->b_flags & (XBF_READ|_XBF_PAGE_LOCKED)) ==
1333               (XBF_READ|_XBF_PAGE_LOCKED)) &&
1334             (blocksize >= PAGE_CACHE_SIZE)) {
1335                 bio = bio_alloc(GFP_NOIO, 1);
1336
1337                 bio->bi_bdev = bp->b_target->bt_bdev;
1338                 bio->bi_sector = sector - (offset >> BBSHIFT);
1339                 bio->bi_end_io = xfs_buf_bio_end_io;
1340                 bio->bi_private = bp;
1341
1342                 bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
1343                 size = 0;
1344
1345                 atomic_inc(&bp->b_io_remaining);
1346
1347                 goto submit_io;
1348         }
1349
1350 next_chunk:
1351         atomic_inc(&bp->b_io_remaining);
1352         nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1353         if (nr_pages > total_nr_pages)
1354                 nr_pages = total_nr_pages;
1355
1356         bio = bio_alloc(GFP_NOIO, nr_pages);
1357         bio->bi_bdev = bp->b_target->bt_bdev;
1358         bio->bi_sector = sector;
1359         bio->bi_end_io = xfs_buf_bio_end_io;
1360         bio->bi_private = bp;
1361
1362         for (; size && nr_pages; nr_pages--, map_i++) {
1363                 int     rbytes, nbytes = PAGE_CACHE_SIZE - offset;
1364
1365                 if (nbytes > size)
1366                         nbytes = size;
1367
1368                 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1369                 if (rbytes < nbytes)
1370                         break;
1371
1372                 offset = 0;
1373                 sector += nbytes >> BBSHIFT;
1374                 size -= nbytes;
1375                 total_nr_pages--;
1376         }
1377
1378 submit_io:
1379         if (likely(bio->bi_size)) {
1380                 if (xfs_buf_is_vmapped(bp)) {
1381                         flush_kernel_vmap_range(bp->b_addr,
1382                                                 xfs_buf_vmap_len(bp));
1383                 }
1384                 submit_bio(rw, bio);
1385                 if (size)
1386                         goto next_chunk;
1387         } else {
1388                 /*
1389                  * if we get here, no pages were added to the bio. However,
1390                  * we can't just error out here - if the pages are locked then
1391                  * we have to unlock them otherwise we can hang on a later
1392                  * access to the page.
1393                  */
1394                 xfs_buf_ioerror(bp, EIO);
1395                 if (bp->b_flags & _XBF_PAGE_LOCKED) {
1396                         int i;
1397                         for (i = 0; i < bp->b_page_count; i++)
1398                                 unlock_page(bp->b_pages[i]);
1399                 }
1400                 bio_put(bio);
1401         }
1402 }
1403
1404 int
1405 xfs_buf_iorequest(
1406         xfs_buf_t               *bp)
1407 {
1408         trace_xfs_buf_iorequest(bp, _RET_IP_);
1409
1410         if (bp->b_flags & XBF_DELWRI) {
1411                 xfs_buf_delwri_queue(bp, 1);
1412                 return 0;
1413         }
1414
1415         if (bp->b_flags & XBF_WRITE) {
1416                 xfs_buf_wait_unpin(bp);
1417         }
1418
1419         xfs_buf_hold(bp);
1420
1421         /* Set the count to 1 initially, this will stop an I/O
1422          * completion callout which happens before we have started
1423          * all the I/O from calling xfs_buf_ioend too early.
1424          */
1425         atomic_set(&bp->b_io_remaining, 1);
1426         _xfs_buf_ioapply(bp);
1427         _xfs_buf_ioend(bp, 0);
1428
1429         xfs_buf_rele(bp);
1430         return 0;
1431 }
1432
1433 /*
1434  *      Waits for I/O to complete on the buffer supplied.
1435  *      It returns immediately if no I/O is pending.
1436  *      It returns the I/O error code, if any, or 0 if there was no error.
1437  */
1438 int
1439 xfs_buf_iowait(
1440         xfs_buf_t               *bp)
1441 {
1442         trace_xfs_buf_iowait(bp, _RET_IP_);
1443
1444         if (atomic_read(&bp->b_io_remaining))
1445                 blk_run_address_space(bp->b_target->bt_mapping);
1446         wait_for_completion(&bp->b_iowait);
1447
1448         trace_xfs_buf_iowait_done(bp, _RET_IP_);
1449         return bp->b_error;
1450 }
1451
1452 xfs_caddr_t
1453 xfs_buf_offset(
1454         xfs_buf_t               *bp,
1455         size_t                  offset)
1456 {
1457         struct page             *page;
1458
1459         if (bp->b_flags & XBF_MAPPED)
1460                 return XFS_BUF_PTR(bp) + offset;
1461
1462         offset += bp->b_offset;
1463         page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
1464         return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
1465 }
1466
1467 /*
1468  *      Move data into or out of a buffer.
1469  */
1470 void
1471 xfs_buf_iomove(
1472         xfs_buf_t               *bp,    /* buffer to process            */
1473         size_t                  boff,   /* starting buffer offset       */
1474         size_t                  bsize,  /* length to copy               */
1475         void                    *data,  /* data address                 */
1476         xfs_buf_rw_t            mode)   /* read/write/zero flag         */
1477 {
1478         size_t                  bend, cpoff, csize;
1479         struct page             *page;
1480
1481         bend = boff + bsize;
1482         while (boff < bend) {
1483                 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1484                 cpoff = xfs_buf_poff(boff + bp->b_offset);
1485                 csize = min_t(size_t,
1486                               PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
1487
1488                 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1489
1490                 switch (mode) {
1491                 case XBRW_ZERO:
1492                         memset(page_address(page) + cpoff, 0, csize);
1493                         break;
1494                 case XBRW_READ:
1495                         memcpy(data, page_address(page) + cpoff, csize);
1496                         break;
1497                 case XBRW_WRITE:
1498                         memcpy(page_address(page) + cpoff, data, csize);
1499                 }
1500
1501                 boff += csize;
1502                 data += csize;
1503         }
1504 }
1505
1506 /*
1507  *      Handling of buffer targets (buftargs).
1508  */
1509
1510 /*
1511  * Wait for any bufs with callbacks that have been submitted but have not yet
1512  * returned. These buffers will have an elevated hold count, so wait on those
1513  * while freeing all the buffers only held by the LRU.
1514  */
1515 void
1516 xfs_wait_buftarg(
1517         struct xfs_buftarg      *btp)
1518 {
1519         struct xfs_buf          *bp;
1520
1521 restart:
1522         spin_lock(&btp->bt_lru_lock);
1523         while (!list_empty(&btp->bt_lru)) {
1524                 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1525                 if (atomic_read(&bp->b_hold) > 1) {
1526                         spin_unlock(&btp->bt_lru_lock);
1527                         delay(100);
1528                         goto restart;
1529                 }
1530                 /*
1531                  * clear the LRU reference count so the bufer doesn't get
1532                  * ignored in xfs_buf_rele().
1533                  */
1534                 atomic_set(&bp->b_lru_ref, 0);
1535                 spin_unlock(&btp->bt_lru_lock);
1536                 xfs_buf_rele(bp);
1537                 spin_lock(&btp->bt_lru_lock);
1538         }
1539         spin_unlock(&btp->bt_lru_lock);
1540 }
1541
1542 int
1543 xfs_buftarg_shrink(
1544         struct shrinker         *shrink,
1545         int                     nr_to_scan,
1546         gfp_t                   mask)
1547 {
1548         struct xfs_buftarg      *btp = container_of(shrink,
1549                                         struct xfs_buftarg, bt_shrinker);
1550         struct xfs_buf          *bp;
1551         LIST_HEAD(dispose);
1552
1553         if (!nr_to_scan)
1554                 return btp->bt_lru_nr;
1555
1556         spin_lock(&btp->bt_lru_lock);
1557         while (!list_empty(&btp->bt_lru)) {
1558                 if (nr_to_scan-- <= 0)
1559                         break;
1560
1561                 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1562
1563                 /*
1564                  * Decrement the b_lru_ref count unless the value is already
1565                  * zero. If the value is already zero, we need to reclaim the
1566                  * buffer, otherwise it gets another trip through the LRU.
1567                  */
1568                 if (!atomic_add_unless(&bp->b_lru_ref, -1, 0)) {
1569                         list_move_tail(&bp->b_lru, &btp->bt_lru);
1570                         continue;
1571                 }
1572
1573                 /*
1574                  * remove the buffer from the LRU now to avoid needing another
1575                  * lock round trip inside xfs_buf_rele().
1576                  */
1577                 list_move(&bp->b_lru, &dispose);
1578                 btp->bt_lru_nr--;
1579         }
1580         spin_unlock(&btp->bt_lru_lock);
1581
1582         while (!list_empty(&dispose)) {
1583                 bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
1584                 list_del_init(&bp->b_lru);
1585                 xfs_buf_rele(bp);
1586         }
1587
1588         return btp->bt_lru_nr;
1589 }
1590
1591 void
1592 xfs_free_buftarg(
1593         struct xfs_mount        *mp,
1594         struct xfs_buftarg      *btp)
1595 {
1596         unregister_shrinker(&btp->bt_shrinker);
1597
1598         xfs_flush_buftarg(btp, 1);
1599         if (mp->m_flags & XFS_MOUNT_BARRIER)
1600                 xfs_blkdev_issue_flush(btp);
1601         iput(btp->bt_mapping->host);
1602
1603         kthread_stop(btp->bt_task);
1604         kmem_free(btp);
1605 }
1606
1607 STATIC int
1608 xfs_setsize_buftarg_flags(
1609         xfs_buftarg_t           *btp,
1610         unsigned int            blocksize,
1611         unsigned int            sectorsize,
1612         int                     verbose)
1613 {
1614         btp->bt_bsize = blocksize;
1615         btp->bt_sshift = ffs(sectorsize) - 1;
1616         btp->bt_smask = sectorsize - 1;
1617
1618         if (set_blocksize(btp->bt_bdev, sectorsize)) {
1619                 xfs_warn(btp->bt_mount,
1620                         "Cannot set_blocksize to %u on device %s\n",
1621                         sectorsize, XFS_BUFTARG_NAME(btp));
1622                 return EINVAL;
1623         }
1624
1625         if (verbose &&
1626             (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1627                 printk(KERN_WARNING
1628                         "XFS: %u byte sectors in use on device %s.  "
1629                         "This is suboptimal; %u or greater is ideal.\n",
1630                         sectorsize, XFS_BUFTARG_NAME(btp),
1631                         (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1632         }
1633
1634         return 0;
1635 }
1636
1637 /*
1638  *      When allocating the initial buffer target we have not yet
1639  *      read in the superblock, so don't know what sized sectors
1640  *      are being used is at this early stage.  Play safe.
1641  */
1642 STATIC int
1643 xfs_setsize_buftarg_early(
1644         xfs_buftarg_t           *btp,
1645         struct block_device     *bdev)
1646 {
1647         return xfs_setsize_buftarg_flags(btp,
1648                         PAGE_CACHE_SIZE, bdev_logical_block_size(bdev), 0);
1649 }
1650
1651 int
1652 xfs_setsize_buftarg(
1653         xfs_buftarg_t           *btp,
1654         unsigned int            blocksize,
1655         unsigned int            sectorsize)
1656 {
1657         return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1658 }
1659
1660 STATIC int
1661 xfs_mapping_buftarg(
1662         xfs_buftarg_t           *btp,
1663         struct block_device     *bdev)
1664 {
1665         struct backing_dev_info *bdi;
1666         struct inode            *inode;
1667         struct address_space    *mapping;
1668         static const struct address_space_operations mapping_aops = {
1669                 .sync_page = block_sync_page,
1670                 .migratepage = fail_migrate_page,
1671         };
1672
1673         inode = new_inode(bdev->bd_inode->i_sb);
1674         if (!inode) {
1675                 printk(KERN_WARNING
1676                         "XFS: Cannot allocate mapping inode for device %s\n",
1677                         XFS_BUFTARG_NAME(btp));
1678                 return ENOMEM;
1679         }
1680         inode->i_ino = get_next_ino();
1681         inode->i_mode = S_IFBLK;
1682         inode->i_bdev = bdev;
1683         inode->i_rdev = bdev->bd_dev;
1684         bdi = blk_get_backing_dev_info(bdev);
1685         if (!bdi)
1686                 bdi = &default_backing_dev_info;
1687         mapping = &inode->i_data;
1688         mapping->a_ops = &mapping_aops;
1689         mapping->backing_dev_info = bdi;
1690         mapping_set_gfp_mask(mapping, GFP_NOFS);
1691         btp->bt_mapping = mapping;
1692         return 0;
1693 }
1694
1695 STATIC int
1696 xfs_alloc_delwrite_queue(
1697         xfs_buftarg_t           *btp,
1698         const char              *fsname)
1699 {
1700         INIT_LIST_HEAD(&btp->bt_delwrite_queue);
1701         spin_lock_init(&btp->bt_delwrite_lock);
1702         btp->bt_flags = 0;
1703         btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd/%s", fsname);
1704         if (IS_ERR(btp->bt_task))
1705                 return PTR_ERR(btp->bt_task);
1706         return 0;
1707 }
1708
1709 xfs_buftarg_t *
1710 xfs_alloc_buftarg(
1711         struct xfs_mount        *mp,
1712         struct block_device     *bdev,
1713         int                     external,
1714         const char              *fsname)
1715 {
1716         xfs_buftarg_t           *btp;
1717
1718         btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1719
1720         btp->bt_mount = mp;
1721         btp->bt_dev =  bdev->bd_dev;
1722         btp->bt_bdev = bdev;
1723         INIT_LIST_HEAD(&btp->bt_lru);
1724         spin_lock_init(&btp->bt_lru_lock);
1725         if (xfs_setsize_buftarg_early(btp, bdev))
1726                 goto error;
1727         if (xfs_mapping_buftarg(btp, bdev))
1728                 goto error;
1729         if (xfs_alloc_delwrite_queue(btp, fsname))
1730                 goto error;
1731         btp->bt_shrinker.shrink = xfs_buftarg_shrink;
1732         btp->bt_shrinker.seeks = DEFAULT_SEEKS;
1733         register_shrinker(&btp->bt_shrinker);
1734         return btp;
1735
1736 error:
1737         kmem_free(btp);
1738         return NULL;
1739 }
1740
1741
1742 /*
1743  *      Delayed write buffer handling
1744  */
1745 STATIC void
1746 xfs_buf_delwri_queue(
1747         xfs_buf_t               *bp,
1748         int                     unlock)
1749 {
1750         struct list_head        *dwq = &bp->b_target->bt_delwrite_queue;
1751         spinlock_t              *dwlk = &bp->b_target->bt_delwrite_lock;
1752
1753         trace_xfs_buf_delwri_queue(bp, _RET_IP_);
1754
1755         ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
1756
1757         spin_lock(dwlk);
1758         /* If already in the queue, dequeue and place at tail */
1759         if (!list_empty(&bp->b_list)) {
1760                 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1761                 if (unlock)
1762                         atomic_dec(&bp->b_hold);
1763                 list_del(&bp->b_list);
1764         }
1765
1766         if (list_empty(dwq)) {
1767                 /* start xfsbufd as it is about to have something to do */
1768                 wake_up_process(bp->b_target->bt_task);
1769         }
1770
1771         bp->b_flags |= _XBF_DELWRI_Q;
1772         list_add_tail(&bp->b_list, dwq);
1773         bp->b_queuetime = jiffies;
1774         spin_unlock(dwlk);
1775
1776         if (unlock)
1777                 xfs_buf_unlock(bp);
1778 }
1779
1780 void
1781 xfs_buf_delwri_dequeue(
1782         xfs_buf_t               *bp)
1783 {
1784         spinlock_t              *dwlk = &bp->b_target->bt_delwrite_lock;
1785         int                     dequeued = 0;
1786
1787         spin_lock(dwlk);
1788         if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1789                 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1790                 list_del_init(&bp->b_list);
1791                 dequeued = 1;
1792         }
1793         bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
1794         spin_unlock(dwlk);
1795
1796         if (dequeued)
1797                 xfs_buf_rele(bp);
1798
1799         trace_xfs_buf_delwri_dequeue(bp, _RET_IP_);
1800 }
1801
1802 /*
1803  * If a delwri buffer needs to be pushed before it has aged out, then promote
1804  * it to the head of the delwri queue so that it will be flushed on the next
1805  * xfsbufd run. We do this by resetting the queuetime of the buffer to be older
1806  * than the age currently needed to flush the buffer. Hence the next time the
1807  * xfsbufd sees it is guaranteed to be considered old enough to flush.
1808  */
1809 void
1810 xfs_buf_delwri_promote(
1811         struct xfs_buf  *bp)
1812 {
1813         struct xfs_buftarg *btp = bp->b_target;
1814         long            age = xfs_buf_age_centisecs * msecs_to_jiffies(10) + 1;
1815
1816         ASSERT(bp->b_flags & XBF_DELWRI);
1817         ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1818
1819         /*
1820          * Check the buffer age before locking the delayed write queue as we
1821          * don't need to promote buffers that are already past the flush age.
1822          */
1823         if (bp->b_queuetime < jiffies - age)
1824                 return;
1825         bp->b_queuetime = jiffies - age;
1826         spin_lock(&btp->bt_delwrite_lock);
1827         list_move(&bp->b_list, &btp->bt_delwrite_queue);
1828         spin_unlock(&btp->bt_delwrite_lock);
1829 }
1830
1831 STATIC void
1832 xfs_buf_runall_queues(
1833         struct workqueue_struct *queue)
1834 {
1835         flush_workqueue(queue);
1836 }
1837
1838 /*
1839  * Move as many buffers as specified to the supplied list
1840  * idicating if we skipped any buffers to prevent deadlocks.
1841  */
1842 STATIC int
1843 xfs_buf_delwri_split(
1844         xfs_buftarg_t   *target,
1845         struct list_head *list,
1846         unsigned long   age)
1847 {
1848         xfs_buf_t       *bp, *n;
1849         struct list_head *dwq = &target->bt_delwrite_queue;
1850         spinlock_t      *dwlk = &target->bt_delwrite_lock;
1851         int             skipped = 0;
1852         int             force;
1853
1854         force = test_and_clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1855         INIT_LIST_HEAD(list);
1856         spin_lock(dwlk);
1857         list_for_each_entry_safe(bp, n, dwq, b_list) {
1858                 ASSERT(bp->b_flags & XBF_DELWRI);
1859
1860                 if (!XFS_BUF_ISPINNED(bp) && !xfs_buf_cond_lock(bp)) {
1861                         if (!force &&
1862                             time_before(jiffies, bp->b_queuetime + age)) {
1863                                 xfs_buf_unlock(bp);
1864                                 break;
1865                         }
1866
1867                         bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q|
1868                                          _XBF_RUN_QUEUES);
1869                         bp->b_flags |= XBF_WRITE;
1870                         list_move_tail(&bp->b_list, list);
1871                         trace_xfs_buf_delwri_split(bp, _RET_IP_);
1872                 } else
1873                         skipped++;
1874         }
1875         spin_unlock(dwlk);
1876
1877         return skipped;
1878
1879 }
1880
1881 /*
1882  * Compare function is more complex than it needs to be because
1883  * the return value is only 32 bits and we are doing comparisons
1884  * on 64 bit values
1885  */
1886 static int
1887 xfs_buf_cmp(
1888         void            *priv,
1889         struct list_head *a,
1890         struct list_head *b)
1891 {
1892         struct xfs_buf  *ap = container_of(a, struct xfs_buf, b_list);
1893         struct xfs_buf  *bp = container_of(b, struct xfs_buf, b_list);
1894         xfs_daddr_t             diff;
1895
1896         diff = ap->b_bn - bp->b_bn;
1897         if (diff < 0)
1898                 return -1;
1899         if (diff > 0)
1900                 return 1;
1901         return 0;
1902 }
1903
1904 void
1905 xfs_buf_delwri_sort(
1906         xfs_buftarg_t   *target,
1907         struct list_head *list)
1908 {
1909         list_sort(NULL, list, xfs_buf_cmp);
1910 }
1911
1912 STATIC int
1913 xfsbufd(
1914         void            *data)
1915 {
1916         xfs_buftarg_t   *target = (xfs_buftarg_t *)data;
1917
1918         current->flags |= PF_MEMALLOC;
1919
1920         set_freezable();
1921
1922         do {
1923                 long    age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
1924                 long    tout = xfs_buf_timer_centisecs * msecs_to_jiffies(10);
1925                 int     count = 0;
1926                 struct list_head tmp;
1927
1928                 if (unlikely(freezing(current))) {
1929                         set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1930                         refrigerator();
1931                 } else {
1932                         clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1933                 }
1934
1935                 /* sleep for a long time if there is nothing to do. */
1936                 if (list_empty(&target->bt_delwrite_queue))
1937                         tout = MAX_SCHEDULE_TIMEOUT;
1938                 schedule_timeout_interruptible(tout);
1939
1940                 xfs_buf_delwri_split(target, &tmp, age);
1941                 list_sort(NULL, &tmp, xfs_buf_cmp);
1942                 while (!list_empty(&tmp)) {
1943                         struct xfs_buf *bp;
1944                         bp = list_first_entry(&tmp, struct xfs_buf, b_list);
1945                         list_del_init(&bp->b_list);
1946                         xfs_bdstrat_cb(bp);
1947                         count++;
1948                 }
1949                 if (count)
1950                         blk_run_address_space(target->bt_mapping);
1951
1952         } while (!kthread_should_stop());
1953
1954         return 0;
1955 }
1956
1957 /*
1958  *      Go through all incore buffers, and release buffers if they belong to
1959  *      the given device. This is used in filesystem error handling to
1960  *      preserve the consistency of its metadata.
1961  */
1962 int
1963 xfs_flush_buftarg(
1964         xfs_buftarg_t   *target,
1965         int             wait)
1966 {
1967         xfs_buf_t       *bp;
1968         int             pincount = 0;
1969         LIST_HEAD(tmp_list);
1970         LIST_HEAD(wait_list);
1971
1972         xfs_buf_runall_queues(xfsconvertd_workqueue);
1973         xfs_buf_runall_queues(xfsdatad_workqueue);
1974         xfs_buf_runall_queues(xfslogd_workqueue);
1975
1976         set_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1977         pincount = xfs_buf_delwri_split(target, &tmp_list, 0);
1978
1979         /*
1980          * Dropped the delayed write list lock, now walk the temporary list.
1981          * All I/O is issued async and then if we need to wait for completion
1982          * we do that after issuing all the IO.
1983          */
1984         list_sort(NULL, &tmp_list, xfs_buf_cmp);
1985         while (!list_empty(&tmp_list)) {
1986                 bp = list_first_entry(&tmp_list, struct xfs_buf, b_list);
1987                 ASSERT(target == bp->b_target);
1988                 list_del_init(&bp->b_list);
1989                 if (wait) {
1990                         bp->b_flags &= ~XBF_ASYNC;
1991                         list_add(&bp->b_list, &wait_list);
1992                 }
1993                 xfs_bdstrat_cb(bp);
1994         }
1995
1996         if (wait) {
1997                 /* Expedite and wait for IO to complete. */
1998                 blk_run_address_space(target->bt_mapping);
1999                 while (!list_empty(&wait_list)) {
2000                         bp = list_first_entry(&wait_list, struct xfs_buf, b_list);
2001
2002                         list_del_init(&bp->b_list);
2003                         xfs_buf_iowait(bp);
2004                         xfs_buf_relse(bp);
2005                 }
2006         }
2007
2008         return pincount;
2009 }
2010
2011 int __init
2012 xfs_buf_init(void)
2013 {
2014         xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
2015                                                 KM_ZONE_HWALIGN, NULL);
2016         if (!xfs_buf_zone)
2017                 goto out;
2018
2019         xfslogd_workqueue = alloc_workqueue("xfslogd",
2020                                         WQ_MEM_RECLAIM | WQ_HIGHPRI, 1);
2021         if (!xfslogd_workqueue)
2022                 goto out_free_buf_zone;
2023
2024         xfsdatad_workqueue = create_workqueue("xfsdatad");
2025         if (!xfsdatad_workqueue)
2026                 goto out_destroy_xfslogd_workqueue;
2027
2028         xfsconvertd_workqueue = create_workqueue("xfsconvertd");
2029         if (!xfsconvertd_workqueue)
2030                 goto out_destroy_xfsdatad_workqueue;
2031
2032         return 0;
2033
2034  out_destroy_xfsdatad_workqueue:
2035         destroy_workqueue(xfsdatad_workqueue);
2036  out_destroy_xfslogd_workqueue:
2037         destroy_workqueue(xfslogd_workqueue);
2038  out_free_buf_zone:
2039         kmem_zone_destroy(xfs_buf_zone);
2040  out:
2041         return -ENOMEM;
2042 }
2043
2044 void
2045 xfs_buf_terminate(void)
2046 {
2047         destroy_workqueue(xfsconvertd_workqueue);
2048         destroy_workqueue(xfsdatad_workqueue);
2049         destroy_workqueue(xfslogd_workqueue);
2050         kmem_zone_destroy(xfs_buf_zone);
2051 }
2052
2053 #ifdef CONFIG_KDB_MODULES
2054 struct list_head *
2055 xfs_get_buftarg_list(void)
2056 {
2057         return &xfs_buftarg_list;
2058 }
2059 #endif