Merge git://git.kernel.org/pub/scm/linux/kernel/git/bunk/trivial
[linux-2.6.git] / fs / buffer.c
1 /*
2  *  linux/fs/buffer.c
3  *
4  *  Copyright (C) 1991, 1992, 2002  Linus Torvalds
5  */
6
7 /*
8  * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9  *
10  * Removed a lot of unnecessary code and simplified things now that
11  * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12  *
13  * Speed up hash, lru, and free list operations.  Use gfp() for allocating
14  * hash table, use SLAB cache for buffer heads. SMP threading.  -DaveM
15  *
16  * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17  *
18  * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/syscalls.h>
23 #include <linux/fs.h>
24 #include <linux/mm.h>
25 #include <linux/percpu.h>
26 #include <linux/slab.h>
27 #include <linux/smp_lock.h>
28 #include <linux/capability.h>
29 #include <linux/blkdev.h>
30 #include <linux/file.h>
31 #include <linux/quotaops.h>
32 #include <linux/highmem.h>
33 #include <linux/module.h>
34 #include <linux/writeback.h>
35 #include <linux/hash.h>
36 #include <linux/suspend.h>
37 #include <linux/buffer_head.h>
38 #include <linux/bio.h>
39 #include <linux/notifier.h>
40 #include <linux/cpu.h>
41 #include <linux/bitops.h>
42 #include <linux/mpage.h>
43 #include <linux/bit_spinlock.h>
44
45 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
46 static void invalidate_bh_lrus(void);
47
48 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
49
50 inline void
51 init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
52 {
53         bh->b_end_io = handler;
54         bh->b_private = private;
55 }
56
57 static int sync_buffer(void *word)
58 {
59         struct block_device *bd;
60         struct buffer_head *bh
61                 = container_of(word, struct buffer_head, b_state);
62
63         smp_mb();
64         bd = bh->b_bdev;
65         if (bd)
66                 blk_run_address_space(bd->bd_inode->i_mapping);
67         io_schedule();
68         return 0;
69 }
70
71 void fastcall __lock_buffer(struct buffer_head *bh)
72 {
73         wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
74                                                         TASK_UNINTERRUPTIBLE);
75 }
76 EXPORT_SYMBOL(__lock_buffer);
77
78 void fastcall unlock_buffer(struct buffer_head *bh)
79 {
80         clear_buffer_locked(bh);
81         smp_mb__after_clear_bit();
82         wake_up_bit(&bh->b_state, BH_Lock);
83 }
84
85 /*
86  * Block until a buffer comes unlocked.  This doesn't stop it
87  * from becoming locked again - you have to lock it yourself
88  * if you want to preserve its state.
89  */
90 void __wait_on_buffer(struct buffer_head * bh)
91 {
92         wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
93 }
94
95 static void
96 __clear_page_buffers(struct page *page)
97 {
98         ClearPagePrivate(page);
99         set_page_private(page, 0);
100         page_cache_release(page);
101 }
102
103 static void buffer_io_error(struct buffer_head *bh)
104 {
105         char b[BDEVNAME_SIZE];
106
107         printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
108                         bdevname(bh->b_bdev, b),
109                         (unsigned long long)bh->b_blocknr);
110 }
111
112 /*
113  * Default synchronous end-of-IO handler..  Just mark it up-to-date and
114  * unlock the buffer. This is what ll_rw_block uses too.
115  */
116 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
117 {
118         if (uptodate) {
119                 set_buffer_uptodate(bh);
120         } else {
121                 /* This happens, due to failed READA attempts. */
122                 clear_buffer_uptodate(bh);
123         }
124         unlock_buffer(bh);
125         put_bh(bh);
126 }
127
128 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
129 {
130         char b[BDEVNAME_SIZE];
131
132         if (uptodate) {
133                 set_buffer_uptodate(bh);
134         } else {
135                 if (!buffer_eopnotsupp(bh) && printk_ratelimit()) {
136                         buffer_io_error(bh);
137                         printk(KERN_WARNING "lost page write due to "
138                                         "I/O error on %s\n",
139                                        bdevname(bh->b_bdev, b));
140                 }
141                 set_buffer_write_io_error(bh);
142                 clear_buffer_uptodate(bh);
143         }
144         unlock_buffer(bh);
145         put_bh(bh);
146 }
147
148 /*
149  * Write out and wait upon all the dirty data associated with a block
150  * device via its mapping.  Does not take the superblock lock.
151  */
152 int sync_blockdev(struct block_device *bdev)
153 {
154         int ret = 0;
155
156         if (bdev)
157                 ret = filemap_write_and_wait(bdev->bd_inode->i_mapping);
158         return ret;
159 }
160 EXPORT_SYMBOL(sync_blockdev);
161
162 static void __fsync_super(struct super_block *sb)
163 {
164         sync_inodes_sb(sb, 0);
165         DQUOT_SYNC(sb);
166         lock_super(sb);
167         if (sb->s_dirt && sb->s_op->write_super)
168                 sb->s_op->write_super(sb);
169         unlock_super(sb);
170         if (sb->s_op->sync_fs)
171                 sb->s_op->sync_fs(sb, 1);
172         sync_blockdev(sb->s_bdev);
173         sync_inodes_sb(sb, 1);
174 }
175
176 /*
177  * Write out and wait upon all dirty data associated with this
178  * superblock.  Filesystem data as well as the underlying block
179  * device.  Takes the superblock lock.
180  */
181 int fsync_super(struct super_block *sb)
182 {
183         __fsync_super(sb);
184         return sync_blockdev(sb->s_bdev);
185 }
186
187 /*
188  * Write out and wait upon all dirty data associated with this
189  * device.   Filesystem data as well as the underlying block
190  * device.  Takes the superblock lock.
191  */
192 int fsync_bdev(struct block_device *bdev)
193 {
194         struct super_block *sb = get_super(bdev);
195         if (sb) {
196                 int res = fsync_super(sb);
197                 drop_super(sb);
198                 return res;
199         }
200         return sync_blockdev(bdev);
201 }
202
203 /**
204  * freeze_bdev  --  lock a filesystem and force it into a consistent state
205  * @bdev:       blockdevice to lock
206  *
207  * This takes the block device bd_mount_mutex to make sure no new mounts
208  * happen on bdev until thaw_bdev() is called.
209  * If a superblock is found on this device, we take the s_umount semaphore
210  * on it to make sure nobody unmounts until the snapshot creation is done.
211  */
212 struct super_block *freeze_bdev(struct block_device *bdev)
213 {
214         struct super_block *sb;
215
216         mutex_lock(&bdev->bd_mount_mutex);
217         sb = get_super(bdev);
218         if (sb && !(sb->s_flags & MS_RDONLY)) {
219                 sb->s_frozen = SB_FREEZE_WRITE;
220                 smp_wmb();
221
222                 __fsync_super(sb);
223
224                 sb->s_frozen = SB_FREEZE_TRANS;
225                 smp_wmb();
226
227                 sync_blockdev(sb->s_bdev);
228
229                 if (sb->s_op->write_super_lockfs)
230                         sb->s_op->write_super_lockfs(sb);
231         }
232
233         sync_blockdev(bdev);
234         return sb;      /* thaw_bdev releases s->s_umount and bd_mount_sem */
235 }
236 EXPORT_SYMBOL(freeze_bdev);
237
238 /**
239  * thaw_bdev  -- unlock filesystem
240  * @bdev:       blockdevice to unlock
241  * @sb:         associated superblock
242  *
243  * Unlocks the filesystem and marks it writeable again after freeze_bdev().
244  */
245 void thaw_bdev(struct block_device *bdev, struct super_block *sb)
246 {
247         if (sb) {
248                 BUG_ON(sb->s_bdev != bdev);
249
250                 if (sb->s_op->unlockfs)
251                         sb->s_op->unlockfs(sb);
252                 sb->s_frozen = SB_UNFROZEN;
253                 smp_wmb();
254                 wake_up(&sb->s_wait_unfrozen);
255                 drop_super(sb);
256         }
257
258         mutex_unlock(&bdev->bd_mount_mutex);
259 }
260 EXPORT_SYMBOL(thaw_bdev);
261
262 /*
263  * sync everything.  Start out by waking pdflush, because that writes back
264  * all queues in parallel.
265  */
266 static void do_sync(unsigned long wait)
267 {
268         wakeup_pdflush(0);
269         sync_inodes(0);         /* All mappings, inodes and their blockdevs */
270         DQUOT_SYNC(NULL);
271         sync_supers();          /* Write the superblocks */
272         sync_filesystems(0);    /* Start syncing the filesystems */
273         sync_filesystems(wait); /* Waitingly sync the filesystems */
274         sync_inodes(wait);      /* Mappings, inodes and blockdevs, again. */
275         if (!wait)
276                 printk("Emergency Sync complete\n");
277         if (unlikely(laptop_mode))
278                 laptop_sync_completion();
279 }
280
281 asmlinkage long sys_sync(void)
282 {
283         do_sync(1);
284         return 0;
285 }
286
287 void emergency_sync(void)
288 {
289         pdflush_operation(do_sync, 0);
290 }
291
292 /*
293  * Generic function to fsync a file.
294  *
295  * filp may be NULL if called via the msync of a vma.
296  */
297  
298 int file_fsync(struct file *filp, struct dentry *dentry, int datasync)
299 {
300         struct inode * inode = dentry->d_inode;
301         struct super_block * sb;
302         int ret, err;
303
304         /* sync the inode to buffers */
305         ret = write_inode_now(inode, 0);
306
307         /* sync the superblock to buffers */
308         sb = inode->i_sb;
309         lock_super(sb);
310         if (sb->s_op->write_super)
311                 sb->s_op->write_super(sb);
312         unlock_super(sb);
313
314         /* .. finally sync the buffers to disk */
315         err = sync_blockdev(sb->s_bdev);
316         if (!ret)
317                 ret = err;
318         return ret;
319 }
320
321 long do_fsync(struct file *file, int datasync)
322 {
323         int ret;
324         int err;
325         struct address_space *mapping = file->f_mapping;
326
327         if (!file->f_op || !file->f_op->fsync) {
328                 /* Why?  We can still call filemap_fdatawrite */
329                 ret = -EINVAL;
330                 goto out;
331         }
332
333         ret = filemap_fdatawrite(mapping);
334
335         /*
336          * We need to protect against concurrent writers, which could cause
337          * livelocks in fsync_buffers_list().
338          */
339         mutex_lock(&mapping->host->i_mutex);
340         err = file->f_op->fsync(file, file->f_dentry, datasync);
341         if (!ret)
342                 ret = err;
343         mutex_unlock(&mapping->host->i_mutex);
344         err = filemap_fdatawait(mapping);
345         if (!ret)
346                 ret = err;
347 out:
348         return ret;
349 }
350
351 static long __do_fsync(unsigned int fd, int datasync)
352 {
353         struct file *file;
354         int ret = -EBADF;
355
356         file = fget(fd);
357         if (file) {
358                 ret = do_fsync(file, datasync);
359                 fput(file);
360         }
361         return ret;
362 }
363
364 asmlinkage long sys_fsync(unsigned int fd)
365 {
366         return __do_fsync(fd, 0);
367 }
368
369 asmlinkage long sys_fdatasync(unsigned int fd)
370 {
371         return __do_fsync(fd, 1);
372 }
373
374 /*
375  * Various filesystems appear to want __find_get_block to be non-blocking.
376  * But it's the page lock which protects the buffers.  To get around this,
377  * we get exclusion from try_to_free_buffers with the blockdev mapping's
378  * private_lock.
379  *
380  * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
381  * may be quite high.  This code could TryLock the page, and if that
382  * succeeds, there is no need to take private_lock. (But if
383  * private_lock is contended then so is mapping->tree_lock).
384  */
385 static struct buffer_head *
386 __find_get_block_slow(struct block_device *bdev, sector_t block)
387 {
388         struct inode *bd_inode = bdev->bd_inode;
389         struct address_space *bd_mapping = bd_inode->i_mapping;
390         struct buffer_head *ret = NULL;
391         pgoff_t index;
392         struct buffer_head *bh;
393         struct buffer_head *head;
394         struct page *page;
395         int all_mapped = 1;
396
397         index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
398         page = find_get_page(bd_mapping, index);
399         if (!page)
400                 goto out;
401
402         spin_lock(&bd_mapping->private_lock);
403         if (!page_has_buffers(page))
404                 goto out_unlock;
405         head = page_buffers(page);
406         bh = head;
407         do {
408                 if (bh->b_blocknr == block) {
409                         ret = bh;
410                         get_bh(bh);
411                         goto out_unlock;
412                 }
413                 if (!buffer_mapped(bh))
414                         all_mapped = 0;
415                 bh = bh->b_this_page;
416         } while (bh != head);
417
418         /* we might be here because some of the buffers on this page are
419          * not mapped.  This is due to various races between
420          * file io on the block device and getblk.  It gets dealt with
421          * elsewhere, don't buffer_error if we had some unmapped buffers
422          */
423         if (all_mapped) {
424                 printk("__find_get_block_slow() failed. "
425                         "block=%llu, b_blocknr=%llu\n",
426                         (unsigned long long)block,
427                         (unsigned long long)bh->b_blocknr);
428                 printk("b_state=0x%08lx, b_size=%zu\n",
429                         bh->b_state, bh->b_size);
430                 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
431         }
432 out_unlock:
433         spin_unlock(&bd_mapping->private_lock);
434         page_cache_release(page);
435 out:
436         return ret;
437 }
438
439 /* If invalidate_buffers() will trash dirty buffers, it means some kind
440    of fs corruption is going on. Trashing dirty data always imply losing
441    information that was supposed to be just stored on the physical layer
442    by the user.
443
444    Thus invalidate_buffers in general usage is not allwowed to trash
445    dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
446    be preserved.  These buffers are simply skipped.
447   
448    We also skip buffers which are still in use.  For example this can
449    happen if a userspace program is reading the block device.
450
451    NOTE: In the case where the user removed a removable-media-disk even if
452    there's still dirty data not synced on disk (due a bug in the device driver
453    or due an error of the user), by not destroying the dirty buffers we could
454    generate corruption also on the next media inserted, thus a parameter is
455    necessary to handle this case in the most safe way possible (trying
456    to not corrupt also the new disk inserted with the data belonging to
457    the old now corrupted disk). Also for the ramdisk the natural thing
458    to do in order to release the ramdisk memory is to destroy dirty buffers.
459
460    These are two special cases. Normal usage imply the device driver
461    to issue a sync on the device (without waiting I/O completion) and
462    then an invalidate_buffers call that doesn't trash dirty buffers.
463
464    For handling cache coherency with the blkdev pagecache the 'update' case
465    is been introduced. It is needed to re-read from disk any pinned
466    buffer. NOTE: re-reading from disk is destructive so we can do it only
467    when we assume nobody is changing the buffercache under our I/O and when
468    we think the disk contains more recent information than the buffercache.
469    The update == 1 pass marks the buffers we need to update, the update == 2
470    pass does the actual I/O. */
471 void invalidate_bdev(struct block_device *bdev, int destroy_dirty_buffers)
472 {
473         invalidate_bh_lrus();
474         /*
475          * FIXME: what about destroy_dirty_buffers?
476          * We really want to use invalidate_inode_pages2() for
477          * that, but not until that's cleaned up.
478          */
479         invalidate_inode_pages(bdev->bd_inode->i_mapping);
480 }
481
482 /*
483  * Kick pdflush then try to free up some ZONE_NORMAL memory.
484  */
485 static void free_more_memory(void)
486 {
487         struct zone **zones;
488         pg_data_t *pgdat;
489
490         wakeup_pdflush(1024);
491         yield();
492
493         for_each_online_pgdat(pgdat) {
494                 zones = pgdat->node_zonelists[gfp_zone(GFP_NOFS)].zones;
495                 if (*zones)
496                         try_to_free_pages(zones, GFP_NOFS);
497         }
498 }
499
500 /*
501  * I/O completion handler for block_read_full_page() - pages
502  * which come unlocked at the end of I/O.
503  */
504 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
505 {
506         unsigned long flags;
507         struct buffer_head *first;
508         struct buffer_head *tmp;
509         struct page *page;
510         int page_uptodate = 1;
511
512         BUG_ON(!buffer_async_read(bh));
513
514         page = bh->b_page;
515         if (uptodate) {
516                 set_buffer_uptodate(bh);
517         } else {
518                 clear_buffer_uptodate(bh);
519                 if (printk_ratelimit())
520                         buffer_io_error(bh);
521                 SetPageError(page);
522         }
523
524         /*
525          * Be _very_ careful from here on. Bad things can happen if
526          * two buffer heads end IO at almost the same time and both
527          * decide that the page is now completely done.
528          */
529         first = page_buffers(page);
530         local_irq_save(flags);
531         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
532         clear_buffer_async_read(bh);
533         unlock_buffer(bh);
534         tmp = bh;
535         do {
536                 if (!buffer_uptodate(tmp))
537                         page_uptodate = 0;
538                 if (buffer_async_read(tmp)) {
539                         BUG_ON(!buffer_locked(tmp));
540                         goto still_busy;
541                 }
542                 tmp = tmp->b_this_page;
543         } while (tmp != bh);
544         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
545         local_irq_restore(flags);
546
547         /*
548          * If none of the buffers had errors and they are all
549          * uptodate then we can set the page uptodate.
550          */
551         if (page_uptodate && !PageError(page))
552                 SetPageUptodate(page);
553         unlock_page(page);
554         return;
555
556 still_busy:
557         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
558         local_irq_restore(flags);
559         return;
560 }
561
562 /*
563  * Completion handler for block_write_full_page() - pages which are unlocked
564  * during I/O, and which have PageWriteback cleared upon I/O completion.
565  */
566 static void end_buffer_async_write(struct buffer_head *bh, int uptodate)
567 {
568         char b[BDEVNAME_SIZE];
569         unsigned long flags;
570         struct buffer_head *first;
571         struct buffer_head *tmp;
572         struct page *page;
573
574         BUG_ON(!buffer_async_write(bh));
575
576         page = bh->b_page;
577         if (uptodate) {
578                 set_buffer_uptodate(bh);
579         } else {
580                 if (printk_ratelimit()) {
581                         buffer_io_error(bh);
582                         printk(KERN_WARNING "lost page write due to "
583                                         "I/O error on %s\n",
584                                bdevname(bh->b_bdev, b));
585                 }
586                 set_bit(AS_EIO, &page->mapping->flags);
587                 clear_buffer_uptodate(bh);
588                 SetPageError(page);
589         }
590
591         first = page_buffers(page);
592         local_irq_save(flags);
593         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
594
595         clear_buffer_async_write(bh);
596         unlock_buffer(bh);
597         tmp = bh->b_this_page;
598         while (tmp != bh) {
599                 if (buffer_async_write(tmp)) {
600                         BUG_ON(!buffer_locked(tmp));
601                         goto still_busy;
602                 }
603                 tmp = tmp->b_this_page;
604         }
605         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
606         local_irq_restore(flags);
607         end_page_writeback(page);
608         return;
609
610 still_busy:
611         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
612         local_irq_restore(flags);
613         return;
614 }
615
616 /*
617  * If a page's buffers are under async readin (end_buffer_async_read
618  * completion) then there is a possibility that another thread of
619  * control could lock one of the buffers after it has completed
620  * but while some of the other buffers have not completed.  This
621  * locked buffer would confuse end_buffer_async_read() into not unlocking
622  * the page.  So the absence of BH_Async_Read tells end_buffer_async_read()
623  * that this buffer is not under async I/O.
624  *
625  * The page comes unlocked when it has no locked buffer_async buffers
626  * left.
627  *
628  * PageLocked prevents anyone starting new async I/O reads any of
629  * the buffers.
630  *
631  * PageWriteback is used to prevent simultaneous writeout of the same
632  * page.
633  *
634  * PageLocked prevents anyone from starting writeback of a page which is
635  * under read I/O (PageWriteback is only ever set against a locked page).
636  */
637 static void mark_buffer_async_read(struct buffer_head *bh)
638 {
639         bh->b_end_io = end_buffer_async_read;
640         set_buffer_async_read(bh);
641 }
642
643 void mark_buffer_async_write(struct buffer_head *bh)
644 {
645         bh->b_end_io = end_buffer_async_write;
646         set_buffer_async_write(bh);
647 }
648 EXPORT_SYMBOL(mark_buffer_async_write);
649
650
651 /*
652  * fs/buffer.c contains helper functions for buffer-backed address space's
653  * fsync functions.  A common requirement for buffer-based filesystems is
654  * that certain data from the backing blockdev needs to be written out for
655  * a successful fsync().  For example, ext2 indirect blocks need to be
656  * written back and waited upon before fsync() returns.
657  *
658  * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
659  * inode_has_buffers() and invalidate_inode_buffers() are provided for the
660  * management of a list of dependent buffers at ->i_mapping->private_list.
661  *
662  * Locking is a little subtle: try_to_free_buffers() will remove buffers
663  * from their controlling inode's queue when they are being freed.  But
664  * try_to_free_buffers() will be operating against the *blockdev* mapping
665  * at the time, not against the S_ISREG file which depends on those buffers.
666  * So the locking for private_list is via the private_lock in the address_space
667  * which backs the buffers.  Which is different from the address_space 
668  * against which the buffers are listed.  So for a particular address_space,
669  * mapping->private_lock does *not* protect mapping->private_list!  In fact,
670  * mapping->private_list will always be protected by the backing blockdev's
671  * ->private_lock.
672  *
673  * Which introduces a requirement: all buffers on an address_space's
674  * ->private_list must be from the same address_space: the blockdev's.
675  *
676  * address_spaces which do not place buffers at ->private_list via these
677  * utility functions are free to use private_lock and private_list for
678  * whatever they want.  The only requirement is that list_empty(private_list)
679  * be true at clear_inode() time.
680  *
681  * FIXME: clear_inode should not call invalidate_inode_buffers().  The
682  * filesystems should do that.  invalidate_inode_buffers() should just go
683  * BUG_ON(!list_empty).
684  *
685  * FIXME: mark_buffer_dirty_inode() is a data-plane operation.  It should
686  * take an address_space, not an inode.  And it should be called
687  * mark_buffer_dirty_fsync() to clearly define why those buffers are being
688  * queued up.
689  *
690  * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
691  * list if it is already on a list.  Because if the buffer is on a list,
692  * it *must* already be on the right one.  If not, the filesystem is being
693  * silly.  This will save a ton of locking.  But first we have to ensure
694  * that buffers are taken *off* the old inode's list when they are freed
695  * (presumably in truncate).  That requires careful auditing of all
696  * filesystems (do it inside bforget()).  It could also be done by bringing
697  * b_inode back.
698  */
699
700 /*
701  * The buffer's backing address_space's private_lock must be held
702  */
703 static inline void __remove_assoc_queue(struct buffer_head *bh)
704 {
705         list_del_init(&bh->b_assoc_buffers);
706 }
707
708 int inode_has_buffers(struct inode *inode)
709 {
710         return !list_empty(&inode->i_data.private_list);
711 }
712
713 /*
714  * osync is designed to support O_SYNC io.  It waits synchronously for
715  * all already-submitted IO to complete, but does not queue any new
716  * writes to the disk.
717  *
718  * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
719  * you dirty the buffers, and then use osync_inode_buffers to wait for
720  * completion.  Any other dirty buffers which are not yet queued for
721  * write will not be flushed to disk by the osync.
722  */
723 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
724 {
725         struct buffer_head *bh;
726         struct list_head *p;
727         int err = 0;
728
729         spin_lock(lock);
730 repeat:
731         list_for_each_prev(p, list) {
732                 bh = BH_ENTRY(p);
733                 if (buffer_locked(bh)) {
734                         get_bh(bh);
735                         spin_unlock(lock);
736                         wait_on_buffer(bh);
737                         if (!buffer_uptodate(bh))
738                                 err = -EIO;
739                         brelse(bh);
740                         spin_lock(lock);
741                         goto repeat;
742                 }
743         }
744         spin_unlock(lock);
745         return err;
746 }
747
748 /**
749  * sync_mapping_buffers - write out and wait upon a mapping's "associated"
750  *                        buffers
751  * @mapping: the mapping which wants those buffers written
752  *
753  * Starts I/O against the buffers at mapping->private_list, and waits upon
754  * that I/O.
755  *
756  * Basically, this is a convenience function for fsync().
757  * @mapping is a file or directory which needs those buffers to be written for
758  * a successful fsync().
759  */
760 int sync_mapping_buffers(struct address_space *mapping)
761 {
762         struct address_space *buffer_mapping = mapping->assoc_mapping;
763
764         if (buffer_mapping == NULL || list_empty(&mapping->private_list))
765                 return 0;
766
767         return fsync_buffers_list(&buffer_mapping->private_lock,
768                                         &mapping->private_list);
769 }
770 EXPORT_SYMBOL(sync_mapping_buffers);
771
772 /*
773  * Called when we've recently written block `bblock', and it is known that
774  * `bblock' was for a buffer_boundary() buffer.  This means that the block at
775  * `bblock + 1' is probably a dirty indirect block.  Hunt it down and, if it's
776  * dirty, schedule it for IO.  So that indirects merge nicely with their data.
777  */
778 void write_boundary_block(struct block_device *bdev,
779                         sector_t bblock, unsigned blocksize)
780 {
781         struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
782         if (bh) {
783                 if (buffer_dirty(bh))
784                         ll_rw_block(WRITE, 1, &bh);
785                 put_bh(bh);
786         }
787 }
788
789 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
790 {
791         struct address_space *mapping = inode->i_mapping;
792         struct address_space *buffer_mapping = bh->b_page->mapping;
793
794         mark_buffer_dirty(bh);
795         if (!mapping->assoc_mapping) {
796                 mapping->assoc_mapping = buffer_mapping;
797         } else {
798                 BUG_ON(mapping->assoc_mapping != buffer_mapping);
799         }
800         if (list_empty(&bh->b_assoc_buffers)) {
801                 spin_lock(&buffer_mapping->private_lock);
802                 list_move_tail(&bh->b_assoc_buffers,
803                                 &mapping->private_list);
804                 spin_unlock(&buffer_mapping->private_lock);
805         }
806 }
807 EXPORT_SYMBOL(mark_buffer_dirty_inode);
808
809 /*
810  * Add a page to the dirty page list.
811  *
812  * It is a sad fact of life that this function is called from several places
813  * deeply under spinlocking.  It may not sleep.
814  *
815  * If the page has buffers, the uptodate buffers are set dirty, to preserve
816  * dirty-state coherency between the page and the buffers.  It the page does
817  * not have buffers then when they are later attached they will all be set
818  * dirty.
819  *
820  * The buffers are dirtied before the page is dirtied.  There's a small race
821  * window in which a writepage caller may see the page cleanness but not the
822  * buffer dirtiness.  That's fine.  If this code were to set the page dirty
823  * before the buffers, a concurrent writepage caller could clear the page dirty
824  * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
825  * page on the dirty page list.
826  *
827  * We use private_lock to lock against try_to_free_buffers while using the
828  * page's buffer list.  Also use this to protect against clean buffers being
829  * added to the page after it was set dirty.
830  *
831  * FIXME: may need to call ->reservepage here as well.  That's rather up to the
832  * address_space though.
833  */
834 int __set_page_dirty_buffers(struct page *page)
835 {
836         struct address_space * const mapping = page->mapping;
837
838         spin_lock(&mapping->private_lock);
839         if (page_has_buffers(page)) {
840                 struct buffer_head *head = page_buffers(page);
841                 struct buffer_head *bh = head;
842
843                 do {
844                         set_buffer_dirty(bh);
845                         bh = bh->b_this_page;
846                 } while (bh != head);
847         }
848         spin_unlock(&mapping->private_lock);
849
850         if (!TestSetPageDirty(page)) {
851                 write_lock_irq(&mapping->tree_lock);
852                 if (page->mapping) {    /* Race with truncate? */
853                         if (mapping_cap_account_dirty(mapping))
854                                 __inc_zone_page_state(page, NR_FILE_DIRTY);
855                         radix_tree_tag_set(&mapping->page_tree,
856                                                 page_index(page),
857                                                 PAGECACHE_TAG_DIRTY);
858                 }
859                 write_unlock_irq(&mapping->tree_lock);
860                 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
861                 return 1;
862         }
863         return 0;
864 }
865 EXPORT_SYMBOL(__set_page_dirty_buffers);
866
867 /*
868  * Write out and wait upon a list of buffers.
869  *
870  * We have conflicting pressures: we want to make sure that all
871  * initially dirty buffers get waited on, but that any subsequently
872  * dirtied buffers don't.  After all, we don't want fsync to last
873  * forever if somebody is actively writing to the file.
874  *
875  * Do this in two main stages: first we copy dirty buffers to a
876  * temporary inode list, queueing the writes as we go.  Then we clean
877  * up, waiting for those writes to complete.
878  * 
879  * During this second stage, any subsequent updates to the file may end
880  * up refiling the buffer on the original inode's dirty list again, so
881  * there is a chance we will end up with a buffer queued for write but
882  * not yet completed on that list.  So, as a final cleanup we go through
883  * the osync code to catch these locked, dirty buffers without requeuing
884  * any newly dirty buffers for write.
885  */
886 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
887 {
888         struct buffer_head *bh;
889         struct list_head tmp;
890         int err = 0, err2;
891
892         INIT_LIST_HEAD(&tmp);
893
894         spin_lock(lock);
895         while (!list_empty(list)) {
896                 bh = BH_ENTRY(list->next);
897                 list_del_init(&bh->b_assoc_buffers);
898                 if (buffer_dirty(bh) || buffer_locked(bh)) {
899                         list_add(&bh->b_assoc_buffers, &tmp);
900                         if (buffer_dirty(bh)) {
901                                 get_bh(bh);
902                                 spin_unlock(lock);
903                                 /*
904                                  * Ensure any pending I/O completes so that
905                                  * ll_rw_block() actually writes the current
906                                  * contents - it is a noop if I/O is still in
907                                  * flight on potentially older contents.
908                                  */
909                                 ll_rw_block(SWRITE, 1, &bh);
910                                 brelse(bh);
911                                 spin_lock(lock);
912                         }
913                 }
914         }
915
916         while (!list_empty(&tmp)) {
917                 bh = BH_ENTRY(tmp.prev);
918                 __remove_assoc_queue(bh);
919                 get_bh(bh);
920                 spin_unlock(lock);
921                 wait_on_buffer(bh);
922                 if (!buffer_uptodate(bh))
923                         err = -EIO;
924                 brelse(bh);
925                 spin_lock(lock);
926         }
927         
928         spin_unlock(lock);
929         err2 = osync_buffers_list(lock, list);
930         if (err)
931                 return err;
932         else
933                 return err2;
934 }
935
936 /*
937  * Invalidate any and all dirty buffers on a given inode.  We are
938  * probably unmounting the fs, but that doesn't mean we have already
939  * done a sync().  Just drop the buffers from the inode list.
940  *
941  * NOTE: we take the inode's blockdev's mapping's private_lock.  Which
942  * assumes that all the buffers are against the blockdev.  Not true
943  * for reiserfs.
944  */
945 void invalidate_inode_buffers(struct inode *inode)
946 {
947         if (inode_has_buffers(inode)) {
948                 struct address_space *mapping = &inode->i_data;
949                 struct list_head *list = &mapping->private_list;
950                 struct address_space *buffer_mapping = mapping->assoc_mapping;
951
952                 spin_lock(&buffer_mapping->private_lock);
953                 while (!list_empty(list))
954                         __remove_assoc_queue(BH_ENTRY(list->next));
955                 spin_unlock(&buffer_mapping->private_lock);
956         }
957 }
958
959 /*
960  * Remove any clean buffers from the inode's buffer list.  This is called
961  * when we're trying to free the inode itself.  Those buffers can pin it.
962  *
963  * Returns true if all buffers were removed.
964  */
965 int remove_inode_buffers(struct inode *inode)
966 {
967         int ret = 1;
968
969         if (inode_has_buffers(inode)) {
970                 struct address_space *mapping = &inode->i_data;
971                 struct list_head *list = &mapping->private_list;
972                 struct address_space *buffer_mapping = mapping->assoc_mapping;
973
974                 spin_lock(&buffer_mapping->private_lock);
975                 while (!list_empty(list)) {
976                         struct buffer_head *bh = BH_ENTRY(list->next);
977                         if (buffer_dirty(bh)) {
978                                 ret = 0;
979                                 break;
980                         }
981                         __remove_assoc_queue(bh);
982                 }
983                 spin_unlock(&buffer_mapping->private_lock);
984         }
985         return ret;
986 }
987
988 /*
989  * Create the appropriate buffers when given a page for data area and
990  * the size of each buffer.. Use the bh->b_this_page linked list to
991  * follow the buffers created.  Return NULL if unable to create more
992  * buffers.
993  *
994  * The retry flag is used to differentiate async IO (paging, swapping)
995  * which may not fail from ordinary buffer allocations.
996  */
997 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
998                 int retry)
999 {
1000         struct buffer_head *bh, *head;
1001         long offset;
1002
1003 try_again:
1004         head = NULL;
1005         offset = PAGE_SIZE;
1006         while ((offset -= size) >= 0) {
1007                 bh = alloc_buffer_head(GFP_NOFS);
1008                 if (!bh)
1009                         goto no_grow;
1010
1011                 bh->b_bdev = NULL;
1012                 bh->b_this_page = head;
1013                 bh->b_blocknr = -1;
1014                 head = bh;
1015
1016                 bh->b_state = 0;
1017                 atomic_set(&bh->b_count, 0);
1018                 bh->b_private = NULL;
1019                 bh->b_size = size;
1020
1021                 /* Link the buffer to its page */
1022                 set_bh_page(bh, page, offset);
1023
1024                 init_buffer(bh, NULL, NULL);
1025         }
1026         return head;
1027 /*
1028  * In case anything failed, we just free everything we got.
1029  */
1030 no_grow:
1031         if (head) {
1032                 do {
1033                         bh = head;
1034                         head = head->b_this_page;
1035                         free_buffer_head(bh);
1036                 } while (head);
1037         }
1038
1039         /*
1040          * Return failure for non-async IO requests.  Async IO requests
1041          * are not allowed to fail, so we have to wait until buffer heads
1042          * become available.  But we don't want tasks sleeping with 
1043          * partially complete buffers, so all were released above.
1044          */
1045         if (!retry)
1046                 return NULL;
1047
1048         /* We're _really_ low on memory. Now we just
1049          * wait for old buffer heads to become free due to
1050          * finishing IO.  Since this is an async request and
1051          * the reserve list is empty, we're sure there are 
1052          * async buffer heads in use.
1053          */
1054         free_more_memory();
1055         goto try_again;
1056 }
1057 EXPORT_SYMBOL_GPL(alloc_page_buffers);
1058
1059 static inline void
1060 link_dev_buffers(struct page *page, struct buffer_head *head)
1061 {
1062         struct buffer_head *bh, *tail;
1063
1064         bh = head;
1065         do {
1066                 tail = bh;
1067                 bh = bh->b_this_page;
1068         } while (bh);
1069         tail->b_this_page = head;
1070         attach_page_buffers(page, head);
1071 }
1072
1073 /*
1074  * Initialise the state of a blockdev page's buffers.
1075  */ 
1076 static void
1077 init_page_buffers(struct page *page, struct block_device *bdev,
1078                         sector_t block, int size)
1079 {
1080         struct buffer_head *head = page_buffers(page);
1081         struct buffer_head *bh = head;
1082         int uptodate = PageUptodate(page);
1083
1084         do {
1085                 if (!buffer_mapped(bh)) {
1086                         init_buffer(bh, NULL, NULL);
1087                         bh->b_bdev = bdev;
1088                         bh->b_blocknr = block;
1089                         if (uptodate)
1090                                 set_buffer_uptodate(bh);
1091                         set_buffer_mapped(bh);
1092                 }
1093                 block++;
1094                 bh = bh->b_this_page;
1095         } while (bh != head);
1096 }
1097
1098 /*
1099  * Create the page-cache page that contains the requested block.
1100  *
1101  * This is user purely for blockdev mappings.
1102  */
1103 static struct page *
1104 grow_dev_page(struct block_device *bdev, sector_t block,
1105                 pgoff_t index, int size)
1106 {
1107         struct inode *inode = bdev->bd_inode;
1108         struct page *page;
1109         struct buffer_head *bh;
1110
1111         page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
1112         if (!page)
1113                 return NULL;
1114
1115         BUG_ON(!PageLocked(page));
1116
1117         if (page_has_buffers(page)) {
1118                 bh = page_buffers(page);
1119                 if (bh->b_size == size) {
1120                         init_page_buffers(page, bdev, block, size);
1121                         return page;
1122                 }
1123                 if (!try_to_free_buffers(page))
1124                         goto failed;
1125         }
1126
1127         /*
1128          * Allocate some buffers for this page
1129          */
1130         bh = alloc_page_buffers(page, size, 0);
1131         if (!bh)
1132                 goto failed;
1133
1134         /*
1135          * Link the page to the buffers and initialise them.  Take the
1136          * lock to be atomic wrt __find_get_block(), which does not
1137          * run under the page lock.
1138          */
1139         spin_lock(&inode->i_mapping->private_lock);
1140         link_dev_buffers(page, bh);
1141         init_page_buffers(page, bdev, block, size);
1142         spin_unlock(&inode->i_mapping->private_lock);
1143         return page;
1144
1145 failed:
1146         BUG();
1147         unlock_page(page);
1148         page_cache_release(page);
1149         return NULL;
1150 }
1151
1152 /*
1153  * Create buffers for the specified block device block's page.  If
1154  * that page was dirty, the buffers are set dirty also.
1155  *
1156  * Except that's a bug.  Attaching dirty buffers to a dirty
1157  * blockdev's page can result in filesystem corruption, because
1158  * some of those buffers may be aliases of filesystem data.
1159  * grow_dev_page() will go BUG() if this happens.
1160  */
1161 static int
1162 grow_buffers(struct block_device *bdev, sector_t block, int size)
1163 {
1164         struct page *page;
1165         pgoff_t index;
1166         int sizebits;
1167
1168         sizebits = -1;
1169         do {
1170                 sizebits++;
1171         } while ((size << sizebits) < PAGE_SIZE);
1172
1173         index = block >> sizebits;
1174         block = index << sizebits;
1175
1176         /* Create a page with the proper size buffers.. */
1177         page = grow_dev_page(bdev, block, index, size);
1178         if (!page)
1179                 return 0;
1180         unlock_page(page);
1181         page_cache_release(page);
1182         return 1;
1183 }
1184
1185 static struct buffer_head *
1186 __getblk_slow(struct block_device *bdev, sector_t block, int size)
1187 {
1188         /* Size must be multiple of hard sectorsize */
1189         if (unlikely(size & (bdev_hardsect_size(bdev)-1) ||
1190                         (size < 512 || size > PAGE_SIZE))) {
1191                 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1192                                         size);
1193                 printk(KERN_ERR "hardsect size: %d\n",
1194                                         bdev_hardsect_size(bdev));
1195
1196                 dump_stack();
1197                 return NULL;
1198         }
1199
1200         for (;;) {
1201                 struct buffer_head * bh;
1202
1203                 bh = __find_get_block(bdev, block, size);
1204                 if (bh)
1205                         return bh;
1206
1207                 if (!grow_buffers(bdev, block, size))
1208                         free_more_memory();
1209         }
1210 }
1211
1212 /*
1213  * The relationship between dirty buffers and dirty pages:
1214  *
1215  * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1216  * the page is tagged dirty in its radix tree.
1217  *
1218  * At all times, the dirtiness of the buffers represents the dirtiness of
1219  * subsections of the page.  If the page has buffers, the page dirty bit is
1220  * merely a hint about the true dirty state.
1221  *
1222  * When a page is set dirty in its entirety, all its buffers are marked dirty
1223  * (if the page has buffers).
1224  *
1225  * When a buffer is marked dirty, its page is dirtied, but the page's other
1226  * buffers are not.
1227  *
1228  * Also.  When blockdev buffers are explicitly read with bread(), they
1229  * individually become uptodate.  But their backing page remains not
1230  * uptodate - even if all of its buffers are uptodate.  A subsequent
1231  * block_read_full_page() against that page will discover all the uptodate
1232  * buffers, will set the page uptodate and will perform no I/O.
1233  */
1234
1235 /**
1236  * mark_buffer_dirty - mark a buffer_head as needing writeout
1237  * @bh: the buffer_head to mark dirty
1238  *
1239  * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1240  * backing page dirty, then tag the page as dirty in its address_space's radix
1241  * tree and then attach the address_space's inode to its superblock's dirty
1242  * inode list.
1243  *
1244  * mark_buffer_dirty() is atomic.  It takes bh->b_page->mapping->private_lock,
1245  * mapping->tree_lock and the global inode_lock.
1246  */
1247 void fastcall mark_buffer_dirty(struct buffer_head *bh)
1248 {
1249         if (!buffer_dirty(bh) && !test_set_buffer_dirty(bh))
1250                 __set_page_dirty_nobuffers(bh->b_page);
1251 }
1252
1253 /*
1254  * Decrement a buffer_head's reference count.  If all buffers against a page
1255  * have zero reference count, are clean and unlocked, and if the page is clean
1256  * and unlocked then try_to_free_buffers() may strip the buffers from the page
1257  * in preparation for freeing it (sometimes, rarely, buffers are removed from
1258  * a page but it ends up not being freed, and buffers may later be reattached).
1259  */
1260 void __brelse(struct buffer_head * buf)
1261 {
1262         if (atomic_read(&buf->b_count)) {
1263                 put_bh(buf);
1264                 return;
1265         }
1266         printk(KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1267         WARN_ON(1);
1268 }
1269
1270 /*
1271  * bforget() is like brelse(), except it discards any
1272  * potentially dirty data.
1273  */
1274 void __bforget(struct buffer_head *bh)
1275 {
1276         clear_buffer_dirty(bh);
1277         if (!list_empty(&bh->b_assoc_buffers)) {
1278                 struct address_space *buffer_mapping = bh->b_page->mapping;
1279
1280                 spin_lock(&buffer_mapping->private_lock);
1281                 list_del_init(&bh->b_assoc_buffers);
1282                 spin_unlock(&buffer_mapping->private_lock);
1283         }
1284         __brelse(bh);
1285 }
1286
1287 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1288 {
1289         lock_buffer(bh);
1290         if (buffer_uptodate(bh)) {
1291                 unlock_buffer(bh);
1292                 return bh;
1293         } else {
1294                 get_bh(bh);
1295                 bh->b_end_io = end_buffer_read_sync;
1296                 submit_bh(READ, bh);
1297                 wait_on_buffer(bh);
1298                 if (buffer_uptodate(bh))
1299                         return bh;
1300         }
1301         brelse(bh);
1302         return NULL;
1303 }
1304
1305 /*
1306  * Per-cpu buffer LRU implementation.  To reduce the cost of __find_get_block().
1307  * The bhs[] array is sorted - newest buffer is at bhs[0].  Buffers have their
1308  * refcount elevated by one when they're in an LRU.  A buffer can only appear
1309  * once in a particular CPU's LRU.  A single buffer can be present in multiple
1310  * CPU's LRUs at the same time.
1311  *
1312  * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1313  * sb_find_get_block().
1314  *
1315  * The LRUs themselves only need locking against invalidate_bh_lrus.  We use
1316  * a local interrupt disable for that.
1317  */
1318
1319 #define BH_LRU_SIZE     8
1320
1321 struct bh_lru {
1322         struct buffer_head *bhs[BH_LRU_SIZE];
1323 };
1324
1325 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1326
1327 #ifdef CONFIG_SMP
1328 #define bh_lru_lock()   local_irq_disable()
1329 #define bh_lru_unlock() local_irq_enable()
1330 #else
1331 #define bh_lru_lock()   preempt_disable()
1332 #define bh_lru_unlock() preempt_enable()
1333 #endif
1334
1335 static inline void check_irqs_on(void)
1336 {
1337 #ifdef irqs_disabled
1338         BUG_ON(irqs_disabled());
1339 #endif
1340 }
1341
1342 /*
1343  * The LRU management algorithm is dopey-but-simple.  Sorry.
1344  */
1345 static void bh_lru_install(struct buffer_head *bh)
1346 {
1347         struct buffer_head *evictee = NULL;
1348         struct bh_lru *lru;
1349
1350         check_irqs_on();
1351         bh_lru_lock();
1352         lru = &__get_cpu_var(bh_lrus);
1353         if (lru->bhs[0] != bh) {
1354                 struct buffer_head *bhs[BH_LRU_SIZE];
1355                 int in;
1356                 int out = 0;
1357
1358                 get_bh(bh);
1359                 bhs[out++] = bh;
1360                 for (in = 0; in < BH_LRU_SIZE; in++) {
1361                         struct buffer_head *bh2 = lru->bhs[in];
1362
1363                         if (bh2 == bh) {
1364                                 __brelse(bh2);
1365                         } else {
1366                                 if (out >= BH_LRU_SIZE) {
1367                                         BUG_ON(evictee != NULL);
1368                                         evictee = bh2;
1369                                 } else {
1370                                         bhs[out++] = bh2;
1371                                 }
1372                         }
1373                 }
1374                 while (out < BH_LRU_SIZE)
1375                         bhs[out++] = NULL;
1376                 memcpy(lru->bhs, bhs, sizeof(bhs));
1377         }
1378         bh_lru_unlock();
1379
1380         if (evictee)
1381                 __brelse(evictee);
1382 }
1383
1384 /*
1385  * Look up the bh in this cpu's LRU.  If it's there, move it to the head.
1386  */
1387 static struct buffer_head *
1388 lookup_bh_lru(struct block_device *bdev, sector_t block, int size)
1389 {
1390         struct buffer_head *ret = NULL;
1391         struct bh_lru *lru;
1392         int i;
1393
1394         check_irqs_on();
1395         bh_lru_lock();
1396         lru = &__get_cpu_var(bh_lrus);
1397         for (i = 0; i < BH_LRU_SIZE; i++) {
1398                 struct buffer_head *bh = lru->bhs[i];
1399
1400                 if (bh && bh->b_bdev == bdev &&
1401                                 bh->b_blocknr == block && bh->b_size == size) {
1402                         if (i) {
1403                                 while (i) {
1404                                         lru->bhs[i] = lru->bhs[i - 1];
1405                                         i--;
1406                                 }
1407                                 lru->bhs[0] = bh;
1408                         }
1409                         get_bh(bh);
1410                         ret = bh;
1411                         break;
1412                 }
1413         }
1414         bh_lru_unlock();
1415         return ret;
1416 }
1417
1418 /*
1419  * Perform a pagecache lookup for the matching buffer.  If it's there, refresh
1420  * it in the LRU and mark it as accessed.  If it is not present then return
1421  * NULL
1422  */
1423 struct buffer_head *
1424 __find_get_block(struct block_device *bdev, sector_t block, int size)
1425 {
1426         struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1427
1428         if (bh == NULL) {
1429                 bh = __find_get_block_slow(bdev, block);
1430                 if (bh)
1431                         bh_lru_install(bh);
1432         }
1433         if (bh)
1434                 touch_buffer(bh);
1435         return bh;
1436 }
1437 EXPORT_SYMBOL(__find_get_block);
1438
1439 /*
1440  * __getblk will locate (and, if necessary, create) the buffer_head
1441  * which corresponds to the passed block_device, block and size. The
1442  * returned buffer has its reference count incremented.
1443  *
1444  * __getblk() cannot fail - it just keeps trying.  If you pass it an
1445  * illegal block number, __getblk() will happily return a buffer_head
1446  * which represents the non-existent block.  Very weird.
1447  *
1448  * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1449  * attempt is failing.  FIXME, perhaps?
1450  */
1451 struct buffer_head *
1452 __getblk(struct block_device *bdev, sector_t block, int size)
1453 {
1454         struct buffer_head *bh = __find_get_block(bdev, block, size);
1455
1456         might_sleep();
1457         if (bh == NULL)
1458                 bh = __getblk_slow(bdev, block, size);
1459         return bh;
1460 }
1461 EXPORT_SYMBOL(__getblk);
1462
1463 /*
1464  * Do async read-ahead on a buffer..
1465  */
1466 void __breadahead(struct block_device *bdev, sector_t block, int size)
1467 {
1468         struct buffer_head *bh = __getblk(bdev, block, size);
1469         if (likely(bh)) {
1470                 ll_rw_block(READA, 1, &bh);
1471                 brelse(bh);
1472         }
1473 }
1474 EXPORT_SYMBOL(__breadahead);
1475
1476 /**
1477  *  __bread() - reads a specified block and returns the bh
1478  *  @bdev: the block_device to read from
1479  *  @block: number of block
1480  *  @size: size (in bytes) to read
1481  * 
1482  *  Reads a specified block, and returns buffer head that contains it.
1483  *  It returns NULL if the block was unreadable.
1484  */
1485 struct buffer_head *
1486 __bread(struct block_device *bdev, sector_t block, int size)
1487 {
1488         struct buffer_head *bh = __getblk(bdev, block, size);
1489
1490         if (likely(bh) && !buffer_uptodate(bh))
1491                 bh = __bread_slow(bh);
1492         return bh;
1493 }
1494 EXPORT_SYMBOL(__bread);
1495
1496 /*
1497  * invalidate_bh_lrus() is called rarely - but not only at unmount.
1498  * This doesn't race because it runs in each cpu either in irq
1499  * or with preempt disabled.
1500  */
1501 static void invalidate_bh_lru(void *arg)
1502 {
1503         struct bh_lru *b = &get_cpu_var(bh_lrus);
1504         int i;
1505
1506         for (i = 0; i < BH_LRU_SIZE; i++) {
1507                 brelse(b->bhs[i]);
1508                 b->bhs[i] = NULL;
1509         }
1510         put_cpu_var(bh_lrus);
1511 }
1512         
1513 static void invalidate_bh_lrus(void)
1514 {
1515         on_each_cpu(invalidate_bh_lru, NULL, 1, 1);
1516 }
1517
1518 void set_bh_page(struct buffer_head *bh,
1519                 struct page *page, unsigned long offset)
1520 {
1521         bh->b_page = page;
1522         BUG_ON(offset >= PAGE_SIZE);
1523         if (PageHighMem(page))
1524                 /*
1525                  * This catches illegal uses and preserves the offset:
1526                  */
1527                 bh->b_data = (char *)(0 + offset);
1528         else
1529                 bh->b_data = page_address(page) + offset;
1530 }
1531 EXPORT_SYMBOL(set_bh_page);
1532
1533 /*
1534  * Called when truncating a buffer on a page completely.
1535  */
1536 static void discard_buffer(struct buffer_head * bh)
1537 {
1538         lock_buffer(bh);
1539         clear_buffer_dirty(bh);
1540         bh->b_bdev = NULL;
1541         clear_buffer_mapped(bh);
1542         clear_buffer_req(bh);
1543         clear_buffer_new(bh);
1544         clear_buffer_delay(bh);
1545         unlock_buffer(bh);
1546 }
1547
1548 /**
1549  * try_to_release_page() - release old fs-specific metadata on a page
1550  *
1551  * @page: the page which the kernel is trying to free
1552  * @gfp_mask: memory allocation flags (and I/O mode)
1553  *
1554  * The address_space is to try to release any data against the page
1555  * (presumably at page->private).  If the release was successful, return `1'.
1556  * Otherwise return zero.
1557  *
1558  * The @gfp_mask argument specifies whether I/O may be performed to release
1559  * this page (__GFP_IO), and whether the call may block (__GFP_WAIT).
1560  *
1561  * NOTE: @gfp_mask may go away, and this function may become non-blocking.
1562  */
1563 int try_to_release_page(struct page *page, gfp_t gfp_mask)
1564 {
1565         struct address_space * const mapping = page->mapping;
1566
1567         BUG_ON(!PageLocked(page));
1568         if (PageWriteback(page))
1569                 return 0;
1570         
1571         if (mapping && mapping->a_ops->releasepage)
1572                 return mapping->a_ops->releasepage(page, gfp_mask);
1573         return try_to_free_buffers(page);
1574 }
1575 EXPORT_SYMBOL(try_to_release_page);
1576
1577 /**
1578  * block_invalidatepage - invalidate part of all of a buffer-backed page
1579  *
1580  * @page: the page which is affected
1581  * @offset: the index of the truncation point
1582  *
1583  * block_invalidatepage() is called when all or part of the page has become
1584  * invalidatedby a truncate operation.
1585  *
1586  * block_invalidatepage() does not have to release all buffers, but it must
1587  * ensure that no dirty buffer is left outside @offset and that no I/O
1588  * is underway against any of the blocks which are outside the truncation
1589  * point.  Because the caller is about to free (and possibly reuse) those
1590  * blocks on-disk.
1591  */
1592 void block_invalidatepage(struct page *page, unsigned long offset)
1593 {
1594         struct buffer_head *head, *bh, *next;
1595         unsigned int curr_off = 0;
1596
1597         BUG_ON(!PageLocked(page));
1598         if (!page_has_buffers(page))
1599                 goto out;
1600
1601         head = page_buffers(page);
1602         bh = head;
1603         do {
1604                 unsigned int next_off = curr_off + bh->b_size;
1605                 next = bh->b_this_page;
1606
1607                 /*
1608                  * is this block fully invalidated?
1609                  */
1610                 if (offset <= curr_off)
1611                         discard_buffer(bh);
1612                 curr_off = next_off;
1613                 bh = next;
1614         } while (bh != head);
1615
1616         /*
1617          * We release buffers only if the entire page is being invalidated.
1618          * The get_block cached value has been unconditionally invalidated,
1619          * so real IO is not possible anymore.
1620          */
1621         if (offset == 0)
1622                 try_to_release_page(page, 0);
1623 out:
1624         return;
1625 }
1626 EXPORT_SYMBOL(block_invalidatepage);
1627
1628 void do_invalidatepage(struct page *page, unsigned long offset)
1629 {
1630         void (*invalidatepage)(struct page *, unsigned long);
1631         invalidatepage = page->mapping->a_ops->invalidatepage ? :
1632                 block_invalidatepage;
1633         (*invalidatepage)(page, offset);
1634 }
1635
1636 /*
1637  * We attach and possibly dirty the buffers atomically wrt
1638  * __set_page_dirty_buffers() via private_lock.  try_to_free_buffers
1639  * is already excluded via the page lock.
1640  */
1641 void create_empty_buffers(struct page *page,
1642                         unsigned long blocksize, unsigned long b_state)
1643 {
1644         struct buffer_head *bh, *head, *tail;
1645
1646         head = alloc_page_buffers(page, blocksize, 1);
1647         bh = head;
1648         do {
1649                 bh->b_state |= b_state;
1650                 tail = bh;
1651                 bh = bh->b_this_page;
1652         } while (bh);
1653         tail->b_this_page = head;
1654
1655         spin_lock(&page->mapping->private_lock);
1656         if (PageUptodate(page) || PageDirty(page)) {
1657                 bh = head;
1658                 do {
1659                         if (PageDirty(page))
1660                                 set_buffer_dirty(bh);
1661                         if (PageUptodate(page))
1662                                 set_buffer_uptodate(bh);
1663                         bh = bh->b_this_page;
1664                 } while (bh != head);
1665         }
1666         attach_page_buffers(page, head);
1667         spin_unlock(&page->mapping->private_lock);
1668 }
1669 EXPORT_SYMBOL(create_empty_buffers);
1670
1671 /*
1672  * We are taking a block for data and we don't want any output from any
1673  * buffer-cache aliases starting from return from that function and
1674  * until the moment when something will explicitly mark the buffer
1675  * dirty (hopefully that will not happen until we will free that block ;-)
1676  * We don't even need to mark it not-uptodate - nobody can expect
1677  * anything from a newly allocated buffer anyway. We used to used
1678  * unmap_buffer() for such invalidation, but that was wrong. We definitely
1679  * don't want to mark the alias unmapped, for example - it would confuse
1680  * anyone who might pick it with bread() afterwards...
1681  *
1682  * Also..  Note that bforget() doesn't lock the buffer.  So there can
1683  * be writeout I/O going on against recently-freed buffers.  We don't
1684  * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1685  * only if we really need to.  That happens here.
1686  */
1687 void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1688 {
1689         struct buffer_head *old_bh;
1690
1691         might_sleep();
1692
1693         old_bh = __find_get_block_slow(bdev, block);
1694         if (old_bh) {
1695                 clear_buffer_dirty(old_bh);
1696                 wait_on_buffer(old_bh);
1697                 clear_buffer_req(old_bh);
1698                 __brelse(old_bh);
1699         }
1700 }
1701 EXPORT_SYMBOL(unmap_underlying_metadata);
1702
1703 /*
1704  * NOTE! All mapped/uptodate combinations are valid:
1705  *
1706  *      Mapped  Uptodate        Meaning
1707  *
1708  *      No      No              "unknown" - must do get_block()
1709  *      No      Yes             "hole" - zero-filled
1710  *      Yes     No              "allocated" - allocated on disk, not read in
1711  *      Yes     Yes             "valid" - allocated and up-to-date in memory.
1712  *
1713  * "Dirty" is valid only with the last case (mapped+uptodate).
1714  */
1715
1716 /*
1717  * While block_write_full_page is writing back the dirty buffers under
1718  * the page lock, whoever dirtied the buffers may decide to clean them
1719  * again at any time.  We handle that by only looking at the buffer
1720  * state inside lock_buffer().
1721  *
1722  * If block_write_full_page() is called for regular writeback
1723  * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1724  * locked buffer.   This only can happen if someone has written the buffer
1725  * directly, with submit_bh().  At the address_space level PageWriteback
1726  * prevents this contention from occurring.
1727  */
1728 static int __block_write_full_page(struct inode *inode, struct page *page,
1729                         get_block_t *get_block, struct writeback_control *wbc)
1730 {
1731         int err;
1732         sector_t block;
1733         sector_t last_block;
1734         struct buffer_head *bh, *head;
1735         const unsigned blocksize = 1 << inode->i_blkbits;
1736         int nr_underway = 0;
1737
1738         BUG_ON(!PageLocked(page));
1739
1740         last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1741
1742         if (!page_has_buffers(page)) {
1743                 create_empty_buffers(page, blocksize,
1744                                         (1 << BH_Dirty)|(1 << BH_Uptodate));
1745         }
1746
1747         /*
1748          * Be very careful.  We have no exclusion from __set_page_dirty_buffers
1749          * here, and the (potentially unmapped) buffers may become dirty at
1750          * any time.  If a buffer becomes dirty here after we've inspected it
1751          * then we just miss that fact, and the page stays dirty.
1752          *
1753          * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1754          * handle that here by just cleaning them.
1755          */
1756
1757         block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1758         head = page_buffers(page);
1759         bh = head;
1760
1761         /*
1762          * Get all the dirty buffers mapped to disk addresses and
1763          * handle any aliases from the underlying blockdev's mapping.
1764          */
1765         do {
1766                 if (block > last_block) {
1767                         /*
1768                          * mapped buffers outside i_size will occur, because
1769                          * this page can be outside i_size when there is a
1770                          * truncate in progress.
1771                          */
1772                         /*
1773                          * The buffer was zeroed by block_write_full_page()
1774                          */
1775                         clear_buffer_dirty(bh);
1776                         set_buffer_uptodate(bh);
1777                 } else if (!buffer_mapped(bh) && buffer_dirty(bh)) {
1778                         WARN_ON(bh->b_size != blocksize);
1779                         err = get_block(inode, block, bh, 1);
1780                         if (err)
1781                                 goto recover;
1782                         if (buffer_new(bh)) {
1783                                 /* blockdev mappings never come here */
1784                                 clear_buffer_new(bh);
1785                                 unmap_underlying_metadata(bh->b_bdev,
1786                                                         bh->b_blocknr);
1787                         }
1788                 }
1789                 bh = bh->b_this_page;
1790                 block++;
1791         } while (bh != head);
1792
1793         do {
1794                 if (!buffer_mapped(bh))
1795                         continue;
1796                 /*
1797                  * If it's a fully non-blocking write attempt and we cannot
1798                  * lock the buffer then redirty the page.  Note that this can
1799                  * potentially cause a busy-wait loop from pdflush and kswapd
1800                  * activity, but those code paths have their own higher-level
1801                  * throttling.
1802                  */
1803                 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1804                         lock_buffer(bh);
1805                 } else if (test_set_buffer_locked(bh)) {
1806                         redirty_page_for_writepage(wbc, page);
1807                         continue;
1808                 }
1809                 if (test_clear_buffer_dirty(bh)) {
1810                         mark_buffer_async_write(bh);
1811                 } else {
1812                         unlock_buffer(bh);
1813                 }
1814         } while ((bh = bh->b_this_page) != head);
1815
1816         /*
1817          * The page and its buffers are protected by PageWriteback(), so we can
1818          * drop the bh refcounts early.
1819          */
1820         BUG_ON(PageWriteback(page));
1821         set_page_writeback(page);
1822
1823         do {
1824                 struct buffer_head *next = bh->b_this_page;
1825                 if (buffer_async_write(bh)) {
1826                         submit_bh(WRITE, bh);
1827                         nr_underway++;
1828                 }
1829                 bh = next;
1830         } while (bh != head);
1831         unlock_page(page);
1832
1833         err = 0;
1834 done:
1835         if (nr_underway == 0) {
1836                 /*
1837                  * The page was marked dirty, but the buffers were
1838                  * clean.  Someone wrote them back by hand with
1839                  * ll_rw_block/submit_bh.  A rare case.
1840                  */
1841                 int uptodate = 1;
1842                 do {
1843                         if (!buffer_uptodate(bh)) {
1844                                 uptodate = 0;
1845                                 break;
1846                         }
1847                         bh = bh->b_this_page;
1848                 } while (bh != head);
1849                 if (uptodate)
1850                         SetPageUptodate(page);
1851                 end_page_writeback(page);
1852                 /*
1853                  * The page and buffer_heads can be released at any time from
1854                  * here on.
1855                  */
1856                 wbc->pages_skipped++;   /* We didn't write this page */
1857         }
1858         return err;
1859
1860 recover:
1861         /*
1862          * ENOSPC, or some other error.  We may already have added some
1863          * blocks to the file, so we need to write these out to avoid
1864          * exposing stale data.
1865          * The page is currently locked and not marked for writeback
1866          */
1867         bh = head;
1868         /* Recovery: lock and submit the mapped buffers */
1869         do {
1870                 if (buffer_mapped(bh) && buffer_dirty(bh)) {
1871                         lock_buffer(bh);
1872                         mark_buffer_async_write(bh);
1873                 } else {
1874                         /*
1875                          * The buffer may have been set dirty during
1876                          * attachment to a dirty page.
1877                          */
1878                         clear_buffer_dirty(bh);
1879                 }
1880         } while ((bh = bh->b_this_page) != head);
1881         SetPageError(page);
1882         BUG_ON(PageWriteback(page));
1883         set_page_writeback(page);
1884         unlock_page(page);
1885         do {
1886                 struct buffer_head *next = bh->b_this_page;
1887                 if (buffer_async_write(bh)) {
1888                         clear_buffer_dirty(bh);
1889                         submit_bh(WRITE, bh);
1890                         nr_underway++;
1891                 }
1892                 bh = next;
1893         } while (bh != head);
1894         goto done;
1895 }
1896
1897 static int __block_prepare_write(struct inode *inode, struct page *page,
1898                 unsigned from, unsigned to, get_block_t *get_block)
1899 {
1900         unsigned block_start, block_end;
1901         sector_t block;
1902         int err = 0;
1903         unsigned blocksize, bbits;
1904         struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1905
1906         BUG_ON(!PageLocked(page));
1907         BUG_ON(from > PAGE_CACHE_SIZE);
1908         BUG_ON(to > PAGE_CACHE_SIZE);
1909         BUG_ON(from > to);
1910
1911         blocksize = 1 << inode->i_blkbits;
1912         if (!page_has_buffers(page))
1913                 create_empty_buffers(page, blocksize, 0);
1914         head = page_buffers(page);
1915
1916         bbits = inode->i_blkbits;
1917         block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1918
1919         for(bh = head, block_start = 0; bh != head || !block_start;
1920             block++, block_start=block_end, bh = bh->b_this_page) {
1921                 block_end = block_start + blocksize;
1922                 if (block_end <= from || block_start >= to) {
1923                         if (PageUptodate(page)) {
1924                                 if (!buffer_uptodate(bh))
1925                                         set_buffer_uptodate(bh);
1926                         }
1927                         continue;
1928                 }
1929                 if (buffer_new(bh))
1930                         clear_buffer_new(bh);
1931                 if (!buffer_mapped(bh)) {
1932                         WARN_ON(bh->b_size != blocksize);
1933                         err = get_block(inode, block, bh, 1);
1934                         if (err)
1935                                 break;
1936                         if (buffer_new(bh)) {
1937                                 unmap_underlying_metadata(bh->b_bdev,
1938                                                         bh->b_blocknr);
1939                                 if (PageUptodate(page)) {
1940                                         set_buffer_uptodate(bh);
1941                                         continue;
1942                                 }
1943                                 if (block_end > to || block_start < from) {
1944                                         void *kaddr;
1945
1946                                         kaddr = kmap_atomic(page, KM_USER0);
1947                                         if (block_end > to)
1948                                                 memset(kaddr+to, 0,
1949                                                         block_end-to);
1950                                         if (block_start < from)
1951                                                 memset(kaddr+block_start,
1952                                                         0, from-block_start);
1953                                         flush_dcache_page(page);
1954                                         kunmap_atomic(kaddr, KM_USER0);
1955                                 }
1956                                 continue;
1957                         }
1958                 }
1959                 if (PageUptodate(page)) {
1960                         if (!buffer_uptodate(bh))
1961                                 set_buffer_uptodate(bh);
1962                         continue; 
1963                 }
1964                 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1965                      (block_start < from || block_end > to)) {
1966                         ll_rw_block(READ, 1, &bh);
1967                         *wait_bh++=bh;
1968                 }
1969         }
1970         /*
1971          * If we issued read requests - let them complete.
1972          */
1973         while(wait_bh > wait) {
1974                 wait_on_buffer(*--wait_bh);
1975                 if (!buffer_uptodate(*wait_bh))
1976                         err = -EIO;
1977         }
1978         if (!err) {
1979                 bh = head;
1980                 do {
1981                         if (buffer_new(bh))
1982                                 clear_buffer_new(bh);
1983                 } while ((bh = bh->b_this_page) != head);
1984                 return 0;
1985         }
1986         /* Error case: */
1987         /*
1988          * Zero out any newly allocated blocks to avoid exposing stale
1989          * data.  If BH_New is set, we know that the block was newly
1990          * allocated in the above loop.
1991          */
1992         bh = head;
1993         block_start = 0;
1994         do {
1995                 block_end = block_start+blocksize;
1996                 if (block_end <= from)
1997                         goto next_bh;
1998                 if (block_start >= to)
1999                         break;
2000                 if (buffer_new(bh)) {
2001                         void *kaddr;
2002
2003                         clear_buffer_new(bh);
2004                         kaddr = kmap_atomic(page, KM_USER0);
2005                         memset(kaddr+block_start, 0, bh->b_size);
2006                         kunmap_atomic(kaddr, KM_USER0);
2007                         set_buffer_uptodate(bh);
2008                         mark_buffer_dirty(bh);
2009                 }
2010 next_bh:
2011                 block_start = block_end;
2012                 bh = bh->b_this_page;
2013         } while (bh != head);
2014         return err;
2015 }
2016
2017 static int __block_commit_write(struct inode *inode, struct page *page,
2018                 unsigned from, unsigned to)
2019 {
2020         unsigned block_start, block_end;
2021         int partial = 0;
2022         unsigned blocksize;
2023         struct buffer_head *bh, *head;
2024
2025         blocksize = 1 << inode->i_blkbits;
2026
2027         for(bh = head = page_buffers(page), block_start = 0;
2028             bh != head || !block_start;
2029             block_start=block_end, bh = bh->b_this_page) {
2030                 block_end = block_start + blocksize;
2031                 if (block_end <= from || block_start >= to) {
2032                         if (!buffer_uptodate(bh))
2033                                 partial = 1;
2034                 } else {
2035                         set_buffer_uptodate(bh);
2036                         mark_buffer_dirty(bh);
2037                 }
2038         }
2039
2040         /*
2041          * If this is a partial write which happened to make all buffers
2042          * uptodate then we can optimize away a bogus readpage() for
2043          * the next read(). Here we 'discover' whether the page went
2044          * uptodate as a result of this (potentially partial) write.
2045          */
2046         if (!partial)
2047                 SetPageUptodate(page);
2048         return 0;
2049 }
2050
2051 /*
2052  * Generic "read page" function for block devices that have the normal
2053  * get_block functionality. This is most of the block device filesystems.
2054  * Reads the page asynchronously --- the unlock_buffer() and
2055  * set/clear_buffer_uptodate() functions propagate buffer state into the
2056  * page struct once IO has completed.
2057  */
2058 int block_read_full_page(struct page *page, get_block_t *get_block)
2059 {
2060         struct inode *inode = page->mapping->host;
2061         sector_t iblock, lblock;
2062         struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2063         unsigned int blocksize;
2064         int nr, i;
2065         int fully_mapped = 1;
2066
2067         BUG_ON(!PageLocked(page));
2068         blocksize = 1 << inode->i_blkbits;
2069         if (!page_has_buffers(page))
2070                 create_empty_buffers(page, blocksize, 0);
2071         head = page_buffers(page);
2072
2073         iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2074         lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
2075         bh = head;
2076         nr = 0;
2077         i = 0;
2078
2079         do {
2080                 if (buffer_uptodate(bh))
2081                         continue;
2082
2083                 if (!buffer_mapped(bh)) {
2084                         int err = 0;
2085
2086                         fully_mapped = 0;
2087                         if (iblock < lblock) {
2088                                 WARN_ON(bh->b_size != blocksize);
2089                                 err = get_block(inode, iblock, bh, 0);
2090                                 if (err)
2091                                         SetPageError(page);
2092                         }
2093                         if (!buffer_mapped(bh)) {
2094                                 void *kaddr = kmap_atomic(page, KM_USER0);
2095                                 memset(kaddr + i * blocksize, 0, blocksize);
2096                                 flush_dcache_page(page);
2097                                 kunmap_atomic(kaddr, KM_USER0);
2098                                 if (!err)
2099                                         set_buffer_uptodate(bh);
2100                                 continue;
2101                         }
2102                         /*
2103                          * get_block() might have updated the buffer
2104                          * synchronously
2105                          */
2106                         if (buffer_uptodate(bh))
2107                                 continue;
2108                 }
2109                 arr[nr++] = bh;
2110         } while (i++, iblock++, (bh = bh->b_this_page) != head);
2111
2112         if (fully_mapped)
2113                 SetPageMappedToDisk(page);
2114
2115         if (!nr) {
2116                 /*
2117                  * All buffers are uptodate - we can set the page uptodate
2118                  * as well. But not if get_block() returned an error.
2119                  */
2120                 if (!PageError(page))
2121                         SetPageUptodate(page);
2122                 unlock_page(page);
2123                 return 0;
2124         }
2125
2126         /* Stage two: lock the buffers */
2127         for (i = 0; i < nr; i++) {
2128                 bh = arr[i];
2129                 lock_buffer(bh);
2130                 mark_buffer_async_read(bh);
2131         }
2132
2133         /*
2134          * Stage 3: start the IO.  Check for uptodateness
2135          * inside the buffer lock in case another process reading
2136          * the underlying blockdev brought it uptodate (the sct fix).
2137          */
2138         for (i = 0; i < nr; i++) {
2139                 bh = arr[i];
2140                 if (buffer_uptodate(bh))
2141                         end_buffer_async_read(bh, 1);
2142                 else
2143                         submit_bh(READ, bh);
2144         }
2145         return 0;
2146 }
2147
2148 /* utility function for filesystems that need to do work on expanding
2149  * truncates.  Uses prepare/commit_write to allow the filesystem to
2150  * deal with the hole.  
2151  */
2152 static int __generic_cont_expand(struct inode *inode, loff_t size,
2153                                  pgoff_t index, unsigned int offset)
2154 {
2155         struct address_space *mapping = inode->i_mapping;
2156         struct page *page;
2157         unsigned long limit;
2158         int err;
2159
2160         err = -EFBIG;
2161         limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
2162         if (limit != RLIM_INFINITY && size > (loff_t)limit) {
2163                 send_sig(SIGXFSZ, current, 0);
2164                 goto out;
2165         }
2166         if (size > inode->i_sb->s_maxbytes)
2167                 goto out;
2168
2169         err = -ENOMEM;
2170         page = grab_cache_page(mapping, index);
2171         if (!page)
2172                 goto out;
2173         err = mapping->a_ops->prepare_write(NULL, page, offset, offset);
2174         if (err) {
2175                 /*
2176                  * ->prepare_write() may have instantiated a few blocks
2177                  * outside i_size.  Trim these off again.
2178                  */
2179                 unlock_page(page);
2180                 page_cache_release(page);
2181                 vmtruncate(inode, inode->i_size);
2182                 goto out;
2183         }
2184
2185         err = mapping->a_ops->commit_write(NULL, page, offset, offset);
2186
2187         unlock_page(page);
2188         page_cache_release(page);
2189         if (err > 0)
2190                 err = 0;
2191 out:
2192         return err;
2193 }
2194
2195 int generic_cont_expand(struct inode *inode, loff_t size)
2196 {
2197         pgoff_t index;
2198         unsigned int offset;
2199
2200         offset = (size & (PAGE_CACHE_SIZE - 1)); /* Within page */
2201
2202         /* ugh.  in prepare/commit_write, if from==to==start of block, we
2203         ** skip the prepare.  make sure we never send an offset for the start
2204         ** of a block
2205         */
2206         if ((offset & (inode->i_sb->s_blocksize - 1)) == 0) {
2207                 /* caller must handle this extra byte. */
2208                 offset++;
2209         }
2210         index = size >> PAGE_CACHE_SHIFT;
2211
2212         return __generic_cont_expand(inode, size, index, offset);
2213 }
2214
2215 int generic_cont_expand_simple(struct inode *inode, loff_t size)
2216 {
2217         loff_t pos = size - 1;
2218         pgoff_t index = pos >> PAGE_CACHE_SHIFT;
2219         unsigned int offset = (pos & (PAGE_CACHE_SIZE - 1)) + 1;
2220
2221         /* prepare/commit_write can handle even if from==to==start of block. */
2222         return __generic_cont_expand(inode, size, index, offset);
2223 }
2224
2225 /*
2226  * For moronic filesystems that do not allow holes in file.
2227  * We may have to extend the file.
2228  */
2229
2230 int cont_prepare_write(struct page *page, unsigned offset,
2231                 unsigned to, get_block_t *get_block, loff_t *bytes)
2232 {
2233         struct address_space *mapping = page->mapping;
2234         struct inode *inode = mapping->host;
2235         struct page *new_page;
2236         pgoff_t pgpos;
2237         long status;
2238         unsigned zerofrom;
2239         unsigned blocksize = 1 << inode->i_blkbits;
2240         void *kaddr;
2241
2242         while(page->index > (pgpos = *bytes>>PAGE_CACHE_SHIFT)) {
2243                 status = -ENOMEM;
2244                 new_page = grab_cache_page(mapping, pgpos);
2245                 if (!new_page)
2246                         goto out;
2247                 /* we might sleep */
2248                 if (*bytes>>PAGE_CACHE_SHIFT != pgpos) {
2249                         unlock_page(new_page);
2250                         page_cache_release(new_page);
2251                         continue;
2252                 }
2253                 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2254                 if (zerofrom & (blocksize-1)) {
2255                         *bytes |= (blocksize-1);
2256                         (*bytes)++;
2257                 }
2258                 status = __block_prepare_write(inode, new_page, zerofrom,
2259                                                 PAGE_CACHE_SIZE, get_block);
2260                 if (status)
2261                         goto out_unmap;
2262                 kaddr = kmap_atomic(new_page, KM_USER0);
2263                 memset(kaddr+zerofrom, 0, PAGE_CACHE_SIZE-zerofrom);
2264                 flush_dcache_page(new_page);
2265                 kunmap_atomic(kaddr, KM_USER0);
2266                 generic_commit_write(NULL, new_page, zerofrom, PAGE_CACHE_SIZE);
2267                 unlock_page(new_page);
2268                 page_cache_release(new_page);
2269         }
2270
2271         if (page->index < pgpos) {
2272                 /* completely inside the area */
2273                 zerofrom = offset;
2274         } else {
2275                 /* page covers the boundary, find the boundary offset */
2276                 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2277
2278                 /* if we will expand the thing last block will be filled */
2279                 if (to > zerofrom && (zerofrom & (blocksize-1))) {
2280                         *bytes |= (blocksize-1);
2281                         (*bytes)++;
2282                 }
2283
2284                 /* starting below the boundary? Nothing to zero out */
2285                 if (offset <= zerofrom)
2286                         zerofrom = offset;
2287         }
2288         status = __block_prepare_write(inode, page, zerofrom, to, get_block);
2289         if (status)
2290                 goto out1;
2291         if (zerofrom < offset) {
2292                 kaddr = kmap_atomic(page, KM_USER0);
2293                 memset(kaddr+zerofrom, 0, offset-zerofrom);
2294                 flush_dcache_page(page);
2295                 kunmap_atomic(kaddr, KM_USER0);
2296                 __block_commit_write(inode, page, zerofrom, offset);
2297         }
2298         return 0;
2299 out1:
2300         ClearPageUptodate(page);
2301         return status;
2302
2303 out_unmap:
2304         ClearPageUptodate(new_page);
2305         unlock_page(new_page);
2306         page_cache_release(new_page);
2307 out:
2308         return status;
2309 }
2310
2311 int block_prepare_write(struct page *page, unsigned from, unsigned to,
2312                         get_block_t *get_block)
2313 {
2314         struct inode *inode = page->mapping->host;
2315         int err = __block_prepare_write(inode, page, from, to, get_block);
2316         if (err)
2317                 ClearPageUptodate(page);
2318         return err;
2319 }
2320
2321 int block_commit_write(struct page *page, unsigned from, unsigned to)
2322 {
2323         struct inode *inode = page->mapping->host;
2324         __block_commit_write(inode,page,from,to);
2325         return 0;
2326 }
2327
2328 int generic_commit_write(struct file *file, struct page *page,
2329                 unsigned from, unsigned to)
2330 {
2331         struct inode *inode = page->mapping->host;
2332         loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2333         __block_commit_write(inode,page,from,to);
2334         /*
2335          * No need to use i_size_read() here, the i_size
2336          * cannot change under us because we hold i_mutex.
2337          */
2338         if (pos > inode->i_size) {
2339                 i_size_write(inode, pos);
2340                 mark_inode_dirty(inode);
2341         }
2342         return 0;
2343 }
2344
2345
2346 /*
2347  * nobh_prepare_write()'s prereads are special: the buffer_heads are freed
2348  * immediately, while under the page lock.  So it needs a special end_io
2349  * handler which does not touch the bh after unlocking it.
2350  *
2351  * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
2352  * a race there is benign: unlock_buffer() only use the bh's address for
2353  * hashing after unlocking the buffer, so it doesn't actually touch the bh
2354  * itself.
2355  */
2356 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2357 {
2358         if (uptodate) {
2359                 set_buffer_uptodate(bh);
2360         } else {
2361                 /* This happens, due to failed READA attempts. */
2362                 clear_buffer_uptodate(bh);
2363         }
2364         unlock_buffer(bh);
2365 }
2366
2367 /*
2368  * On entry, the page is fully not uptodate.
2369  * On exit the page is fully uptodate in the areas outside (from,to)
2370  */
2371 int nobh_prepare_write(struct page *page, unsigned from, unsigned to,
2372                         get_block_t *get_block)
2373 {
2374         struct inode *inode = page->mapping->host;
2375         const unsigned blkbits = inode->i_blkbits;
2376         const unsigned blocksize = 1 << blkbits;
2377         struct buffer_head map_bh;
2378         struct buffer_head *read_bh[MAX_BUF_PER_PAGE];
2379         unsigned block_in_page;
2380         unsigned block_start;
2381         sector_t block_in_file;
2382         char *kaddr;
2383         int nr_reads = 0;
2384         int i;
2385         int ret = 0;
2386         int is_mapped_to_disk = 1;
2387         int dirtied_it = 0;
2388
2389         if (PageMappedToDisk(page))
2390                 return 0;
2391
2392         block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2393         map_bh.b_page = page;
2394
2395         /*
2396          * We loop across all blocks in the page, whether or not they are
2397          * part of the affected region.  This is so we can discover if the
2398          * page is fully mapped-to-disk.
2399          */
2400         for (block_start = 0, block_in_page = 0;
2401                   block_start < PAGE_CACHE_SIZE;
2402                   block_in_page++, block_start += blocksize) {
2403                 unsigned block_end = block_start + blocksize;
2404                 int create;
2405
2406                 map_bh.b_state = 0;
2407                 create = 1;
2408                 if (block_start >= to)
2409                         create = 0;
2410                 map_bh.b_size = blocksize;
2411                 ret = get_block(inode, block_in_file + block_in_page,
2412                                         &map_bh, create);
2413                 if (ret)
2414                         goto failed;
2415                 if (!buffer_mapped(&map_bh))
2416                         is_mapped_to_disk = 0;
2417                 if (buffer_new(&map_bh))
2418                         unmap_underlying_metadata(map_bh.b_bdev,
2419                                                         map_bh.b_blocknr);
2420                 if (PageUptodate(page))
2421                         continue;
2422                 if (buffer_new(&map_bh) || !buffer_mapped(&map_bh)) {
2423                         kaddr = kmap_atomic(page, KM_USER0);
2424                         if (block_start < from) {
2425                                 memset(kaddr+block_start, 0, from-block_start);
2426                                 dirtied_it = 1;
2427                         }
2428                         if (block_end > to) {
2429                                 memset(kaddr + to, 0, block_end - to);
2430                                 dirtied_it = 1;
2431                         }
2432                         flush_dcache_page(page);
2433                         kunmap_atomic(kaddr, KM_USER0);
2434                         continue;
2435                 }
2436                 if (buffer_uptodate(&map_bh))
2437                         continue;       /* reiserfs does this */
2438                 if (block_start < from || block_end > to) {
2439                         struct buffer_head *bh = alloc_buffer_head(GFP_NOFS);
2440
2441                         if (!bh) {
2442                                 ret = -ENOMEM;
2443                                 goto failed;
2444                         }
2445                         bh->b_state = map_bh.b_state;
2446                         atomic_set(&bh->b_count, 0);
2447                         bh->b_this_page = NULL;
2448                         bh->b_page = page;
2449                         bh->b_blocknr = map_bh.b_blocknr;
2450                         bh->b_size = blocksize;
2451                         bh->b_data = (char *)(long)block_start;
2452                         bh->b_bdev = map_bh.b_bdev;
2453                         bh->b_private = NULL;
2454                         read_bh[nr_reads++] = bh;
2455                 }
2456         }
2457
2458         if (nr_reads) {
2459                 struct buffer_head *bh;
2460
2461                 /*
2462                  * The page is locked, so these buffers are protected from
2463                  * any VM or truncate activity.  Hence we don't need to care
2464                  * for the buffer_head refcounts.
2465                  */
2466                 for (i = 0; i < nr_reads; i++) {
2467                         bh = read_bh[i];
2468                         lock_buffer(bh);
2469                         bh->b_end_io = end_buffer_read_nobh;
2470                         submit_bh(READ, bh);
2471                 }
2472                 for (i = 0; i < nr_reads; i++) {
2473                         bh = read_bh[i];
2474                         wait_on_buffer(bh);
2475                         if (!buffer_uptodate(bh))
2476                                 ret = -EIO;
2477                         free_buffer_head(bh);
2478                         read_bh[i] = NULL;
2479                 }
2480                 if (ret)
2481                         goto failed;
2482         }
2483
2484         if (is_mapped_to_disk)
2485                 SetPageMappedToDisk(page);
2486         SetPageUptodate(page);
2487
2488         /*
2489          * Setting the page dirty here isn't necessary for the prepare_write
2490          * function - commit_write will do that.  But if/when this function is
2491          * used within the pagefault handler to ensure that all mmapped pages
2492          * have backing space in the filesystem, we will need to dirty the page
2493          * if its contents were altered.
2494          */
2495         if (dirtied_it)
2496                 set_page_dirty(page);
2497
2498         return 0;
2499
2500 failed:
2501         for (i = 0; i < nr_reads; i++) {
2502                 if (read_bh[i])
2503                         free_buffer_head(read_bh[i]);
2504         }
2505
2506         /*
2507          * Error recovery is pretty slack.  Clear the page and mark it dirty
2508          * so we'll later zero out any blocks which _were_ allocated.
2509          */
2510         kaddr = kmap_atomic(page, KM_USER0);
2511         memset(kaddr, 0, PAGE_CACHE_SIZE);
2512         kunmap_atomic(kaddr, KM_USER0);
2513         SetPageUptodate(page);
2514         set_page_dirty(page);
2515         return ret;
2516 }
2517 EXPORT_SYMBOL(nobh_prepare_write);
2518
2519 int nobh_commit_write(struct file *file, struct page *page,
2520                 unsigned from, unsigned to)
2521 {
2522         struct inode *inode = page->mapping->host;
2523         loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2524
2525         set_page_dirty(page);
2526         if (pos > inode->i_size) {
2527                 i_size_write(inode, pos);
2528                 mark_inode_dirty(inode);
2529         }
2530         return 0;
2531 }
2532 EXPORT_SYMBOL(nobh_commit_write);
2533
2534 /*
2535  * nobh_writepage() - based on block_full_write_page() except
2536  * that it tries to operate without attaching bufferheads to
2537  * the page.
2538  */
2539 int nobh_writepage(struct page *page, get_block_t *get_block,
2540                         struct writeback_control *wbc)
2541 {
2542         struct inode * const inode = page->mapping->host;
2543         loff_t i_size = i_size_read(inode);
2544         const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2545         unsigned offset;
2546         void *kaddr;
2547         int ret;
2548
2549         /* Is the page fully inside i_size? */
2550         if (page->index < end_index)
2551                 goto out;
2552
2553         /* Is the page fully outside i_size? (truncate in progress) */
2554         offset = i_size & (PAGE_CACHE_SIZE-1);
2555         if (page->index >= end_index+1 || !offset) {
2556                 /*
2557                  * The page may have dirty, unmapped buffers.  For example,
2558                  * they may have been added in ext3_writepage().  Make them
2559                  * freeable here, so the page does not leak.
2560                  */
2561 #if 0
2562                 /* Not really sure about this  - do we need this ? */
2563                 if (page->mapping->a_ops->invalidatepage)
2564                         page->mapping->a_ops->invalidatepage(page, offset);
2565 #endif
2566                 unlock_page(page);
2567                 return 0; /* don't care */
2568         }
2569
2570         /*
2571          * The page straddles i_size.  It must be zeroed out on each and every
2572          * writepage invocation because it may be mmapped.  "A file is mapped
2573          * in multiples of the page size.  For a file that is not a multiple of
2574          * the  page size, the remaining memory is zeroed when mapped, and
2575          * writes to that region are not written out to the file."
2576          */
2577         kaddr = kmap_atomic(page, KM_USER0);
2578         memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2579         flush_dcache_page(page);
2580         kunmap_atomic(kaddr, KM_USER0);
2581 out:
2582         ret = mpage_writepage(page, get_block, wbc);
2583         if (ret == -EAGAIN)
2584                 ret = __block_write_full_page(inode, page, get_block, wbc);
2585         return ret;
2586 }
2587 EXPORT_SYMBOL(nobh_writepage);
2588
2589 /*
2590  * This function assumes that ->prepare_write() uses nobh_prepare_write().
2591  */
2592 int nobh_truncate_page(struct address_space *mapping, loff_t from)
2593 {
2594         struct inode *inode = mapping->host;
2595         unsigned blocksize = 1 << inode->i_blkbits;
2596         pgoff_t index = from >> PAGE_CACHE_SHIFT;
2597         unsigned offset = from & (PAGE_CACHE_SIZE-1);
2598         unsigned to;
2599         struct page *page;
2600         const struct address_space_operations *a_ops = mapping->a_ops;
2601         char *kaddr;
2602         int ret = 0;
2603
2604         if ((offset & (blocksize - 1)) == 0)
2605                 goto out;
2606
2607         ret = -ENOMEM;
2608         page = grab_cache_page(mapping, index);
2609         if (!page)
2610                 goto out;
2611
2612         to = (offset + blocksize) & ~(blocksize - 1);
2613         ret = a_ops->prepare_write(NULL, page, offset, to);
2614         if (ret == 0) {
2615                 kaddr = kmap_atomic(page, KM_USER0);
2616                 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2617                 flush_dcache_page(page);
2618                 kunmap_atomic(kaddr, KM_USER0);
2619                 set_page_dirty(page);
2620         }
2621         unlock_page(page);
2622         page_cache_release(page);
2623 out:
2624         return ret;
2625 }
2626 EXPORT_SYMBOL(nobh_truncate_page);
2627
2628 int block_truncate_page(struct address_space *mapping,
2629                         loff_t from, get_block_t *get_block)
2630 {
2631         pgoff_t index = from >> PAGE_CACHE_SHIFT;
2632         unsigned offset = from & (PAGE_CACHE_SIZE-1);
2633         unsigned blocksize;
2634         sector_t iblock;
2635         unsigned length, pos;
2636         struct inode *inode = mapping->host;
2637         struct page *page;
2638         struct buffer_head *bh;
2639         void *kaddr;
2640         int err;
2641
2642         blocksize = 1 << inode->i_blkbits;
2643         length = offset & (blocksize - 1);
2644
2645         /* Block boundary? Nothing to do */
2646         if (!length)
2647                 return 0;
2648
2649         length = blocksize - length;
2650         iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2651         
2652         page = grab_cache_page(mapping, index);
2653         err = -ENOMEM;
2654         if (!page)
2655                 goto out;
2656
2657         if (!page_has_buffers(page))
2658                 create_empty_buffers(page, blocksize, 0);
2659
2660         /* Find the buffer that contains "offset" */
2661         bh = page_buffers(page);
2662         pos = blocksize;
2663         while (offset >= pos) {
2664                 bh = bh->b_this_page;
2665                 iblock++;
2666                 pos += blocksize;
2667         }
2668
2669         err = 0;
2670         if (!buffer_mapped(bh)) {
2671                 WARN_ON(bh->b_size != blocksize);
2672                 err = get_block(inode, iblock, bh, 0);
2673                 if (err)
2674                         goto unlock;
2675                 /* unmapped? It's a hole - nothing to do */
2676                 if (!buffer_mapped(bh))
2677                         goto unlock;
2678         }
2679
2680         /* Ok, it's mapped. Make sure it's up-to-date */
2681         if (PageUptodate(page))
2682                 set_buffer_uptodate(bh);
2683
2684         if (!buffer_uptodate(bh) && !buffer_delay(bh)) {
2685                 err = -EIO;
2686                 ll_rw_block(READ, 1, &bh);
2687                 wait_on_buffer(bh);
2688                 /* Uhhuh. Read error. Complain and punt. */
2689                 if (!buffer_uptodate(bh))
2690                         goto unlock;
2691         }
2692
2693         kaddr = kmap_atomic(page, KM_USER0);
2694         memset(kaddr + offset, 0, length);
2695         flush_dcache_page(page);
2696         kunmap_atomic(kaddr, KM_USER0);
2697
2698         mark_buffer_dirty(bh);
2699         err = 0;
2700
2701 unlock:
2702         unlock_page(page);
2703         page_cache_release(page);
2704 out:
2705         return err;
2706 }
2707
2708 /*
2709  * The generic ->writepage function for buffer-backed address_spaces
2710  */
2711 int block_write_full_page(struct page *page, get_block_t *get_block,
2712                         struct writeback_control *wbc)
2713 {
2714         struct inode * const inode = page->mapping->host;
2715         loff_t i_size = i_size_read(inode);
2716         const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2717         unsigned offset;
2718         void *kaddr;
2719
2720         /* Is the page fully inside i_size? */
2721         if (page->index < end_index)
2722                 return __block_write_full_page(inode, page, get_block, wbc);
2723
2724         /* Is the page fully outside i_size? (truncate in progress) */
2725         offset = i_size & (PAGE_CACHE_SIZE-1);
2726         if (page->index >= end_index+1 || !offset) {
2727                 /*
2728                  * The page may have dirty, unmapped buffers.  For example,
2729                  * they may have been added in ext3_writepage().  Make them
2730                  * freeable here, so the page does not leak.
2731                  */
2732                 do_invalidatepage(page, 0);
2733                 unlock_page(page);
2734                 return 0; /* don't care */
2735         }
2736
2737         /*
2738          * The page straddles i_size.  It must be zeroed out on each and every
2739          * writepage invokation because it may be mmapped.  "A file is mapped
2740          * in multiples of the page size.  For a file that is not a multiple of
2741          * the  page size, the remaining memory is zeroed when mapped, and
2742          * writes to that region are not written out to the file."
2743          */
2744         kaddr = kmap_atomic(page, KM_USER0);
2745         memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2746         flush_dcache_page(page);
2747         kunmap_atomic(kaddr, KM_USER0);
2748         return __block_write_full_page(inode, page, get_block, wbc);
2749 }
2750
2751 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2752                             get_block_t *get_block)
2753 {
2754         struct buffer_head tmp;
2755         struct inode *inode = mapping->host;
2756         tmp.b_state = 0;
2757         tmp.b_blocknr = 0;
2758         tmp.b_size = 1 << inode->i_blkbits;
2759         get_block(inode, block, &tmp, 0);
2760         return tmp.b_blocknr;
2761 }
2762
2763 static int end_bio_bh_io_sync(struct bio *bio, unsigned int bytes_done, int err)
2764 {
2765         struct buffer_head *bh = bio->bi_private;
2766
2767         if (bio->bi_size)
2768                 return 1;
2769
2770         if (err == -EOPNOTSUPP) {
2771                 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2772                 set_bit(BH_Eopnotsupp, &bh->b_state);
2773         }
2774
2775         bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2776         bio_put(bio);
2777         return 0;
2778 }
2779
2780 int submit_bh(int rw, struct buffer_head * bh)
2781 {
2782         struct bio *bio;
2783         int ret = 0;
2784
2785         BUG_ON(!buffer_locked(bh));
2786         BUG_ON(!buffer_mapped(bh));
2787         BUG_ON(!bh->b_end_io);
2788
2789         if (buffer_ordered(bh) && (rw == WRITE))
2790                 rw = WRITE_BARRIER;
2791
2792         /*
2793          * Only clear out a write error when rewriting, should this
2794          * include WRITE_SYNC as well?
2795          */
2796         if (test_set_buffer_req(bh) && (rw == WRITE || rw == WRITE_BARRIER))
2797                 clear_buffer_write_io_error(bh);
2798
2799         /*
2800          * from here on down, it's all bio -- do the initial mapping,
2801          * submit_bio -> generic_make_request may further map this bio around
2802          */
2803         bio = bio_alloc(GFP_NOIO, 1);
2804
2805         bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2806         bio->bi_bdev = bh->b_bdev;
2807         bio->bi_io_vec[0].bv_page = bh->b_page;
2808         bio->bi_io_vec[0].bv_len = bh->b_size;
2809         bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2810
2811         bio->bi_vcnt = 1;
2812         bio->bi_idx = 0;
2813         bio->bi_size = bh->b_size;
2814
2815         bio->bi_end_io = end_bio_bh_io_sync;
2816         bio->bi_private = bh;
2817
2818         bio_get(bio);
2819         submit_bio(rw, bio);
2820
2821         if (bio_flagged(bio, BIO_EOPNOTSUPP))
2822                 ret = -EOPNOTSUPP;
2823
2824         bio_put(bio);
2825         return ret;
2826 }
2827
2828 /**
2829  * ll_rw_block: low-level access to block devices (DEPRECATED)
2830  * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
2831  * @nr: number of &struct buffer_heads in the array
2832  * @bhs: array of pointers to &struct buffer_head
2833  *
2834  * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
2835  * requests an I/O operation on them, either a %READ or a %WRITE.  The third
2836  * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
2837  * are sent to disk. The fourth %READA option is described in the documentation
2838  * for generic_make_request() which ll_rw_block() calls.
2839  *
2840  * This function drops any buffer that it cannot get a lock on (with the
2841  * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
2842  * clean when doing a write request, and any buffer that appears to be
2843  * up-to-date when doing read request.  Further it marks as clean buffers that
2844  * are processed for writing (the buffer cache won't assume that they are
2845  * actually clean until the buffer gets unlocked).
2846  *
2847  * ll_rw_block sets b_end_io to simple completion handler that marks
2848  * the buffer up-to-date (if approriate), unlocks the buffer and wakes
2849  * any waiters. 
2850  *
2851  * All of the buffers must be for the same device, and must also be a
2852  * multiple of the current approved size for the device.
2853  */
2854 void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
2855 {
2856         int i;
2857
2858         for (i = 0; i < nr; i++) {
2859                 struct buffer_head *bh = bhs[i];
2860
2861                 if (rw == SWRITE)
2862                         lock_buffer(bh);
2863                 else if (test_set_buffer_locked(bh))
2864                         continue;
2865
2866                 if (rw == WRITE || rw == SWRITE) {
2867                         if (test_clear_buffer_dirty(bh)) {
2868                                 bh->b_end_io = end_buffer_write_sync;
2869                                 get_bh(bh);
2870                                 submit_bh(WRITE, bh);
2871                                 continue;
2872                         }
2873                 } else {
2874                         if (!buffer_uptodate(bh)) {
2875                                 bh->b_end_io = end_buffer_read_sync;
2876                                 get_bh(bh);
2877                                 submit_bh(rw, bh);
2878                                 continue;
2879                         }
2880                 }
2881                 unlock_buffer(bh);
2882         }
2883 }
2884
2885 /*
2886  * For a data-integrity writeout, we need to wait upon any in-progress I/O
2887  * and then start new I/O and then wait upon it.  The caller must have a ref on
2888  * the buffer_head.
2889  */
2890 int sync_dirty_buffer(struct buffer_head *bh)
2891 {
2892         int ret = 0;
2893
2894         WARN_ON(atomic_read(&bh->b_count) < 1);
2895         lock_buffer(bh);
2896         if (test_clear_buffer_dirty(bh)) {
2897                 get_bh(bh);
2898                 bh->b_end_io = end_buffer_write_sync;
2899                 ret = submit_bh(WRITE, bh);
2900                 wait_on_buffer(bh);
2901                 if (buffer_eopnotsupp(bh)) {
2902                         clear_buffer_eopnotsupp(bh);
2903                         ret = -EOPNOTSUPP;
2904                 }
2905                 if (!ret && !buffer_uptodate(bh))
2906                         ret = -EIO;
2907         } else {
2908                 unlock_buffer(bh);
2909         }
2910         return ret;
2911 }
2912
2913 /*
2914  * try_to_free_buffers() checks if all the buffers on this particular page
2915  * are unused, and releases them if so.
2916  *
2917  * Exclusion against try_to_free_buffers may be obtained by either
2918  * locking the page or by holding its mapping's private_lock.
2919  *
2920  * If the page is dirty but all the buffers are clean then we need to
2921  * be sure to mark the page clean as well.  This is because the page
2922  * may be against a block device, and a later reattachment of buffers
2923  * to a dirty page will set *all* buffers dirty.  Which would corrupt
2924  * filesystem data on the same device.
2925  *
2926  * The same applies to regular filesystem pages: if all the buffers are
2927  * clean then we set the page clean and proceed.  To do that, we require
2928  * total exclusion from __set_page_dirty_buffers().  That is obtained with
2929  * private_lock.
2930  *
2931  * try_to_free_buffers() is non-blocking.
2932  */
2933 static inline int buffer_busy(struct buffer_head *bh)
2934 {
2935         return atomic_read(&bh->b_count) |
2936                 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
2937 }
2938
2939 static int
2940 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
2941 {
2942         struct buffer_head *head = page_buffers(page);
2943         struct buffer_head *bh;
2944
2945         bh = head;
2946         do {
2947                 if (buffer_write_io_error(bh) && page->mapping)
2948                         set_bit(AS_EIO, &page->mapping->flags);
2949                 if (buffer_busy(bh))
2950                         goto failed;
2951                 bh = bh->b_this_page;
2952         } while (bh != head);
2953
2954         do {
2955                 struct buffer_head *next = bh->b_this_page;
2956
2957                 if (!list_empty(&bh->b_assoc_buffers))
2958                         __remove_assoc_queue(bh);
2959                 bh = next;
2960         } while (bh != head);
2961         *buffers_to_free = head;
2962         __clear_page_buffers(page);
2963         return 1;
2964 failed:
2965         return 0;
2966 }
2967
2968 int try_to_free_buffers(struct page *page)
2969 {
2970         struct address_space * const mapping = page->mapping;
2971         struct buffer_head *buffers_to_free = NULL;
2972         int ret = 0;
2973
2974         BUG_ON(!PageLocked(page));
2975         if (PageWriteback(page))
2976                 return 0;
2977
2978         if (mapping == NULL) {          /* can this still happen? */
2979                 ret = drop_buffers(page, &buffers_to_free);
2980                 goto out;
2981         }
2982
2983         spin_lock(&mapping->private_lock);
2984         ret = drop_buffers(page, &buffers_to_free);
2985         if (ret) {
2986                 /*
2987                  * If the filesystem writes its buffers by hand (eg ext3)
2988                  * then we can have clean buffers against a dirty page.  We
2989                  * clean the page here; otherwise later reattachment of buffers
2990                  * could encounter a non-uptodate page, which is unresolvable.
2991                  * This only applies in the rare case where try_to_free_buffers
2992                  * succeeds but the page is not freed.
2993                  */
2994                 clear_page_dirty(page);
2995         }
2996         spin_unlock(&mapping->private_lock);
2997 out:
2998         if (buffers_to_free) {
2999                 struct buffer_head *bh = buffers_to_free;
3000
3001                 do {
3002                         struct buffer_head *next = bh->b_this_page;
3003                         free_buffer_head(bh);
3004                         bh = next;
3005                 } while (bh != buffers_to_free);
3006         }
3007         return ret;
3008 }
3009 EXPORT_SYMBOL(try_to_free_buffers);
3010
3011 void block_sync_page(struct page *page)
3012 {
3013         struct address_space *mapping;
3014
3015         smp_mb();
3016         mapping = page_mapping(page);
3017         if (mapping)
3018                 blk_run_backing_dev(mapping->backing_dev_info, page);
3019 }
3020
3021 /*
3022  * There are no bdflush tunables left.  But distributions are
3023  * still running obsolete flush daemons, so we terminate them here.
3024  *
3025  * Use of bdflush() is deprecated and will be removed in a future kernel.
3026  * The `pdflush' kernel threads fully replace bdflush daemons and this call.
3027  */
3028 asmlinkage long sys_bdflush(int func, long data)
3029 {
3030         static int msg_count;
3031
3032         if (!capable(CAP_SYS_ADMIN))
3033                 return -EPERM;
3034
3035         if (msg_count < 5) {
3036                 msg_count++;
3037                 printk(KERN_INFO
3038                         "warning: process `%s' used the obsolete bdflush"
3039                         " system call\n", current->comm);
3040                 printk(KERN_INFO "Fix your initscripts?\n");
3041         }
3042
3043         if (func == 1)
3044                 do_exit(0);
3045         return 0;
3046 }
3047
3048 /*
3049  * Buffer-head allocation
3050  */
3051 static kmem_cache_t *bh_cachep;
3052
3053 /*
3054  * Once the number of bh's in the machine exceeds this level, we start
3055  * stripping them in writeback.
3056  */
3057 static int max_buffer_heads;
3058
3059 int buffer_heads_over_limit;
3060
3061 struct bh_accounting {
3062         int nr;                 /* Number of live bh's */
3063         int ratelimit;          /* Limit cacheline bouncing */
3064 };
3065
3066 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3067
3068 static void recalc_bh_state(void)
3069 {
3070         int i;
3071         int tot = 0;
3072
3073         if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
3074                 return;
3075         __get_cpu_var(bh_accounting).ratelimit = 0;
3076         for_each_online_cpu(i)
3077                 tot += per_cpu(bh_accounting, i).nr;
3078         buffer_heads_over_limit = (tot > max_buffer_heads);
3079 }
3080         
3081 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3082 {
3083         struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags);
3084         if (ret) {
3085                 get_cpu_var(bh_accounting).nr++;
3086                 recalc_bh_state();
3087                 put_cpu_var(bh_accounting);
3088         }
3089         return ret;
3090 }
3091 EXPORT_SYMBOL(alloc_buffer_head);
3092
3093 void free_buffer_head(struct buffer_head *bh)
3094 {
3095         BUG_ON(!list_empty(&bh->b_assoc_buffers));
3096         kmem_cache_free(bh_cachep, bh);
3097         get_cpu_var(bh_accounting).nr--;
3098         recalc_bh_state();
3099         put_cpu_var(bh_accounting);
3100 }
3101 EXPORT_SYMBOL(free_buffer_head);
3102
3103 static void
3104 init_buffer_head(void *data, kmem_cache_t *cachep, unsigned long flags)
3105 {
3106         if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
3107                             SLAB_CTOR_CONSTRUCTOR) {
3108                 struct buffer_head * bh = (struct buffer_head *)data;
3109
3110                 memset(bh, 0, sizeof(*bh));
3111                 INIT_LIST_HEAD(&bh->b_assoc_buffers);
3112         }
3113 }
3114
3115 #ifdef CONFIG_HOTPLUG_CPU
3116 static void buffer_exit_cpu(int cpu)
3117 {
3118         int i;
3119         struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3120
3121         for (i = 0; i < BH_LRU_SIZE; i++) {
3122                 brelse(b->bhs[i]);
3123                 b->bhs[i] = NULL;
3124         }
3125         get_cpu_var(bh_accounting).nr += per_cpu(bh_accounting, cpu).nr;
3126         per_cpu(bh_accounting, cpu).nr = 0;
3127         put_cpu_var(bh_accounting);
3128 }
3129
3130 static int buffer_cpu_notify(struct notifier_block *self,
3131                               unsigned long action, void *hcpu)
3132 {
3133         if (action == CPU_DEAD)
3134                 buffer_exit_cpu((unsigned long)hcpu);
3135         return NOTIFY_OK;
3136 }
3137 #endif /* CONFIG_HOTPLUG_CPU */
3138
3139 void __init buffer_init(void)
3140 {
3141         int nrpages;
3142
3143         bh_cachep = kmem_cache_create("buffer_head",
3144                                         sizeof(struct buffer_head), 0,
3145                                         (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3146                                         SLAB_MEM_SPREAD),
3147                                         init_buffer_head,
3148                                         NULL);
3149
3150         /*
3151          * Limit the bh occupancy to 10% of ZONE_NORMAL
3152          */
3153         nrpages = (nr_free_buffer_pages() * 10) / 100;
3154         max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3155         hotcpu_notifier(buffer_cpu_notify, 0);
3156 }
3157
3158 EXPORT_SYMBOL(__bforget);
3159 EXPORT_SYMBOL(__brelse);
3160 EXPORT_SYMBOL(__wait_on_buffer);
3161 EXPORT_SYMBOL(block_commit_write);
3162 EXPORT_SYMBOL(block_prepare_write);
3163 EXPORT_SYMBOL(block_read_full_page);
3164 EXPORT_SYMBOL(block_sync_page);
3165 EXPORT_SYMBOL(block_truncate_page);
3166 EXPORT_SYMBOL(block_write_full_page);
3167 EXPORT_SYMBOL(cont_prepare_write);
3168 EXPORT_SYMBOL(end_buffer_read_sync);
3169 EXPORT_SYMBOL(end_buffer_write_sync);
3170 EXPORT_SYMBOL(file_fsync);
3171 EXPORT_SYMBOL(fsync_bdev);
3172 EXPORT_SYMBOL(generic_block_bmap);
3173 EXPORT_SYMBOL(generic_commit_write);
3174 EXPORT_SYMBOL(generic_cont_expand);
3175 EXPORT_SYMBOL(generic_cont_expand_simple);
3176 EXPORT_SYMBOL(init_buffer);
3177 EXPORT_SYMBOL(invalidate_bdev);
3178 EXPORT_SYMBOL(ll_rw_block);
3179 EXPORT_SYMBOL(mark_buffer_dirty);
3180 EXPORT_SYMBOL(submit_bh);
3181 EXPORT_SYMBOL(sync_dirty_buffer);
3182 EXPORT_SYMBOL(unlock_buffer);