1216c0d3c8ce179adecc466a772d06ea26f4ed8d
[linux-2.6.git] / fs / buffer.c
1 /*
2  *  linux/fs/buffer.c
3  *
4  *  Copyright (C) 1991, 1992, 2002  Linus Torvalds
5  */
6
7 /*
8  * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9  *
10  * Removed a lot of unnecessary code and simplified things now that
11  * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12  *
13  * Speed up hash, lru, and free list operations.  Use gfp() for allocating
14  * hash table, use SLAB cache for buffer heads. SMP threading.  -DaveM
15  *
16  * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17  *
18  * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19  */
20
21 #include <linux/config.h>
22 #include <linux/kernel.h>
23 #include <linux/syscalls.h>
24 #include <linux/fs.h>
25 #include <linux/mm.h>
26 #include <linux/percpu.h>
27 #include <linux/slab.h>
28 #include <linux/smp_lock.h>
29 #include <linux/blkdev.h>
30 #include <linux/file.h>
31 #include <linux/quotaops.h>
32 #include <linux/highmem.h>
33 #include <linux/module.h>
34 #include <linux/writeback.h>
35 #include <linux/hash.h>
36 #include <linux/suspend.h>
37 #include <linux/buffer_head.h>
38 #include <linux/bio.h>
39 #include <linux/notifier.h>
40 #include <linux/cpu.h>
41 #include <linux/bitops.h>
42 #include <linux/mpage.h>
43 #include <linux/bit_spinlock.h>
44
45 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
46 static void invalidate_bh_lrus(void);
47
48 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
49
50 inline void
51 init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
52 {
53         bh->b_end_io = handler;
54         bh->b_private = private;
55 }
56
57 static int sync_buffer(void *word)
58 {
59         struct block_device *bd;
60         struct buffer_head *bh
61                 = container_of(word, struct buffer_head, b_state);
62
63         smp_mb();
64         bd = bh->b_bdev;
65         if (bd)
66                 blk_run_address_space(bd->bd_inode->i_mapping);
67         io_schedule();
68         return 0;
69 }
70
71 void fastcall __lock_buffer(struct buffer_head *bh)
72 {
73         wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
74                                                         TASK_UNINTERRUPTIBLE);
75 }
76 EXPORT_SYMBOL(__lock_buffer);
77
78 void fastcall unlock_buffer(struct buffer_head *bh)
79 {
80         clear_buffer_locked(bh);
81         smp_mb__after_clear_bit();
82         wake_up_bit(&bh->b_state, BH_Lock);
83 }
84
85 /*
86  * Block until a buffer comes unlocked.  This doesn't stop it
87  * from becoming locked again - you have to lock it yourself
88  * if you want to preserve its state.
89  */
90 void __wait_on_buffer(struct buffer_head * bh)
91 {
92         wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
93 }
94
95 static void
96 __clear_page_buffers(struct page *page)
97 {
98         ClearPagePrivate(page);
99         page->private = 0;
100         page_cache_release(page);
101 }
102
103 static void buffer_io_error(struct buffer_head *bh)
104 {
105         char b[BDEVNAME_SIZE];
106
107         printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
108                         bdevname(bh->b_bdev, b),
109                         (unsigned long long)bh->b_blocknr);
110 }
111
112 /*
113  * Default synchronous end-of-IO handler..  Just mark it up-to-date and
114  * unlock the buffer. This is what ll_rw_block uses too.
115  */
116 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
117 {
118         if (uptodate) {
119                 set_buffer_uptodate(bh);
120         } else {
121                 /* This happens, due to failed READA attempts. */
122                 clear_buffer_uptodate(bh);
123         }
124         unlock_buffer(bh);
125         put_bh(bh);
126 }
127
128 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
129 {
130         char b[BDEVNAME_SIZE];
131
132         if (uptodate) {
133                 set_buffer_uptodate(bh);
134         } else {
135                 if (!buffer_eopnotsupp(bh) && printk_ratelimit()) {
136                         buffer_io_error(bh);
137                         printk(KERN_WARNING "lost page write due to "
138                                         "I/O error on %s\n",
139                                        bdevname(bh->b_bdev, b));
140                 }
141                 set_buffer_write_io_error(bh);
142                 clear_buffer_uptodate(bh);
143         }
144         unlock_buffer(bh);
145         put_bh(bh);
146 }
147
148 /*
149  * Write out and wait upon all the dirty data associated with a block
150  * device via its mapping.  Does not take the superblock lock.
151  */
152 int sync_blockdev(struct block_device *bdev)
153 {
154         int ret = 0;
155
156         if (bdev) {
157                 int err;
158
159                 ret = filemap_fdatawrite(bdev->bd_inode->i_mapping);
160                 err = filemap_fdatawait(bdev->bd_inode->i_mapping);
161                 if (!ret)
162                         ret = err;
163         }
164         return ret;
165 }
166 EXPORT_SYMBOL(sync_blockdev);
167
168 /*
169  * Write out and wait upon all dirty data associated with this
170  * superblock.  Filesystem data as well as the underlying block
171  * device.  Takes the superblock lock.
172  */
173 int fsync_super(struct super_block *sb)
174 {
175         sync_inodes_sb(sb, 0);
176         DQUOT_SYNC(sb);
177         lock_super(sb);
178         if (sb->s_dirt && sb->s_op->write_super)
179                 sb->s_op->write_super(sb);
180         unlock_super(sb);
181         if (sb->s_op->sync_fs)
182                 sb->s_op->sync_fs(sb, 1);
183         sync_blockdev(sb->s_bdev);
184         sync_inodes_sb(sb, 1);
185
186         return sync_blockdev(sb->s_bdev);
187 }
188
189 /*
190  * Write out and wait upon all dirty data associated with this
191  * device.   Filesystem data as well as the underlying block
192  * device.  Takes the superblock lock.
193  */
194 int fsync_bdev(struct block_device *bdev)
195 {
196         struct super_block *sb = get_super(bdev);
197         if (sb) {
198                 int res = fsync_super(sb);
199                 drop_super(sb);
200                 return res;
201         }
202         return sync_blockdev(bdev);
203 }
204
205 /**
206  * freeze_bdev  --  lock a filesystem and force it into a consistent state
207  * @bdev:       blockdevice to lock
208  *
209  * This takes the block device bd_mount_sem to make sure no new mounts
210  * happen on bdev until thaw_bdev() is called.
211  * If a superblock is found on this device, we take the s_umount semaphore
212  * on it to make sure nobody unmounts until the snapshot creation is done.
213  */
214 struct super_block *freeze_bdev(struct block_device *bdev)
215 {
216         struct super_block *sb;
217
218         down(&bdev->bd_mount_sem);
219         sb = get_super(bdev);
220         if (sb && !(sb->s_flags & MS_RDONLY)) {
221                 sb->s_frozen = SB_FREEZE_WRITE;
222                 smp_wmb();
223
224                 sync_inodes_sb(sb, 0);
225                 DQUOT_SYNC(sb);
226
227                 lock_super(sb);
228                 if (sb->s_dirt && sb->s_op->write_super)
229                         sb->s_op->write_super(sb);
230                 unlock_super(sb);
231
232                 if (sb->s_op->sync_fs)
233                         sb->s_op->sync_fs(sb, 1);
234
235                 sync_blockdev(sb->s_bdev);
236                 sync_inodes_sb(sb, 1);
237
238                 sb->s_frozen = SB_FREEZE_TRANS;
239                 smp_wmb();
240
241                 sync_blockdev(sb->s_bdev);
242
243                 if (sb->s_op->write_super_lockfs)
244                         sb->s_op->write_super_lockfs(sb);
245         }
246
247         sync_blockdev(bdev);
248         return sb;      /* thaw_bdev releases s->s_umount and bd_mount_sem */
249 }
250 EXPORT_SYMBOL(freeze_bdev);
251
252 /**
253  * thaw_bdev  -- unlock filesystem
254  * @bdev:       blockdevice to unlock
255  * @sb:         associated superblock
256  *
257  * Unlocks the filesystem and marks it writeable again after freeze_bdev().
258  */
259 void thaw_bdev(struct block_device *bdev, struct super_block *sb)
260 {
261         if (sb) {
262                 BUG_ON(sb->s_bdev != bdev);
263
264                 if (sb->s_op->unlockfs)
265                         sb->s_op->unlockfs(sb);
266                 sb->s_frozen = SB_UNFROZEN;
267                 smp_wmb();
268                 wake_up(&sb->s_wait_unfrozen);
269                 drop_super(sb);
270         }
271
272         up(&bdev->bd_mount_sem);
273 }
274 EXPORT_SYMBOL(thaw_bdev);
275
276 /*
277  * sync everything.  Start out by waking pdflush, because that writes back
278  * all queues in parallel.
279  */
280 static void do_sync(unsigned long wait)
281 {
282         wakeup_pdflush(0);
283         sync_inodes(0);         /* All mappings, inodes and their blockdevs */
284         DQUOT_SYNC(NULL);
285         sync_supers();          /* Write the superblocks */
286         sync_filesystems(0);    /* Start syncing the filesystems */
287         sync_filesystems(wait); /* Waitingly sync the filesystems */
288         sync_inodes(wait);      /* Mappings, inodes and blockdevs, again. */
289         if (!wait)
290                 printk("Emergency Sync complete\n");
291         if (unlikely(laptop_mode))
292                 laptop_sync_completion();
293 }
294
295 asmlinkage long sys_sync(void)
296 {
297         do_sync(1);
298         return 0;
299 }
300
301 void emergency_sync(void)
302 {
303         pdflush_operation(do_sync, 0);
304 }
305
306 /*
307  * Generic function to fsync a file.
308  *
309  * filp may be NULL if called via the msync of a vma.
310  */
311  
312 int file_fsync(struct file *filp, struct dentry *dentry, int datasync)
313 {
314         struct inode * inode = dentry->d_inode;
315         struct super_block * sb;
316         int ret, err;
317
318         /* sync the inode to buffers */
319         ret = write_inode_now(inode, 0);
320
321         /* sync the superblock to buffers */
322         sb = inode->i_sb;
323         lock_super(sb);
324         if (sb->s_op->write_super)
325                 sb->s_op->write_super(sb);
326         unlock_super(sb);
327
328         /* .. finally sync the buffers to disk */
329         err = sync_blockdev(sb->s_bdev);
330         if (!ret)
331                 ret = err;
332         return ret;
333 }
334
335 static long do_fsync(unsigned int fd, int datasync)
336 {
337         struct file * file;
338         struct address_space *mapping;
339         int ret, err;
340
341         ret = -EBADF;
342         file = fget(fd);
343         if (!file)
344                 goto out;
345
346         ret = -EINVAL;
347         if (!file->f_op || !file->f_op->fsync) {
348                 /* Why?  We can still call filemap_fdatawrite */
349                 goto out_putf;
350         }
351
352         mapping = file->f_mapping;
353
354         current->flags |= PF_SYNCWRITE;
355         ret = filemap_fdatawrite(mapping);
356
357         /*
358          * We need to protect against concurrent writers,
359          * which could cause livelocks in fsync_buffers_list
360          */
361         down(&mapping->host->i_sem);
362         err = file->f_op->fsync(file, file->f_dentry, datasync);
363         if (!ret)
364                 ret = err;
365         up(&mapping->host->i_sem);
366         err = filemap_fdatawait(mapping);
367         if (!ret)
368                 ret = err;
369         current->flags &= ~PF_SYNCWRITE;
370
371 out_putf:
372         fput(file);
373 out:
374         return ret;
375 }
376
377 asmlinkage long sys_fsync(unsigned int fd)
378 {
379         return do_fsync(fd, 0);
380 }
381
382 asmlinkage long sys_fdatasync(unsigned int fd)
383 {
384         return do_fsync(fd, 1);
385 }
386
387 /*
388  * Various filesystems appear to want __find_get_block to be non-blocking.
389  * But it's the page lock which protects the buffers.  To get around this,
390  * we get exclusion from try_to_free_buffers with the blockdev mapping's
391  * private_lock.
392  *
393  * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
394  * may be quite high.  This code could TryLock the page, and if that
395  * succeeds, there is no need to take private_lock. (But if
396  * private_lock is contended then so is mapping->tree_lock).
397  */
398 static struct buffer_head *
399 __find_get_block_slow(struct block_device *bdev, sector_t block, int unused)
400 {
401         struct inode *bd_inode = bdev->bd_inode;
402         struct address_space *bd_mapping = bd_inode->i_mapping;
403         struct buffer_head *ret = NULL;
404         pgoff_t index;
405         struct buffer_head *bh;
406         struct buffer_head *head;
407         struct page *page;
408         int all_mapped = 1;
409
410         index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
411         page = find_get_page(bd_mapping, index);
412         if (!page)
413                 goto out;
414
415         spin_lock(&bd_mapping->private_lock);
416         if (!page_has_buffers(page))
417                 goto out_unlock;
418         head = page_buffers(page);
419         bh = head;
420         do {
421                 if (bh->b_blocknr == block) {
422                         ret = bh;
423                         get_bh(bh);
424                         goto out_unlock;
425                 }
426                 if (!buffer_mapped(bh))
427                         all_mapped = 0;
428                 bh = bh->b_this_page;
429         } while (bh != head);
430
431         /* we might be here because some of the buffers on this page are
432          * not mapped.  This is due to various races between
433          * file io on the block device and getblk.  It gets dealt with
434          * elsewhere, don't buffer_error if we had some unmapped buffers
435          */
436         if (all_mapped) {
437                 printk("__find_get_block_slow() failed. "
438                         "block=%llu, b_blocknr=%llu\n",
439                         (unsigned long long)block, (unsigned long long)bh->b_blocknr);
440                 printk("b_state=0x%08lx, b_size=%u\n", bh->b_state, bh->b_size);
441                 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
442         }
443 out_unlock:
444         spin_unlock(&bd_mapping->private_lock);
445         page_cache_release(page);
446 out:
447         return ret;
448 }
449
450 /* If invalidate_buffers() will trash dirty buffers, it means some kind
451    of fs corruption is going on. Trashing dirty data always imply losing
452    information that was supposed to be just stored on the physical layer
453    by the user.
454
455    Thus invalidate_buffers in general usage is not allwowed to trash
456    dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
457    be preserved.  These buffers are simply skipped.
458   
459    We also skip buffers which are still in use.  For example this can
460    happen if a userspace program is reading the block device.
461
462    NOTE: In the case where the user removed a removable-media-disk even if
463    there's still dirty data not synced on disk (due a bug in the device driver
464    or due an error of the user), by not destroying the dirty buffers we could
465    generate corruption also on the next media inserted, thus a parameter is
466    necessary to handle this case in the most safe way possible (trying
467    to not corrupt also the new disk inserted with the data belonging to
468    the old now corrupted disk). Also for the ramdisk the natural thing
469    to do in order to release the ramdisk memory is to destroy dirty buffers.
470
471    These are two special cases. Normal usage imply the device driver
472    to issue a sync on the device (without waiting I/O completion) and
473    then an invalidate_buffers call that doesn't trash dirty buffers.
474
475    For handling cache coherency with the blkdev pagecache the 'update' case
476    is been introduced. It is needed to re-read from disk any pinned
477    buffer. NOTE: re-reading from disk is destructive so we can do it only
478    when we assume nobody is changing the buffercache under our I/O and when
479    we think the disk contains more recent information than the buffercache.
480    The update == 1 pass marks the buffers we need to update, the update == 2
481    pass does the actual I/O. */
482 void invalidate_bdev(struct block_device *bdev, int destroy_dirty_buffers)
483 {
484         invalidate_bh_lrus();
485         /*
486          * FIXME: what about destroy_dirty_buffers?
487          * We really want to use invalidate_inode_pages2() for
488          * that, but not until that's cleaned up.
489          */
490         invalidate_inode_pages(bdev->bd_inode->i_mapping);
491 }
492
493 /*
494  * Kick pdflush then try to free up some ZONE_NORMAL memory.
495  */
496 static void free_more_memory(void)
497 {
498         struct zone **zones;
499         pg_data_t *pgdat;
500
501         wakeup_pdflush(1024);
502         yield();
503
504         for_each_pgdat(pgdat) {
505                 zones = pgdat->node_zonelists[GFP_NOFS&GFP_ZONEMASK].zones;
506                 if (*zones)
507                         try_to_free_pages(zones, GFP_NOFS);
508         }
509 }
510
511 /*
512  * I/O completion handler for block_read_full_page() - pages
513  * which come unlocked at the end of I/O.
514  */
515 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
516 {
517         unsigned long flags;
518         struct buffer_head *first;
519         struct buffer_head *tmp;
520         struct page *page;
521         int page_uptodate = 1;
522
523         BUG_ON(!buffer_async_read(bh));
524
525         page = bh->b_page;
526         if (uptodate) {
527                 set_buffer_uptodate(bh);
528         } else {
529                 clear_buffer_uptodate(bh);
530                 if (printk_ratelimit())
531                         buffer_io_error(bh);
532                 SetPageError(page);
533         }
534
535         /*
536          * Be _very_ careful from here on. Bad things can happen if
537          * two buffer heads end IO at almost the same time and both
538          * decide that the page is now completely done.
539          */
540         first = page_buffers(page);
541         local_irq_save(flags);
542         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
543         clear_buffer_async_read(bh);
544         unlock_buffer(bh);
545         tmp = bh;
546         do {
547                 if (!buffer_uptodate(tmp))
548                         page_uptodate = 0;
549                 if (buffer_async_read(tmp)) {
550                         BUG_ON(!buffer_locked(tmp));
551                         goto still_busy;
552                 }
553                 tmp = tmp->b_this_page;
554         } while (tmp != bh);
555         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
556         local_irq_restore(flags);
557
558         /*
559          * If none of the buffers had errors and they are all
560          * uptodate then we can set the page uptodate.
561          */
562         if (page_uptodate && !PageError(page))
563                 SetPageUptodate(page);
564         unlock_page(page);
565         return;
566
567 still_busy:
568         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
569         local_irq_restore(flags);
570         return;
571 }
572
573 /*
574  * Completion handler for block_write_full_page() - pages which are unlocked
575  * during I/O, and which have PageWriteback cleared upon I/O completion.
576  */
577 void end_buffer_async_write(struct buffer_head *bh, int uptodate)
578 {
579         char b[BDEVNAME_SIZE];
580         unsigned long flags;
581         struct buffer_head *first;
582         struct buffer_head *tmp;
583         struct page *page;
584
585         BUG_ON(!buffer_async_write(bh));
586
587         page = bh->b_page;
588         if (uptodate) {
589                 set_buffer_uptodate(bh);
590         } else {
591                 if (printk_ratelimit()) {
592                         buffer_io_error(bh);
593                         printk(KERN_WARNING "lost page write due to "
594                                         "I/O error on %s\n",
595                                bdevname(bh->b_bdev, b));
596                 }
597                 set_bit(AS_EIO, &page->mapping->flags);
598                 clear_buffer_uptodate(bh);
599                 SetPageError(page);
600         }
601
602         first = page_buffers(page);
603         local_irq_save(flags);
604         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
605
606         clear_buffer_async_write(bh);
607         unlock_buffer(bh);
608         tmp = bh->b_this_page;
609         while (tmp != bh) {
610                 if (buffer_async_write(tmp)) {
611                         BUG_ON(!buffer_locked(tmp));
612                         goto still_busy;
613                 }
614                 tmp = tmp->b_this_page;
615         }
616         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
617         local_irq_restore(flags);
618         end_page_writeback(page);
619         return;
620
621 still_busy:
622         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
623         local_irq_restore(flags);
624         return;
625 }
626
627 /*
628  * If a page's buffers are under async readin (end_buffer_async_read
629  * completion) then there is a possibility that another thread of
630  * control could lock one of the buffers after it has completed
631  * but while some of the other buffers have not completed.  This
632  * locked buffer would confuse end_buffer_async_read() into not unlocking
633  * the page.  So the absence of BH_Async_Read tells end_buffer_async_read()
634  * that this buffer is not under async I/O.
635  *
636  * The page comes unlocked when it has no locked buffer_async buffers
637  * left.
638  *
639  * PageLocked prevents anyone starting new async I/O reads any of
640  * the buffers.
641  *
642  * PageWriteback is used to prevent simultaneous writeout of the same
643  * page.
644  *
645  * PageLocked prevents anyone from starting writeback of a page which is
646  * under read I/O (PageWriteback is only ever set against a locked page).
647  */
648 static void mark_buffer_async_read(struct buffer_head *bh)
649 {
650         bh->b_end_io = end_buffer_async_read;
651         set_buffer_async_read(bh);
652 }
653
654 void mark_buffer_async_write(struct buffer_head *bh)
655 {
656         bh->b_end_io = end_buffer_async_write;
657         set_buffer_async_write(bh);
658 }
659 EXPORT_SYMBOL(mark_buffer_async_write);
660
661
662 /*
663  * fs/buffer.c contains helper functions for buffer-backed address space's
664  * fsync functions.  A common requirement for buffer-based filesystems is
665  * that certain data from the backing blockdev needs to be written out for
666  * a successful fsync().  For example, ext2 indirect blocks need to be
667  * written back and waited upon before fsync() returns.
668  *
669  * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
670  * inode_has_buffers() and invalidate_inode_buffers() are provided for the
671  * management of a list of dependent buffers at ->i_mapping->private_list.
672  *
673  * Locking is a little subtle: try_to_free_buffers() will remove buffers
674  * from their controlling inode's queue when they are being freed.  But
675  * try_to_free_buffers() will be operating against the *blockdev* mapping
676  * at the time, not against the S_ISREG file which depends on those buffers.
677  * So the locking for private_list is via the private_lock in the address_space
678  * which backs the buffers.  Which is different from the address_space 
679  * against which the buffers are listed.  So for a particular address_space,
680  * mapping->private_lock does *not* protect mapping->private_list!  In fact,
681  * mapping->private_list will always be protected by the backing blockdev's
682  * ->private_lock.
683  *
684  * Which introduces a requirement: all buffers on an address_space's
685  * ->private_list must be from the same address_space: the blockdev's.
686  *
687  * address_spaces which do not place buffers at ->private_list via these
688  * utility functions are free to use private_lock and private_list for
689  * whatever they want.  The only requirement is that list_empty(private_list)
690  * be true at clear_inode() time.
691  *
692  * FIXME: clear_inode should not call invalidate_inode_buffers().  The
693  * filesystems should do that.  invalidate_inode_buffers() should just go
694  * BUG_ON(!list_empty).
695  *
696  * FIXME: mark_buffer_dirty_inode() is a data-plane operation.  It should
697  * take an address_space, not an inode.  And it should be called
698  * mark_buffer_dirty_fsync() to clearly define why those buffers are being
699  * queued up.
700  *
701  * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
702  * list if it is already on a list.  Because if the buffer is on a list,
703  * it *must* already be on the right one.  If not, the filesystem is being
704  * silly.  This will save a ton of locking.  But first we have to ensure
705  * that buffers are taken *off* the old inode's list when they are freed
706  * (presumably in truncate).  That requires careful auditing of all
707  * filesystems (do it inside bforget()).  It could also be done by bringing
708  * b_inode back.
709  */
710
711 /*
712  * The buffer's backing address_space's private_lock must be held
713  */
714 static inline void __remove_assoc_queue(struct buffer_head *bh)
715 {
716         list_del_init(&bh->b_assoc_buffers);
717 }
718
719 int inode_has_buffers(struct inode *inode)
720 {
721         return !list_empty(&inode->i_data.private_list);
722 }
723
724 /*
725  * osync is designed to support O_SYNC io.  It waits synchronously for
726  * all already-submitted IO to complete, but does not queue any new
727  * writes to the disk.
728  *
729  * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
730  * you dirty the buffers, and then use osync_inode_buffers to wait for
731  * completion.  Any other dirty buffers which are not yet queued for
732  * write will not be flushed to disk by the osync.
733  */
734 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
735 {
736         struct buffer_head *bh;
737         struct list_head *p;
738         int err = 0;
739
740         spin_lock(lock);
741 repeat:
742         list_for_each_prev(p, list) {
743                 bh = BH_ENTRY(p);
744                 if (buffer_locked(bh)) {
745                         get_bh(bh);
746                         spin_unlock(lock);
747                         wait_on_buffer(bh);
748                         if (!buffer_uptodate(bh))
749                                 err = -EIO;
750                         brelse(bh);
751                         spin_lock(lock);
752                         goto repeat;
753                 }
754         }
755         spin_unlock(lock);
756         return err;
757 }
758
759 /**
760  * sync_mapping_buffers - write out and wait upon a mapping's "associated"
761  *                        buffers
762  * @mapping: the mapping which wants those buffers written
763  *
764  * Starts I/O against the buffers at mapping->private_list, and waits upon
765  * that I/O.
766  *
767  * Basically, this is a convenience function for fsync().
768  * @mapping is a file or directory which needs those buffers to be written for
769  * a successful fsync().
770  */
771 int sync_mapping_buffers(struct address_space *mapping)
772 {
773         struct address_space *buffer_mapping = mapping->assoc_mapping;
774
775         if (buffer_mapping == NULL || list_empty(&mapping->private_list))
776                 return 0;
777
778         return fsync_buffers_list(&buffer_mapping->private_lock,
779                                         &mapping->private_list);
780 }
781 EXPORT_SYMBOL(sync_mapping_buffers);
782
783 /*
784  * Called when we've recently written block `bblock', and it is known that
785  * `bblock' was for a buffer_boundary() buffer.  This means that the block at
786  * `bblock + 1' is probably a dirty indirect block.  Hunt it down and, if it's
787  * dirty, schedule it for IO.  So that indirects merge nicely with their data.
788  */
789 void write_boundary_block(struct block_device *bdev,
790                         sector_t bblock, unsigned blocksize)
791 {
792         struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
793         if (bh) {
794                 if (buffer_dirty(bh))
795                         ll_rw_block(WRITE, 1, &bh);
796                 put_bh(bh);
797         }
798 }
799
800 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
801 {
802         struct address_space *mapping = inode->i_mapping;
803         struct address_space *buffer_mapping = bh->b_page->mapping;
804
805         mark_buffer_dirty(bh);
806         if (!mapping->assoc_mapping) {
807                 mapping->assoc_mapping = buffer_mapping;
808         } else {
809                 if (mapping->assoc_mapping != buffer_mapping)
810                         BUG();
811         }
812         if (list_empty(&bh->b_assoc_buffers)) {
813                 spin_lock(&buffer_mapping->private_lock);
814                 list_move_tail(&bh->b_assoc_buffers,
815                                 &mapping->private_list);
816                 spin_unlock(&buffer_mapping->private_lock);
817         }
818 }
819 EXPORT_SYMBOL(mark_buffer_dirty_inode);
820
821 /*
822  * Add a page to the dirty page list.
823  *
824  * It is a sad fact of life that this function is called from several places
825  * deeply under spinlocking.  It may not sleep.
826  *
827  * If the page has buffers, the uptodate buffers are set dirty, to preserve
828  * dirty-state coherency between the page and the buffers.  It the page does
829  * not have buffers then when they are later attached they will all be set
830  * dirty.
831  *
832  * The buffers are dirtied before the page is dirtied.  There's a small race
833  * window in which a writepage caller may see the page cleanness but not the
834  * buffer dirtiness.  That's fine.  If this code were to set the page dirty
835  * before the buffers, a concurrent writepage caller could clear the page dirty
836  * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
837  * page on the dirty page list.
838  *
839  * We use private_lock to lock against try_to_free_buffers while using the
840  * page's buffer list.  Also use this to protect against clean buffers being
841  * added to the page after it was set dirty.
842  *
843  * FIXME: may need to call ->reservepage here as well.  That's rather up to the
844  * address_space though.
845  */
846 int __set_page_dirty_buffers(struct page *page)
847 {
848         struct address_space * const mapping = page->mapping;
849
850         spin_lock(&mapping->private_lock);
851         if (page_has_buffers(page)) {
852                 struct buffer_head *head = page_buffers(page);
853                 struct buffer_head *bh = head;
854
855                 do {
856                         set_buffer_dirty(bh);
857                         bh = bh->b_this_page;
858                 } while (bh != head);
859         }
860         spin_unlock(&mapping->private_lock);
861
862         if (!TestSetPageDirty(page)) {
863                 write_lock_irq(&mapping->tree_lock);
864                 if (page->mapping) {    /* Race with truncate? */
865                         if (mapping_cap_account_dirty(mapping))
866                                 inc_page_state(nr_dirty);
867                         radix_tree_tag_set(&mapping->page_tree,
868                                                 page_index(page),
869                                                 PAGECACHE_TAG_DIRTY);
870                 }
871                 write_unlock_irq(&mapping->tree_lock);
872                 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
873         }
874         
875         return 0;
876 }
877 EXPORT_SYMBOL(__set_page_dirty_buffers);
878
879 /*
880  * Write out and wait upon a list of buffers.
881  *
882  * We have conflicting pressures: we want to make sure that all
883  * initially dirty buffers get waited on, but that any subsequently
884  * dirtied buffers don't.  After all, we don't want fsync to last
885  * forever if somebody is actively writing to the file.
886  *
887  * Do this in two main stages: first we copy dirty buffers to a
888  * temporary inode list, queueing the writes as we go.  Then we clean
889  * up, waiting for those writes to complete.
890  * 
891  * During this second stage, any subsequent updates to the file may end
892  * up refiling the buffer on the original inode's dirty list again, so
893  * there is a chance we will end up with a buffer queued for write but
894  * not yet completed on that list.  So, as a final cleanup we go through
895  * the osync code to catch these locked, dirty buffers without requeuing
896  * any newly dirty buffers for write.
897  */
898 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
899 {
900         struct buffer_head *bh;
901         struct list_head tmp;
902         int err = 0, err2;
903
904         INIT_LIST_HEAD(&tmp);
905
906         spin_lock(lock);
907         while (!list_empty(list)) {
908                 bh = BH_ENTRY(list->next);
909                 list_del_init(&bh->b_assoc_buffers);
910                 if (buffer_dirty(bh) || buffer_locked(bh)) {
911                         list_add(&bh->b_assoc_buffers, &tmp);
912                         if (buffer_dirty(bh)) {
913                                 get_bh(bh);
914                                 spin_unlock(lock);
915                                 /*
916                                  * Ensure any pending I/O completes so that
917                                  * ll_rw_block() actually writes the current
918                                  * contents - it is a noop if I/O is still in
919                                  * flight on potentially older contents.
920                                  */
921                                 ll_rw_block(SWRITE, 1, &bh);
922                                 brelse(bh);
923                                 spin_lock(lock);
924                         }
925                 }
926         }
927
928         while (!list_empty(&tmp)) {
929                 bh = BH_ENTRY(tmp.prev);
930                 __remove_assoc_queue(bh);
931                 get_bh(bh);
932                 spin_unlock(lock);
933                 wait_on_buffer(bh);
934                 if (!buffer_uptodate(bh))
935                         err = -EIO;
936                 brelse(bh);
937                 spin_lock(lock);
938         }
939         
940         spin_unlock(lock);
941         err2 = osync_buffers_list(lock, list);
942         if (err)
943                 return err;
944         else
945                 return err2;
946 }
947
948 /*
949  * Invalidate any and all dirty buffers on a given inode.  We are
950  * probably unmounting the fs, but that doesn't mean we have already
951  * done a sync().  Just drop the buffers from the inode list.
952  *
953  * NOTE: we take the inode's blockdev's mapping's private_lock.  Which
954  * assumes that all the buffers are against the blockdev.  Not true
955  * for reiserfs.
956  */
957 void invalidate_inode_buffers(struct inode *inode)
958 {
959         if (inode_has_buffers(inode)) {
960                 struct address_space *mapping = &inode->i_data;
961                 struct list_head *list = &mapping->private_list;
962                 struct address_space *buffer_mapping = mapping->assoc_mapping;
963
964                 spin_lock(&buffer_mapping->private_lock);
965                 while (!list_empty(list))
966                         __remove_assoc_queue(BH_ENTRY(list->next));
967                 spin_unlock(&buffer_mapping->private_lock);
968         }
969 }
970
971 /*
972  * Remove any clean buffers from the inode's buffer list.  This is called
973  * when we're trying to free the inode itself.  Those buffers can pin it.
974  *
975  * Returns true if all buffers were removed.
976  */
977 int remove_inode_buffers(struct inode *inode)
978 {
979         int ret = 1;
980
981         if (inode_has_buffers(inode)) {
982                 struct address_space *mapping = &inode->i_data;
983                 struct list_head *list = &mapping->private_list;
984                 struct address_space *buffer_mapping = mapping->assoc_mapping;
985
986                 spin_lock(&buffer_mapping->private_lock);
987                 while (!list_empty(list)) {
988                         struct buffer_head *bh = BH_ENTRY(list->next);
989                         if (buffer_dirty(bh)) {
990                                 ret = 0;
991                                 break;
992                         }
993                         __remove_assoc_queue(bh);
994                 }
995                 spin_unlock(&buffer_mapping->private_lock);
996         }
997         return ret;
998 }
999
1000 /*
1001  * Create the appropriate buffers when given a page for data area and
1002  * the size of each buffer.. Use the bh->b_this_page linked list to
1003  * follow the buffers created.  Return NULL if unable to create more
1004  * buffers.
1005  *
1006  * The retry flag is used to differentiate async IO (paging, swapping)
1007  * which may not fail from ordinary buffer allocations.
1008  */
1009 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
1010                 int retry)
1011 {
1012         struct buffer_head *bh, *head;
1013         long offset;
1014
1015 try_again:
1016         head = NULL;
1017         offset = PAGE_SIZE;
1018         while ((offset -= size) >= 0) {
1019                 bh = alloc_buffer_head(GFP_NOFS);
1020                 if (!bh)
1021                         goto no_grow;
1022
1023                 bh->b_bdev = NULL;
1024                 bh->b_this_page = head;
1025                 bh->b_blocknr = -1;
1026                 head = bh;
1027
1028                 bh->b_state = 0;
1029                 atomic_set(&bh->b_count, 0);
1030                 bh->b_size = size;
1031
1032                 /* Link the buffer to its page */
1033                 set_bh_page(bh, page, offset);
1034
1035                 bh->b_end_io = NULL;
1036         }
1037         return head;
1038 /*
1039  * In case anything failed, we just free everything we got.
1040  */
1041 no_grow:
1042         if (head) {
1043                 do {
1044                         bh = head;
1045                         head = head->b_this_page;
1046                         free_buffer_head(bh);
1047                 } while (head);
1048         }
1049
1050         /*
1051          * Return failure for non-async IO requests.  Async IO requests
1052          * are not allowed to fail, so we have to wait until buffer heads
1053          * become available.  But we don't want tasks sleeping with 
1054          * partially complete buffers, so all were released above.
1055          */
1056         if (!retry)
1057                 return NULL;
1058
1059         /* We're _really_ low on memory. Now we just
1060          * wait for old buffer heads to become free due to
1061          * finishing IO.  Since this is an async request and
1062          * the reserve list is empty, we're sure there are 
1063          * async buffer heads in use.
1064          */
1065         free_more_memory();
1066         goto try_again;
1067 }
1068 EXPORT_SYMBOL_GPL(alloc_page_buffers);
1069
1070 static inline void
1071 link_dev_buffers(struct page *page, struct buffer_head *head)
1072 {
1073         struct buffer_head *bh, *tail;
1074
1075         bh = head;
1076         do {
1077                 tail = bh;
1078                 bh = bh->b_this_page;
1079         } while (bh);
1080         tail->b_this_page = head;
1081         attach_page_buffers(page, head);
1082 }
1083
1084 /*
1085  * Initialise the state of a blockdev page's buffers.
1086  */ 
1087 static void
1088 init_page_buffers(struct page *page, struct block_device *bdev,
1089                         sector_t block, int size)
1090 {
1091         struct buffer_head *head = page_buffers(page);
1092         struct buffer_head *bh = head;
1093         int uptodate = PageUptodate(page);
1094
1095         do {
1096                 if (!buffer_mapped(bh)) {
1097                         init_buffer(bh, NULL, NULL);
1098                         bh->b_bdev = bdev;
1099                         bh->b_blocknr = block;
1100                         if (uptodate)
1101                                 set_buffer_uptodate(bh);
1102                         set_buffer_mapped(bh);
1103                 }
1104                 block++;
1105                 bh = bh->b_this_page;
1106         } while (bh != head);
1107 }
1108
1109 /*
1110  * Create the page-cache page that contains the requested block.
1111  *
1112  * This is user purely for blockdev mappings.
1113  */
1114 static struct page *
1115 grow_dev_page(struct block_device *bdev, sector_t block,
1116                 pgoff_t index, int size)
1117 {
1118         struct inode *inode = bdev->bd_inode;
1119         struct page *page;
1120         struct buffer_head *bh;
1121
1122         page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
1123         if (!page)
1124                 return NULL;
1125
1126         if (!PageLocked(page))
1127                 BUG();
1128
1129         if (page_has_buffers(page)) {
1130                 bh = page_buffers(page);
1131                 if (bh->b_size == size) {
1132                         init_page_buffers(page, bdev, block, size);
1133                         return page;
1134                 }
1135                 if (!try_to_free_buffers(page))
1136                         goto failed;
1137         }
1138
1139         /*
1140          * Allocate some buffers for this page
1141          */
1142         bh = alloc_page_buffers(page, size, 0);
1143         if (!bh)
1144                 goto failed;
1145
1146         /*
1147          * Link the page to the buffers and initialise them.  Take the
1148          * lock to be atomic wrt __find_get_block(), which does not
1149          * run under the page lock.
1150          */
1151         spin_lock(&inode->i_mapping->private_lock);
1152         link_dev_buffers(page, bh);
1153         init_page_buffers(page, bdev, block, size);
1154         spin_unlock(&inode->i_mapping->private_lock);
1155         return page;
1156
1157 failed:
1158         BUG();
1159         unlock_page(page);
1160         page_cache_release(page);
1161         return NULL;
1162 }
1163
1164 /*
1165  * Create buffers for the specified block device block's page.  If
1166  * that page was dirty, the buffers are set dirty also.
1167  *
1168  * Except that's a bug.  Attaching dirty buffers to a dirty
1169  * blockdev's page can result in filesystem corruption, because
1170  * some of those buffers may be aliases of filesystem data.
1171  * grow_dev_page() will go BUG() if this happens.
1172  */
1173 static inline int
1174 grow_buffers(struct block_device *bdev, sector_t block, int size)
1175 {
1176         struct page *page;
1177         pgoff_t index;
1178         int sizebits;
1179
1180         sizebits = -1;
1181         do {
1182                 sizebits++;
1183         } while ((size << sizebits) < PAGE_SIZE);
1184
1185         index = block >> sizebits;
1186         block = index << sizebits;
1187
1188         /* Create a page with the proper size buffers.. */
1189         page = grow_dev_page(bdev, block, index, size);
1190         if (!page)
1191                 return 0;
1192         unlock_page(page);
1193         page_cache_release(page);
1194         return 1;
1195 }
1196
1197 static struct buffer_head *
1198 __getblk_slow(struct block_device *bdev, sector_t block, int size)
1199 {
1200         /* Size must be multiple of hard sectorsize */
1201         if (unlikely(size & (bdev_hardsect_size(bdev)-1) ||
1202                         (size < 512 || size > PAGE_SIZE))) {
1203                 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1204                                         size);
1205                 printk(KERN_ERR "hardsect size: %d\n",
1206                                         bdev_hardsect_size(bdev));
1207
1208                 dump_stack();
1209                 return NULL;
1210         }
1211
1212         for (;;) {
1213                 struct buffer_head * bh;
1214
1215                 bh = __find_get_block(bdev, block, size);
1216                 if (bh)
1217                         return bh;
1218
1219                 if (!grow_buffers(bdev, block, size))
1220                         free_more_memory();
1221         }
1222 }
1223
1224 /*
1225  * The relationship between dirty buffers and dirty pages:
1226  *
1227  * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1228  * the page is tagged dirty in its radix tree.
1229  *
1230  * At all times, the dirtiness of the buffers represents the dirtiness of
1231  * subsections of the page.  If the page has buffers, the page dirty bit is
1232  * merely a hint about the true dirty state.
1233  *
1234  * When a page is set dirty in its entirety, all its buffers are marked dirty
1235  * (if the page has buffers).
1236  *
1237  * When a buffer is marked dirty, its page is dirtied, but the page's other
1238  * buffers are not.
1239  *
1240  * Also.  When blockdev buffers are explicitly read with bread(), they
1241  * individually become uptodate.  But their backing page remains not
1242  * uptodate - even if all of its buffers are uptodate.  A subsequent
1243  * block_read_full_page() against that page will discover all the uptodate
1244  * buffers, will set the page uptodate and will perform no I/O.
1245  */
1246
1247 /**
1248  * mark_buffer_dirty - mark a buffer_head as needing writeout
1249  * @bh: the buffer_head to mark dirty
1250  *
1251  * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1252  * backing page dirty, then tag the page as dirty in its address_space's radix
1253  * tree and then attach the address_space's inode to its superblock's dirty
1254  * inode list.
1255  *
1256  * mark_buffer_dirty() is atomic.  It takes bh->b_page->mapping->private_lock,
1257  * mapping->tree_lock and the global inode_lock.
1258  */
1259 void fastcall mark_buffer_dirty(struct buffer_head *bh)
1260 {
1261         if (!buffer_dirty(bh) && !test_set_buffer_dirty(bh))
1262                 __set_page_dirty_nobuffers(bh->b_page);
1263 }
1264
1265 /*
1266  * Decrement a buffer_head's reference count.  If all buffers against a page
1267  * have zero reference count, are clean and unlocked, and if the page is clean
1268  * and unlocked then try_to_free_buffers() may strip the buffers from the page
1269  * in preparation for freeing it (sometimes, rarely, buffers are removed from
1270  * a page but it ends up not being freed, and buffers may later be reattached).
1271  */
1272 void __brelse(struct buffer_head * buf)
1273 {
1274         if (atomic_read(&buf->b_count)) {
1275                 put_bh(buf);
1276                 return;
1277         }
1278         printk(KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1279         WARN_ON(1);
1280 }
1281
1282 /*
1283  * bforget() is like brelse(), except it discards any
1284  * potentially dirty data.
1285  */
1286 void __bforget(struct buffer_head *bh)
1287 {
1288         clear_buffer_dirty(bh);
1289         if (!list_empty(&bh->b_assoc_buffers)) {
1290                 struct address_space *buffer_mapping = bh->b_page->mapping;
1291
1292                 spin_lock(&buffer_mapping->private_lock);
1293                 list_del_init(&bh->b_assoc_buffers);
1294                 spin_unlock(&buffer_mapping->private_lock);
1295         }
1296         __brelse(bh);
1297 }
1298
1299 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1300 {
1301         lock_buffer(bh);
1302         if (buffer_uptodate(bh)) {
1303                 unlock_buffer(bh);
1304                 return bh;
1305         } else {
1306                 get_bh(bh);
1307                 bh->b_end_io = end_buffer_read_sync;
1308                 submit_bh(READ, bh);
1309                 wait_on_buffer(bh);
1310                 if (buffer_uptodate(bh))
1311                         return bh;
1312         }
1313         brelse(bh);
1314         return NULL;
1315 }
1316
1317 /*
1318  * Per-cpu buffer LRU implementation.  To reduce the cost of __find_get_block().
1319  * The bhs[] array is sorted - newest buffer is at bhs[0].  Buffers have their
1320  * refcount elevated by one when they're in an LRU.  A buffer can only appear
1321  * once in a particular CPU's LRU.  A single buffer can be present in multiple
1322  * CPU's LRUs at the same time.
1323  *
1324  * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1325  * sb_find_get_block().
1326  *
1327  * The LRUs themselves only need locking against invalidate_bh_lrus.  We use
1328  * a local interrupt disable for that.
1329  */
1330
1331 #define BH_LRU_SIZE     8
1332
1333 struct bh_lru {
1334         struct buffer_head *bhs[BH_LRU_SIZE];
1335 };
1336
1337 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1338
1339 #ifdef CONFIG_SMP
1340 #define bh_lru_lock()   local_irq_disable()
1341 #define bh_lru_unlock() local_irq_enable()
1342 #else
1343 #define bh_lru_lock()   preempt_disable()
1344 #define bh_lru_unlock() preempt_enable()
1345 #endif
1346
1347 static inline void check_irqs_on(void)
1348 {
1349 #ifdef irqs_disabled
1350         BUG_ON(irqs_disabled());
1351 #endif
1352 }
1353
1354 /*
1355  * The LRU management algorithm is dopey-but-simple.  Sorry.
1356  */
1357 static void bh_lru_install(struct buffer_head *bh)
1358 {
1359         struct buffer_head *evictee = NULL;
1360         struct bh_lru *lru;
1361
1362         check_irqs_on();
1363         bh_lru_lock();
1364         lru = &__get_cpu_var(bh_lrus);
1365         if (lru->bhs[0] != bh) {
1366                 struct buffer_head *bhs[BH_LRU_SIZE];
1367                 int in;
1368                 int out = 0;
1369
1370                 get_bh(bh);
1371                 bhs[out++] = bh;
1372                 for (in = 0; in < BH_LRU_SIZE; in++) {
1373                         struct buffer_head *bh2 = lru->bhs[in];
1374
1375                         if (bh2 == bh) {
1376                                 __brelse(bh2);
1377                         } else {
1378                                 if (out >= BH_LRU_SIZE) {
1379                                         BUG_ON(evictee != NULL);
1380                                         evictee = bh2;
1381                                 } else {
1382                                         bhs[out++] = bh2;
1383                                 }
1384                         }
1385                 }
1386                 while (out < BH_LRU_SIZE)
1387                         bhs[out++] = NULL;
1388                 memcpy(lru->bhs, bhs, sizeof(bhs));
1389         }
1390         bh_lru_unlock();
1391
1392         if (evictee)
1393                 __brelse(evictee);
1394 }
1395
1396 /*
1397  * Look up the bh in this cpu's LRU.  If it's there, move it to the head.
1398  */
1399 static inline struct buffer_head *
1400 lookup_bh_lru(struct block_device *bdev, sector_t block, int size)
1401 {
1402         struct buffer_head *ret = NULL;
1403         struct bh_lru *lru;
1404         int i;
1405
1406         check_irqs_on();
1407         bh_lru_lock();
1408         lru = &__get_cpu_var(bh_lrus);
1409         for (i = 0; i < BH_LRU_SIZE; i++) {
1410                 struct buffer_head *bh = lru->bhs[i];
1411
1412                 if (bh && bh->b_bdev == bdev &&
1413                                 bh->b_blocknr == block && bh->b_size == size) {
1414                         if (i) {
1415                                 while (i) {
1416                                         lru->bhs[i] = lru->bhs[i - 1];
1417                                         i--;
1418                                 }
1419                                 lru->bhs[0] = bh;
1420                         }
1421                         get_bh(bh);
1422                         ret = bh;
1423                         break;
1424                 }
1425         }
1426         bh_lru_unlock();
1427         return ret;
1428 }
1429
1430 /*
1431  * Perform a pagecache lookup for the matching buffer.  If it's there, refresh
1432  * it in the LRU and mark it as accessed.  If it is not present then return
1433  * NULL
1434  */
1435 struct buffer_head *
1436 __find_get_block(struct block_device *bdev, sector_t block, int size)
1437 {
1438         struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1439
1440         if (bh == NULL) {
1441                 bh = __find_get_block_slow(bdev, block, size);
1442                 if (bh)
1443                         bh_lru_install(bh);
1444         }
1445         if (bh)
1446                 touch_buffer(bh);
1447         return bh;
1448 }
1449 EXPORT_SYMBOL(__find_get_block);
1450
1451 /*
1452  * __getblk will locate (and, if necessary, create) the buffer_head
1453  * which corresponds to the passed block_device, block and size. The
1454  * returned buffer has its reference count incremented.
1455  *
1456  * __getblk() cannot fail - it just keeps trying.  If you pass it an
1457  * illegal block number, __getblk() will happily return a buffer_head
1458  * which represents the non-existent block.  Very weird.
1459  *
1460  * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1461  * attempt is failing.  FIXME, perhaps?
1462  */
1463 struct buffer_head *
1464 __getblk(struct block_device *bdev, sector_t block, int size)
1465 {
1466         struct buffer_head *bh = __find_get_block(bdev, block, size);
1467
1468         might_sleep();
1469         if (bh == NULL)
1470                 bh = __getblk_slow(bdev, block, size);
1471         return bh;
1472 }
1473 EXPORT_SYMBOL(__getblk);
1474
1475 /*
1476  * Do async read-ahead on a buffer..
1477  */
1478 void __breadahead(struct block_device *bdev, sector_t block, int size)
1479 {
1480         struct buffer_head *bh = __getblk(bdev, block, size);
1481         ll_rw_block(READA, 1, &bh);
1482         brelse(bh);
1483 }
1484 EXPORT_SYMBOL(__breadahead);
1485
1486 /**
1487  *  __bread() - reads a specified block and returns the bh
1488  *  @bdev: the block_device to read from
1489  *  @block: number of block
1490  *  @size: size (in bytes) to read
1491  * 
1492  *  Reads a specified block, and returns buffer head that contains it.
1493  *  It returns NULL if the block was unreadable.
1494  */
1495 struct buffer_head *
1496 __bread(struct block_device *bdev, sector_t block, int size)
1497 {
1498         struct buffer_head *bh = __getblk(bdev, block, size);
1499
1500         if (!buffer_uptodate(bh))
1501                 bh = __bread_slow(bh);
1502         return bh;
1503 }
1504 EXPORT_SYMBOL(__bread);
1505
1506 /*
1507  * invalidate_bh_lrus() is called rarely - but not only at unmount.
1508  * This doesn't race because it runs in each cpu either in irq
1509  * or with preempt disabled.
1510  */
1511 static void invalidate_bh_lru(void *arg)
1512 {
1513         struct bh_lru *b = &get_cpu_var(bh_lrus);
1514         int i;
1515
1516         for (i = 0; i < BH_LRU_SIZE; i++) {
1517                 brelse(b->bhs[i]);
1518                 b->bhs[i] = NULL;
1519         }
1520         put_cpu_var(bh_lrus);
1521 }
1522         
1523 static void invalidate_bh_lrus(void)
1524 {
1525         on_each_cpu(invalidate_bh_lru, NULL, 1, 1);
1526 }
1527
1528 void set_bh_page(struct buffer_head *bh,
1529                 struct page *page, unsigned long offset)
1530 {
1531         bh->b_page = page;
1532         if (offset >= PAGE_SIZE)
1533                 BUG();
1534         if (PageHighMem(page))
1535                 /*
1536                  * This catches illegal uses and preserves the offset:
1537                  */
1538                 bh->b_data = (char *)(0 + offset);
1539         else
1540                 bh->b_data = page_address(page) + offset;
1541 }
1542 EXPORT_SYMBOL(set_bh_page);
1543
1544 /*
1545  * Called when truncating a buffer on a page completely.
1546  */
1547 static inline void discard_buffer(struct buffer_head * bh)
1548 {
1549         lock_buffer(bh);
1550         clear_buffer_dirty(bh);
1551         bh->b_bdev = NULL;
1552         clear_buffer_mapped(bh);
1553         clear_buffer_req(bh);
1554         clear_buffer_new(bh);
1555         clear_buffer_delay(bh);
1556         unlock_buffer(bh);
1557 }
1558
1559 /**
1560  * try_to_release_page() - release old fs-specific metadata on a page
1561  *
1562  * @page: the page which the kernel is trying to free
1563  * @gfp_mask: memory allocation flags (and I/O mode)
1564  *
1565  * The address_space is to try to release any data against the page
1566  * (presumably at page->private).  If the release was successful, return `1'.
1567  * Otherwise return zero.
1568  *
1569  * The @gfp_mask argument specifies whether I/O may be performed to release
1570  * this page (__GFP_IO), and whether the call may block (__GFP_WAIT).
1571  *
1572  * NOTE: @gfp_mask may go away, and this function may become non-blocking.
1573  */
1574 int try_to_release_page(struct page *page, int gfp_mask)
1575 {
1576         struct address_space * const mapping = page->mapping;
1577
1578         BUG_ON(!PageLocked(page));
1579         if (PageWriteback(page))
1580                 return 0;
1581         
1582         if (mapping && mapping->a_ops->releasepage)
1583                 return mapping->a_ops->releasepage(page, gfp_mask);
1584         return try_to_free_buffers(page);
1585 }
1586 EXPORT_SYMBOL(try_to_release_page);
1587
1588 /**
1589  * block_invalidatepage - invalidate part of all of a buffer-backed page
1590  *
1591  * @page: the page which is affected
1592  * @offset: the index of the truncation point
1593  *
1594  * block_invalidatepage() is called when all or part of the page has become
1595  * invalidatedby a truncate operation.
1596  *
1597  * block_invalidatepage() does not have to release all buffers, but it must
1598  * ensure that no dirty buffer is left outside @offset and that no I/O
1599  * is underway against any of the blocks which are outside the truncation
1600  * point.  Because the caller is about to free (and possibly reuse) those
1601  * blocks on-disk.
1602  */
1603 int block_invalidatepage(struct page *page, unsigned long offset)
1604 {
1605         struct buffer_head *head, *bh, *next;
1606         unsigned int curr_off = 0;
1607         int ret = 1;
1608
1609         BUG_ON(!PageLocked(page));
1610         if (!page_has_buffers(page))
1611                 goto out;
1612
1613         head = page_buffers(page);
1614         bh = head;
1615         do {
1616                 unsigned int next_off = curr_off + bh->b_size;
1617                 next = bh->b_this_page;
1618
1619                 /*
1620                  * is this block fully invalidated?
1621                  */
1622                 if (offset <= curr_off)
1623                         discard_buffer(bh);
1624                 curr_off = next_off;
1625                 bh = next;
1626         } while (bh != head);
1627
1628         /*
1629          * We release buffers only if the entire page is being invalidated.
1630          * The get_block cached value has been unconditionally invalidated,
1631          * so real IO is not possible anymore.
1632          */
1633         if (offset == 0)
1634                 ret = try_to_release_page(page, 0);
1635 out:
1636         return ret;
1637 }
1638 EXPORT_SYMBOL(block_invalidatepage);
1639
1640 /*
1641  * We attach and possibly dirty the buffers atomically wrt
1642  * __set_page_dirty_buffers() via private_lock.  try_to_free_buffers
1643  * is already excluded via the page lock.
1644  */
1645 void create_empty_buffers(struct page *page,
1646                         unsigned long blocksize, unsigned long b_state)
1647 {
1648         struct buffer_head *bh, *head, *tail;
1649
1650         head = alloc_page_buffers(page, blocksize, 1);
1651         bh = head;
1652         do {
1653                 bh->b_state |= b_state;
1654                 tail = bh;
1655                 bh = bh->b_this_page;
1656         } while (bh);
1657         tail->b_this_page = head;
1658
1659         spin_lock(&page->mapping->private_lock);
1660         if (PageUptodate(page) || PageDirty(page)) {
1661                 bh = head;
1662                 do {
1663                         if (PageDirty(page))
1664                                 set_buffer_dirty(bh);
1665                         if (PageUptodate(page))
1666                                 set_buffer_uptodate(bh);
1667                         bh = bh->b_this_page;
1668                 } while (bh != head);
1669         }
1670         attach_page_buffers(page, head);
1671         spin_unlock(&page->mapping->private_lock);
1672 }
1673 EXPORT_SYMBOL(create_empty_buffers);
1674
1675 /*
1676  * We are taking a block for data and we don't want any output from any
1677  * buffer-cache aliases starting from return from that function and
1678  * until the moment when something will explicitly mark the buffer
1679  * dirty (hopefully that will not happen until we will free that block ;-)
1680  * We don't even need to mark it not-uptodate - nobody can expect
1681  * anything from a newly allocated buffer anyway. We used to used
1682  * unmap_buffer() for such invalidation, but that was wrong. We definitely
1683  * don't want to mark the alias unmapped, for example - it would confuse
1684  * anyone who might pick it with bread() afterwards...
1685  *
1686  * Also..  Note that bforget() doesn't lock the buffer.  So there can
1687  * be writeout I/O going on against recently-freed buffers.  We don't
1688  * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1689  * only if we really need to.  That happens here.
1690  */
1691 void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1692 {
1693         struct buffer_head *old_bh;
1694
1695         might_sleep();
1696
1697         old_bh = __find_get_block_slow(bdev, block, 0);
1698         if (old_bh) {
1699                 clear_buffer_dirty(old_bh);
1700                 wait_on_buffer(old_bh);
1701                 clear_buffer_req(old_bh);
1702                 __brelse(old_bh);
1703         }
1704 }
1705 EXPORT_SYMBOL(unmap_underlying_metadata);
1706
1707 /*
1708  * NOTE! All mapped/uptodate combinations are valid:
1709  *
1710  *      Mapped  Uptodate        Meaning
1711  *
1712  *      No      No              "unknown" - must do get_block()
1713  *      No      Yes             "hole" - zero-filled
1714  *      Yes     No              "allocated" - allocated on disk, not read in
1715  *      Yes     Yes             "valid" - allocated and up-to-date in memory.
1716  *
1717  * "Dirty" is valid only with the last case (mapped+uptodate).
1718  */
1719
1720 /*
1721  * While block_write_full_page is writing back the dirty buffers under
1722  * the page lock, whoever dirtied the buffers may decide to clean them
1723  * again at any time.  We handle that by only looking at the buffer
1724  * state inside lock_buffer().
1725  *
1726  * If block_write_full_page() is called for regular writeback
1727  * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1728  * locked buffer.   This only can happen if someone has written the buffer
1729  * directly, with submit_bh().  At the address_space level PageWriteback
1730  * prevents this contention from occurring.
1731  */
1732 static int __block_write_full_page(struct inode *inode, struct page *page,
1733                         get_block_t *get_block, struct writeback_control *wbc)
1734 {
1735         int err;
1736         sector_t block;
1737         sector_t last_block;
1738         struct buffer_head *bh, *head;
1739         int nr_underway = 0;
1740
1741         BUG_ON(!PageLocked(page));
1742
1743         last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1744
1745         if (!page_has_buffers(page)) {
1746                 create_empty_buffers(page, 1 << inode->i_blkbits,
1747                                         (1 << BH_Dirty)|(1 << BH_Uptodate));
1748         }
1749
1750         /*
1751          * Be very careful.  We have no exclusion from __set_page_dirty_buffers
1752          * here, and the (potentially unmapped) buffers may become dirty at
1753          * any time.  If a buffer becomes dirty here after we've inspected it
1754          * then we just miss that fact, and the page stays dirty.
1755          *
1756          * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1757          * handle that here by just cleaning them.
1758          */
1759
1760         block = page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1761         head = page_buffers(page);
1762         bh = head;
1763
1764         /*
1765          * Get all the dirty buffers mapped to disk addresses and
1766          * handle any aliases from the underlying blockdev's mapping.
1767          */
1768         do {
1769                 if (block > last_block) {
1770                         /*
1771                          * mapped buffers outside i_size will occur, because
1772                          * this page can be outside i_size when there is a
1773                          * truncate in progress.
1774                          */
1775                         /*
1776                          * The buffer was zeroed by block_write_full_page()
1777                          */
1778                         clear_buffer_dirty(bh);
1779                         set_buffer_uptodate(bh);
1780                 } else if (!buffer_mapped(bh) && buffer_dirty(bh)) {
1781                         err = get_block(inode, block, bh, 1);
1782                         if (err)
1783                                 goto recover;
1784                         if (buffer_new(bh)) {
1785                                 /* blockdev mappings never come here */
1786                                 clear_buffer_new(bh);
1787                                 unmap_underlying_metadata(bh->b_bdev,
1788                                                         bh->b_blocknr);
1789                         }
1790                 }
1791                 bh = bh->b_this_page;
1792                 block++;
1793         } while (bh != head);
1794
1795         do {
1796                 if (!buffer_mapped(bh))
1797                         continue;
1798                 /*
1799                  * If it's a fully non-blocking write attempt and we cannot
1800                  * lock the buffer then redirty the page.  Note that this can
1801                  * potentially cause a busy-wait loop from pdflush and kswapd
1802                  * activity, but those code paths have their own higher-level
1803                  * throttling.
1804                  */
1805                 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1806                         lock_buffer(bh);
1807                 } else if (test_set_buffer_locked(bh)) {
1808                         redirty_page_for_writepage(wbc, page);
1809                         continue;
1810                 }
1811                 if (test_clear_buffer_dirty(bh)) {
1812                         mark_buffer_async_write(bh);
1813                 } else {
1814                         unlock_buffer(bh);
1815                 }
1816         } while ((bh = bh->b_this_page) != head);
1817
1818         /*
1819          * The page and its buffers are protected by PageWriteback(), so we can
1820          * drop the bh refcounts early.
1821          */
1822         BUG_ON(PageWriteback(page));
1823         set_page_writeback(page);
1824
1825         do {
1826                 struct buffer_head *next = bh->b_this_page;
1827                 if (buffer_async_write(bh)) {
1828                         submit_bh(WRITE, bh);
1829                         nr_underway++;
1830                 }
1831                 bh = next;
1832         } while (bh != head);
1833         unlock_page(page);
1834
1835         err = 0;
1836 done:
1837         if (nr_underway == 0) {
1838                 /*
1839                  * The page was marked dirty, but the buffers were
1840                  * clean.  Someone wrote them back by hand with
1841                  * ll_rw_block/submit_bh.  A rare case.
1842                  */
1843                 int uptodate = 1;
1844                 do {
1845                         if (!buffer_uptodate(bh)) {
1846                                 uptodate = 0;
1847                                 break;
1848                         }
1849                         bh = bh->b_this_page;
1850                 } while (bh != head);
1851                 if (uptodate)
1852                         SetPageUptodate(page);
1853                 end_page_writeback(page);
1854                 /*
1855                  * The page and buffer_heads can be released at any time from
1856                  * here on.
1857                  */
1858                 wbc->pages_skipped++;   /* We didn't write this page */
1859         }
1860         return err;
1861
1862 recover:
1863         /*
1864          * ENOSPC, or some other error.  We may already have added some
1865          * blocks to the file, so we need to write these out to avoid
1866          * exposing stale data.
1867          * The page is currently locked and not marked for writeback
1868          */
1869         bh = head;
1870         /* Recovery: lock and submit the mapped buffers */
1871         do {
1872                 if (buffer_mapped(bh) && buffer_dirty(bh)) {
1873                         lock_buffer(bh);
1874                         mark_buffer_async_write(bh);
1875                 } else {
1876                         /*
1877                          * The buffer may have been set dirty during
1878                          * attachment to a dirty page.
1879                          */
1880                         clear_buffer_dirty(bh);
1881                 }
1882         } while ((bh = bh->b_this_page) != head);
1883         SetPageError(page);
1884         BUG_ON(PageWriteback(page));
1885         set_page_writeback(page);
1886         unlock_page(page);
1887         do {
1888                 struct buffer_head *next = bh->b_this_page;
1889                 if (buffer_async_write(bh)) {
1890                         clear_buffer_dirty(bh);
1891                         submit_bh(WRITE, bh);
1892                         nr_underway++;
1893                 }
1894                 bh = next;
1895         } while (bh != head);
1896         goto done;
1897 }
1898
1899 static int __block_prepare_write(struct inode *inode, struct page *page,
1900                 unsigned from, unsigned to, get_block_t *get_block)
1901 {
1902         unsigned block_start, block_end;
1903         sector_t block;
1904         int err = 0;
1905         unsigned blocksize, bbits;
1906         struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1907
1908         BUG_ON(!PageLocked(page));
1909         BUG_ON(from > PAGE_CACHE_SIZE);
1910         BUG_ON(to > PAGE_CACHE_SIZE);
1911         BUG_ON(from > to);
1912
1913         blocksize = 1 << inode->i_blkbits;
1914         if (!page_has_buffers(page))
1915                 create_empty_buffers(page, blocksize, 0);
1916         head = page_buffers(page);
1917
1918         bbits = inode->i_blkbits;
1919         block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1920
1921         for(bh = head, block_start = 0; bh != head || !block_start;
1922             block++, block_start=block_end, bh = bh->b_this_page) {
1923                 block_end = block_start + blocksize;
1924                 if (block_end <= from || block_start >= to) {
1925                         if (PageUptodate(page)) {
1926                                 if (!buffer_uptodate(bh))
1927                                         set_buffer_uptodate(bh);
1928                         }
1929                         continue;
1930                 }
1931                 if (buffer_new(bh))
1932                         clear_buffer_new(bh);
1933                 if (!buffer_mapped(bh)) {
1934                         err = get_block(inode, block, bh, 1);
1935                         if (err)
1936                                 break;
1937                         if (buffer_new(bh)) {
1938                                 unmap_underlying_metadata(bh->b_bdev,
1939                                                         bh->b_blocknr);
1940                                 if (PageUptodate(page)) {
1941                                         set_buffer_uptodate(bh);
1942                                         continue;
1943                                 }
1944                                 if (block_end > to || block_start < from) {
1945                                         void *kaddr;
1946
1947                                         kaddr = kmap_atomic(page, KM_USER0);
1948                                         if (block_end > to)
1949                                                 memset(kaddr+to, 0,
1950                                                         block_end-to);
1951                                         if (block_start < from)
1952                                                 memset(kaddr+block_start,
1953                                                         0, from-block_start);
1954                                         flush_dcache_page(page);
1955                                         kunmap_atomic(kaddr, KM_USER0);
1956                                 }
1957                                 continue;
1958                         }
1959                 }
1960                 if (PageUptodate(page)) {
1961                         if (!buffer_uptodate(bh))
1962                                 set_buffer_uptodate(bh);
1963                         continue; 
1964                 }
1965                 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1966                      (block_start < from || block_end > to)) {
1967                         ll_rw_block(READ, 1, &bh);
1968                         *wait_bh++=bh;
1969                 }
1970         }
1971         /*
1972          * If we issued read requests - let them complete.
1973          */
1974         while(wait_bh > wait) {
1975                 wait_on_buffer(*--wait_bh);
1976                 if (!buffer_uptodate(*wait_bh))
1977                         err = -EIO;
1978         }
1979         if (!err) {
1980                 bh = head;
1981                 do {
1982                         if (buffer_new(bh))
1983                                 clear_buffer_new(bh);
1984                 } while ((bh = bh->b_this_page) != head);
1985                 return 0;
1986         }
1987         /* Error case: */
1988         /*
1989          * Zero out any newly allocated blocks to avoid exposing stale
1990          * data.  If BH_New is set, we know that the block was newly
1991          * allocated in the above loop.
1992          */
1993         bh = head;
1994         block_start = 0;
1995         do {
1996                 block_end = block_start+blocksize;
1997                 if (block_end <= from)
1998                         goto next_bh;
1999                 if (block_start >= to)
2000                         break;
2001                 if (buffer_new(bh)) {
2002                         void *kaddr;
2003
2004                         clear_buffer_new(bh);
2005                         kaddr = kmap_atomic(page, KM_USER0);
2006                         memset(kaddr+block_start, 0, bh->b_size);
2007                         kunmap_atomic(kaddr, KM_USER0);
2008                         set_buffer_uptodate(bh);
2009                         mark_buffer_dirty(bh);
2010                 }
2011 next_bh:
2012                 block_start = block_end;
2013                 bh = bh->b_this_page;
2014         } while (bh != head);
2015         return err;
2016 }
2017
2018 static int __block_commit_write(struct inode *inode, struct page *page,
2019                 unsigned from, unsigned to)
2020 {
2021         unsigned block_start, block_end;
2022         int partial = 0;
2023         unsigned blocksize;
2024         struct buffer_head *bh, *head;
2025
2026         blocksize = 1 << inode->i_blkbits;
2027
2028         for(bh = head = page_buffers(page), block_start = 0;
2029             bh != head || !block_start;
2030             block_start=block_end, bh = bh->b_this_page) {
2031                 block_end = block_start + blocksize;
2032                 if (block_end <= from || block_start >= to) {
2033                         if (!buffer_uptodate(bh))
2034                                 partial = 1;
2035                 } else {
2036                         set_buffer_uptodate(bh);
2037                         mark_buffer_dirty(bh);
2038                 }
2039         }
2040
2041         /*
2042          * If this is a partial write which happened to make all buffers
2043          * uptodate then we can optimize away a bogus readpage() for
2044          * the next read(). Here we 'discover' whether the page went
2045          * uptodate as a result of this (potentially partial) write.
2046          */
2047         if (!partial)
2048                 SetPageUptodate(page);
2049         return 0;
2050 }
2051
2052 /*
2053  * Generic "read page" function for block devices that have the normal
2054  * get_block functionality. This is most of the block device filesystems.
2055  * Reads the page asynchronously --- the unlock_buffer() and
2056  * set/clear_buffer_uptodate() functions propagate buffer state into the
2057  * page struct once IO has completed.
2058  */
2059 int block_read_full_page(struct page *page, get_block_t *get_block)
2060 {
2061         struct inode *inode = page->mapping->host;
2062         sector_t iblock, lblock;
2063         struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2064         unsigned int blocksize;
2065         int nr, i;
2066         int fully_mapped = 1;
2067
2068         BUG_ON(!PageLocked(page));
2069         blocksize = 1 << inode->i_blkbits;
2070         if (!page_has_buffers(page))
2071                 create_empty_buffers(page, blocksize, 0);
2072         head = page_buffers(page);
2073
2074         iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2075         lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
2076         bh = head;
2077         nr = 0;
2078         i = 0;
2079
2080         do {
2081                 if (buffer_uptodate(bh))
2082                         continue;
2083
2084                 if (!buffer_mapped(bh)) {
2085                         int err = 0;
2086
2087                         fully_mapped = 0;
2088                         if (iblock < lblock) {
2089                                 err = get_block(inode, iblock, bh, 0);
2090                                 if (err)
2091                                         SetPageError(page);
2092                         }
2093                         if (!buffer_mapped(bh)) {
2094                                 void *kaddr = kmap_atomic(page, KM_USER0);
2095                                 memset(kaddr + i * blocksize, 0, blocksize);
2096                                 flush_dcache_page(page);
2097                                 kunmap_atomic(kaddr, KM_USER0);
2098                                 if (!err)
2099                                         set_buffer_uptodate(bh);
2100                                 continue;
2101                         }
2102                         /*
2103                          * get_block() might have updated the buffer
2104                          * synchronously
2105                          */
2106                         if (buffer_uptodate(bh))
2107                                 continue;
2108                 }
2109                 arr[nr++] = bh;
2110         } while (i++, iblock++, (bh = bh->b_this_page) != head);
2111
2112         if (fully_mapped)
2113                 SetPageMappedToDisk(page);
2114
2115         if (!nr) {
2116                 /*
2117                  * All buffers are uptodate - we can set the page uptodate
2118                  * as well. But not if get_block() returned an error.
2119                  */
2120                 if (!PageError(page))
2121                         SetPageUptodate(page);
2122                 unlock_page(page);
2123                 return 0;
2124         }
2125
2126         /* Stage two: lock the buffers */
2127         for (i = 0; i < nr; i++) {
2128                 bh = arr[i];
2129                 lock_buffer(bh);
2130                 mark_buffer_async_read(bh);
2131         }
2132
2133         /*
2134          * Stage 3: start the IO.  Check for uptodateness
2135          * inside the buffer lock in case another process reading
2136          * the underlying blockdev brought it uptodate (the sct fix).
2137          */
2138         for (i = 0; i < nr; i++) {
2139                 bh = arr[i];
2140                 if (buffer_uptodate(bh))
2141                         end_buffer_async_read(bh, 1);
2142                 else
2143                         submit_bh(READ, bh);
2144         }
2145         return 0;
2146 }
2147
2148 /* utility function for filesystems that need to do work on expanding
2149  * truncates.  Uses prepare/commit_write to allow the filesystem to
2150  * deal with the hole.  
2151  */
2152 int generic_cont_expand(struct inode *inode, loff_t size)
2153 {
2154         struct address_space *mapping = inode->i_mapping;
2155         struct page *page;
2156         unsigned long index, offset, limit;
2157         int err;
2158
2159         err = -EFBIG;
2160         limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
2161         if (limit != RLIM_INFINITY && size > (loff_t)limit) {
2162                 send_sig(SIGXFSZ, current, 0);
2163                 goto out;
2164         }
2165         if (size > inode->i_sb->s_maxbytes)
2166                 goto out;
2167
2168         offset = (size & (PAGE_CACHE_SIZE-1)); /* Within page */
2169
2170         /* ugh.  in prepare/commit_write, if from==to==start of block, we 
2171         ** skip the prepare.  make sure we never send an offset for the start
2172         ** of a block
2173         */
2174         if ((offset & (inode->i_sb->s_blocksize - 1)) == 0) {
2175                 offset++;
2176         }
2177         index = size >> PAGE_CACHE_SHIFT;
2178         err = -ENOMEM;
2179         page = grab_cache_page(mapping, index);
2180         if (!page)
2181                 goto out;
2182         err = mapping->a_ops->prepare_write(NULL, page, offset, offset);
2183         if (!err) {
2184                 err = mapping->a_ops->commit_write(NULL, page, offset, offset);
2185         }
2186         unlock_page(page);
2187         page_cache_release(page);
2188         if (err > 0)
2189                 err = 0;
2190 out:
2191         return err;
2192 }
2193
2194 /*
2195  * For moronic filesystems that do not allow holes in file.
2196  * We may have to extend the file.
2197  */
2198
2199 int cont_prepare_write(struct page *page, unsigned offset,
2200                 unsigned to, get_block_t *get_block, loff_t *bytes)
2201 {
2202         struct address_space *mapping = page->mapping;
2203         struct inode *inode = mapping->host;
2204         struct page *new_page;
2205         pgoff_t pgpos;
2206         long status;
2207         unsigned zerofrom;
2208         unsigned blocksize = 1 << inode->i_blkbits;
2209         void *kaddr;
2210
2211         while(page->index > (pgpos = *bytes>>PAGE_CACHE_SHIFT)) {
2212                 status = -ENOMEM;
2213                 new_page = grab_cache_page(mapping, pgpos);
2214                 if (!new_page)
2215                         goto out;
2216                 /* we might sleep */
2217                 if (*bytes>>PAGE_CACHE_SHIFT != pgpos) {
2218                         unlock_page(new_page);
2219                         page_cache_release(new_page);
2220                         continue;
2221                 }
2222                 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2223                 if (zerofrom & (blocksize-1)) {
2224                         *bytes |= (blocksize-1);
2225                         (*bytes)++;
2226                 }
2227                 status = __block_prepare_write(inode, new_page, zerofrom,
2228                                                 PAGE_CACHE_SIZE, get_block);
2229                 if (status)
2230                         goto out_unmap;
2231                 kaddr = kmap_atomic(new_page, KM_USER0);
2232                 memset(kaddr+zerofrom, 0, PAGE_CACHE_SIZE-zerofrom);
2233                 flush_dcache_page(new_page);
2234                 kunmap_atomic(kaddr, KM_USER0);
2235                 generic_commit_write(NULL, new_page, zerofrom, PAGE_CACHE_SIZE);
2236                 unlock_page(new_page);
2237                 page_cache_release(new_page);
2238         }
2239
2240         if (page->index < pgpos) {
2241                 /* completely inside the area */
2242                 zerofrom = offset;
2243         } else {
2244                 /* page covers the boundary, find the boundary offset */
2245                 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2246
2247                 /* if we will expand the thing last block will be filled */
2248                 if (to > zerofrom && (zerofrom & (blocksize-1))) {
2249                         *bytes |= (blocksize-1);
2250                         (*bytes)++;
2251                 }
2252
2253                 /* starting below the boundary? Nothing to zero out */
2254                 if (offset <= zerofrom)
2255                         zerofrom = offset;
2256         }
2257         status = __block_prepare_write(inode, page, zerofrom, to, get_block);
2258         if (status)
2259                 goto out1;
2260         if (zerofrom < offset) {
2261                 kaddr = kmap_atomic(page, KM_USER0);
2262                 memset(kaddr+zerofrom, 0, offset-zerofrom);
2263                 flush_dcache_page(page);
2264                 kunmap_atomic(kaddr, KM_USER0);
2265                 __block_commit_write(inode, page, zerofrom, offset);
2266         }
2267         return 0;
2268 out1:
2269         ClearPageUptodate(page);
2270         return status;
2271
2272 out_unmap:
2273         ClearPageUptodate(new_page);
2274         unlock_page(new_page);
2275         page_cache_release(new_page);
2276 out:
2277         return status;
2278 }
2279
2280 int block_prepare_write(struct page *page, unsigned from, unsigned to,
2281                         get_block_t *get_block)
2282 {
2283         struct inode *inode = page->mapping->host;
2284         int err = __block_prepare_write(inode, page, from, to, get_block);
2285         if (err)
2286                 ClearPageUptodate(page);
2287         return err;
2288 }
2289
2290 int block_commit_write(struct page *page, unsigned from, unsigned to)
2291 {
2292         struct inode *inode = page->mapping->host;
2293         __block_commit_write(inode,page,from,to);
2294         return 0;
2295 }
2296
2297 int generic_commit_write(struct file *file, struct page *page,
2298                 unsigned from, unsigned to)
2299 {
2300         struct inode *inode = page->mapping->host;
2301         loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2302         __block_commit_write(inode,page,from,to);
2303         /*
2304          * No need to use i_size_read() here, the i_size
2305          * cannot change under us because we hold i_sem.
2306          */
2307         if (pos > inode->i_size) {
2308                 i_size_write(inode, pos);
2309                 mark_inode_dirty(inode);
2310         }
2311         return 0;
2312 }
2313
2314
2315 /*
2316  * nobh_prepare_write()'s prereads are special: the buffer_heads are freed
2317  * immediately, while under the page lock.  So it needs a special end_io
2318  * handler which does not touch the bh after unlocking it.
2319  *
2320  * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
2321  * a race there is benign: unlock_buffer() only use the bh's address for
2322  * hashing after unlocking the buffer, so it doesn't actually touch the bh
2323  * itself.
2324  */
2325 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2326 {
2327         if (uptodate) {
2328                 set_buffer_uptodate(bh);
2329         } else {
2330                 /* This happens, due to failed READA attempts. */
2331                 clear_buffer_uptodate(bh);
2332         }
2333         unlock_buffer(bh);
2334 }
2335
2336 /*
2337  * On entry, the page is fully not uptodate.
2338  * On exit the page is fully uptodate in the areas outside (from,to)
2339  */
2340 int nobh_prepare_write(struct page *page, unsigned from, unsigned to,
2341                         get_block_t *get_block)
2342 {
2343         struct inode *inode = page->mapping->host;
2344         const unsigned blkbits = inode->i_blkbits;
2345         const unsigned blocksize = 1 << blkbits;
2346         struct buffer_head map_bh;
2347         struct buffer_head *read_bh[MAX_BUF_PER_PAGE];
2348         unsigned block_in_page;
2349         unsigned block_start;
2350         sector_t block_in_file;
2351         char *kaddr;
2352         int nr_reads = 0;
2353         int i;
2354         int ret = 0;
2355         int is_mapped_to_disk = 1;
2356         int dirtied_it = 0;
2357
2358         if (PageMappedToDisk(page))
2359                 return 0;
2360
2361         block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2362         map_bh.b_page = page;
2363
2364         /*
2365          * We loop across all blocks in the page, whether or not they are
2366          * part of the affected region.  This is so we can discover if the
2367          * page is fully mapped-to-disk.
2368          */
2369         for (block_start = 0, block_in_page = 0;
2370                   block_start < PAGE_CACHE_SIZE;
2371                   block_in_page++, block_start += blocksize) {
2372                 unsigned block_end = block_start + blocksize;
2373                 int create;
2374
2375                 map_bh.b_state = 0;
2376                 create = 1;
2377                 if (block_start >= to)
2378                         create = 0;
2379                 ret = get_block(inode, block_in_file + block_in_page,
2380                                         &map_bh, create);
2381                 if (ret)
2382                         goto failed;
2383                 if (!buffer_mapped(&map_bh))
2384                         is_mapped_to_disk = 0;
2385                 if (buffer_new(&map_bh))
2386                         unmap_underlying_metadata(map_bh.b_bdev,
2387                                                         map_bh.b_blocknr);
2388                 if (PageUptodate(page))
2389                         continue;
2390                 if (buffer_new(&map_bh) || !buffer_mapped(&map_bh)) {
2391                         kaddr = kmap_atomic(page, KM_USER0);
2392                         if (block_start < from) {
2393                                 memset(kaddr+block_start, 0, from-block_start);
2394                                 dirtied_it = 1;
2395                         }
2396                         if (block_end > to) {
2397                                 memset(kaddr + to, 0, block_end - to);
2398                                 dirtied_it = 1;
2399                         }
2400                         flush_dcache_page(page);
2401                         kunmap_atomic(kaddr, KM_USER0);
2402                         continue;
2403                 }
2404                 if (buffer_uptodate(&map_bh))
2405                         continue;       /* reiserfs does this */
2406                 if (block_start < from || block_end > to) {
2407                         struct buffer_head *bh = alloc_buffer_head(GFP_NOFS);
2408
2409                         if (!bh) {
2410                                 ret = -ENOMEM;
2411                                 goto failed;
2412                         }
2413                         bh->b_state = map_bh.b_state;
2414                         atomic_set(&bh->b_count, 0);
2415                         bh->b_this_page = NULL;
2416                         bh->b_page = page;
2417                         bh->b_blocknr = map_bh.b_blocknr;
2418                         bh->b_size = blocksize;
2419                         bh->b_data = (char *)(long)block_start;
2420                         bh->b_bdev = map_bh.b_bdev;
2421                         bh->b_private = NULL;
2422                         read_bh[nr_reads++] = bh;
2423                 }
2424         }
2425
2426         if (nr_reads) {
2427                 struct buffer_head *bh;
2428
2429                 /*
2430                  * The page is locked, so these buffers are protected from
2431                  * any VM or truncate activity.  Hence we don't need to care
2432                  * for the buffer_head refcounts.
2433                  */
2434                 for (i = 0; i < nr_reads; i++) {
2435                         bh = read_bh[i];
2436                         lock_buffer(bh);
2437                         bh->b_end_io = end_buffer_read_nobh;
2438                         submit_bh(READ, bh);
2439                 }
2440                 for (i = 0; i < nr_reads; i++) {
2441                         bh = read_bh[i];
2442                         wait_on_buffer(bh);
2443                         if (!buffer_uptodate(bh))
2444                                 ret = -EIO;
2445                         free_buffer_head(bh);
2446                         read_bh[i] = NULL;
2447                 }
2448                 if (ret)
2449                         goto failed;
2450         }
2451
2452         if (is_mapped_to_disk)
2453                 SetPageMappedToDisk(page);
2454         SetPageUptodate(page);
2455
2456         /*
2457          * Setting the page dirty here isn't necessary for the prepare_write
2458          * function - commit_write will do that.  But if/when this function is
2459          * used within the pagefault handler to ensure that all mmapped pages
2460          * have backing space in the filesystem, we will need to dirty the page
2461          * if its contents were altered.
2462          */
2463         if (dirtied_it)
2464                 set_page_dirty(page);
2465
2466         return 0;
2467
2468 failed:
2469         for (i = 0; i < nr_reads; i++) {
2470                 if (read_bh[i])
2471                         free_buffer_head(read_bh[i]);
2472         }
2473
2474         /*
2475          * Error recovery is pretty slack.  Clear the page and mark it dirty
2476          * so we'll later zero out any blocks which _were_ allocated.
2477          */
2478         kaddr = kmap_atomic(page, KM_USER0);
2479         memset(kaddr, 0, PAGE_CACHE_SIZE);
2480         kunmap_atomic(kaddr, KM_USER0);
2481         SetPageUptodate(page);
2482         set_page_dirty(page);
2483         return ret;
2484 }
2485 EXPORT_SYMBOL(nobh_prepare_write);
2486
2487 int nobh_commit_write(struct file *file, struct page *page,
2488                 unsigned from, unsigned to)
2489 {
2490         struct inode *inode = page->mapping->host;
2491         loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2492
2493         set_page_dirty(page);
2494         if (pos > inode->i_size) {
2495                 i_size_write(inode, pos);
2496                 mark_inode_dirty(inode);
2497         }
2498         return 0;
2499 }
2500 EXPORT_SYMBOL(nobh_commit_write);
2501
2502 /*
2503  * nobh_writepage() - based on block_full_write_page() except
2504  * that it tries to operate without attaching bufferheads to
2505  * the page.
2506  */
2507 int nobh_writepage(struct page *page, get_block_t *get_block,
2508                         struct writeback_control *wbc)
2509 {
2510         struct inode * const inode = page->mapping->host;
2511         loff_t i_size = i_size_read(inode);
2512         const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2513         unsigned offset;
2514         void *kaddr;
2515         int ret;
2516
2517         /* Is the page fully inside i_size? */
2518         if (page->index < end_index)
2519                 goto out;
2520
2521         /* Is the page fully outside i_size? (truncate in progress) */
2522         offset = i_size & (PAGE_CACHE_SIZE-1);
2523         if (page->index >= end_index+1 || !offset) {
2524                 /*
2525                  * The page may have dirty, unmapped buffers.  For example,
2526                  * they may have been added in ext3_writepage().  Make them
2527                  * freeable here, so the page does not leak.
2528                  */
2529 #if 0
2530                 /* Not really sure about this  - do we need this ? */
2531                 if (page->mapping->a_ops->invalidatepage)
2532                         page->mapping->a_ops->invalidatepage(page, offset);
2533 #endif
2534                 unlock_page(page);
2535                 return 0; /* don't care */
2536         }
2537
2538         /*
2539          * The page straddles i_size.  It must be zeroed out on each and every
2540          * writepage invocation because it may be mmapped.  "A file is mapped
2541          * in multiples of the page size.  For a file that is not a multiple of
2542          * the  page size, the remaining memory is zeroed when mapped, and
2543          * writes to that region are not written out to the file."
2544          */
2545         kaddr = kmap_atomic(page, KM_USER0);
2546         memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2547         flush_dcache_page(page);
2548         kunmap_atomic(kaddr, KM_USER0);
2549 out:
2550         ret = mpage_writepage(page, get_block, wbc);
2551         if (ret == -EAGAIN)
2552                 ret = __block_write_full_page(inode, page, get_block, wbc);
2553         return ret;
2554 }
2555 EXPORT_SYMBOL(nobh_writepage);
2556
2557 /*
2558  * This function assumes that ->prepare_write() uses nobh_prepare_write().
2559  */
2560 int nobh_truncate_page(struct address_space *mapping, loff_t from)
2561 {
2562         struct inode *inode = mapping->host;
2563         unsigned blocksize = 1 << inode->i_blkbits;
2564         pgoff_t index = from >> PAGE_CACHE_SHIFT;
2565         unsigned offset = from & (PAGE_CACHE_SIZE-1);
2566         unsigned to;
2567         struct page *page;
2568         struct address_space_operations *a_ops = mapping->a_ops;
2569         char *kaddr;
2570         int ret = 0;
2571
2572         if ((offset & (blocksize - 1)) == 0)
2573                 goto out;
2574
2575         ret = -ENOMEM;
2576         page = grab_cache_page(mapping, index);
2577         if (!page)
2578                 goto out;
2579
2580         to = (offset + blocksize) & ~(blocksize - 1);
2581         ret = a_ops->prepare_write(NULL, page, offset, to);
2582         if (ret == 0) {
2583                 kaddr = kmap_atomic(page, KM_USER0);
2584                 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2585                 flush_dcache_page(page);
2586                 kunmap_atomic(kaddr, KM_USER0);
2587                 set_page_dirty(page);
2588         }
2589         unlock_page(page);
2590         page_cache_release(page);
2591 out:
2592         return ret;
2593 }
2594 EXPORT_SYMBOL(nobh_truncate_page);
2595
2596 int block_truncate_page(struct address_space *mapping,
2597                         loff_t from, get_block_t *get_block)
2598 {
2599         pgoff_t index = from >> PAGE_CACHE_SHIFT;
2600         unsigned offset = from & (PAGE_CACHE_SIZE-1);
2601         unsigned blocksize;
2602         pgoff_t iblock;
2603         unsigned length, pos;
2604         struct inode *inode = mapping->host;
2605         struct page *page;
2606         struct buffer_head *bh;
2607         void *kaddr;
2608         int err;
2609
2610         blocksize = 1 << inode->i_blkbits;
2611         length = offset & (blocksize - 1);
2612
2613         /* Block boundary? Nothing to do */
2614         if (!length)
2615                 return 0;
2616
2617         length = blocksize - length;
2618         iblock = index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2619         
2620         page = grab_cache_page(mapping, index);
2621         err = -ENOMEM;
2622         if (!page)
2623                 goto out;
2624
2625         if (!page_has_buffers(page))
2626                 create_empty_buffers(page, blocksize, 0);
2627
2628         /* Find the buffer that contains "offset" */
2629         bh = page_buffers(page);
2630         pos = blocksize;
2631         while (offset >= pos) {
2632                 bh = bh->b_this_page;
2633                 iblock++;
2634                 pos += blocksize;
2635         }
2636
2637         err = 0;
2638         if (!buffer_mapped(bh)) {
2639                 err = get_block(inode, iblock, bh, 0);
2640                 if (err)
2641                         goto unlock;
2642                 /* unmapped? It's a hole - nothing to do */
2643                 if (!buffer_mapped(bh))
2644                         goto unlock;
2645         }
2646
2647         /* Ok, it's mapped. Make sure it's up-to-date */
2648         if (PageUptodate(page))
2649                 set_buffer_uptodate(bh);
2650
2651         if (!buffer_uptodate(bh) && !buffer_delay(bh)) {
2652                 err = -EIO;
2653                 ll_rw_block(READ, 1, &bh);
2654                 wait_on_buffer(bh);
2655                 /* Uhhuh. Read error. Complain and punt. */
2656                 if (!buffer_uptodate(bh))
2657                         goto unlock;
2658         }
2659
2660         kaddr = kmap_atomic(page, KM_USER0);
2661         memset(kaddr + offset, 0, length);
2662         flush_dcache_page(page);
2663         kunmap_atomic(kaddr, KM_USER0);
2664
2665         mark_buffer_dirty(bh);
2666         err = 0;
2667
2668 unlock:
2669         unlock_page(page);
2670         page_cache_release(page);
2671 out:
2672         return err;
2673 }
2674
2675 /*
2676  * The generic ->writepage function for buffer-backed address_spaces
2677  */
2678 int block_write_full_page(struct page *page, get_block_t *get_block,
2679                         struct writeback_control *wbc)
2680 {
2681         struct inode * const inode = page->mapping->host;
2682         loff_t i_size = i_size_read(inode);
2683         const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2684         unsigned offset;
2685         void *kaddr;
2686
2687         /* Is the page fully inside i_size? */
2688         if (page->index < end_index)
2689                 return __block_write_full_page(inode, page, get_block, wbc);
2690
2691         /* Is the page fully outside i_size? (truncate in progress) */
2692         offset = i_size & (PAGE_CACHE_SIZE-1);
2693         if (page->index >= end_index+1 || !offset) {
2694                 /*
2695                  * The page may have dirty, unmapped buffers.  For example,
2696                  * they may have been added in ext3_writepage().  Make them
2697                  * freeable here, so the page does not leak.
2698                  */
2699                 block_invalidatepage(page, 0);
2700                 unlock_page(page);
2701                 return 0; /* don't care */
2702         }
2703
2704         /*
2705          * The page straddles i_size.  It must be zeroed out on each and every
2706          * writepage invokation because it may be mmapped.  "A file is mapped
2707          * in multiples of the page size.  For a file that is not a multiple of
2708          * the  page size, the remaining memory is zeroed when mapped, and
2709          * writes to that region are not written out to the file."
2710          */
2711         kaddr = kmap_atomic(page, KM_USER0);
2712         memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2713         flush_dcache_page(page);
2714         kunmap_atomic(kaddr, KM_USER0);
2715         return __block_write_full_page(inode, page, get_block, wbc);
2716 }
2717
2718 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2719                             get_block_t *get_block)
2720 {
2721         struct buffer_head tmp;
2722         struct inode *inode = mapping->host;
2723         tmp.b_state = 0;
2724         tmp.b_blocknr = 0;
2725         get_block(inode, block, &tmp, 0);
2726         return tmp.b_blocknr;
2727 }
2728
2729 static int end_bio_bh_io_sync(struct bio *bio, unsigned int bytes_done, int err)
2730 {
2731         struct buffer_head *bh = bio->bi_private;
2732
2733         if (bio->bi_size)
2734                 return 1;
2735
2736         if (err == -EOPNOTSUPP) {
2737                 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2738                 set_bit(BH_Eopnotsupp, &bh->b_state);
2739         }
2740
2741         bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2742         bio_put(bio);
2743         return 0;
2744 }
2745
2746 int submit_bh(int rw, struct buffer_head * bh)
2747 {
2748         struct bio *bio;
2749         int ret = 0;
2750
2751         BUG_ON(!buffer_locked(bh));
2752         BUG_ON(!buffer_mapped(bh));
2753         BUG_ON(!bh->b_end_io);
2754
2755         if (buffer_ordered(bh) && (rw == WRITE))
2756                 rw = WRITE_BARRIER;
2757
2758         /*
2759          * Only clear out a write error when rewriting, should this
2760          * include WRITE_SYNC as well?
2761          */
2762         if (test_set_buffer_req(bh) && (rw == WRITE || rw == WRITE_BARRIER))
2763                 clear_buffer_write_io_error(bh);
2764
2765         /*
2766          * from here on down, it's all bio -- do the initial mapping,
2767          * submit_bio -> generic_make_request may further map this bio around
2768          */
2769         bio = bio_alloc(GFP_NOIO, 1);
2770
2771         bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2772         bio->bi_bdev = bh->b_bdev;
2773         bio->bi_io_vec[0].bv_page = bh->b_page;
2774         bio->bi_io_vec[0].bv_len = bh->b_size;
2775         bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2776
2777         bio->bi_vcnt = 1;
2778         bio->bi_idx = 0;
2779         bio->bi_size = bh->b_size;
2780
2781         bio->bi_end_io = end_bio_bh_io_sync;
2782         bio->bi_private = bh;
2783
2784         bio_get(bio);
2785         submit_bio(rw, bio);
2786
2787         if (bio_flagged(bio, BIO_EOPNOTSUPP))
2788                 ret = -EOPNOTSUPP;
2789
2790         bio_put(bio);
2791         return ret;
2792 }
2793
2794 /**
2795  * ll_rw_block: low-level access to block devices (DEPRECATED)
2796  * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
2797  * @nr: number of &struct buffer_heads in the array
2798  * @bhs: array of pointers to &struct buffer_head
2799  *
2800  * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
2801  * requests an I/O operation on them, either a %READ or a %WRITE.  The third
2802  * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
2803  * are sent to disk. The fourth %READA option is described in the documentation
2804  * for generic_make_request() which ll_rw_block() calls.
2805  *
2806  * This function drops any buffer that it cannot get a lock on (with the
2807  * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
2808  * clean when doing a write request, and any buffer that appears to be
2809  * up-to-date when doing read request.  Further it marks as clean buffers that
2810  * are processed for writing (the buffer cache won't assume that they are
2811  * actually clean until the buffer gets unlocked).
2812  *
2813  * ll_rw_block sets b_end_io to simple completion handler that marks
2814  * the buffer up-to-date (if approriate), unlocks the buffer and wakes
2815  * any waiters. 
2816  *
2817  * All of the buffers must be for the same device, and must also be a
2818  * multiple of the current approved size for the device.
2819  */
2820 void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
2821 {
2822         int i;
2823
2824         for (i = 0; i < nr; i++) {
2825                 struct buffer_head *bh = bhs[i];
2826
2827                 if (rw == SWRITE)
2828                         lock_buffer(bh);
2829                 else if (test_set_buffer_locked(bh))
2830                         continue;
2831
2832                 get_bh(bh);
2833                 if (rw == WRITE || rw == SWRITE) {
2834                         if (test_clear_buffer_dirty(bh)) {
2835                                 bh->b_end_io = end_buffer_write_sync;
2836                                 submit_bh(WRITE, bh);
2837                                 continue;
2838                         }
2839                 } else {
2840                         if (!buffer_uptodate(bh)) {
2841                                 bh->b_end_io = end_buffer_read_sync;
2842                                 submit_bh(rw, bh);
2843                                 continue;
2844                         }
2845                 }
2846                 unlock_buffer(bh);
2847                 put_bh(bh);
2848         }
2849 }
2850
2851 /*
2852  * For a data-integrity writeout, we need to wait upon any in-progress I/O
2853  * and then start new I/O and then wait upon it.  The caller must have a ref on
2854  * the buffer_head.
2855  */
2856 int sync_dirty_buffer(struct buffer_head *bh)
2857 {
2858         int ret = 0;
2859
2860         WARN_ON(atomic_read(&bh->b_count) < 1);
2861         lock_buffer(bh);
2862         if (test_clear_buffer_dirty(bh)) {
2863                 get_bh(bh);
2864                 bh->b_end_io = end_buffer_write_sync;
2865                 ret = submit_bh(WRITE, bh);
2866                 wait_on_buffer(bh);
2867                 if (buffer_eopnotsupp(bh)) {
2868                         clear_buffer_eopnotsupp(bh);
2869                         ret = -EOPNOTSUPP;
2870                 }
2871                 if (!ret && !buffer_uptodate(bh))
2872                         ret = -EIO;
2873         } else {
2874                 unlock_buffer(bh);
2875         }
2876         return ret;
2877 }
2878
2879 /*
2880  * try_to_free_buffers() checks if all the buffers on this particular page
2881  * are unused, and releases them if so.
2882  *
2883  * Exclusion against try_to_free_buffers may be obtained by either
2884  * locking the page or by holding its mapping's private_lock.
2885  *
2886  * If the page is dirty but all the buffers are clean then we need to
2887  * be sure to mark the page clean as well.  This is because the page
2888  * may be against a block device, and a later reattachment of buffers
2889  * to a dirty page will set *all* buffers dirty.  Which would corrupt
2890  * filesystem data on the same device.
2891  *
2892  * The same applies to regular filesystem pages: if all the buffers are
2893  * clean then we set the page clean and proceed.  To do that, we require
2894  * total exclusion from __set_page_dirty_buffers().  That is obtained with
2895  * private_lock.
2896  *
2897  * try_to_free_buffers() is non-blocking.
2898  */
2899 static inline int buffer_busy(struct buffer_head *bh)
2900 {
2901         return atomic_read(&bh->b_count) |
2902                 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
2903 }
2904
2905 static int
2906 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
2907 {
2908         struct buffer_head *head = page_buffers(page);
2909         struct buffer_head *bh;
2910
2911         bh = head;
2912         do {
2913                 if (buffer_write_io_error(bh) && page->mapping)
2914                         set_bit(AS_EIO, &page->mapping->flags);
2915                 if (buffer_busy(bh))
2916                         goto failed;
2917                 bh = bh->b_this_page;
2918         } while (bh != head);
2919
2920         do {
2921                 struct buffer_head *next = bh->b_this_page;
2922
2923                 if (!list_empty(&bh->b_assoc_buffers))
2924                         __remove_assoc_queue(bh);
2925                 bh = next;
2926         } while (bh != head);
2927         *buffers_to_free = head;
2928         __clear_page_buffers(page);
2929         return 1;
2930 failed:
2931         return 0;
2932 }
2933
2934 int try_to_free_buffers(struct page *page)
2935 {
2936         struct address_space * const mapping = page->mapping;
2937         struct buffer_head *buffers_to_free = NULL;
2938         int ret = 0;
2939
2940         BUG_ON(!PageLocked(page));
2941         if (PageWriteback(page))
2942                 return 0;
2943
2944         if (mapping == NULL) {          /* can this still happen? */
2945                 ret = drop_buffers(page, &buffers_to_free);
2946                 goto out;
2947         }
2948
2949         spin_lock(&mapping->private_lock);
2950         ret = drop_buffers(page, &buffers_to_free);
2951         if (ret) {
2952                 /*
2953                  * If the filesystem writes its buffers by hand (eg ext3)
2954                  * then we can have clean buffers against a dirty page.  We
2955                  * clean the page here; otherwise later reattachment of buffers
2956                  * could encounter a non-uptodate page, which is unresolvable.
2957                  * This only applies in the rare case where try_to_free_buffers
2958                  * succeeds but the page is not freed.
2959                  */
2960                 clear_page_dirty(page);
2961         }
2962         spin_unlock(&mapping->private_lock);
2963 out:
2964         if (buffers_to_free) {
2965                 struct buffer_head *bh = buffers_to_free;
2966
2967                 do {
2968                         struct buffer_head *next = bh->b_this_page;
2969                         free_buffer_head(bh);
2970                         bh = next;
2971                 } while (bh != buffers_to_free);
2972         }
2973         return ret;
2974 }
2975 EXPORT_SYMBOL(try_to_free_buffers);
2976
2977 int block_sync_page(struct page *page)
2978 {
2979         struct address_space *mapping;
2980
2981         smp_mb();
2982         mapping = page_mapping(page);
2983         if (mapping)
2984                 blk_run_backing_dev(mapping->backing_dev_info, page);
2985         return 0;
2986 }
2987
2988 /*
2989  * There are no bdflush tunables left.  But distributions are
2990  * still running obsolete flush daemons, so we terminate them here.
2991  *
2992  * Use of bdflush() is deprecated and will be removed in a future kernel.
2993  * The `pdflush' kernel threads fully replace bdflush daemons and this call.
2994  */
2995 asmlinkage long sys_bdflush(int func, long data)
2996 {
2997         static int msg_count;
2998
2999         if (!capable(CAP_SYS_ADMIN))
3000                 return -EPERM;
3001
3002         if (msg_count < 5) {
3003                 msg_count++;
3004                 printk(KERN_INFO
3005                         "warning: process `%s' used the obsolete bdflush"
3006                         " system call\n", current->comm);
3007                 printk(KERN_INFO "Fix your initscripts?\n");
3008         }
3009
3010         if (func == 1)
3011                 do_exit(0);
3012         return 0;
3013 }
3014
3015 /*
3016  * Buffer-head allocation
3017  */
3018 static kmem_cache_t *bh_cachep;
3019
3020 /*
3021  * Once the number of bh's in the machine exceeds this level, we start
3022  * stripping them in writeback.
3023  */
3024 static int max_buffer_heads;
3025
3026 int buffer_heads_over_limit;
3027
3028 struct bh_accounting {
3029         int nr;                 /* Number of live bh's */
3030         int ratelimit;          /* Limit cacheline bouncing */
3031 };
3032
3033 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3034
3035 static void recalc_bh_state(void)
3036 {
3037         int i;
3038         int tot = 0;
3039
3040         if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
3041                 return;
3042         __get_cpu_var(bh_accounting).ratelimit = 0;
3043         for_each_cpu(i)
3044                 tot += per_cpu(bh_accounting, i).nr;
3045         buffer_heads_over_limit = (tot > max_buffer_heads);
3046 }
3047         
3048 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3049 {
3050         struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags);
3051         if (ret) {
3052                 get_cpu_var(bh_accounting).nr++;
3053                 recalc_bh_state();
3054                 put_cpu_var(bh_accounting);
3055         }
3056         return ret;
3057 }
3058 EXPORT_SYMBOL(alloc_buffer_head);
3059
3060 void free_buffer_head(struct buffer_head *bh)
3061 {
3062         BUG_ON(!list_empty(&bh->b_assoc_buffers));
3063         kmem_cache_free(bh_cachep, bh);
3064         get_cpu_var(bh_accounting).nr--;
3065         recalc_bh_state();
3066         put_cpu_var(bh_accounting);
3067 }
3068 EXPORT_SYMBOL(free_buffer_head);
3069
3070 static void
3071 init_buffer_head(void *data, kmem_cache_t *cachep, unsigned long flags)
3072 {
3073         if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
3074                             SLAB_CTOR_CONSTRUCTOR) {
3075                 struct buffer_head * bh = (struct buffer_head *)data;
3076
3077                 memset(bh, 0, sizeof(*bh));
3078                 INIT_LIST_HEAD(&bh->b_assoc_buffers);
3079         }
3080 }
3081
3082 #ifdef CONFIG_HOTPLUG_CPU
3083 static void buffer_exit_cpu(int cpu)
3084 {
3085         int i;
3086         struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3087
3088         for (i = 0; i < BH_LRU_SIZE; i++) {
3089                 brelse(b->bhs[i]);
3090                 b->bhs[i] = NULL;
3091         }
3092 }
3093
3094 static int buffer_cpu_notify(struct notifier_block *self,
3095                               unsigned long action, void *hcpu)
3096 {
3097         if (action == CPU_DEAD)
3098                 buffer_exit_cpu((unsigned long)hcpu);
3099         return NOTIFY_OK;
3100 }
3101 #endif /* CONFIG_HOTPLUG_CPU */
3102
3103 void __init buffer_init(void)
3104 {
3105         int nrpages;
3106
3107         bh_cachep = kmem_cache_create("buffer_head",
3108                         sizeof(struct buffer_head), 0,
3109                         SLAB_RECLAIM_ACCOUNT|SLAB_PANIC, init_buffer_head, NULL);
3110
3111         /*
3112          * Limit the bh occupancy to 10% of ZONE_NORMAL
3113          */
3114         nrpages = (nr_free_buffer_pages() * 10) / 100;
3115         max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3116         hotcpu_notifier(buffer_cpu_notify, 0);
3117 }
3118
3119 EXPORT_SYMBOL(__bforget);
3120 EXPORT_SYMBOL(__brelse);
3121 EXPORT_SYMBOL(__wait_on_buffer);
3122 EXPORT_SYMBOL(block_commit_write);
3123 EXPORT_SYMBOL(block_prepare_write);
3124 EXPORT_SYMBOL(block_read_full_page);
3125 EXPORT_SYMBOL(block_sync_page);
3126 EXPORT_SYMBOL(block_truncate_page);
3127 EXPORT_SYMBOL(block_write_full_page);
3128 EXPORT_SYMBOL(cont_prepare_write);
3129 EXPORT_SYMBOL(end_buffer_async_write);
3130 EXPORT_SYMBOL(end_buffer_read_sync);
3131 EXPORT_SYMBOL(end_buffer_write_sync);
3132 EXPORT_SYMBOL(file_fsync);
3133 EXPORT_SYMBOL(fsync_bdev);
3134 EXPORT_SYMBOL(generic_block_bmap);
3135 EXPORT_SYMBOL(generic_commit_write);
3136 EXPORT_SYMBOL(generic_cont_expand);
3137 EXPORT_SYMBOL(init_buffer);
3138 EXPORT_SYMBOL(invalidate_bdev);
3139 EXPORT_SYMBOL(ll_rw_block);
3140 EXPORT_SYMBOL(mark_buffer_dirty);
3141 EXPORT_SYMBOL(submit_bh);
3142 EXPORT_SYMBOL(sync_dirty_buffer);
3143 EXPORT_SYMBOL(unlock_buffer);