USB: musb: do not work if no gadget driver is loaded
[linux-2.6.git] / fs / buffer.c
1 /*
2  *  linux/fs/buffer.c
3  *
4  *  Copyright (C) 1991, 1992, 2002  Linus Torvalds
5  */
6
7 /*
8  * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9  *
10  * Removed a lot of unnecessary code and simplified things now that
11  * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12  *
13  * Speed up hash, lru, and free list operations.  Use gfp() for allocating
14  * hash table, use SLAB cache for buffer heads. SMP threading.  -DaveM
15  *
16  * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17  *
18  * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/syscalls.h>
23 #include <linux/fs.h>
24 #include <linux/mm.h>
25 #include <linux/percpu.h>
26 #include <linux/slab.h>
27 #include <linux/capability.h>
28 #include <linux/blkdev.h>
29 #include <linux/file.h>
30 #include <linux/quotaops.h>
31 #include <linux/highmem.h>
32 #include <linux/module.h>
33 #include <linux/writeback.h>
34 #include <linux/hash.h>
35 #include <linux/suspend.h>
36 #include <linux/buffer_head.h>
37 #include <linux/task_io_accounting_ops.h>
38 #include <linux/bio.h>
39 #include <linux/notifier.h>
40 #include <linux/cpu.h>
41 #include <linux/bitops.h>
42 #include <linux/mpage.h>
43 #include <linux/bit_spinlock.h>
44
45 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
46
47 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
48
49 inline void
50 init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
51 {
52         bh->b_end_io = handler;
53         bh->b_private = private;
54 }
55 EXPORT_SYMBOL(init_buffer);
56
57 static int sync_buffer(void *word)
58 {
59         struct block_device *bd;
60         struct buffer_head *bh
61                 = container_of(word, struct buffer_head, b_state);
62
63         smp_mb();
64         bd = bh->b_bdev;
65         if (bd)
66                 blk_run_address_space(bd->bd_inode->i_mapping);
67         io_schedule();
68         return 0;
69 }
70
71 void __lock_buffer(struct buffer_head *bh)
72 {
73         wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
74                                                         TASK_UNINTERRUPTIBLE);
75 }
76 EXPORT_SYMBOL(__lock_buffer);
77
78 void unlock_buffer(struct buffer_head *bh)
79 {
80         clear_bit_unlock(BH_Lock, &bh->b_state);
81         smp_mb__after_clear_bit();
82         wake_up_bit(&bh->b_state, BH_Lock);
83 }
84 EXPORT_SYMBOL(unlock_buffer);
85
86 /*
87  * Block until a buffer comes unlocked.  This doesn't stop it
88  * from becoming locked again - you have to lock it yourself
89  * if you want to preserve its state.
90  */
91 void __wait_on_buffer(struct buffer_head * bh)
92 {
93         wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
94 }
95 EXPORT_SYMBOL(__wait_on_buffer);
96
97 static void
98 __clear_page_buffers(struct page *page)
99 {
100         ClearPagePrivate(page);
101         set_page_private(page, 0);
102         page_cache_release(page);
103 }
104
105
106 static int quiet_error(struct buffer_head *bh)
107 {
108         if (!test_bit(BH_Quiet, &bh->b_state) && printk_ratelimit())
109                 return 0;
110         return 1;
111 }
112
113
114 static void buffer_io_error(struct buffer_head *bh)
115 {
116         char b[BDEVNAME_SIZE];
117         printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
118                         bdevname(bh->b_bdev, b),
119                         (unsigned long long)bh->b_blocknr);
120 }
121
122 /*
123  * End-of-IO handler helper function which does not touch the bh after
124  * unlocking it.
125  * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
126  * a race there is benign: unlock_buffer() only use the bh's address for
127  * hashing after unlocking the buffer, so it doesn't actually touch the bh
128  * itself.
129  */
130 static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
131 {
132         if (uptodate) {
133                 set_buffer_uptodate(bh);
134         } else {
135                 /* This happens, due to failed READA attempts. */
136                 clear_buffer_uptodate(bh);
137         }
138         unlock_buffer(bh);
139 }
140
141 /*
142  * Default synchronous end-of-IO handler..  Just mark it up-to-date and
143  * unlock the buffer. This is what ll_rw_block uses too.
144  */
145 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
146 {
147         __end_buffer_read_notouch(bh, uptodate);
148         put_bh(bh);
149 }
150 EXPORT_SYMBOL(end_buffer_read_sync);
151
152 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
153 {
154         char b[BDEVNAME_SIZE];
155
156         if (uptodate) {
157                 set_buffer_uptodate(bh);
158         } else {
159                 if (!buffer_eopnotsupp(bh) && !quiet_error(bh)) {
160                         buffer_io_error(bh);
161                         printk(KERN_WARNING "lost page write due to "
162                                         "I/O error on %s\n",
163                                        bdevname(bh->b_bdev, b));
164                 }
165                 set_buffer_write_io_error(bh);
166                 clear_buffer_uptodate(bh);
167         }
168         unlock_buffer(bh);
169         put_bh(bh);
170 }
171 EXPORT_SYMBOL(end_buffer_write_sync);
172
173 /*
174  * Various filesystems appear to want __find_get_block to be non-blocking.
175  * But it's the page lock which protects the buffers.  To get around this,
176  * we get exclusion from try_to_free_buffers with the blockdev mapping's
177  * private_lock.
178  *
179  * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
180  * may be quite high.  This code could TryLock the page, and if that
181  * succeeds, there is no need to take private_lock. (But if
182  * private_lock is contended then so is mapping->tree_lock).
183  */
184 static struct buffer_head *
185 __find_get_block_slow(struct block_device *bdev, sector_t block)
186 {
187         struct inode *bd_inode = bdev->bd_inode;
188         struct address_space *bd_mapping = bd_inode->i_mapping;
189         struct buffer_head *ret = NULL;
190         pgoff_t index;
191         struct buffer_head *bh;
192         struct buffer_head *head;
193         struct page *page;
194         int all_mapped = 1;
195
196         index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
197         page = find_get_page(bd_mapping, index);
198         if (!page)
199                 goto out;
200
201         spin_lock(&bd_mapping->private_lock);
202         if (!page_has_buffers(page))
203                 goto out_unlock;
204         head = page_buffers(page);
205         bh = head;
206         do {
207                 if (!buffer_mapped(bh))
208                         all_mapped = 0;
209                 else if (bh->b_blocknr == block) {
210                         ret = bh;
211                         get_bh(bh);
212                         goto out_unlock;
213                 }
214                 bh = bh->b_this_page;
215         } while (bh != head);
216
217         /* we might be here because some of the buffers on this page are
218          * not mapped.  This is due to various races between
219          * file io on the block device and getblk.  It gets dealt with
220          * elsewhere, don't buffer_error if we had some unmapped buffers
221          */
222         if (all_mapped) {
223                 printk("__find_get_block_slow() failed. "
224                         "block=%llu, b_blocknr=%llu\n",
225                         (unsigned long long)block,
226                         (unsigned long long)bh->b_blocknr);
227                 printk("b_state=0x%08lx, b_size=%zu\n",
228                         bh->b_state, bh->b_size);
229                 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
230         }
231 out_unlock:
232         spin_unlock(&bd_mapping->private_lock);
233         page_cache_release(page);
234 out:
235         return ret;
236 }
237
238 /* If invalidate_buffers() will trash dirty buffers, it means some kind
239    of fs corruption is going on. Trashing dirty data always imply losing
240    information that was supposed to be just stored on the physical layer
241    by the user.
242
243    Thus invalidate_buffers in general usage is not allwowed to trash
244    dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
245    be preserved.  These buffers are simply skipped.
246   
247    We also skip buffers which are still in use.  For example this can
248    happen if a userspace program is reading the block device.
249
250    NOTE: In the case where the user removed a removable-media-disk even if
251    there's still dirty data not synced on disk (due a bug in the device driver
252    or due an error of the user), by not destroying the dirty buffers we could
253    generate corruption also on the next media inserted, thus a parameter is
254    necessary to handle this case in the most safe way possible (trying
255    to not corrupt also the new disk inserted with the data belonging to
256    the old now corrupted disk). Also for the ramdisk the natural thing
257    to do in order to release the ramdisk memory is to destroy dirty buffers.
258
259    These are two special cases. Normal usage imply the device driver
260    to issue a sync on the device (without waiting I/O completion) and
261    then an invalidate_buffers call that doesn't trash dirty buffers.
262
263    For handling cache coherency with the blkdev pagecache the 'update' case
264    is been introduced. It is needed to re-read from disk any pinned
265    buffer. NOTE: re-reading from disk is destructive so we can do it only
266    when we assume nobody is changing the buffercache under our I/O and when
267    we think the disk contains more recent information than the buffercache.
268    The update == 1 pass marks the buffers we need to update, the update == 2
269    pass does the actual I/O. */
270 void invalidate_bdev(struct block_device *bdev)
271 {
272         struct address_space *mapping = bdev->bd_inode->i_mapping;
273
274         if (mapping->nrpages == 0)
275                 return;
276
277         invalidate_bh_lrus();
278         invalidate_mapping_pages(mapping, 0, -1);
279 }
280 EXPORT_SYMBOL(invalidate_bdev);
281
282 /*
283  * Kick the writeback threads then try to free up some ZONE_NORMAL memory.
284  */
285 static void free_more_memory(void)
286 {
287         struct zone *zone;
288         int nid;
289
290         wakeup_flusher_threads(1024);
291         yield();
292
293         for_each_online_node(nid) {
294                 (void)first_zones_zonelist(node_zonelist(nid, GFP_NOFS),
295                                                 gfp_zone(GFP_NOFS), NULL,
296                                                 &zone);
297                 if (zone)
298                         try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0,
299                                                 GFP_NOFS, NULL);
300         }
301 }
302
303 /*
304  * I/O completion handler for block_read_full_page() - pages
305  * which come unlocked at the end of I/O.
306  */
307 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
308 {
309         unsigned long flags;
310         struct buffer_head *first;
311         struct buffer_head *tmp;
312         struct page *page;
313         int page_uptodate = 1;
314
315         BUG_ON(!buffer_async_read(bh));
316
317         page = bh->b_page;
318         if (uptodate) {
319                 set_buffer_uptodate(bh);
320         } else {
321                 clear_buffer_uptodate(bh);
322                 if (!quiet_error(bh))
323                         buffer_io_error(bh);
324                 SetPageError(page);
325         }
326
327         /*
328          * Be _very_ careful from here on. Bad things can happen if
329          * two buffer heads end IO at almost the same time and both
330          * decide that the page is now completely done.
331          */
332         first = page_buffers(page);
333         local_irq_save(flags);
334         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
335         clear_buffer_async_read(bh);
336         unlock_buffer(bh);
337         tmp = bh;
338         do {
339                 if (!buffer_uptodate(tmp))
340                         page_uptodate = 0;
341                 if (buffer_async_read(tmp)) {
342                         BUG_ON(!buffer_locked(tmp));
343                         goto still_busy;
344                 }
345                 tmp = tmp->b_this_page;
346         } while (tmp != bh);
347         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
348         local_irq_restore(flags);
349
350         /*
351          * If none of the buffers had errors and they are all
352          * uptodate then we can set the page uptodate.
353          */
354         if (page_uptodate && !PageError(page))
355                 SetPageUptodate(page);
356         unlock_page(page);
357         return;
358
359 still_busy:
360         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
361         local_irq_restore(flags);
362         return;
363 }
364
365 /*
366  * Completion handler for block_write_full_page() - pages which are unlocked
367  * during I/O, and which have PageWriteback cleared upon I/O completion.
368  */
369 void end_buffer_async_write(struct buffer_head *bh, int uptodate)
370 {
371         char b[BDEVNAME_SIZE];
372         unsigned long flags;
373         struct buffer_head *first;
374         struct buffer_head *tmp;
375         struct page *page;
376
377         BUG_ON(!buffer_async_write(bh));
378
379         page = bh->b_page;
380         if (uptodate) {
381                 set_buffer_uptodate(bh);
382         } else {
383                 if (!quiet_error(bh)) {
384                         buffer_io_error(bh);
385                         printk(KERN_WARNING "lost page write due to "
386                                         "I/O error on %s\n",
387                                bdevname(bh->b_bdev, b));
388                 }
389                 set_bit(AS_EIO, &page->mapping->flags);
390                 set_buffer_write_io_error(bh);
391                 clear_buffer_uptodate(bh);
392                 SetPageError(page);
393         }
394
395         first = page_buffers(page);
396         local_irq_save(flags);
397         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
398
399         clear_buffer_async_write(bh);
400         unlock_buffer(bh);
401         tmp = bh->b_this_page;
402         while (tmp != bh) {
403                 if (buffer_async_write(tmp)) {
404                         BUG_ON(!buffer_locked(tmp));
405                         goto still_busy;
406                 }
407                 tmp = tmp->b_this_page;
408         }
409         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
410         local_irq_restore(flags);
411         end_page_writeback(page);
412         return;
413
414 still_busy:
415         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
416         local_irq_restore(flags);
417         return;
418 }
419 EXPORT_SYMBOL(end_buffer_async_write);
420
421 /*
422  * If a page's buffers are under async readin (end_buffer_async_read
423  * completion) then there is a possibility that another thread of
424  * control could lock one of the buffers after it has completed
425  * but while some of the other buffers have not completed.  This
426  * locked buffer would confuse end_buffer_async_read() into not unlocking
427  * the page.  So the absence of BH_Async_Read tells end_buffer_async_read()
428  * that this buffer is not under async I/O.
429  *
430  * The page comes unlocked when it has no locked buffer_async buffers
431  * left.
432  *
433  * PageLocked prevents anyone starting new async I/O reads any of
434  * the buffers.
435  *
436  * PageWriteback is used to prevent simultaneous writeout of the same
437  * page.
438  *
439  * PageLocked prevents anyone from starting writeback of a page which is
440  * under read I/O (PageWriteback is only ever set against a locked page).
441  */
442 static void mark_buffer_async_read(struct buffer_head *bh)
443 {
444         bh->b_end_io = end_buffer_async_read;
445         set_buffer_async_read(bh);
446 }
447
448 static void mark_buffer_async_write_endio(struct buffer_head *bh,
449                                           bh_end_io_t *handler)
450 {
451         bh->b_end_io = handler;
452         set_buffer_async_write(bh);
453 }
454
455 void mark_buffer_async_write(struct buffer_head *bh)
456 {
457         mark_buffer_async_write_endio(bh, end_buffer_async_write);
458 }
459 EXPORT_SYMBOL(mark_buffer_async_write);
460
461
462 /*
463  * fs/buffer.c contains helper functions for buffer-backed address space's
464  * fsync functions.  A common requirement for buffer-based filesystems is
465  * that certain data from the backing blockdev needs to be written out for
466  * a successful fsync().  For example, ext2 indirect blocks need to be
467  * written back and waited upon before fsync() returns.
468  *
469  * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
470  * inode_has_buffers() and invalidate_inode_buffers() are provided for the
471  * management of a list of dependent buffers at ->i_mapping->private_list.
472  *
473  * Locking is a little subtle: try_to_free_buffers() will remove buffers
474  * from their controlling inode's queue when they are being freed.  But
475  * try_to_free_buffers() will be operating against the *blockdev* mapping
476  * at the time, not against the S_ISREG file which depends on those buffers.
477  * So the locking for private_list is via the private_lock in the address_space
478  * which backs the buffers.  Which is different from the address_space 
479  * against which the buffers are listed.  So for a particular address_space,
480  * mapping->private_lock does *not* protect mapping->private_list!  In fact,
481  * mapping->private_list will always be protected by the backing blockdev's
482  * ->private_lock.
483  *
484  * Which introduces a requirement: all buffers on an address_space's
485  * ->private_list must be from the same address_space: the blockdev's.
486  *
487  * address_spaces which do not place buffers at ->private_list via these
488  * utility functions are free to use private_lock and private_list for
489  * whatever they want.  The only requirement is that list_empty(private_list)
490  * be true at clear_inode() time.
491  *
492  * FIXME: clear_inode should not call invalidate_inode_buffers().  The
493  * filesystems should do that.  invalidate_inode_buffers() should just go
494  * BUG_ON(!list_empty).
495  *
496  * FIXME: mark_buffer_dirty_inode() is a data-plane operation.  It should
497  * take an address_space, not an inode.  And it should be called
498  * mark_buffer_dirty_fsync() to clearly define why those buffers are being
499  * queued up.
500  *
501  * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
502  * list if it is already on a list.  Because if the buffer is on a list,
503  * it *must* already be on the right one.  If not, the filesystem is being
504  * silly.  This will save a ton of locking.  But first we have to ensure
505  * that buffers are taken *off* the old inode's list when they are freed
506  * (presumably in truncate).  That requires careful auditing of all
507  * filesystems (do it inside bforget()).  It could also be done by bringing
508  * b_inode back.
509  */
510
511 /*
512  * The buffer's backing address_space's private_lock must be held
513  */
514 static void __remove_assoc_queue(struct buffer_head *bh)
515 {
516         list_del_init(&bh->b_assoc_buffers);
517         WARN_ON(!bh->b_assoc_map);
518         if (buffer_write_io_error(bh))
519                 set_bit(AS_EIO, &bh->b_assoc_map->flags);
520         bh->b_assoc_map = NULL;
521 }
522
523 int inode_has_buffers(struct inode *inode)
524 {
525         return !list_empty(&inode->i_data.private_list);
526 }
527
528 /*
529  * osync is designed to support O_SYNC io.  It waits synchronously for
530  * all already-submitted IO to complete, but does not queue any new
531  * writes to the disk.
532  *
533  * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
534  * you dirty the buffers, and then use osync_inode_buffers to wait for
535  * completion.  Any other dirty buffers which are not yet queued for
536  * write will not be flushed to disk by the osync.
537  */
538 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
539 {
540         struct buffer_head *bh;
541         struct list_head *p;
542         int err = 0;
543
544         spin_lock(lock);
545 repeat:
546         list_for_each_prev(p, list) {
547                 bh = BH_ENTRY(p);
548                 if (buffer_locked(bh)) {
549                         get_bh(bh);
550                         spin_unlock(lock);
551                         wait_on_buffer(bh);
552                         if (!buffer_uptodate(bh))
553                                 err = -EIO;
554                         brelse(bh);
555                         spin_lock(lock);
556                         goto repeat;
557                 }
558         }
559         spin_unlock(lock);
560         return err;
561 }
562
563 static void do_thaw_all(struct work_struct *work)
564 {
565         struct super_block *sb;
566         char b[BDEVNAME_SIZE];
567
568         spin_lock(&sb_lock);
569 restart:
570         list_for_each_entry(sb, &super_blocks, s_list) {
571                 sb->s_count++;
572                 spin_unlock(&sb_lock);
573                 down_read(&sb->s_umount);
574                 while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb))
575                         printk(KERN_WARNING "Emergency Thaw on %s\n",
576                                bdevname(sb->s_bdev, b));
577                 up_read(&sb->s_umount);
578                 spin_lock(&sb_lock);
579                 if (__put_super_and_need_restart(sb))
580                         goto restart;
581         }
582         spin_unlock(&sb_lock);
583         kfree(work);
584         printk(KERN_WARNING "Emergency Thaw complete\n");
585 }
586
587 /**
588  * emergency_thaw_all -- forcibly thaw every frozen filesystem
589  *
590  * Used for emergency unfreeze of all filesystems via SysRq
591  */
592 void emergency_thaw_all(void)
593 {
594         struct work_struct *work;
595
596         work = kmalloc(sizeof(*work), GFP_ATOMIC);
597         if (work) {
598                 INIT_WORK(work, do_thaw_all);
599                 schedule_work(work);
600         }
601 }
602
603 /**
604  * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
605  * @mapping: the mapping which wants those buffers written
606  *
607  * Starts I/O against the buffers at mapping->private_list, and waits upon
608  * that I/O.
609  *
610  * Basically, this is a convenience function for fsync().
611  * @mapping is a file or directory which needs those buffers to be written for
612  * a successful fsync().
613  */
614 int sync_mapping_buffers(struct address_space *mapping)
615 {
616         struct address_space *buffer_mapping = mapping->assoc_mapping;
617
618         if (buffer_mapping == NULL || list_empty(&mapping->private_list))
619                 return 0;
620
621         return fsync_buffers_list(&buffer_mapping->private_lock,
622                                         &mapping->private_list);
623 }
624 EXPORT_SYMBOL(sync_mapping_buffers);
625
626 /*
627  * Called when we've recently written block `bblock', and it is known that
628  * `bblock' was for a buffer_boundary() buffer.  This means that the block at
629  * `bblock + 1' is probably a dirty indirect block.  Hunt it down and, if it's
630  * dirty, schedule it for IO.  So that indirects merge nicely with their data.
631  */
632 void write_boundary_block(struct block_device *bdev,
633                         sector_t bblock, unsigned blocksize)
634 {
635         struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
636         if (bh) {
637                 if (buffer_dirty(bh))
638                         ll_rw_block(WRITE, 1, &bh);
639                 put_bh(bh);
640         }
641 }
642
643 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
644 {
645         struct address_space *mapping = inode->i_mapping;
646         struct address_space *buffer_mapping = bh->b_page->mapping;
647
648         mark_buffer_dirty(bh);
649         if (!mapping->assoc_mapping) {
650                 mapping->assoc_mapping = buffer_mapping;
651         } else {
652                 BUG_ON(mapping->assoc_mapping != buffer_mapping);
653         }
654         if (!bh->b_assoc_map) {
655                 spin_lock(&buffer_mapping->private_lock);
656                 list_move_tail(&bh->b_assoc_buffers,
657                                 &mapping->private_list);
658                 bh->b_assoc_map = mapping;
659                 spin_unlock(&buffer_mapping->private_lock);
660         }
661 }
662 EXPORT_SYMBOL(mark_buffer_dirty_inode);
663
664 /*
665  * Mark the page dirty, and set it dirty in the radix tree, and mark the inode
666  * dirty.
667  *
668  * If warn is true, then emit a warning if the page is not uptodate and has
669  * not been truncated.
670  */
671 static void __set_page_dirty(struct page *page,
672                 struct address_space *mapping, int warn)
673 {
674         spin_lock_irq(&mapping->tree_lock);
675         if (page->mapping) {    /* Race with truncate? */
676                 WARN_ON_ONCE(warn && !PageUptodate(page));
677                 account_page_dirtied(page, mapping);
678                 radix_tree_tag_set(&mapping->page_tree,
679                                 page_index(page), PAGECACHE_TAG_DIRTY);
680         }
681         spin_unlock_irq(&mapping->tree_lock);
682         __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
683 }
684
685 /*
686  * Add a page to the dirty page list.
687  *
688  * It is a sad fact of life that this function is called from several places
689  * deeply under spinlocking.  It may not sleep.
690  *
691  * If the page has buffers, the uptodate buffers are set dirty, to preserve
692  * dirty-state coherency between the page and the buffers.  It the page does
693  * not have buffers then when they are later attached they will all be set
694  * dirty.
695  *
696  * The buffers are dirtied before the page is dirtied.  There's a small race
697  * window in which a writepage caller may see the page cleanness but not the
698  * buffer dirtiness.  That's fine.  If this code were to set the page dirty
699  * before the buffers, a concurrent writepage caller could clear the page dirty
700  * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
701  * page on the dirty page list.
702  *
703  * We use private_lock to lock against try_to_free_buffers while using the
704  * page's buffer list.  Also use this to protect against clean buffers being
705  * added to the page after it was set dirty.
706  *
707  * FIXME: may need to call ->reservepage here as well.  That's rather up to the
708  * address_space though.
709  */
710 int __set_page_dirty_buffers(struct page *page)
711 {
712         int newly_dirty;
713         struct address_space *mapping = page_mapping(page);
714
715         if (unlikely(!mapping))
716                 return !TestSetPageDirty(page);
717
718         spin_lock(&mapping->private_lock);
719         if (page_has_buffers(page)) {
720                 struct buffer_head *head = page_buffers(page);
721                 struct buffer_head *bh = head;
722
723                 do {
724                         set_buffer_dirty(bh);
725                         bh = bh->b_this_page;
726                 } while (bh != head);
727         }
728         newly_dirty = !TestSetPageDirty(page);
729         spin_unlock(&mapping->private_lock);
730
731         if (newly_dirty)
732                 __set_page_dirty(page, mapping, 1);
733         return newly_dirty;
734 }
735 EXPORT_SYMBOL(__set_page_dirty_buffers);
736
737 /*
738  * Write out and wait upon a list of buffers.
739  *
740  * We have conflicting pressures: we want to make sure that all
741  * initially dirty buffers get waited on, but that any subsequently
742  * dirtied buffers don't.  After all, we don't want fsync to last
743  * forever if somebody is actively writing to the file.
744  *
745  * Do this in two main stages: first we copy dirty buffers to a
746  * temporary inode list, queueing the writes as we go.  Then we clean
747  * up, waiting for those writes to complete.
748  * 
749  * During this second stage, any subsequent updates to the file may end
750  * up refiling the buffer on the original inode's dirty list again, so
751  * there is a chance we will end up with a buffer queued for write but
752  * not yet completed on that list.  So, as a final cleanup we go through
753  * the osync code to catch these locked, dirty buffers without requeuing
754  * any newly dirty buffers for write.
755  */
756 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
757 {
758         struct buffer_head *bh;
759         struct list_head tmp;
760         struct address_space *mapping, *prev_mapping = NULL;
761         int err = 0, err2;
762
763         INIT_LIST_HEAD(&tmp);
764
765         spin_lock(lock);
766         while (!list_empty(list)) {
767                 bh = BH_ENTRY(list->next);
768                 mapping = bh->b_assoc_map;
769                 __remove_assoc_queue(bh);
770                 /* Avoid race with mark_buffer_dirty_inode() which does
771                  * a lockless check and we rely on seeing the dirty bit */
772                 smp_mb();
773                 if (buffer_dirty(bh) || buffer_locked(bh)) {
774                         list_add(&bh->b_assoc_buffers, &tmp);
775                         bh->b_assoc_map = mapping;
776                         if (buffer_dirty(bh)) {
777                                 get_bh(bh);
778                                 spin_unlock(lock);
779                                 /*
780                                  * Ensure any pending I/O completes so that
781                                  * ll_rw_block() actually writes the current
782                                  * contents - it is a noop if I/O is still in
783                                  * flight on potentially older contents.
784                                  */
785                                 ll_rw_block(SWRITE_SYNC_PLUG, 1, &bh);
786
787                                 /*
788                                  * Kick off IO for the previous mapping. Note
789                                  * that we will not run the very last mapping,
790                                  * wait_on_buffer() will do that for us
791                                  * through sync_buffer().
792                                  */
793                                 if (prev_mapping && prev_mapping != mapping)
794                                         blk_run_address_space(prev_mapping);
795                                 prev_mapping = mapping;
796
797                                 brelse(bh);
798                                 spin_lock(lock);
799                         }
800                 }
801         }
802
803         while (!list_empty(&tmp)) {
804                 bh = BH_ENTRY(tmp.prev);
805                 get_bh(bh);
806                 mapping = bh->b_assoc_map;
807                 __remove_assoc_queue(bh);
808                 /* Avoid race with mark_buffer_dirty_inode() which does
809                  * a lockless check and we rely on seeing the dirty bit */
810                 smp_mb();
811                 if (buffer_dirty(bh)) {
812                         list_add(&bh->b_assoc_buffers,
813                                  &mapping->private_list);
814                         bh->b_assoc_map = mapping;
815                 }
816                 spin_unlock(lock);
817                 wait_on_buffer(bh);
818                 if (!buffer_uptodate(bh))
819                         err = -EIO;
820                 brelse(bh);
821                 spin_lock(lock);
822         }
823         
824         spin_unlock(lock);
825         err2 = osync_buffers_list(lock, list);
826         if (err)
827                 return err;
828         else
829                 return err2;
830 }
831
832 /*
833  * Invalidate any and all dirty buffers on a given inode.  We are
834  * probably unmounting the fs, but that doesn't mean we have already
835  * done a sync().  Just drop the buffers from the inode list.
836  *
837  * NOTE: we take the inode's blockdev's mapping's private_lock.  Which
838  * assumes that all the buffers are against the blockdev.  Not true
839  * for reiserfs.
840  */
841 void invalidate_inode_buffers(struct inode *inode)
842 {
843         if (inode_has_buffers(inode)) {
844                 struct address_space *mapping = &inode->i_data;
845                 struct list_head *list = &mapping->private_list;
846                 struct address_space *buffer_mapping = mapping->assoc_mapping;
847
848                 spin_lock(&buffer_mapping->private_lock);
849                 while (!list_empty(list))
850                         __remove_assoc_queue(BH_ENTRY(list->next));
851                 spin_unlock(&buffer_mapping->private_lock);
852         }
853 }
854 EXPORT_SYMBOL(invalidate_inode_buffers);
855
856 /*
857  * Remove any clean buffers from the inode's buffer list.  This is called
858  * when we're trying to free the inode itself.  Those buffers can pin it.
859  *
860  * Returns true if all buffers were removed.
861  */
862 int remove_inode_buffers(struct inode *inode)
863 {
864         int ret = 1;
865
866         if (inode_has_buffers(inode)) {
867                 struct address_space *mapping = &inode->i_data;
868                 struct list_head *list = &mapping->private_list;
869                 struct address_space *buffer_mapping = mapping->assoc_mapping;
870
871                 spin_lock(&buffer_mapping->private_lock);
872                 while (!list_empty(list)) {
873                         struct buffer_head *bh = BH_ENTRY(list->next);
874                         if (buffer_dirty(bh)) {
875                                 ret = 0;
876                                 break;
877                         }
878                         __remove_assoc_queue(bh);
879                 }
880                 spin_unlock(&buffer_mapping->private_lock);
881         }
882         return ret;
883 }
884
885 /*
886  * Create the appropriate buffers when given a page for data area and
887  * the size of each buffer.. Use the bh->b_this_page linked list to
888  * follow the buffers created.  Return NULL if unable to create more
889  * buffers.
890  *
891  * The retry flag is used to differentiate async IO (paging, swapping)
892  * which may not fail from ordinary buffer allocations.
893  */
894 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
895                 int retry)
896 {
897         struct buffer_head *bh, *head;
898         long offset;
899
900 try_again:
901         head = NULL;
902         offset = PAGE_SIZE;
903         while ((offset -= size) >= 0) {
904                 bh = alloc_buffer_head(GFP_NOFS);
905                 if (!bh)
906                         goto no_grow;
907
908                 bh->b_bdev = NULL;
909                 bh->b_this_page = head;
910                 bh->b_blocknr = -1;
911                 head = bh;
912
913                 bh->b_state = 0;
914                 atomic_set(&bh->b_count, 0);
915                 bh->b_private = NULL;
916                 bh->b_size = size;
917
918                 /* Link the buffer to its page */
919                 set_bh_page(bh, page, offset);
920
921                 init_buffer(bh, NULL, NULL);
922         }
923         return head;
924 /*
925  * In case anything failed, we just free everything we got.
926  */
927 no_grow:
928         if (head) {
929                 do {
930                         bh = head;
931                         head = head->b_this_page;
932                         free_buffer_head(bh);
933                 } while (head);
934         }
935
936         /*
937          * Return failure for non-async IO requests.  Async IO requests
938          * are not allowed to fail, so we have to wait until buffer heads
939          * become available.  But we don't want tasks sleeping with 
940          * partially complete buffers, so all were released above.
941          */
942         if (!retry)
943                 return NULL;
944
945         /* We're _really_ low on memory. Now we just
946          * wait for old buffer heads to become free due to
947          * finishing IO.  Since this is an async request and
948          * the reserve list is empty, we're sure there are 
949          * async buffer heads in use.
950          */
951         free_more_memory();
952         goto try_again;
953 }
954 EXPORT_SYMBOL_GPL(alloc_page_buffers);
955
956 static inline void
957 link_dev_buffers(struct page *page, struct buffer_head *head)
958 {
959         struct buffer_head *bh, *tail;
960
961         bh = head;
962         do {
963                 tail = bh;
964                 bh = bh->b_this_page;
965         } while (bh);
966         tail->b_this_page = head;
967         attach_page_buffers(page, head);
968 }
969
970 /*
971  * Initialise the state of a blockdev page's buffers.
972  */ 
973 static void
974 init_page_buffers(struct page *page, struct block_device *bdev,
975                         sector_t block, int size)
976 {
977         struct buffer_head *head = page_buffers(page);
978         struct buffer_head *bh = head;
979         int uptodate = PageUptodate(page);
980
981         do {
982                 if (!buffer_mapped(bh)) {
983                         init_buffer(bh, NULL, NULL);
984                         bh->b_bdev = bdev;
985                         bh->b_blocknr = block;
986                         if (uptodate)
987                                 set_buffer_uptodate(bh);
988                         set_buffer_mapped(bh);
989                 }
990                 block++;
991                 bh = bh->b_this_page;
992         } while (bh != head);
993 }
994
995 /*
996  * Create the page-cache page that contains the requested block.
997  *
998  * This is user purely for blockdev mappings.
999  */
1000 static struct page *
1001 grow_dev_page(struct block_device *bdev, sector_t block,
1002                 pgoff_t index, int size)
1003 {
1004         struct inode *inode = bdev->bd_inode;
1005         struct page *page;
1006         struct buffer_head *bh;
1007
1008         page = find_or_create_page(inode->i_mapping, index,
1009                 (mapping_gfp_mask(inode->i_mapping) & ~__GFP_FS)|__GFP_MOVABLE);
1010         if (!page)
1011                 return NULL;
1012
1013         BUG_ON(!PageLocked(page));
1014
1015         if (page_has_buffers(page)) {
1016                 bh = page_buffers(page);
1017                 if (bh->b_size == size) {
1018                         init_page_buffers(page, bdev, block, size);
1019                         return page;
1020                 }
1021                 if (!try_to_free_buffers(page))
1022                         goto failed;
1023         }
1024
1025         /*
1026          * Allocate some buffers for this page
1027          */
1028         bh = alloc_page_buffers(page, size, 0);
1029         if (!bh)
1030                 goto failed;
1031
1032         /*
1033          * Link the page to the buffers and initialise them.  Take the
1034          * lock to be atomic wrt __find_get_block(), which does not
1035          * run under the page lock.
1036          */
1037         spin_lock(&inode->i_mapping->private_lock);
1038         link_dev_buffers(page, bh);
1039         init_page_buffers(page, bdev, block, size);
1040         spin_unlock(&inode->i_mapping->private_lock);
1041         return page;
1042
1043 failed:
1044         BUG();
1045         unlock_page(page);
1046         page_cache_release(page);
1047         return NULL;
1048 }
1049
1050 /*
1051  * Create buffers for the specified block device block's page.  If
1052  * that page was dirty, the buffers are set dirty also.
1053  */
1054 static int
1055 grow_buffers(struct block_device *bdev, sector_t block, int size)
1056 {
1057         struct page *page;
1058         pgoff_t index;
1059         int sizebits;
1060
1061         sizebits = -1;
1062         do {
1063                 sizebits++;
1064         } while ((size << sizebits) < PAGE_SIZE);
1065
1066         index = block >> sizebits;
1067
1068         /*
1069          * Check for a block which wants to lie outside our maximum possible
1070          * pagecache index.  (this comparison is done using sector_t types).
1071          */
1072         if (unlikely(index != block >> sizebits)) {
1073                 char b[BDEVNAME_SIZE];
1074
1075                 printk(KERN_ERR "%s: requested out-of-range block %llu for "
1076                         "device %s\n",
1077                         __func__, (unsigned long long)block,
1078                         bdevname(bdev, b));
1079                 return -EIO;
1080         }
1081         block = index << sizebits;
1082         /* Create a page with the proper size buffers.. */
1083         page = grow_dev_page(bdev, block, index, size);
1084         if (!page)
1085                 return 0;
1086         unlock_page(page);
1087         page_cache_release(page);
1088         return 1;
1089 }
1090
1091 static struct buffer_head *
1092 __getblk_slow(struct block_device *bdev, sector_t block, int size)
1093 {
1094         /* Size must be multiple of hard sectorsize */
1095         if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
1096                         (size < 512 || size > PAGE_SIZE))) {
1097                 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1098                                         size);
1099                 printk(KERN_ERR "logical block size: %d\n",
1100                                         bdev_logical_block_size(bdev));
1101
1102                 dump_stack();
1103                 return NULL;
1104         }
1105
1106         for (;;) {
1107                 struct buffer_head * bh;
1108                 int ret;
1109
1110                 bh = __find_get_block(bdev, block, size);
1111                 if (bh)
1112                         return bh;
1113
1114                 ret = grow_buffers(bdev, block, size);
1115                 if (ret < 0)
1116                         return NULL;
1117                 if (ret == 0)
1118                         free_more_memory();
1119         }
1120 }
1121
1122 /*
1123  * The relationship between dirty buffers and dirty pages:
1124  *
1125  * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1126  * the page is tagged dirty in its radix tree.
1127  *
1128  * At all times, the dirtiness of the buffers represents the dirtiness of
1129  * subsections of the page.  If the page has buffers, the page dirty bit is
1130  * merely a hint about the true dirty state.
1131  *
1132  * When a page is set dirty in its entirety, all its buffers are marked dirty
1133  * (if the page has buffers).
1134  *
1135  * When a buffer is marked dirty, its page is dirtied, but the page's other
1136  * buffers are not.
1137  *
1138  * Also.  When blockdev buffers are explicitly read with bread(), they
1139  * individually become uptodate.  But their backing page remains not
1140  * uptodate - even if all of its buffers are uptodate.  A subsequent
1141  * block_read_full_page() against that page will discover all the uptodate
1142  * buffers, will set the page uptodate and will perform no I/O.
1143  */
1144
1145 /**
1146  * mark_buffer_dirty - mark a buffer_head as needing writeout
1147  * @bh: the buffer_head to mark dirty
1148  *
1149  * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1150  * backing page dirty, then tag the page as dirty in its address_space's radix
1151  * tree and then attach the address_space's inode to its superblock's dirty
1152  * inode list.
1153  *
1154  * mark_buffer_dirty() is atomic.  It takes bh->b_page->mapping->private_lock,
1155  * mapping->tree_lock and the global inode_lock.
1156  */
1157 void mark_buffer_dirty(struct buffer_head *bh)
1158 {
1159         WARN_ON_ONCE(!buffer_uptodate(bh));
1160
1161         /*
1162          * Very *carefully* optimize the it-is-already-dirty case.
1163          *
1164          * Don't let the final "is it dirty" escape to before we
1165          * perhaps modified the buffer.
1166          */
1167         if (buffer_dirty(bh)) {
1168                 smp_mb();
1169                 if (buffer_dirty(bh))
1170                         return;
1171         }
1172
1173         if (!test_set_buffer_dirty(bh)) {
1174                 struct page *page = bh->b_page;
1175                 if (!TestSetPageDirty(page)) {
1176                         struct address_space *mapping = page_mapping(page);
1177                         if (mapping)
1178                                 __set_page_dirty(page, mapping, 0);
1179                 }
1180         }
1181 }
1182 EXPORT_SYMBOL(mark_buffer_dirty);
1183
1184 /*
1185  * Decrement a buffer_head's reference count.  If all buffers against a page
1186  * have zero reference count, are clean and unlocked, and if the page is clean
1187  * and unlocked then try_to_free_buffers() may strip the buffers from the page
1188  * in preparation for freeing it (sometimes, rarely, buffers are removed from
1189  * a page but it ends up not being freed, and buffers may later be reattached).
1190  */
1191 void __brelse(struct buffer_head * buf)
1192 {
1193         if (atomic_read(&buf->b_count)) {
1194                 put_bh(buf);
1195                 return;
1196         }
1197         WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1198 }
1199 EXPORT_SYMBOL(__brelse);
1200
1201 /*
1202  * bforget() is like brelse(), except it discards any
1203  * potentially dirty data.
1204  */
1205 void __bforget(struct buffer_head *bh)
1206 {
1207         clear_buffer_dirty(bh);
1208         if (bh->b_assoc_map) {
1209                 struct address_space *buffer_mapping = bh->b_page->mapping;
1210
1211                 spin_lock(&buffer_mapping->private_lock);
1212                 list_del_init(&bh->b_assoc_buffers);
1213                 bh->b_assoc_map = NULL;
1214                 spin_unlock(&buffer_mapping->private_lock);
1215         }
1216         __brelse(bh);
1217 }
1218 EXPORT_SYMBOL(__bforget);
1219
1220 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1221 {
1222         lock_buffer(bh);
1223         if (buffer_uptodate(bh)) {
1224                 unlock_buffer(bh);
1225                 return bh;
1226         } else {
1227                 get_bh(bh);
1228                 bh->b_end_io = end_buffer_read_sync;
1229                 submit_bh(READ, bh);
1230                 wait_on_buffer(bh);
1231                 if (buffer_uptodate(bh))
1232                         return bh;
1233         }
1234         brelse(bh);
1235         return NULL;
1236 }
1237
1238 /*
1239  * Per-cpu buffer LRU implementation.  To reduce the cost of __find_get_block().
1240  * The bhs[] array is sorted - newest buffer is at bhs[0].  Buffers have their
1241  * refcount elevated by one when they're in an LRU.  A buffer can only appear
1242  * once in a particular CPU's LRU.  A single buffer can be present in multiple
1243  * CPU's LRUs at the same time.
1244  *
1245  * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1246  * sb_find_get_block().
1247  *
1248  * The LRUs themselves only need locking against invalidate_bh_lrus.  We use
1249  * a local interrupt disable for that.
1250  */
1251
1252 #define BH_LRU_SIZE     8
1253
1254 struct bh_lru {
1255         struct buffer_head *bhs[BH_LRU_SIZE];
1256 };
1257
1258 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1259
1260 #ifdef CONFIG_SMP
1261 #define bh_lru_lock()   local_irq_disable()
1262 #define bh_lru_unlock() local_irq_enable()
1263 #else
1264 #define bh_lru_lock()   preempt_disable()
1265 #define bh_lru_unlock() preempt_enable()
1266 #endif
1267
1268 static inline void check_irqs_on(void)
1269 {
1270 #ifdef irqs_disabled
1271         BUG_ON(irqs_disabled());
1272 #endif
1273 }
1274
1275 /*
1276  * The LRU management algorithm is dopey-but-simple.  Sorry.
1277  */
1278 static void bh_lru_install(struct buffer_head *bh)
1279 {
1280         struct buffer_head *evictee = NULL;
1281         struct bh_lru *lru;
1282
1283         check_irqs_on();
1284         bh_lru_lock();
1285         lru = &__get_cpu_var(bh_lrus);
1286         if (lru->bhs[0] != bh) {
1287                 struct buffer_head *bhs[BH_LRU_SIZE];
1288                 int in;
1289                 int out = 0;
1290
1291                 get_bh(bh);
1292                 bhs[out++] = bh;
1293                 for (in = 0; in < BH_LRU_SIZE; in++) {
1294                         struct buffer_head *bh2 = lru->bhs[in];
1295
1296                         if (bh2 == bh) {
1297                                 __brelse(bh2);
1298                         } else {
1299                                 if (out >= BH_LRU_SIZE) {
1300                                         BUG_ON(evictee != NULL);
1301                                         evictee = bh2;
1302                                 } else {
1303                                         bhs[out++] = bh2;
1304                                 }
1305                         }
1306                 }
1307                 while (out < BH_LRU_SIZE)
1308                         bhs[out++] = NULL;
1309                 memcpy(lru->bhs, bhs, sizeof(bhs));
1310         }
1311         bh_lru_unlock();
1312
1313         if (evictee)
1314                 __brelse(evictee);
1315 }
1316
1317 /*
1318  * Look up the bh in this cpu's LRU.  If it's there, move it to the head.
1319  */
1320 static struct buffer_head *
1321 lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1322 {
1323         struct buffer_head *ret = NULL;
1324         struct bh_lru *lru;
1325         unsigned int i;
1326
1327         check_irqs_on();
1328         bh_lru_lock();
1329         lru = &__get_cpu_var(bh_lrus);
1330         for (i = 0; i < BH_LRU_SIZE; i++) {
1331                 struct buffer_head *bh = lru->bhs[i];
1332
1333                 if (bh && bh->b_bdev == bdev &&
1334                                 bh->b_blocknr == block && bh->b_size == size) {
1335                         if (i) {
1336                                 while (i) {
1337                                         lru->bhs[i] = lru->bhs[i - 1];
1338                                         i--;
1339                                 }
1340                                 lru->bhs[0] = bh;
1341                         }
1342                         get_bh(bh);
1343                         ret = bh;
1344                         break;
1345                 }
1346         }
1347         bh_lru_unlock();
1348         return ret;
1349 }
1350
1351 /*
1352  * Perform a pagecache lookup for the matching buffer.  If it's there, refresh
1353  * it in the LRU and mark it as accessed.  If it is not present then return
1354  * NULL
1355  */
1356 struct buffer_head *
1357 __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1358 {
1359         struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1360
1361         if (bh == NULL) {
1362                 bh = __find_get_block_slow(bdev, block);
1363                 if (bh)
1364                         bh_lru_install(bh);
1365         }
1366         if (bh)
1367                 touch_buffer(bh);
1368         return bh;
1369 }
1370 EXPORT_SYMBOL(__find_get_block);
1371
1372 /*
1373  * __getblk will locate (and, if necessary, create) the buffer_head
1374  * which corresponds to the passed block_device, block and size. The
1375  * returned buffer has its reference count incremented.
1376  *
1377  * __getblk() cannot fail - it just keeps trying.  If you pass it an
1378  * illegal block number, __getblk() will happily return a buffer_head
1379  * which represents the non-existent block.  Very weird.
1380  *
1381  * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1382  * attempt is failing.  FIXME, perhaps?
1383  */
1384 struct buffer_head *
1385 __getblk(struct block_device *bdev, sector_t block, unsigned size)
1386 {
1387         struct buffer_head *bh = __find_get_block(bdev, block, size);
1388
1389         might_sleep();
1390         if (bh == NULL)
1391                 bh = __getblk_slow(bdev, block, size);
1392         return bh;
1393 }
1394 EXPORT_SYMBOL(__getblk);
1395
1396 /*
1397  * Do async read-ahead on a buffer..
1398  */
1399 void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1400 {
1401         struct buffer_head *bh = __getblk(bdev, block, size);
1402         if (likely(bh)) {
1403                 ll_rw_block(READA, 1, &bh);
1404                 brelse(bh);
1405         }
1406 }
1407 EXPORT_SYMBOL(__breadahead);
1408
1409 /**
1410  *  __bread() - reads a specified block and returns the bh
1411  *  @bdev: the block_device to read from
1412  *  @block: number of block
1413  *  @size: size (in bytes) to read
1414  * 
1415  *  Reads a specified block, and returns buffer head that contains it.
1416  *  It returns NULL if the block was unreadable.
1417  */
1418 struct buffer_head *
1419 __bread(struct block_device *bdev, sector_t block, unsigned size)
1420 {
1421         struct buffer_head *bh = __getblk(bdev, block, size);
1422
1423         if (likely(bh) && !buffer_uptodate(bh))
1424                 bh = __bread_slow(bh);
1425         return bh;
1426 }
1427 EXPORT_SYMBOL(__bread);
1428
1429 /*
1430  * invalidate_bh_lrus() is called rarely - but not only at unmount.
1431  * This doesn't race because it runs in each cpu either in irq
1432  * or with preempt disabled.
1433  */
1434 static void invalidate_bh_lru(void *arg)
1435 {
1436         struct bh_lru *b = &get_cpu_var(bh_lrus);
1437         int i;
1438
1439         for (i = 0; i < BH_LRU_SIZE; i++) {
1440                 brelse(b->bhs[i]);
1441                 b->bhs[i] = NULL;
1442         }
1443         put_cpu_var(bh_lrus);
1444 }
1445         
1446 void invalidate_bh_lrus(void)
1447 {
1448         on_each_cpu(invalidate_bh_lru, NULL, 1);
1449 }
1450 EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1451
1452 void set_bh_page(struct buffer_head *bh,
1453                 struct page *page, unsigned long offset)
1454 {
1455         bh->b_page = page;
1456         BUG_ON(offset >= PAGE_SIZE);
1457         if (PageHighMem(page))
1458                 /*
1459                  * This catches illegal uses and preserves the offset:
1460                  */
1461                 bh->b_data = (char *)(0 + offset);
1462         else
1463                 bh->b_data = page_address(page) + offset;
1464 }
1465 EXPORT_SYMBOL(set_bh_page);
1466
1467 /*
1468  * Called when truncating a buffer on a page completely.
1469  */
1470 static void discard_buffer(struct buffer_head * bh)
1471 {
1472         lock_buffer(bh);
1473         clear_buffer_dirty(bh);
1474         bh->b_bdev = NULL;
1475         clear_buffer_mapped(bh);
1476         clear_buffer_req(bh);
1477         clear_buffer_new(bh);
1478         clear_buffer_delay(bh);
1479         clear_buffer_unwritten(bh);
1480         unlock_buffer(bh);
1481 }
1482
1483 /**
1484  * block_invalidatepage - invalidate part of all of a buffer-backed page
1485  *
1486  * @page: the page which is affected
1487  * @offset: the index of the truncation point
1488  *
1489  * block_invalidatepage() is called when all or part of the page has become
1490  * invalidatedby a truncate operation.
1491  *
1492  * block_invalidatepage() does not have to release all buffers, but it must
1493  * ensure that no dirty buffer is left outside @offset and that no I/O
1494  * is underway against any of the blocks which are outside the truncation
1495  * point.  Because the caller is about to free (and possibly reuse) those
1496  * blocks on-disk.
1497  */
1498 void block_invalidatepage(struct page *page, unsigned long offset)
1499 {
1500         struct buffer_head *head, *bh, *next;
1501         unsigned int curr_off = 0;
1502
1503         BUG_ON(!PageLocked(page));
1504         if (!page_has_buffers(page))
1505                 goto out;
1506
1507         head = page_buffers(page);
1508         bh = head;
1509         do {
1510                 unsigned int next_off = curr_off + bh->b_size;
1511                 next = bh->b_this_page;
1512
1513                 /*
1514                  * is this block fully invalidated?
1515                  */
1516                 if (offset <= curr_off)
1517                         discard_buffer(bh);
1518                 curr_off = next_off;
1519                 bh = next;
1520         } while (bh != head);
1521
1522         /*
1523          * We release buffers only if the entire page is being invalidated.
1524          * The get_block cached value has been unconditionally invalidated,
1525          * so real IO is not possible anymore.
1526          */
1527         if (offset == 0)
1528                 try_to_release_page(page, 0);
1529 out:
1530         return;
1531 }
1532 EXPORT_SYMBOL(block_invalidatepage);
1533
1534 /*
1535  * We attach and possibly dirty the buffers atomically wrt
1536  * __set_page_dirty_buffers() via private_lock.  try_to_free_buffers
1537  * is already excluded via the page lock.
1538  */
1539 void create_empty_buffers(struct page *page,
1540                         unsigned long blocksize, unsigned long b_state)
1541 {
1542         struct buffer_head *bh, *head, *tail;
1543
1544         head = alloc_page_buffers(page, blocksize, 1);
1545         bh = head;
1546         do {
1547                 bh->b_state |= b_state;
1548                 tail = bh;
1549                 bh = bh->b_this_page;
1550         } while (bh);
1551         tail->b_this_page = head;
1552
1553         spin_lock(&page->mapping->private_lock);
1554         if (PageUptodate(page) || PageDirty(page)) {
1555                 bh = head;
1556                 do {
1557                         if (PageDirty(page))
1558                                 set_buffer_dirty(bh);
1559                         if (PageUptodate(page))
1560                                 set_buffer_uptodate(bh);
1561                         bh = bh->b_this_page;
1562                 } while (bh != head);
1563         }
1564         attach_page_buffers(page, head);
1565         spin_unlock(&page->mapping->private_lock);
1566 }
1567 EXPORT_SYMBOL(create_empty_buffers);
1568
1569 /*
1570  * We are taking a block for data and we don't want any output from any
1571  * buffer-cache aliases starting from return from that function and
1572  * until the moment when something will explicitly mark the buffer
1573  * dirty (hopefully that will not happen until we will free that block ;-)
1574  * We don't even need to mark it not-uptodate - nobody can expect
1575  * anything from a newly allocated buffer anyway. We used to used
1576  * unmap_buffer() for such invalidation, but that was wrong. We definitely
1577  * don't want to mark the alias unmapped, for example - it would confuse
1578  * anyone who might pick it with bread() afterwards...
1579  *
1580  * Also..  Note that bforget() doesn't lock the buffer.  So there can
1581  * be writeout I/O going on against recently-freed buffers.  We don't
1582  * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1583  * only if we really need to.  That happens here.
1584  */
1585 void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1586 {
1587         struct buffer_head *old_bh;
1588
1589         might_sleep();
1590
1591         old_bh = __find_get_block_slow(bdev, block);
1592         if (old_bh) {
1593                 clear_buffer_dirty(old_bh);
1594                 wait_on_buffer(old_bh);
1595                 clear_buffer_req(old_bh);
1596                 __brelse(old_bh);
1597         }
1598 }
1599 EXPORT_SYMBOL(unmap_underlying_metadata);
1600
1601 /*
1602  * NOTE! All mapped/uptodate combinations are valid:
1603  *
1604  *      Mapped  Uptodate        Meaning
1605  *
1606  *      No      No              "unknown" - must do get_block()
1607  *      No      Yes             "hole" - zero-filled
1608  *      Yes     No              "allocated" - allocated on disk, not read in
1609  *      Yes     Yes             "valid" - allocated and up-to-date in memory.
1610  *
1611  * "Dirty" is valid only with the last case (mapped+uptodate).
1612  */
1613
1614 /*
1615  * While block_write_full_page is writing back the dirty buffers under
1616  * the page lock, whoever dirtied the buffers may decide to clean them
1617  * again at any time.  We handle that by only looking at the buffer
1618  * state inside lock_buffer().
1619  *
1620  * If block_write_full_page() is called for regular writeback
1621  * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1622  * locked buffer.   This only can happen if someone has written the buffer
1623  * directly, with submit_bh().  At the address_space level PageWriteback
1624  * prevents this contention from occurring.
1625  *
1626  * If block_write_full_page() is called with wbc->sync_mode ==
1627  * WB_SYNC_ALL, the writes are posted using WRITE_SYNC_PLUG; this
1628  * causes the writes to be flagged as synchronous writes, but the
1629  * block device queue will NOT be unplugged, since usually many pages
1630  * will be pushed to the out before the higher-level caller actually
1631  * waits for the writes to be completed.  The various wait functions,
1632  * such as wait_on_writeback_range() will ultimately call sync_page()
1633  * which will ultimately call blk_run_backing_dev(), which will end up
1634  * unplugging the device queue.
1635  */
1636 static int __block_write_full_page(struct inode *inode, struct page *page,
1637                         get_block_t *get_block, struct writeback_control *wbc,
1638                         bh_end_io_t *handler)
1639 {
1640         int err;
1641         sector_t block;
1642         sector_t last_block;
1643         struct buffer_head *bh, *head;
1644         const unsigned blocksize = 1 << inode->i_blkbits;
1645         int nr_underway = 0;
1646         int write_op = (wbc->sync_mode == WB_SYNC_ALL ?
1647                         WRITE_SYNC_PLUG : WRITE);
1648
1649         BUG_ON(!PageLocked(page));
1650
1651         last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1652
1653         if (!page_has_buffers(page)) {
1654                 create_empty_buffers(page, blocksize,
1655                                         (1 << BH_Dirty)|(1 << BH_Uptodate));
1656         }
1657
1658         /*
1659          * Be very careful.  We have no exclusion from __set_page_dirty_buffers
1660          * here, and the (potentially unmapped) buffers may become dirty at
1661          * any time.  If a buffer becomes dirty here after we've inspected it
1662          * then we just miss that fact, and the page stays dirty.
1663          *
1664          * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1665          * handle that here by just cleaning them.
1666          */
1667
1668         block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1669         head = page_buffers(page);
1670         bh = head;
1671
1672         /*
1673          * Get all the dirty buffers mapped to disk addresses and
1674          * handle any aliases from the underlying blockdev's mapping.
1675          */
1676         do {
1677                 if (block > last_block) {
1678                         /*
1679                          * mapped buffers outside i_size will occur, because
1680                          * this page can be outside i_size when there is a
1681                          * truncate in progress.
1682                          */
1683                         /*
1684                          * The buffer was zeroed by block_write_full_page()
1685                          */
1686                         clear_buffer_dirty(bh);
1687                         set_buffer_uptodate(bh);
1688                 } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
1689                            buffer_dirty(bh)) {
1690                         WARN_ON(bh->b_size != blocksize);
1691                         err = get_block(inode, block, bh, 1);
1692                         if (err)
1693                                 goto recover;
1694                         clear_buffer_delay(bh);
1695                         if (buffer_new(bh)) {
1696                                 /* blockdev mappings never come here */
1697                                 clear_buffer_new(bh);
1698                                 unmap_underlying_metadata(bh->b_bdev,
1699                                                         bh->b_blocknr);
1700                         }
1701                 }
1702                 bh = bh->b_this_page;
1703                 block++;
1704         } while (bh != head);
1705
1706         do {
1707                 if (!buffer_mapped(bh))
1708                         continue;
1709                 /*
1710                  * If it's a fully non-blocking write attempt and we cannot
1711                  * lock the buffer then redirty the page.  Note that this can
1712                  * potentially cause a busy-wait loop from writeback threads
1713                  * and kswapd activity, but those code paths have their own
1714                  * higher-level throttling.
1715                  */
1716                 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1717                         lock_buffer(bh);
1718                 } else if (!trylock_buffer(bh)) {
1719                         redirty_page_for_writepage(wbc, page);
1720                         continue;
1721                 }
1722                 if (test_clear_buffer_dirty(bh)) {
1723                         mark_buffer_async_write_endio(bh, handler);
1724                 } else {
1725                         unlock_buffer(bh);
1726                 }
1727         } while ((bh = bh->b_this_page) != head);
1728
1729         /*
1730          * The page and its buffers are protected by PageWriteback(), so we can
1731          * drop the bh refcounts early.
1732          */
1733         BUG_ON(PageWriteback(page));
1734         set_page_writeback(page);
1735
1736         do {
1737                 struct buffer_head *next = bh->b_this_page;
1738                 if (buffer_async_write(bh)) {
1739                         submit_bh(write_op, bh);
1740                         nr_underway++;
1741                 }
1742                 bh = next;
1743         } while (bh != head);
1744         unlock_page(page);
1745
1746         err = 0;
1747 done:
1748         if (nr_underway == 0) {
1749                 /*
1750                  * The page was marked dirty, but the buffers were
1751                  * clean.  Someone wrote them back by hand with
1752                  * ll_rw_block/submit_bh.  A rare case.
1753                  */
1754                 end_page_writeback(page);
1755
1756                 /*
1757                  * The page and buffer_heads can be released at any time from
1758                  * here on.
1759                  */
1760         }
1761         return err;
1762
1763 recover:
1764         /*
1765          * ENOSPC, or some other error.  We may already have added some
1766          * blocks to the file, so we need to write these out to avoid
1767          * exposing stale data.
1768          * The page is currently locked and not marked for writeback
1769          */
1770         bh = head;
1771         /* Recovery: lock and submit the mapped buffers */
1772         do {
1773                 if (buffer_mapped(bh) && buffer_dirty(bh) &&
1774                     !buffer_delay(bh)) {
1775                         lock_buffer(bh);
1776                         mark_buffer_async_write_endio(bh, handler);
1777                 } else {
1778                         /*
1779                          * The buffer may have been set dirty during
1780                          * attachment to a dirty page.
1781                          */
1782                         clear_buffer_dirty(bh);
1783                 }
1784         } while ((bh = bh->b_this_page) != head);
1785         SetPageError(page);
1786         BUG_ON(PageWriteback(page));
1787         mapping_set_error(page->mapping, err);
1788         set_page_writeback(page);
1789         do {
1790                 struct buffer_head *next = bh->b_this_page;
1791                 if (buffer_async_write(bh)) {
1792                         clear_buffer_dirty(bh);
1793                         submit_bh(write_op, bh);
1794                         nr_underway++;
1795                 }
1796                 bh = next;
1797         } while (bh != head);
1798         unlock_page(page);
1799         goto done;
1800 }
1801
1802 /*
1803  * If a page has any new buffers, zero them out here, and mark them uptodate
1804  * and dirty so they'll be written out (in order to prevent uninitialised
1805  * block data from leaking). And clear the new bit.
1806  */
1807 void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
1808 {
1809         unsigned int block_start, block_end;
1810         struct buffer_head *head, *bh;
1811
1812         BUG_ON(!PageLocked(page));
1813         if (!page_has_buffers(page))
1814                 return;
1815
1816         bh = head = page_buffers(page);
1817         block_start = 0;
1818         do {
1819                 block_end = block_start + bh->b_size;
1820
1821                 if (buffer_new(bh)) {
1822                         if (block_end > from && block_start < to) {
1823                                 if (!PageUptodate(page)) {
1824                                         unsigned start, size;
1825
1826                                         start = max(from, block_start);
1827                                         size = min(to, block_end) - start;
1828
1829                                         zero_user(page, start, size);
1830                                         set_buffer_uptodate(bh);
1831                                 }
1832
1833                                 clear_buffer_new(bh);
1834                                 mark_buffer_dirty(bh);
1835                         }
1836                 }
1837
1838                 block_start = block_end;
1839                 bh = bh->b_this_page;
1840         } while (bh != head);
1841 }
1842 EXPORT_SYMBOL(page_zero_new_buffers);
1843
1844 static int __block_prepare_write(struct inode *inode, struct page *page,
1845                 unsigned from, unsigned to, get_block_t *get_block)
1846 {
1847         unsigned block_start, block_end;
1848         sector_t block;
1849         int err = 0;
1850         unsigned blocksize, bbits;
1851         struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1852
1853         BUG_ON(!PageLocked(page));
1854         BUG_ON(from > PAGE_CACHE_SIZE);
1855         BUG_ON(to > PAGE_CACHE_SIZE);
1856         BUG_ON(from > to);
1857
1858         blocksize = 1 << inode->i_blkbits;
1859         if (!page_has_buffers(page))
1860                 create_empty_buffers(page, blocksize, 0);
1861         head = page_buffers(page);
1862
1863         bbits = inode->i_blkbits;
1864         block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1865
1866         for(bh = head, block_start = 0; bh != head || !block_start;
1867             block++, block_start=block_end, bh = bh->b_this_page) {
1868                 block_end = block_start + blocksize;
1869                 if (block_end <= from || block_start >= to) {
1870                         if (PageUptodate(page)) {
1871                                 if (!buffer_uptodate(bh))
1872                                         set_buffer_uptodate(bh);
1873                         }
1874                         continue;
1875                 }
1876                 if (buffer_new(bh))
1877                         clear_buffer_new(bh);
1878                 if (!buffer_mapped(bh)) {
1879                         WARN_ON(bh->b_size != blocksize);
1880                         err = get_block(inode, block, bh, 1);
1881                         if (err)
1882                                 break;
1883                         if (buffer_new(bh)) {
1884                                 unmap_underlying_metadata(bh->b_bdev,
1885                                                         bh->b_blocknr);
1886                                 if (PageUptodate(page)) {
1887                                         clear_buffer_new(bh);
1888                                         set_buffer_uptodate(bh);
1889                                         mark_buffer_dirty(bh);
1890                                         continue;
1891                                 }
1892                                 if (block_end > to || block_start < from)
1893                                         zero_user_segments(page,
1894                                                 to, block_end,
1895                                                 block_start, from);
1896                                 continue;
1897                         }
1898                 }
1899                 if (PageUptodate(page)) {
1900                         if (!buffer_uptodate(bh))
1901                                 set_buffer_uptodate(bh);
1902                         continue; 
1903                 }
1904                 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1905                     !buffer_unwritten(bh) &&
1906                      (block_start < from || block_end > to)) {
1907                         ll_rw_block(READ, 1, &bh);
1908                         *wait_bh++=bh;
1909                 }
1910         }
1911         /*
1912          * If we issued read requests - let them complete.
1913          */
1914         while(wait_bh > wait) {
1915                 wait_on_buffer(*--wait_bh);
1916                 if (!buffer_uptodate(*wait_bh))
1917                         err = -EIO;
1918         }
1919         if (unlikely(err))
1920                 page_zero_new_buffers(page, from, to);
1921         return err;
1922 }
1923
1924 static int __block_commit_write(struct inode *inode, struct page *page,
1925                 unsigned from, unsigned to)
1926 {
1927         unsigned block_start, block_end;
1928         int partial = 0;
1929         unsigned blocksize;
1930         struct buffer_head *bh, *head;
1931
1932         blocksize = 1 << inode->i_blkbits;
1933
1934         for(bh = head = page_buffers(page), block_start = 0;
1935             bh != head || !block_start;
1936             block_start=block_end, bh = bh->b_this_page) {
1937                 block_end = block_start + blocksize;
1938                 if (block_end <= from || block_start >= to) {
1939                         if (!buffer_uptodate(bh))
1940                                 partial = 1;
1941                 } else {
1942                         set_buffer_uptodate(bh);
1943                         mark_buffer_dirty(bh);
1944                 }
1945                 clear_buffer_new(bh);
1946         }
1947
1948         /*
1949          * If this is a partial write which happened to make all buffers
1950          * uptodate then we can optimize away a bogus readpage() for
1951          * the next read(). Here we 'discover' whether the page went
1952          * uptodate as a result of this (potentially partial) write.
1953          */
1954         if (!partial)
1955                 SetPageUptodate(page);
1956         return 0;
1957 }
1958
1959 /*
1960  * block_write_begin takes care of the basic task of block allocation and
1961  * bringing partial write blocks uptodate first.
1962  *
1963  * If *pagep is not NULL, then block_write_begin uses the locked page
1964  * at *pagep rather than allocating its own. In this case, the page will
1965  * not be unlocked or deallocated on failure.
1966  */
1967 int block_write_begin(struct file *file, struct address_space *mapping,
1968                         loff_t pos, unsigned len, unsigned flags,
1969                         struct page **pagep, void **fsdata,
1970                         get_block_t *get_block)
1971 {
1972         struct inode *inode = mapping->host;
1973         int status = 0;
1974         struct page *page;
1975         pgoff_t index;
1976         unsigned start, end;
1977         int ownpage = 0;
1978
1979         index = pos >> PAGE_CACHE_SHIFT;
1980         start = pos & (PAGE_CACHE_SIZE - 1);
1981         end = start + len;
1982
1983         page = *pagep;
1984         if (page == NULL) {
1985                 ownpage = 1;
1986                 page = grab_cache_page_write_begin(mapping, index, flags);
1987                 if (!page) {
1988                         status = -ENOMEM;
1989                         goto out;
1990                 }
1991                 *pagep = page;
1992         } else
1993                 BUG_ON(!PageLocked(page));
1994
1995         status = __block_prepare_write(inode, page, start, end, get_block);
1996         if (unlikely(status)) {
1997                 ClearPageUptodate(page);
1998
1999                 if (ownpage) {
2000                         unlock_page(page);
2001                         page_cache_release(page);
2002                         *pagep = NULL;
2003
2004                         /*
2005                          * prepare_write() may have instantiated a few blocks
2006                          * outside i_size.  Trim these off again. Don't need
2007                          * i_size_read because we hold i_mutex.
2008                          */
2009                         if (pos + len > inode->i_size)
2010                                 vmtruncate(inode, inode->i_size);
2011                 }
2012         }
2013
2014 out:
2015         return status;
2016 }
2017 EXPORT_SYMBOL(block_write_begin);
2018
2019 int block_write_end(struct file *file, struct address_space *mapping,
2020                         loff_t pos, unsigned len, unsigned copied,
2021                         struct page *page, void *fsdata)
2022 {
2023         struct inode *inode = mapping->host;
2024         unsigned start;
2025
2026         start = pos & (PAGE_CACHE_SIZE - 1);
2027
2028         if (unlikely(copied < len)) {
2029                 /*
2030                  * The buffers that were written will now be uptodate, so we
2031                  * don't have to worry about a readpage reading them and
2032                  * overwriting a partial write. However if we have encountered
2033                  * a short write and only partially written into a buffer, it
2034                  * will not be marked uptodate, so a readpage might come in and
2035                  * destroy our partial write.
2036                  *
2037                  * Do the simplest thing, and just treat any short write to a
2038                  * non uptodate page as a zero-length write, and force the
2039                  * caller to redo the whole thing.
2040                  */
2041                 if (!PageUptodate(page))
2042                         copied = 0;
2043
2044                 page_zero_new_buffers(page, start+copied, start+len);
2045         }
2046         flush_dcache_page(page);
2047
2048         /* This could be a short (even 0-length) commit */
2049         __block_commit_write(inode, page, start, start+copied);
2050
2051         return copied;
2052 }
2053 EXPORT_SYMBOL(block_write_end);
2054
2055 int generic_write_end(struct file *file, struct address_space *mapping,
2056                         loff_t pos, unsigned len, unsigned copied,
2057                         struct page *page, void *fsdata)
2058 {
2059         struct inode *inode = mapping->host;
2060         int i_size_changed = 0;
2061
2062         copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
2063
2064         /*
2065          * No need to use i_size_read() here, the i_size
2066          * cannot change under us because we hold i_mutex.
2067          *
2068          * But it's important to update i_size while still holding page lock:
2069          * page writeout could otherwise come in and zero beyond i_size.
2070          */
2071         if (pos+copied > inode->i_size) {
2072                 i_size_write(inode, pos+copied);
2073                 i_size_changed = 1;
2074         }
2075
2076         unlock_page(page);
2077         page_cache_release(page);
2078
2079         /*
2080          * Don't mark the inode dirty under page lock. First, it unnecessarily
2081          * makes the holding time of page lock longer. Second, it forces lock
2082          * ordering of page lock and transaction start for journaling
2083          * filesystems.
2084          */
2085         if (i_size_changed)
2086                 mark_inode_dirty(inode);
2087
2088         return copied;
2089 }
2090 EXPORT_SYMBOL(generic_write_end);
2091
2092 /*
2093  * block_is_partially_uptodate checks whether buffers within a page are
2094  * uptodate or not.
2095  *
2096  * Returns true if all buffers which correspond to a file portion
2097  * we want to read are uptodate.
2098  */
2099 int block_is_partially_uptodate(struct page *page, read_descriptor_t *desc,
2100                                         unsigned long from)
2101 {
2102         struct inode *inode = page->mapping->host;
2103         unsigned block_start, block_end, blocksize;
2104         unsigned to;
2105         struct buffer_head *bh, *head;
2106         int ret = 1;
2107
2108         if (!page_has_buffers(page))
2109                 return 0;
2110
2111         blocksize = 1 << inode->i_blkbits;
2112         to = min_t(unsigned, PAGE_CACHE_SIZE - from, desc->count);
2113         to = from + to;
2114         if (from < blocksize && to > PAGE_CACHE_SIZE - blocksize)
2115                 return 0;
2116
2117         head = page_buffers(page);
2118         bh = head;
2119         block_start = 0;
2120         do {
2121                 block_end = block_start + blocksize;
2122                 if (block_end > from && block_start < to) {
2123                         if (!buffer_uptodate(bh)) {
2124                                 ret = 0;
2125                                 break;
2126                         }
2127                         if (block_end >= to)
2128                                 break;
2129                 }
2130                 block_start = block_end;
2131                 bh = bh->b_this_page;
2132         } while (bh != head);
2133
2134         return ret;
2135 }
2136 EXPORT_SYMBOL(block_is_partially_uptodate);
2137
2138 /*
2139  * Generic "read page" function for block devices that have the normal
2140  * get_block functionality. This is most of the block device filesystems.
2141  * Reads the page asynchronously --- the unlock_buffer() and
2142  * set/clear_buffer_uptodate() functions propagate buffer state into the
2143  * page struct once IO has completed.
2144  */
2145 int block_read_full_page(struct page *page, get_block_t *get_block)
2146 {
2147         struct inode *inode = page->mapping->host;
2148         sector_t iblock, lblock;
2149         struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2150         unsigned int blocksize;
2151         int nr, i;
2152         int fully_mapped = 1;
2153
2154         BUG_ON(!PageLocked(page));
2155         blocksize = 1 << inode->i_blkbits;
2156         if (!page_has_buffers(page))
2157                 create_empty_buffers(page, blocksize, 0);
2158         head = page_buffers(page);
2159
2160         iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2161         lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
2162         bh = head;
2163         nr = 0;
2164         i = 0;
2165
2166         do {
2167                 if (buffer_uptodate(bh))
2168                         continue;
2169
2170                 if (!buffer_mapped(bh)) {
2171                         int err = 0;
2172
2173                         fully_mapped = 0;
2174                         if (iblock < lblock) {
2175                                 WARN_ON(bh->b_size != blocksize);
2176                                 err = get_block(inode, iblock, bh, 0);
2177                                 if (err)
2178                                         SetPageError(page);
2179                         }
2180                         if (!buffer_mapped(bh)) {
2181                                 zero_user(page, i * blocksize, blocksize);
2182                                 if (!err)
2183                                         set_buffer_uptodate(bh);
2184                                 continue;
2185                         }
2186                         /*
2187                          * get_block() might have updated the buffer
2188                          * synchronously
2189                          */
2190                         if (buffer_uptodate(bh))
2191                                 continue;
2192                 }
2193                 arr[nr++] = bh;
2194         } while (i++, iblock++, (bh = bh->b_this_page) != head);
2195
2196         if (fully_mapped)
2197                 SetPageMappedToDisk(page);
2198
2199         if (!nr) {
2200                 /*
2201                  * All buffers are uptodate - we can set the page uptodate
2202                  * as well. But not if get_block() returned an error.
2203                  */
2204                 if (!PageError(page))
2205                         SetPageUptodate(page);
2206                 unlock_page(page);
2207                 return 0;
2208         }
2209
2210         /* Stage two: lock the buffers */
2211         for (i = 0; i < nr; i++) {
2212                 bh = arr[i];
2213                 lock_buffer(bh);
2214                 mark_buffer_async_read(bh);
2215         }
2216
2217         /*
2218          * Stage 3: start the IO.  Check for uptodateness
2219          * inside the buffer lock in case another process reading
2220          * the underlying blockdev brought it uptodate (the sct fix).
2221          */
2222         for (i = 0; i < nr; i++) {
2223                 bh = arr[i];
2224                 if (buffer_uptodate(bh))
2225                         end_buffer_async_read(bh, 1);
2226                 else
2227                         submit_bh(READ, bh);
2228         }
2229         return 0;
2230 }
2231 EXPORT_SYMBOL(block_read_full_page);
2232
2233 /* utility function for filesystems that need to do work on expanding
2234  * truncates.  Uses filesystem pagecache writes to allow the filesystem to
2235  * deal with the hole.  
2236  */
2237 int generic_cont_expand_simple(struct inode *inode, loff_t size)
2238 {
2239         struct address_space *mapping = inode->i_mapping;
2240         struct page *page;
2241         void *fsdata;
2242         int err;
2243
2244         err = inode_newsize_ok(inode, size);
2245         if (err)
2246                 goto out;
2247
2248         err = pagecache_write_begin(NULL, mapping, size, 0,
2249                                 AOP_FLAG_UNINTERRUPTIBLE|AOP_FLAG_CONT_EXPAND,
2250                                 &page, &fsdata);
2251         if (err)
2252                 goto out;
2253
2254         err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
2255         BUG_ON(err > 0);
2256
2257 out:
2258         return err;
2259 }
2260 EXPORT_SYMBOL(generic_cont_expand_simple);
2261
2262 static int cont_expand_zero(struct file *file, struct address_space *mapping,
2263                             loff_t pos, loff_t *bytes)
2264 {
2265         struct inode *inode = mapping->host;
2266         unsigned blocksize = 1 << inode->i_blkbits;
2267         struct page *page;
2268         void *fsdata;
2269         pgoff_t index, curidx;
2270         loff_t curpos;
2271         unsigned zerofrom, offset, len;
2272         int err = 0;
2273
2274         index = pos >> PAGE_CACHE_SHIFT;
2275         offset = pos & ~PAGE_CACHE_MASK;
2276
2277         while (index > (curidx = (curpos = *bytes)>>PAGE_CACHE_SHIFT)) {
2278                 zerofrom = curpos & ~PAGE_CACHE_MASK;
2279                 if (zerofrom & (blocksize-1)) {
2280                         *bytes |= (blocksize-1);
2281                         (*bytes)++;
2282                 }
2283                 len = PAGE_CACHE_SIZE - zerofrom;
2284
2285                 err = pagecache_write_begin(file, mapping, curpos, len,
2286                                                 AOP_FLAG_UNINTERRUPTIBLE,
2287                                                 &page, &fsdata);
2288                 if (err)
2289                         goto out;
2290                 zero_user(page, zerofrom, len);
2291                 err = pagecache_write_end(file, mapping, curpos, len, len,
2292                                                 page, fsdata);
2293                 if (err < 0)
2294                         goto out;
2295                 BUG_ON(err != len);
2296                 err = 0;
2297
2298                 balance_dirty_pages_ratelimited(mapping);
2299         }
2300
2301         /* page covers the boundary, find the boundary offset */
2302         if (index == curidx) {
2303                 zerofrom = curpos & ~PAGE_CACHE_MASK;
2304                 /* if we will expand the thing last block will be filled */
2305                 if (offset <= zerofrom) {
2306                         goto out;
2307                 }
2308                 if (zerofrom & (blocksize-1)) {
2309                         *bytes |= (blocksize-1);
2310                         (*bytes)++;
2311                 }
2312                 len = offset - zerofrom;
2313
2314                 err = pagecache_write_begin(file, mapping, curpos, len,
2315                                                 AOP_FLAG_UNINTERRUPTIBLE,
2316                                                 &page, &fsdata);
2317                 if (err)
2318                         goto out;
2319                 zero_user(page, zerofrom, len);
2320                 err = pagecache_write_end(file, mapping, curpos, len, len,
2321                                                 page, fsdata);
2322                 if (err < 0)
2323                         goto out;
2324                 BUG_ON(err != len);
2325                 err = 0;
2326         }
2327 out:
2328         return err;
2329 }
2330
2331 /*
2332  * For moronic filesystems that do not allow holes in file.
2333  * We may have to extend the file.
2334  */
2335 int cont_write_begin(struct file *file, struct address_space *mapping,
2336                         loff_t pos, unsigned len, unsigned flags,
2337                         struct page **pagep, void **fsdata,
2338                         get_block_t *get_block, loff_t *bytes)
2339 {
2340         struct inode *inode = mapping->host;
2341         unsigned blocksize = 1 << inode->i_blkbits;
2342         unsigned zerofrom;
2343         int err;
2344
2345         err = cont_expand_zero(file, mapping, pos, bytes);
2346         if (err)
2347                 goto out;
2348
2349         zerofrom = *bytes & ~PAGE_CACHE_MASK;
2350         if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2351                 *bytes |= (blocksize-1);
2352                 (*bytes)++;
2353         }
2354
2355         *pagep = NULL;
2356         err = block_write_begin(file, mapping, pos, len,
2357                                 flags, pagep, fsdata, get_block);
2358 out:
2359         return err;
2360 }
2361 EXPORT_SYMBOL(cont_write_begin);
2362
2363 int block_prepare_write(struct page *page, unsigned from, unsigned to,
2364                         get_block_t *get_block)
2365 {
2366         struct inode *inode = page->mapping->host;
2367         int err = __block_prepare_write(inode, page, from, to, get_block);
2368         if (err)
2369                 ClearPageUptodate(page);
2370         return err;
2371 }
2372 EXPORT_SYMBOL(block_prepare_write);
2373
2374 int block_commit_write(struct page *page, unsigned from, unsigned to)
2375 {
2376         struct inode *inode = page->mapping->host;
2377         __block_commit_write(inode,page,from,to);
2378         return 0;
2379 }
2380 EXPORT_SYMBOL(block_commit_write);
2381
2382 /*
2383  * block_page_mkwrite() is not allowed to change the file size as it gets
2384  * called from a page fault handler when a page is first dirtied. Hence we must
2385  * be careful to check for EOF conditions here. We set the page up correctly
2386  * for a written page which means we get ENOSPC checking when writing into
2387  * holes and correct delalloc and unwritten extent mapping on filesystems that
2388  * support these features.
2389  *
2390  * We are not allowed to take the i_mutex here so we have to play games to
2391  * protect against truncate races as the page could now be beyond EOF.  Because
2392  * vmtruncate() writes the inode size before removing pages, once we have the
2393  * page lock we can determine safely if the page is beyond EOF. If it is not
2394  * beyond EOF, then the page is guaranteed safe against truncation until we
2395  * unlock the page.
2396  */
2397 int
2398 block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
2399                    get_block_t get_block)
2400 {
2401         struct page *page = vmf->page;
2402         struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
2403         unsigned long end;
2404         loff_t size;
2405         int ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */
2406
2407         lock_page(page);
2408         size = i_size_read(inode);
2409         if ((page->mapping != inode->i_mapping) ||
2410             (page_offset(page) > size)) {
2411                 /* page got truncated out from underneath us */
2412                 unlock_page(page);
2413                 goto out;
2414         }
2415
2416         /* page is wholly or partially inside EOF */
2417         if (((page->index + 1) << PAGE_CACHE_SHIFT) > size)
2418                 end = size & ~PAGE_CACHE_MASK;
2419         else
2420                 end = PAGE_CACHE_SIZE;
2421
2422         ret = block_prepare_write(page, 0, end, get_block);
2423         if (!ret)
2424                 ret = block_commit_write(page, 0, end);
2425
2426         if (unlikely(ret)) {
2427                 unlock_page(page);
2428                 if (ret == -ENOMEM)
2429                         ret = VM_FAULT_OOM;
2430                 else /* -ENOSPC, -EIO, etc */
2431                         ret = VM_FAULT_SIGBUS;
2432         } else
2433                 ret = VM_FAULT_LOCKED;
2434
2435 out:
2436         return ret;
2437 }
2438 EXPORT_SYMBOL(block_page_mkwrite);
2439
2440 /*
2441  * nobh_write_begin()'s prereads are special: the buffer_heads are freed
2442  * immediately, while under the page lock.  So it needs a special end_io
2443  * handler which does not touch the bh after unlocking it.
2444  */
2445 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2446 {
2447         __end_buffer_read_notouch(bh, uptodate);
2448 }
2449
2450 /*
2451  * Attach the singly-linked list of buffers created by nobh_write_begin, to
2452  * the page (converting it to circular linked list and taking care of page
2453  * dirty races).
2454  */
2455 static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
2456 {
2457         struct buffer_head *bh;
2458
2459         BUG_ON(!PageLocked(page));
2460
2461         spin_lock(&page->mapping->private_lock);
2462         bh = head;
2463         do {
2464                 if (PageDirty(page))
2465                         set_buffer_dirty(bh);
2466                 if (!bh->b_this_page)
2467                         bh->b_this_page = head;
2468                 bh = bh->b_this_page;
2469         } while (bh != head);
2470         attach_page_buffers(page, head);
2471         spin_unlock(&page->mapping->private_lock);
2472 }
2473
2474 /*
2475  * On entry, the page is fully not uptodate.
2476  * On exit the page is fully uptodate in the areas outside (from,to)
2477  */
2478 int nobh_write_begin(struct file *file, struct address_space *mapping,
2479                         loff_t pos, unsigned len, unsigned flags,
2480                         struct page **pagep, void **fsdata,
2481                         get_block_t *get_block)
2482 {
2483         struct inode *inode = mapping->host;
2484         const unsigned blkbits = inode->i_blkbits;
2485         const unsigned blocksize = 1 << blkbits;
2486         struct buffer_head *head, *bh;
2487         struct page *page;
2488         pgoff_t index;
2489         unsigned from, to;
2490         unsigned block_in_page;
2491         unsigned block_start, block_end;
2492         sector_t block_in_file;
2493         int nr_reads = 0;
2494         int ret = 0;
2495         int is_mapped_to_disk = 1;
2496
2497         index = pos >> PAGE_CACHE_SHIFT;
2498         from = pos & (PAGE_CACHE_SIZE - 1);
2499         to = from + len;
2500
2501         page = grab_cache_page_write_begin(mapping, index, flags);
2502         if (!page)
2503                 return -ENOMEM;
2504         *pagep = page;
2505         *fsdata = NULL;
2506
2507         if (page_has_buffers(page)) {
2508                 unlock_page(page);
2509                 page_cache_release(page);
2510                 *pagep = NULL;
2511                 return block_write_begin(file, mapping, pos, len, flags, pagep,
2512                                         fsdata, get_block);
2513         }
2514
2515         if (PageMappedToDisk(page))
2516                 return 0;
2517
2518         /*
2519          * Allocate buffers so that we can keep track of state, and potentially
2520          * attach them to the page if an error occurs. In the common case of
2521          * no error, they will just be freed again without ever being attached
2522          * to the page (which is all OK, because we're under the page lock).
2523          *
2524          * Be careful: the buffer linked list is a NULL terminated one, rather
2525          * than the circular one we're used to.
2526          */
2527         head = alloc_page_buffers(page, blocksize, 0);
2528         if (!head) {
2529                 ret = -ENOMEM;
2530                 goto out_release;
2531         }
2532
2533         block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2534
2535         /*
2536          * We loop across all blocks in the page, whether or not they are
2537          * part of the affected region.  This is so we can discover if the
2538          * page is fully mapped-to-disk.
2539          */
2540         for (block_start = 0, block_in_page = 0, bh = head;
2541                   block_start < PAGE_CACHE_SIZE;
2542                   block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
2543                 int create;
2544
2545                 block_end = block_start + blocksize;
2546                 bh->b_state = 0;
2547                 create = 1;
2548                 if (block_start >= to)
2549                         create = 0;
2550                 ret = get_block(inode, block_in_file + block_in_page,
2551                                         bh, create);
2552                 if (ret)
2553                         goto failed;
2554                 if (!buffer_mapped(bh))
2555                         is_mapped_to_disk = 0;
2556                 if (buffer_new(bh))
2557                         unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
2558                 if (PageUptodate(page)) {
2559                         set_buffer_uptodate(bh);
2560                         continue;
2561                 }
2562                 if (buffer_new(bh) || !buffer_mapped(bh)) {
2563                         zero_user_segments(page, block_start, from,
2564                                                         to, block_end);
2565                         continue;
2566                 }
2567                 if (buffer_uptodate(bh))
2568                         continue;       /* reiserfs does this */
2569                 if (block_start < from || block_end > to) {
2570                         lock_buffer(bh);
2571                         bh->b_end_io = end_buffer_read_nobh;
2572                         submit_bh(READ, bh);
2573                         nr_reads++;
2574                 }
2575         }
2576
2577         if (nr_reads) {
2578                 /*
2579                  * The page is locked, so these buffers are protected from
2580                  * any VM or truncate activity.  Hence we don't need to care
2581                  * for the buffer_head refcounts.
2582                  */
2583                 for (bh = head; bh; bh = bh->b_this_page) {
2584                         wait_on_buffer(bh);
2585                         if (!buffer_uptodate(bh))
2586                                 ret = -EIO;
2587                 }
2588                 if (ret)
2589                         goto failed;
2590         }
2591
2592         if (is_mapped_to_disk)
2593                 SetPageMappedToDisk(page);
2594
2595         *fsdata = head; /* to be released by nobh_write_end */
2596
2597         return 0;
2598
2599 failed:
2600         BUG_ON(!ret);
2601         /*
2602          * Error recovery is a bit difficult. We need to zero out blocks that
2603          * were newly allocated, and dirty them to ensure they get written out.
2604          * Buffers need to be attached to the page at this point, otherwise
2605          * the handling of potential IO errors during writeout would be hard
2606          * (could try doing synchronous writeout, but what if that fails too?)
2607          */
2608         attach_nobh_buffers(page, head);
2609         page_zero_new_buffers(page, from, to);
2610
2611 out_release:
2612         unlock_page(page);
2613         page_cache_release(page);
2614         *pagep = NULL;
2615
2616         if (pos + len > inode->i_size)
2617                 vmtruncate(inode, inode->i_size);
2618
2619         return ret;
2620 }
2621 EXPORT_SYMBOL(nobh_write_begin);
2622
2623 int nobh_write_end(struct file *file, struct address_space *mapping,
2624                         loff_t pos, unsigned len, unsigned copied,
2625                         struct page *page, void *fsdata)
2626 {
2627         struct inode *inode = page->mapping->host;
2628         struct buffer_head *head = fsdata;
2629         struct buffer_head *bh;
2630         BUG_ON(fsdata != NULL && page_has_buffers(page));
2631
2632         if (unlikely(copied < len) && head)
2633                 attach_nobh_buffers(page, head);
2634         if (page_has_buffers(page))
2635                 return generic_write_end(file, mapping, pos, len,
2636                                         copied, page, fsdata);
2637
2638         SetPageUptodate(page);
2639         set_page_dirty(page);
2640         if (pos+copied > inode->i_size) {
2641                 i_size_write(inode, pos+copied);
2642                 mark_inode_dirty(inode);
2643         }
2644
2645         unlock_page(page);
2646         page_cache_release(page);
2647
2648         while (head) {
2649                 bh = head;
2650                 head = head->b_this_page;
2651                 free_buffer_head(bh);
2652         }
2653
2654         return copied;
2655 }
2656 EXPORT_SYMBOL(nobh_write_end);
2657
2658 /*
2659  * nobh_writepage() - based on block_full_write_page() except
2660  * that it tries to operate without attaching bufferheads to
2661  * the page.
2662  */
2663 int nobh_writepage(struct page *page, get_block_t *get_block,
2664                         struct writeback_control *wbc)
2665 {
2666         struct inode * const inode = page->mapping->host;
2667         loff_t i_size = i_size_read(inode);
2668         const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2669         unsigned offset;
2670         int ret;
2671
2672         /* Is the page fully inside i_size? */
2673         if (page->index < end_index)
2674                 goto out;
2675
2676         /* Is the page fully outside i_size? (truncate in progress) */
2677         offset = i_size & (PAGE_CACHE_SIZE-1);
2678         if (page->index >= end_index+1 || !offset) {
2679                 /*
2680                  * The page may have dirty, unmapped buffers.  For example,
2681                  * they may have been added in ext3_writepage().  Make them
2682                  * freeable here, so the page does not leak.
2683                  */
2684 #if 0
2685                 /* Not really sure about this  - do we need this ? */
2686                 if (page->mapping->a_ops->invalidatepage)
2687                         page->mapping->a_ops->invalidatepage(page, offset);
2688 #endif
2689                 unlock_page(page);
2690                 return 0; /* don't care */
2691         }
2692
2693         /*
2694          * The page straddles i_size.  It must be zeroed out on each and every
2695          * writepage invocation because it may be mmapped.  "A file is mapped
2696          * in multiples of the page size.  For a file that is not a multiple of
2697          * the  page size, the remaining memory is zeroed when mapped, and
2698          * writes to that region are not written out to the file."
2699          */
2700         zero_user_segment(page, offset, PAGE_CACHE_SIZE);
2701 out:
2702         ret = mpage_writepage(page, get_block, wbc);
2703         if (ret == -EAGAIN)
2704                 ret = __block_write_full_page(inode, page, get_block, wbc,
2705                                               end_buffer_async_write);
2706         return ret;
2707 }
2708 EXPORT_SYMBOL(nobh_writepage);
2709
2710 int nobh_truncate_page(struct address_space *mapping,
2711                         loff_t from, get_block_t *get_block)
2712 {
2713         pgoff_t index = from >> PAGE_CACHE_SHIFT;
2714         unsigned offset = from & (PAGE_CACHE_SIZE-1);
2715         unsigned blocksize;
2716         sector_t iblock;
2717         unsigned length, pos;
2718         struct inode *inode = mapping->host;
2719         struct page *page;
2720         struct buffer_head map_bh;
2721         int err;
2722
2723         blocksize = 1 << inode->i_blkbits;
2724         length = offset & (blocksize - 1);
2725
2726         /* Block boundary? Nothing to do */
2727         if (!length)
2728                 return 0;
2729
2730         length = blocksize - length;
2731         iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2732
2733         page = grab_cache_page(mapping, index);
2734         err = -ENOMEM;
2735         if (!page)
2736                 goto out;
2737
2738         if (page_has_buffers(page)) {
2739 has_buffers:
2740                 unlock_page(page);
2741                 page_cache_release(page);
2742                 return block_truncate_page(mapping, from, get_block);
2743         }
2744
2745         /* Find the buffer that contains "offset" */
2746         pos = blocksize;
2747         while (offset >= pos) {
2748                 iblock++;
2749                 pos += blocksize;
2750         }
2751
2752         map_bh.b_size = blocksize;
2753         map_bh.b_state = 0;
2754         err = get_block(inode, iblock, &map_bh, 0);
2755         if (err)
2756                 goto unlock;
2757         /* unmapped? It's a hole - nothing to do */
2758         if (!buffer_mapped(&map_bh))
2759                 goto unlock;
2760
2761         /* Ok, it's mapped. Make sure it's up-to-date */
2762         if (!PageUptodate(page)) {
2763                 err = mapping->a_ops->readpage(NULL, page);
2764                 if (err) {
2765                         page_cache_release(page);
2766                         goto out;
2767                 }
2768                 lock_page(page);
2769                 if (!PageUptodate(page)) {
2770                         err = -EIO;
2771                         goto unlock;
2772                 }
2773                 if (page_has_buffers(page))
2774                         goto has_buffers;
2775         }
2776         zero_user(page, offset, length);
2777         set_page_dirty(page);
2778         err = 0;
2779
2780 unlock:
2781         unlock_page(page);
2782         page_cache_release(page);
2783 out:
2784         return err;
2785 }
2786 EXPORT_SYMBOL(nobh_truncate_page);
2787
2788 int block_truncate_page(struct address_space *mapping,
2789                         loff_t from, get_block_t *get_block)
2790 {
2791         pgoff_t index = from >> PAGE_CACHE_SHIFT;
2792         unsigned offset = from & (PAGE_CACHE_SIZE-1);
2793         unsigned blocksize;
2794         sector_t iblock;
2795         unsigned length, pos;
2796         struct inode *inode = mapping->host;
2797         struct page *page;
2798         struct buffer_head *bh;
2799         int err;
2800
2801         blocksize = 1 << inode->i_blkbits;
2802         length = offset & (blocksize - 1);
2803
2804         /* Block boundary? Nothing to do */
2805         if (!length)
2806                 return 0;
2807
2808         length = blocksize - length;
2809         iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2810         
2811         page = grab_cache_page(mapping, index);
2812         err = -ENOMEM;
2813         if (!page)
2814                 goto out;
2815
2816         if (!page_has_buffers(page))
2817                 create_empty_buffers(page, blocksize, 0);
2818
2819         /* Find the buffer that contains "offset" */
2820         bh = page_buffers(page);
2821         pos = blocksize;
2822         while (offset >= pos) {
2823                 bh = bh->b_this_page;
2824                 iblock++;
2825                 pos += blocksize;
2826         }
2827
2828         err = 0;
2829         if (!buffer_mapped(bh)) {
2830                 WARN_ON(bh->b_size != blocksize);
2831                 err = get_block(inode, iblock, bh, 0);
2832                 if (err)
2833                         goto unlock;
2834                 /* unmapped? It's a hole - nothing to do */
2835                 if (!buffer_mapped(bh))
2836                         goto unlock;
2837         }
2838
2839         /* Ok, it's mapped. Make sure it's up-to-date */
2840         if (PageUptodate(page))
2841                 set_buffer_uptodate(bh);
2842
2843         if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
2844                 err = -EIO;
2845                 ll_rw_block(READ, 1, &bh);
2846                 wait_on_buffer(bh);
2847                 /* Uhhuh. Read error. Complain and punt. */
2848                 if (!buffer_uptodate(bh))
2849                         goto unlock;
2850         }
2851
2852         zero_user(page, offset, length);
2853         mark_buffer_dirty(bh);
2854         err = 0;
2855
2856 unlock:
2857         unlock_page(page);
2858         page_cache_release(page);
2859 out:
2860         return err;
2861 }
2862 EXPORT_SYMBOL(block_truncate_page);
2863
2864 /*
2865  * The generic ->writepage function for buffer-backed address_spaces
2866  * this form passes in the end_io handler used to finish the IO.
2867  */
2868 int block_write_full_page_endio(struct page *page, get_block_t *get_block,
2869                         struct writeback_control *wbc, bh_end_io_t *handler)
2870 {
2871         struct inode * const inode = page->mapping->host;
2872         loff_t i_size = i_size_read(inode);
2873         const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2874         unsigned offset;
2875
2876         /* Is the page fully inside i_size? */
2877         if (page->index < end_index)
2878                 return __block_write_full_page(inode, page, get_block, wbc,
2879                                                handler);
2880
2881         /* Is the page fully outside i_size? (truncate in progress) */
2882         offset = i_size & (PAGE_CACHE_SIZE-1);
2883         if (page->index >= end_index+1 || !offset) {
2884                 /*
2885                  * The page may have dirty, unmapped buffers.  For example,
2886                  * they may have been added in ext3_writepage().  Make them
2887                  * freeable here, so the page does not leak.
2888                  */
2889                 do_invalidatepage(page, 0);
2890                 unlock_page(page);
2891                 return 0; /* don't care */
2892         }
2893
2894         /*
2895          * The page straddles i_size.  It must be zeroed out on each and every
2896          * writepage invokation because it may be mmapped.  "A file is mapped
2897          * in multiples of the page size.  For a file that is not a multiple of
2898          * the  page size, the remaining memory is zeroed when mapped, and
2899          * writes to that region are not written out to the file."
2900          */
2901         zero_user_segment(page, offset, PAGE_CACHE_SIZE);
2902         return __block_write_full_page(inode, page, get_block, wbc, handler);
2903 }
2904 EXPORT_SYMBOL(block_write_full_page_endio);
2905
2906 /*
2907  * The generic ->writepage function for buffer-backed address_spaces
2908  */
2909 int block_write_full_page(struct page *page, get_block_t *get_block,
2910                         struct writeback_control *wbc)
2911 {
2912         return block_write_full_page_endio(page, get_block, wbc,
2913                                            end_buffer_async_write);
2914 }
2915 EXPORT_SYMBOL(block_write_full_page);
2916
2917 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2918                             get_block_t *get_block)
2919 {
2920         struct buffer_head tmp;
2921         struct inode *inode = mapping->host;
2922         tmp.b_state = 0;
2923         tmp.b_blocknr = 0;
2924         tmp.b_size = 1 << inode->i_blkbits;
2925         get_block(inode, block, &tmp, 0);
2926         return tmp.b_blocknr;
2927 }
2928 EXPORT_SYMBOL(generic_block_bmap);
2929
2930 static void end_bio_bh_io_sync(struct bio *bio, int err)
2931 {
2932         struct buffer_head *bh = bio->bi_private;
2933
2934         if (err == -EOPNOTSUPP) {
2935                 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2936                 set_bit(BH_Eopnotsupp, &bh->b_state);
2937         }
2938
2939         if (unlikely (test_bit(BIO_QUIET,&bio->bi_flags)))
2940                 set_bit(BH_Quiet, &bh->b_state);
2941
2942         bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2943         bio_put(bio);
2944 }
2945
2946 int submit_bh(int rw, struct buffer_head * bh)
2947 {
2948         struct bio *bio;
2949         int ret = 0;
2950
2951         BUG_ON(!buffer_locked(bh));
2952         BUG_ON(!buffer_mapped(bh));
2953         BUG_ON(!bh->b_end_io);
2954         BUG_ON(buffer_delay(bh));
2955         BUG_ON(buffer_unwritten(bh));
2956
2957         /*
2958          * Mask in barrier bit for a write (could be either a WRITE or a
2959          * WRITE_SYNC
2960          */
2961         if (buffer_ordered(bh) && (rw & WRITE))
2962                 rw |= WRITE_BARRIER;
2963
2964         /*
2965          * Only clear out a write error when rewriting
2966          */
2967         if (test_set_buffer_req(bh) && (rw & WRITE))
2968                 clear_buffer_write_io_error(bh);
2969
2970         /*
2971          * from here on down, it's all bio -- do the initial mapping,
2972          * submit_bio -> generic_make_request may further map this bio around
2973          */
2974         bio = bio_alloc(GFP_NOIO, 1);
2975
2976         bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2977         bio->bi_bdev = bh->b_bdev;
2978         bio->bi_io_vec[0].bv_page = bh->b_page;
2979         bio->bi_io_vec[0].bv_len = bh->b_size;
2980         bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2981
2982         bio->bi_vcnt = 1;
2983         bio->bi_idx = 0;
2984         bio->bi_size = bh->b_size;
2985
2986         bio->bi_end_io = end_bio_bh_io_sync;
2987         bio->bi_private = bh;
2988
2989         bio_get(bio);
2990         submit_bio(rw, bio);
2991
2992         if (bio_flagged(bio, BIO_EOPNOTSUPP))
2993                 ret = -EOPNOTSUPP;
2994
2995         bio_put(bio);
2996         return ret;
2997 }
2998 EXPORT_SYMBOL(submit_bh);
2999
3000 /**
3001  * ll_rw_block: low-level access to block devices (DEPRECATED)
3002  * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
3003  * @nr: number of &struct buffer_heads in the array
3004  * @bhs: array of pointers to &struct buffer_head
3005  *
3006  * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
3007  * requests an I/O operation on them, either a %READ or a %WRITE.  The third
3008  * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
3009  * are sent to disk. The fourth %READA option is described in the documentation
3010  * for generic_make_request() which ll_rw_block() calls.
3011  *
3012  * This function drops any buffer that it cannot get a lock on (with the
3013  * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
3014  * clean when doing a write request, and any buffer that appears to be
3015  * up-to-date when doing read request.  Further it marks as clean buffers that
3016  * are processed for writing (the buffer cache won't assume that they are
3017  * actually clean until the buffer gets unlocked).
3018  *
3019  * ll_rw_block sets b_end_io to simple completion handler that marks
3020  * the buffer up-to-date (if approriate), unlocks the buffer and wakes
3021  * any waiters. 
3022  *
3023  * All of the buffers must be for the same device, and must also be a
3024  * multiple of the current approved size for the device.
3025  */
3026 void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
3027 {
3028         int i;
3029
3030         for (i = 0; i < nr; i++) {
3031                 struct buffer_head *bh = bhs[i];
3032
3033                 if (rw == SWRITE || rw == SWRITE_SYNC || rw == SWRITE_SYNC_PLUG)
3034                         lock_buffer(bh);
3035                 else if (!trylock_buffer(bh))
3036                         continue;
3037
3038                 if (rw == WRITE || rw == SWRITE || rw == SWRITE_SYNC ||
3039                     rw == SWRITE_SYNC_PLUG) {
3040                         if (test_clear_buffer_dirty(bh)) {
3041                                 bh->b_end_io = end_buffer_write_sync;
3042                                 get_bh(bh);
3043                                 if (rw == SWRITE_SYNC)
3044                                         submit_bh(WRITE_SYNC, bh);
3045                                 else
3046                                         submit_bh(WRITE, bh);
3047                                 continue;
3048                         }
3049                 } else {
3050                         if (!buffer_uptodate(bh)) {
3051                                 bh->b_end_io = end_buffer_read_sync;
3052                                 get_bh(bh);
3053                                 submit_bh(rw, bh);
3054                                 continue;
3055                         }
3056                 }
3057                 unlock_buffer(bh);
3058         }
3059 }
3060 EXPORT_SYMBOL(ll_rw_block);
3061
3062 /*
3063  * For a data-integrity writeout, we need to wait upon any in-progress I/O
3064  * and then start new I/O and then wait upon it.  The caller must have a ref on
3065  * the buffer_head.
3066  */
3067 int sync_dirty_buffer(struct buffer_head *bh)
3068 {
3069         int ret = 0;
3070
3071         WARN_ON(atomic_read(&bh->b_count) < 1);
3072         lock_buffer(bh);
3073         if (test_clear_buffer_dirty(bh)) {
3074                 get_bh(bh);
3075                 bh->b_end_io = end_buffer_write_sync;
3076                 ret = submit_bh(WRITE_SYNC, bh);
3077                 wait_on_buffer(bh);
3078                 if (buffer_eopnotsupp(bh)) {
3079                         clear_buffer_eopnotsupp(bh);
3080                         ret = -EOPNOTSUPP;
3081                 }
3082                 if (!ret && !buffer_uptodate(bh))
3083                         ret = -EIO;
3084         } else {
3085                 unlock_buffer(bh);
3086         }
3087         return ret;
3088 }
3089 EXPORT_SYMBOL(sync_dirty_buffer);
3090
3091 /*
3092  * try_to_free_buffers() checks if all the buffers on this particular page
3093  * are unused, and releases them if so.
3094  *
3095  * Exclusion against try_to_free_buffers may be obtained by either
3096  * locking the page or by holding its mapping's private_lock.
3097  *
3098  * If the page is dirty but all the buffers are clean then we need to
3099  * be sure to mark the page clean as well.  This is because the page
3100  * may be against a block device, and a later reattachment of buffers
3101  * to a dirty page will set *all* buffers dirty.  Which would corrupt
3102  * filesystem data on the same device.
3103  *
3104  * The same applies to regular filesystem pages: if all the buffers are
3105  * clean then we set the page clean and proceed.  To do that, we require
3106  * total exclusion from __set_page_dirty_buffers().  That is obtained with
3107  * private_lock.
3108  *
3109  * try_to_free_buffers() is non-blocking.
3110  */
3111 static inline int buffer_busy(struct buffer_head *bh)
3112 {
3113         return atomic_read(&bh->b_count) |
3114                 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
3115 }
3116
3117 static int
3118 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
3119 {
3120         struct buffer_head *head = page_buffers(page);
3121         struct buffer_head *bh;
3122
3123         bh = head;
3124         do {
3125                 if (buffer_write_io_error(bh) && page->mapping)
3126                         set_bit(AS_EIO, &page->mapping->flags);
3127                 if (buffer_busy(bh))
3128                         goto failed;
3129                 bh = bh->b_this_page;
3130         } while (bh != head);
3131
3132         do {
3133                 struct buffer_head *next = bh->b_this_page;
3134
3135                 if (bh->b_assoc_map)
3136                         __remove_assoc_queue(bh);
3137                 bh = next;
3138         } while (bh != head);
3139         *buffers_to_free = head;
3140         __clear_page_buffers(page);
3141         return 1;
3142 failed:
3143         return 0;
3144 }
3145
3146 int try_to_free_buffers(struct page *page)
3147 {
3148         struct address_space * const mapping = page->mapping;
3149         struct buffer_head *buffers_to_free = NULL;
3150         int ret = 0;
3151
3152         BUG_ON(!PageLocked(page));
3153         if (PageWriteback(page))
3154                 return 0;
3155
3156         if (mapping == NULL) {          /* can this still happen? */
3157                 ret = drop_buffers(page, &buffers_to_free);
3158                 goto out;
3159         }
3160
3161         spin_lock(&mapping->private_lock);
3162         ret = drop_buffers(page, &buffers_to_free);
3163
3164         /*
3165          * If the filesystem writes its buffers by hand (eg ext3)
3166          * then we can have clean buffers against a dirty page.  We
3167          * clean the page here; otherwise the VM will never notice
3168          * that the filesystem did any IO at all.
3169          *
3170          * Also, during truncate, discard_buffer will have marked all
3171          * the page's buffers clean.  We discover that here and clean
3172          * the page also.
3173          *
3174          * private_lock must be held over this entire operation in order
3175          * to synchronise against __set_page_dirty_buffers and prevent the
3176          * dirty bit from being lost.
3177          */
3178         if (ret)
3179                 cancel_dirty_page(page, PAGE_CACHE_SIZE);
3180         spin_unlock(&mapping->private_lock);
3181 out:
3182         if (buffers_to_free) {
3183                 struct buffer_head *bh = buffers_to_free;
3184
3185                 do {
3186                         struct buffer_head *next = bh->b_this_page;
3187                         free_buffer_head(bh);
3188                         bh = next;
3189                 } while (bh != buffers_to_free);
3190         }
3191         return ret;
3192 }
3193 EXPORT_SYMBOL(try_to_free_buffers);
3194
3195 void block_sync_page(struct page *page)
3196 {
3197         struct address_space *mapping;
3198
3199         smp_mb();
3200         mapping = page_mapping(page);
3201         if (mapping)
3202                 blk_run_backing_dev(mapping->backing_dev_info, page);
3203 }
3204 EXPORT_SYMBOL(block_sync_page);
3205
3206 /*
3207  * There are no bdflush tunables left.  But distributions are
3208  * still running obsolete flush daemons, so we terminate them here.
3209  *
3210  * Use of bdflush() is deprecated and will be removed in a future kernel.
3211  * The `flush-X' kernel threads fully replace bdflush daemons and this call.
3212  */
3213 SYSCALL_DEFINE2(bdflush, int, func, long, data)
3214 {
3215         static int msg_count;
3216
3217         if (!capable(CAP_SYS_ADMIN))
3218                 return -EPERM;
3219
3220         if (msg_count < 5) {
3221                 msg_count++;
3222                 printk(KERN_INFO
3223                         "warning: process `%s' used the obsolete bdflush"
3224                         " system call\n", current->comm);
3225                 printk(KERN_INFO "Fix your initscripts?\n");
3226         }
3227
3228         if (func == 1)
3229                 do_exit(0);
3230         return 0;
3231 }
3232
3233 /*
3234  * Buffer-head allocation
3235  */
3236 static struct kmem_cache *bh_cachep;
3237
3238 /*
3239  * Once the number of bh's in the machine exceeds this level, we start
3240  * stripping them in writeback.
3241  */
3242 static int max_buffer_heads;
3243
3244 int buffer_heads_over_limit;
3245
3246 struct bh_accounting {
3247         int nr;                 /* Number of live bh's */
3248         int ratelimit;          /* Limit cacheline bouncing */
3249 };
3250
3251 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3252
3253 static void recalc_bh_state(void)
3254 {
3255         int i;
3256         int tot = 0;
3257
3258         if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
3259                 return;
3260         __get_cpu_var(bh_accounting).ratelimit = 0;
3261         for_each_online_cpu(i)
3262                 tot += per_cpu(bh_accounting, i).nr;
3263         buffer_heads_over_limit = (tot > max_buffer_heads);
3264 }
3265         
3266 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3267 {
3268         struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags);
3269         if (ret) {
3270                 INIT_LIST_HEAD(&ret->b_assoc_buffers);
3271                 get_cpu_var(bh_accounting).nr++;
3272                 recalc_bh_state();
3273                 put_cpu_var(bh_accounting);
3274         }
3275         return ret;
3276 }
3277 EXPORT_SYMBOL(alloc_buffer_head);
3278
3279 void free_buffer_head(struct buffer_head *bh)
3280 {
3281         BUG_ON(!list_empty(&bh->b_assoc_buffers));
3282         kmem_cache_free(bh_cachep, bh);
3283         get_cpu_var(bh_accounting).nr--;
3284         recalc_bh_state();
3285         put_cpu_var(bh_accounting);
3286 }
3287 EXPORT_SYMBOL(free_buffer_head);
3288
3289 static void buffer_exit_cpu(int cpu)
3290 {
3291         int i;
3292         struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3293
3294         for (i = 0; i < BH_LRU_SIZE; i++) {
3295                 brelse(b->bhs[i]);
3296                 b->bhs[i] = NULL;
3297         }
3298         get_cpu_var(bh_accounting).nr += per_cpu(bh_accounting, cpu).nr;
3299         per_cpu(bh_accounting, cpu).nr = 0;
3300         put_cpu_var(bh_accounting);
3301 }
3302
3303 static int buffer_cpu_notify(struct notifier_block *self,
3304                               unsigned long action, void *hcpu)
3305 {
3306         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
3307                 buffer_exit_cpu((unsigned long)hcpu);
3308         return NOTIFY_OK;
3309 }
3310
3311 /**
3312  * bh_uptodate_or_lock - Test whether the buffer is uptodate
3313  * @bh: struct buffer_head
3314  *
3315  * Return true if the buffer is up-to-date and false,
3316  * with the buffer locked, if not.
3317  */
3318 int bh_uptodate_or_lock(struct buffer_head *bh)
3319 {
3320         if (!buffer_uptodate(bh)) {
3321                 lock_buffer(bh);
3322                 if (!buffer_uptodate(bh))
3323                         return 0;
3324                 unlock_buffer(bh);
3325         }
3326         return 1;
3327 }
3328 EXPORT_SYMBOL(bh_uptodate_or_lock);
3329
3330 /**
3331  * bh_submit_read - Submit a locked buffer for reading
3332  * @bh: struct buffer_head
3333  *
3334  * Returns zero on success and -EIO on error.
3335  */
3336 int bh_submit_read(struct buffer_head *bh)
3337 {
3338         BUG_ON(!buffer_locked(bh));
3339
3340         if (buffer_uptodate(bh)) {
3341                 unlock_buffer(bh);
3342                 return 0;
3343         }
3344
3345         get_bh(bh);
3346         bh->b_end_io = end_buffer_read_sync;
3347         submit_bh(READ, bh);
3348         wait_on_buffer(bh);
3349         if (buffer_uptodate(bh))
3350                 return 0;
3351         return -EIO;
3352 }
3353 EXPORT_SYMBOL(bh_submit_read);
3354
3355 static void
3356 init_buffer_head(void *data)
3357 {
3358         struct buffer_head *bh = data;
3359
3360         memset(bh, 0, sizeof(*bh));
3361         INIT_LIST_HEAD(&bh->b_assoc_buffers);
3362 }
3363
3364 void __init buffer_init(void)
3365 {
3366         int nrpages;
3367
3368         bh_cachep = kmem_cache_create("buffer_head",
3369                         sizeof(struct buffer_head), 0,
3370                                 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3371                                 SLAB_MEM_SPREAD),
3372                                 init_buffer_head);
3373
3374         /*
3375          * Limit the bh occupancy to 10% of ZONE_NORMAL
3376          */
3377         nrpages = (nr_free_buffer_pages() * 10) / 100;
3378         max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3379         hotcpu_notifier(buffer_cpu_notify, 0);
3380 }