* async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
*/
-#include <linux/config.h>
#include <linux/kernel.h>
#include <linux/syscalls.h>
#include <linux/fs.h>
#include <linux/percpu.h>
#include <linux/slab.h>
#include <linux/smp_lock.h>
+#include <linux/capability.h>
#include <linux/blkdev.h>
#include <linux/file.h>
#include <linux/quotaops.h>
#include <linux/hash.h>
#include <linux/suspend.h>
#include <linux/buffer_head.h>
+#include <linux/task_io_accounting_ops.h>
#include <linux/bio.h>
#include <linux/notifier.h>
#include <linux/cpu.h>
#include <linux/bitops.h>
#include <linux/mpage.h>
+#include <linux/bit_spinlock.h>
static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
static void invalidate_bh_lrus(void);
__clear_page_buffers(struct page *page)
{
ClearPagePrivate(page);
- page->private = 0;
+ set_page_private(page, 0);
page_cache_release(page);
}
{
int ret = 0;
- if (bdev) {
- int err;
-
- ret = filemap_fdatawrite(bdev->bd_inode->i_mapping);
- err = filemap_fdatawait(bdev->bd_inode->i_mapping);
- if (!ret)
- ret = err;
- }
+ if (bdev)
+ ret = filemap_write_and_wait(bdev->bd_inode->i_mapping);
return ret;
}
EXPORT_SYMBOL(sync_blockdev);
-/*
- * Write out and wait upon all dirty data associated with this
- * superblock. Filesystem data as well as the underlying block
- * device. Takes the superblock lock.
- */
-int fsync_super(struct super_block *sb)
-{
- sync_inodes_sb(sb, 0);
- DQUOT_SYNC(sb);
- lock_super(sb);
- if (sb->s_dirt && sb->s_op->write_super)
- sb->s_op->write_super(sb);
- unlock_super(sb);
- if (sb->s_op->sync_fs)
- sb->s_op->sync_fs(sb, 1);
- sync_blockdev(sb->s_bdev);
- sync_inodes_sb(sb, 1);
-
- return sync_blockdev(sb->s_bdev);
-}
-
/*
* Write out and wait upon all dirty data associated with this
* device. Filesystem data as well as the underlying block
* freeze_bdev -- lock a filesystem and force it into a consistent state
* @bdev: blockdevice to lock
*
- * This takes the block device bd_mount_sem to make sure no new mounts
+ * This takes the block device bd_mount_mutex to make sure no new mounts
* happen on bdev until thaw_bdev() is called.
* If a superblock is found on this device, we take the s_umount semaphore
* on it to make sure nobody unmounts until the snapshot creation is done.
{
struct super_block *sb;
- down(&bdev->bd_mount_sem);
+ mutex_lock(&bdev->bd_mount_mutex);
sb = get_super(bdev);
if (sb && !(sb->s_flags & MS_RDONLY)) {
sb->s_frozen = SB_FREEZE_WRITE;
smp_wmb();
- sync_inodes_sb(sb, 0);
- DQUOT_SYNC(sb);
-
- lock_super(sb);
- if (sb->s_dirt && sb->s_op->write_super)
- sb->s_op->write_super(sb);
- unlock_super(sb);
-
- if (sb->s_op->sync_fs)
- sb->s_op->sync_fs(sb, 1);
-
- sync_blockdev(sb->s_bdev);
- sync_inodes_sb(sb, 1);
+ __fsync_super(sb);
sb->s_frozen = SB_FREEZE_TRANS;
smp_wmb();
drop_super(sb);
}
- up(&bdev->bd_mount_sem);
+ mutex_unlock(&bdev->bd_mount_mutex);
}
EXPORT_SYMBOL(thaw_bdev);
-/*
- * sync everything. Start out by waking pdflush, because that writes back
- * all queues in parallel.
- */
-static void do_sync(unsigned long wait)
-{
- wakeup_bdflush(0);
- sync_inodes(0); /* All mappings, inodes and their blockdevs */
- DQUOT_SYNC(NULL);
- sync_supers(); /* Write the superblocks */
- sync_filesystems(0); /* Start syncing the filesystems */
- sync_filesystems(wait); /* Waitingly sync the filesystems */
- sync_inodes(wait); /* Mappings, inodes and blockdevs, again. */
- if (!wait)
- printk("Emergency Sync complete\n");
- if (unlikely(laptop_mode))
- laptop_sync_completion();
-}
-
-asmlinkage long sys_sync(void)
-{
- do_sync(1);
- return 0;
-}
-
-void emergency_sync(void)
-{
- pdflush_operation(do_sync, 0);
-}
-
-/*
- * Generic function to fsync a file.
- *
- * filp may be NULL if called via the msync of a vma.
- */
-
-int file_fsync(struct file *filp, struct dentry *dentry, int datasync)
-{
- struct inode * inode = dentry->d_inode;
- struct super_block * sb;
- int ret, err;
-
- /* sync the inode to buffers */
- ret = write_inode_now(inode, 0);
-
- /* sync the superblock to buffers */
- sb = inode->i_sb;
- lock_super(sb);
- if (sb->s_op->write_super)
- sb->s_op->write_super(sb);
- unlock_super(sb);
-
- /* .. finally sync the buffers to disk */
- err = sync_blockdev(sb->s_bdev);
- if (!ret)
- ret = err;
- return ret;
-}
-
-asmlinkage long sys_fsync(unsigned int fd)
-{
- struct file * file;
- struct address_space *mapping;
- int ret, err;
-
- ret = -EBADF;
- file = fget(fd);
- if (!file)
- goto out;
-
- mapping = file->f_mapping;
-
- ret = -EINVAL;
- if (!file->f_op || !file->f_op->fsync) {
- /* Why? We can still call filemap_fdatawrite */
- goto out_putf;
- }
-
- current->flags |= PF_SYNCWRITE;
- ret = filemap_fdatawrite(mapping);
-
- /*
- * We need to protect against concurrent writers,
- * which could cause livelocks in fsync_buffers_list
- */
- down(&mapping->host->i_sem);
- err = file->f_op->fsync(file, file->f_dentry, 0);
- if (!ret)
- ret = err;
- up(&mapping->host->i_sem);
- err = filemap_fdatawait(mapping);
- if (!ret)
- ret = err;
- current->flags &= ~PF_SYNCWRITE;
-
-out_putf:
- fput(file);
-out:
- return ret;
-}
-
-asmlinkage long sys_fdatasync(unsigned int fd)
-{
- struct file * file;
- struct address_space *mapping;
- int ret, err;
-
- ret = -EBADF;
- file = fget(fd);
- if (!file)
- goto out;
-
- ret = -EINVAL;
- if (!file->f_op || !file->f_op->fsync)
- goto out_putf;
-
- mapping = file->f_mapping;
-
- current->flags |= PF_SYNCWRITE;
- ret = filemap_fdatawrite(mapping);
- down(&mapping->host->i_sem);
- err = file->f_op->fsync(file, file->f_dentry, 1);
- if (!ret)
- ret = err;
- up(&mapping->host->i_sem);
- err = filemap_fdatawait(mapping);
- if (!ret)
- ret = err;
- current->flags &= ~PF_SYNCWRITE;
-
-out_putf:
- fput(file);
-out:
- return ret;
-}
-
/*
* Various filesystems appear to want __find_get_block to be non-blocking.
* But it's the page lock which protects the buffers. To get around this,
* private_lock is contended then so is mapping->tree_lock).
*/
static struct buffer_head *
-__find_get_block_slow(struct block_device *bdev, sector_t block, int unused)
+__find_get_block_slow(struct block_device *bdev, sector_t block)
{
struct inode *bd_inode = bdev->bd_inode;
struct address_space *bd_mapping = bd_inode->i_mapping;
if (all_mapped) {
printk("__find_get_block_slow() failed. "
"block=%llu, b_blocknr=%llu\n",
- (unsigned long long)block, (unsigned long long)bh->b_blocknr);
- printk("b_state=0x%08lx, b_size=%u\n", bh->b_state, bh->b_size);
+ (unsigned long long)block,
+ (unsigned long long)bh->b_blocknr);
+ printk("b_state=0x%08lx, b_size=%zu\n",
+ bh->b_state, bh->b_size);
printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
}
out_unlock:
pass does the actual I/O. */
void invalidate_bdev(struct block_device *bdev, int destroy_dirty_buffers)
{
+ struct address_space *mapping = bdev->bd_inode->i_mapping;
+
+ if (mapping->nrpages == 0)
+ return;
+
invalidate_bh_lrus();
/*
* FIXME: what about destroy_dirty_buffers?
* We really want to use invalidate_inode_pages2() for
* that, but not until that's cleaned up.
*/
- invalidate_inode_pages(bdev->bd_inode->i_mapping);
+ invalidate_inode_pages(mapping);
}
/*
struct zone **zones;
pg_data_t *pgdat;
- wakeup_bdflush(1024);
+ wakeup_pdflush(1024);
yield();
- for_each_pgdat(pgdat) {
- zones = pgdat->node_zonelists[GFP_NOFS&GFP_ZONEMASK].zones;
+ for_each_online_pgdat(pgdat) {
+ zones = pgdat->node_zonelists[gfp_zone(GFP_NOFS)].zones;
if (*zones)
- try_to_free_pages(zones, GFP_NOFS, 0);
+ try_to_free_pages(zones, GFP_NOFS);
}
}
*/
static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
{
- static DEFINE_SPINLOCK(page_uptodate_lock);
unsigned long flags;
+ struct buffer_head *first;
struct buffer_head *tmp;
struct page *page;
int page_uptodate = 1;
* two buffer heads end IO at almost the same time and both
* decide that the page is now completely done.
*/
- spin_lock_irqsave(&page_uptodate_lock, flags);
+ first = page_buffers(page);
+ local_irq_save(flags);
+ bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
clear_buffer_async_read(bh);
unlock_buffer(bh);
tmp = bh;
}
tmp = tmp->b_this_page;
} while (tmp != bh);
- spin_unlock_irqrestore(&page_uptodate_lock, flags);
+ bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
+ local_irq_restore(flags);
/*
* If none of the buffers had errors and they are all
return;
still_busy:
- spin_unlock_irqrestore(&page_uptodate_lock, flags);
+ bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
+ local_irq_restore(flags);
return;
}
* Completion handler for block_write_full_page() - pages which are unlocked
* during I/O, and which have PageWriteback cleared upon I/O completion.
*/
-void end_buffer_async_write(struct buffer_head *bh, int uptodate)
+static void end_buffer_async_write(struct buffer_head *bh, int uptodate)
{
char b[BDEVNAME_SIZE];
- static DEFINE_SPINLOCK(page_uptodate_lock);
unsigned long flags;
+ struct buffer_head *first;
struct buffer_head *tmp;
struct page *page;
bdevname(bh->b_bdev, b));
}
set_bit(AS_EIO, &page->mapping->flags);
+ set_buffer_write_io_error(bh);
clear_buffer_uptodate(bh);
SetPageError(page);
}
- spin_lock_irqsave(&page_uptodate_lock, flags);
+ first = page_buffers(page);
+ local_irq_save(flags);
+ bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
+
clear_buffer_async_write(bh);
unlock_buffer(bh);
tmp = bh->b_this_page;
}
tmp = tmp->b_this_page;
}
- spin_unlock_irqrestore(&page_uptodate_lock, flags);
+ bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
+ local_irq_restore(flags);
end_page_writeback(page);
return;
still_busy:
- spin_unlock_irqrestore(&page_uptodate_lock, flags);
+ bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
+ local_irq_restore(flags);
return;
}
static inline void __remove_assoc_queue(struct buffer_head *bh)
{
list_del_init(&bh->b_assoc_buffers);
+ WARN_ON(!bh->b_assoc_map);
+ if (buffer_write_io_error(bh))
+ set_bit(AS_EIO, &bh->b_assoc_map->flags);
+ bh->b_assoc_map = NULL;
}
int inode_has_buffers(struct inode *inode)
if (!mapping->assoc_mapping) {
mapping->assoc_mapping = buffer_mapping;
} else {
- if (mapping->assoc_mapping != buffer_mapping)
- BUG();
+ BUG_ON(mapping->assoc_mapping != buffer_mapping);
}
if (list_empty(&bh->b_assoc_buffers)) {
spin_lock(&buffer_mapping->private_lock);
list_move_tail(&bh->b_assoc_buffers,
&mapping->private_list);
+ bh->b_assoc_map = mapping;
spin_unlock(&buffer_mapping->private_lock);
}
}
*/
int __set_page_dirty_buffers(struct page *page)
{
- struct address_space * const mapping = page->mapping;
+ struct address_space * const mapping = page_mapping(page);
+
+ if (unlikely(!mapping))
+ return !TestSetPageDirty(page);
spin_lock(&mapping->private_lock);
if (page_has_buffers(page)) {
}
spin_unlock(&mapping->private_lock);
- if (!TestSetPageDirty(page)) {
- write_lock_irq(&mapping->tree_lock);
- if (page->mapping) { /* Race with truncate? */
- if (mapping_cap_account_dirty(mapping))
- inc_page_state(nr_dirty);
- radix_tree_tag_set(&mapping->page_tree,
- page_index(page),
- PAGECACHE_TAG_DIRTY);
+ if (TestSetPageDirty(page))
+ return 0;
+
+ write_lock_irq(&mapping->tree_lock);
+ if (page->mapping) { /* Race with truncate? */
+ if (mapping_cap_account_dirty(mapping)) {
+ __inc_zone_page_state(page, NR_FILE_DIRTY);
+ task_io_account_write(PAGE_CACHE_SIZE);
}
- write_unlock_irq(&mapping->tree_lock);
- __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
+ radix_tree_tag_set(&mapping->page_tree,
+ page_index(page), PAGECACHE_TAG_DIRTY);
}
-
- return 0;
+ write_unlock_irq(&mapping->tree_lock);
+ __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
+ return 1;
}
EXPORT_SYMBOL(__set_page_dirty_buffers);
spin_lock(lock);
while (!list_empty(list)) {
bh = BH_ENTRY(list->next);
- list_del_init(&bh->b_assoc_buffers);
+ __remove_assoc_queue(bh);
if (buffer_dirty(bh) || buffer_locked(bh)) {
list_add(&bh->b_assoc_buffers, &tmp);
if (buffer_dirty(bh)) {
* contents - it is a noop if I/O is still in
* flight on potentially older contents.
*/
- wait_on_buffer(bh);
- ll_rw_block(WRITE, 1, &bh);
+ ll_rw_block(SWRITE, 1, &bh);
brelse(bh);
spin_lock(lock);
}
while (!list_empty(&tmp)) {
bh = BH_ENTRY(tmp.prev);
- __remove_assoc_queue(bh);
+ list_del_init(&bh->b_assoc_buffers);
get_bh(bh);
spin_unlock(lock);
wait_on_buffer(bh);
bh->b_state = 0;
atomic_set(&bh->b_count, 0);
+ bh->b_private = NULL;
bh->b_size = size;
/* Link the buffer to its page */
set_bh_page(bh, page, offset);
- bh->b_end_io = NULL;
+ init_buffer(bh, NULL, NULL);
}
return head;
/*
if (!page)
return NULL;
- if (!PageLocked(page))
- BUG();
+ BUG_ON(!PageLocked(page));
if (page_has_buffers(page)) {
bh = page_buffers(page);
* some of those buffers may be aliases of filesystem data.
* grow_dev_page() will go BUG() if this happens.
*/
-static inline int
+static int
grow_buffers(struct block_device *bdev, sector_t block, int size)
{
struct page *page;
} while ((size << sizebits) < PAGE_SIZE);
index = block >> sizebits;
- block = index << sizebits;
+ /*
+ * Check for a block which wants to lie outside our maximum possible
+ * pagecache index. (this comparison is done using sector_t types).
+ */
+ if (unlikely(index != block >> sizebits)) {
+ char b[BDEVNAME_SIZE];
+
+ printk(KERN_ERR "%s: requested out-of-range block %llu for "
+ "device %s\n",
+ __FUNCTION__, (unsigned long long)block,
+ bdevname(bdev, b));
+ return -EIO;
+ }
+ block = index << sizebits;
/* Create a page with the proper size buffers.. */
page = grow_dev_page(bdev, block, index, size);
if (!page)
return 1;
}
-struct buffer_head *
+static struct buffer_head *
__getblk_slow(struct block_device *bdev, sector_t block, int size)
{
/* Size must be multiple of hard sectorsize */
for (;;) {
struct buffer_head * bh;
+ int ret;
bh = __find_get_block(bdev, block, size);
if (bh)
return bh;
- if (!grow_buffers(bdev, block, size))
+ ret = grow_buffers(bdev, block, size);
+ if (ret < 0)
+ return NULL;
+ if (ret == 0)
free_more_memory();
}
}
spin_lock(&buffer_mapping->private_lock);
list_del_init(&bh->b_assoc_buffers);
+ bh->b_assoc_map = NULL;
spin_unlock(&buffer_mapping->private_lock);
}
__brelse(bh);
/*
* Look up the bh in this cpu's LRU. If it's there, move it to the head.
*/
-static inline struct buffer_head *
+static struct buffer_head *
lookup_bh_lru(struct block_device *bdev, sector_t block, int size)
{
struct buffer_head *ret = NULL;
struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
if (bh == NULL) {
- bh = __find_get_block_slow(bdev, block, size);
+ bh = __find_get_block_slow(bdev, block);
if (bh)
bh_lru_install(bh);
}
void __breadahead(struct block_device *bdev, sector_t block, int size)
{
struct buffer_head *bh = __getblk(bdev, block, size);
- ll_rw_block(READA, 1, &bh);
- brelse(bh);
+ if (likely(bh)) {
+ ll_rw_block(READA, 1, &bh);
+ brelse(bh);
+ }
}
EXPORT_SYMBOL(__breadahead);
{
struct buffer_head *bh = __getblk(bdev, block, size);
- if (!buffer_uptodate(bh))
+ if (likely(bh) && !buffer_uptodate(bh))
bh = __bread_slow(bh);
return bh;
}
struct page *page, unsigned long offset)
{
bh->b_page = page;
- if (offset >= PAGE_SIZE)
- BUG();
+ BUG_ON(offset >= PAGE_SIZE);
if (PageHighMem(page))
/*
* This catches illegal uses and preserves the offset:
/*
* Called when truncating a buffer on a page completely.
*/
-static inline void discard_buffer(struct buffer_head * bh)
+static void discard_buffer(struct buffer_head * bh)
{
lock_buffer(bh);
clear_buffer_dirty(bh);
unlock_buffer(bh);
}
-/**
- * try_to_release_page() - release old fs-specific metadata on a page
- *
- * @page: the page which the kernel is trying to free
- * @gfp_mask: memory allocation flags (and I/O mode)
- *
- * The address_space is to try to release any data against the page
- * (presumably at page->private). If the release was successful, return `1'.
- * Otherwise return zero.
- *
- * The @gfp_mask argument specifies whether I/O may be performed to release
- * this page (__GFP_IO), and whether the call may block (__GFP_WAIT).
- *
- * NOTE: @gfp_mask may go away, and this function may become non-blocking.
- */
-int try_to_release_page(struct page *page, int gfp_mask)
-{
- struct address_space * const mapping = page->mapping;
-
- BUG_ON(!PageLocked(page));
- if (PageWriteback(page))
- return 0;
-
- if (mapping && mapping->a_ops->releasepage)
- return mapping->a_ops->releasepage(page, gfp_mask);
- return try_to_free_buffers(page);
-}
-EXPORT_SYMBOL(try_to_release_page);
-
/**
* block_invalidatepage - invalidate part of all of a buffer-backed page
*
* point. Because the caller is about to free (and possibly reuse) those
* blocks on-disk.
*/
-int block_invalidatepage(struct page *page, unsigned long offset)
+void block_invalidatepage(struct page *page, unsigned long offset)
{
struct buffer_head *head, *bh, *next;
unsigned int curr_off = 0;
- int ret = 1;
BUG_ON(!PageLocked(page));
if (!page_has_buffers(page))
* so real IO is not possible anymore.
*/
if (offset == 0)
- ret = try_to_release_page(page, 0);
+ try_to_release_page(page, 0);
out:
- return ret;
+ return;
}
EXPORT_SYMBOL(block_invalidatepage);
might_sleep();
- old_bh = __find_get_block_slow(bdev, block, 0);
+ old_bh = __find_get_block_slow(bdev, block);
if (old_bh) {
clear_buffer_dirty(old_bh);
wait_on_buffer(old_bh);
int err;
sector_t block;
sector_t last_block;
- struct buffer_head *bh, *head, *last_bh = NULL;
+ struct buffer_head *bh, *head;
+ const unsigned blocksize = 1 << inode->i_blkbits;
int nr_underway = 0;
BUG_ON(!PageLocked(page));
last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
if (!page_has_buffers(page)) {
- create_empty_buffers(page, 1 << inode->i_blkbits,
+ create_empty_buffers(page, blocksize,
(1 << BH_Dirty)|(1 << BH_Uptodate));
}
* handle that here by just cleaning them.
*/
- block = page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
+ block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
head = page_buffers(page);
bh = head;
clear_buffer_dirty(bh);
set_buffer_uptodate(bh);
} else if (!buffer_mapped(bh) && buffer_dirty(bh)) {
+ WARN_ON(bh->b_size != blocksize);
err = get_block(inode, block, bh, 1);
if (err)
goto recover;
}
if (test_clear_buffer_dirty(bh)) {
mark_buffer_async_write(bh);
- get_bh(bh);
- last_bh = bh;
} else {
unlock_buffer(bh);
}
*/
BUG_ON(PageWriteback(page));
set_page_writeback(page);
- unlock_page(page);
do {
struct buffer_head *next = bh->b_this_page;
if (buffer_async_write(bh)) {
submit_bh(WRITE, bh);
nr_underway++;
- put_bh(bh);
- if (bh == last_bh)
- break;
}
bh = next;
} while (bh != head);
- bh = head;
+ unlock_page(page);
err = 0;
done:
if (buffer_mapped(bh) && buffer_dirty(bh)) {
lock_buffer(bh);
mark_buffer_async_write(bh);
- get_bh(bh);
- last_bh = bh;
} else {
/*
* The buffer may have been set dirty during
clear_buffer_dirty(bh);
submit_bh(WRITE, bh);
nr_underway++;
- put_bh(bh);
- if (bh == last_bh)
- break;
}
bh = next;
} while (bh != head);
- bh = head;
goto done;
}
if (buffer_new(bh))
clear_buffer_new(bh);
if (!buffer_mapped(bh)) {
+ WARN_ON(bh->b_size != blocksize);
err = get_block(inode, block, bh, 1);
if (err)
break;
if (buffer_new(bh)) {
- clear_buffer_new(bh);
unmap_underlying_metadata(bh->b_bdev,
bh->b_blocknr);
if (PageUptodate(page)) {
if (!buffer_uptodate(*wait_bh))
err = -EIO;
}
- if (!err)
- return err;
-
+ if (!err) {
+ bh = head;
+ do {
+ if (buffer_new(bh))
+ clear_buffer_new(bh);
+ } while ((bh = bh->b_this_page) != head);
+ return 0;
+ }
/* Error case: */
/*
* Zero out any newly allocated blocks to avoid exposing stale
clear_buffer_new(bh);
kaddr = kmap_atomic(page, KM_USER0);
memset(kaddr+block_start, 0, bh->b_size);
+ flush_dcache_page(page);
kunmap_atomic(kaddr, KM_USER0);
set_buffer_uptodate(bh);
mark_buffer_dirty(bh);
continue;
if (!buffer_mapped(bh)) {
+ int err = 0;
+
fully_mapped = 0;
if (iblock < lblock) {
- if (get_block(inode, iblock, bh, 0))
+ WARN_ON(bh->b_size != blocksize);
+ err = get_block(inode, iblock, bh, 0);
+ if (err)
SetPageError(page);
}
if (!buffer_mapped(bh)) {
memset(kaddr + i * blocksize, 0, blocksize);
flush_dcache_page(page);
kunmap_atomic(kaddr, KM_USER0);
- set_buffer_uptodate(bh);
+ if (!err)
+ set_buffer_uptodate(bh);
continue;
}
/*
* truncates. Uses prepare/commit_write to allow the filesystem to
* deal with the hole.
*/
-int generic_cont_expand(struct inode *inode, loff_t size)
+static int __generic_cont_expand(struct inode *inode, loff_t size,
+ pgoff_t index, unsigned int offset)
{
struct address_space *mapping = inode->i_mapping;
struct page *page;
- unsigned long index, offset, limit;
+ unsigned long limit;
int err;
err = -EFBIG;
if (size > inode->i_sb->s_maxbytes)
goto out;
- offset = (size & (PAGE_CACHE_SIZE-1)); /* Within page */
-
- /* ugh. in prepare/commit_write, if from==to==start of block, we
- ** skip the prepare. make sure we never send an offset for the start
- ** of a block
- */
- if ((offset & (inode->i_sb->s_blocksize - 1)) == 0) {
- offset++;
- }
- index = size >> PAGE_CACHE_SHIFT;
err = -ENOMEM;
page = grab_cache_page(mapping, index);
if (!page)
goto out;
err = mapping->a_ops->prepare_write(NULL, page, offset, offset);
- if (!err) {
- err = mapping->a_ops->commit_write(NULL, page, offset, offset);
+ if (err) {
+ /*
+ * ->prepare_write() may have instantiated a few blocks
+ * outside i_size. Trim these off again.
+ */
+ unlock_page(page);
+ page_cache_release(page);
+ vmtruncate(inode, inode->i_size);
+ goto out;
}
+
+ err = mapping->a_ops->commit_write(NULL, page, offset, offset);
+
unlock_page(page);
page_cache_release(page);
if (err > 0)
return err;
}
+int generic_cont_expand(struct inode *inode, loff_t size)
+{
+ pgoff_t index;
+ unsigned int offset;
+
+ offset = (size & (PAGE_CACHE_SIZE - 1)); /* Within page */
+
+ /* ugh. in prepare/commit_write, if from==to==start of block, we
+ ** skip the prepare. make sure we never send an offset for the start
+ ** of a block
+ */
+ if ((offset & (inode->i_sb->s_blocksize - 1)) == 0) {
+ /* caller must handle this extra byte. */
+ offset++;
+ }
+ index = size >> PAGE_CACHE_SHIFT;
+
+ return __generic_cont_expand(inode, size, index, offset);
+}
+
+int generic_cont_expand_simple(struct inode *inode, loff_t size)
+{
+ loff_t pos = size - 1;
+ pgoff_t index = pos >> PAGE_CACHE_SHIFT;
+ unsigned int offset = (pos & (PAGE_CACHE_SIZE - 1)) + 1;
+
+ /* prepare/commit_write can handle even if from==to==start of block. */
+ return __generic_cont_expand(inode, size, index, offset);
+}
+
/*
* For moronic filesystems that do not allow holes in file.
* We may have to extend the file.
__block_commit_write(inode,page,from,to);
/*
* No need to use i_size_read() here, the i_size
- * cannot change under us because we hold i_sem.
+ * cannot change under us because we hold i_mutex.
*/
if (pos > inode->i_size) {
i_size_write(inode, pos);
create = 1;
if (block_start >= to)
create = 0;
+ map_bh.b_size = blocksize;
ret = get_block(inode, block_in_file + block_in_page,
&map_bh, create);
if (ret)
*/
kaddr = kmap_atomic(page, KM_USER0);
memset(kaddr, 0, PAGE_CACHE_SIZE);
+ flush_dcache_page(page);
kunmap_atomic(kaddr, KM_USER0);
SetPageUptodate(page);
set_page_dirty(page);
unsigned offset = from & (PAGE_CACHE_SIZE-1);
unsigned to;
struct page *page;
- struct address_space_operations *a_ops = mapping->a_ops;
+ const struct address_space_operations *a_ops = mapping->a_ops;
char *kaddr;
int ret = 0;
pgoff_t index = from >> PAGE_CACHE_SHIFT;
unsigned offset = from & (PAGE_CACHE_SIZE-1);
unsigned blocksize;
- pgoff_t iblock;
+ sector_t iblock;
unsigned length, pos;
struct inode *inode = mapping->host;
struct page *page;
return 0;
length = blocksize - length;
- iblock = index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
+ iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
page = grab_cache_page(mapping, index);
err = -ENOMEM;
err = 0;
if (!buffer_mapped(bh)) {
+ WARN_ON(bh->b_size != blocksize);
err = get_block(inode, iblock, bh, 0);
if (err)
goto unlock;
* they may have been added in ext3_writepage(). Make them
* freeable here, so the page does not leak.
*/
- block_invalidatepage(page, 0);
+ do_invalidatepage(page, 0);
unlock_page(page);
return 0; /* don't care */
}
struct inode *inode = mapping->host;
tmp.b_state = 0;
tmp.b_blocknr = 0;
+ tmp.b_size = 1 << inode->i_blkbits;
get_block(inode, block, &tmp, 0);
return tmp.b_blocknr;
}
/**
* ll_rw_block: low-level access to block devices (DEPRECATED)
- * @rw: whether to %READ or %WRITE or maybe %READA (readahead)
+ * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
* @nr: number of &struct buffer_heads in the array
* @bhs: array of pointers to &struct buffer_head
*
- * ll_rw_block() takes an array of pointers to &struct buffer_heads,
- * and requests an I/O operation on them, either a %READ or a %WRITE.
- * The third %READA option is described in the documentation for
- * generic_make_request() which ll_rw_block() calls.
+ * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
+ * requests an I/O operation on them, either a %READ or a %WRITE. The third
+ * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
+ * are sent to disk. The fourth %READA option is described in the documentation
+ * for generic_make_request() which ll_rw_block() calls.
*
* This function drops any buffer that it cannot get a lock on (with the
- * BH_Lock state bit), any buffer that appears to be clean when doing a
- * write request, and any buffer that appears to be up-to-date when doing
- * read request. Further it marks as clean buffers that are processed for
- * writing (the buffer cache won't assume that they are actually clean until
- * the buffer gets unlocked).
+ * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
+ * clean when doing a write request, and any buffer that appears to be
+ * up-to-date when doing read request. Further it marks as clean buffers that
+ * are processed for writing (the buffer cache won't assume that they are
+ * actually clean until the buffer gets unlocked).
*
* ll_rw_block sets b_end_io to simple completion handler that marks
* the buffer up-to-date (if approriate), unlocks the buffer and wakes
for (i = 0; i < nr; i++) {
struct buffer_head *bh = bhs[i];
- if (test_set_buffer_locked(bh))
+ if (rw == SWRITE)
+ lock_buffer(bh);
+ else if (test_set_buffer_locked(bh))
continue;
- get_bh(bh);
- if (rw == WRITE) {
+ if (rw == WRITE || rw == SWRITE) {
if (test_clear_buffer_dirty(bh)) {
bh->b_end_io = end_buffer_write_sync;
+ get_bh(bh);
submit_bh(WRITE, bh);
continue;
}
} else {
if (!buffer_uptodate(bh)) {
bh->b_end_io = end_buffer_read_sync;
+ get_bh(bh);
submit_bh(rw, bh);
continue;
}
}
unlock_buffer(bh);
- put_bh(bh);
}
}
spin_lock(&mapping->private_lock);
ret = drop_buffers(page, &buffers_to_free);
+ spin_unlock(&mapping->private_lock);
if (ret) {
/*
* If the filesystem writes its buffers by hand (eg ext3)
* could encounter a non-uptodate page, which is unresolvable.
* This only applies in the rare case where try_to_free_buffers
* succeeds but the page is not freed.
+ *
+ * Also, during truncate, discard_buffer will have marked all
+ * the page's buffers clean. We discover that here and clean
+ * the page also.
*/
- clear_page_dirty(page);
+ if (test_clear_page_dirty(page))
+ task_io_account_cancelled_write(PAGE_CACHE_SIZE);
}
- spin_unlock(&mapping->private_lock);
out:
if (buffers_to_free) {
struct buffer_head *bh = buffers_to_free;
}
EXPORT_SYMBOL(try_to_free_buffers);
-int block_sync_page(struct page *page)
+void block_sync_page(struct page *page)
{
struct address_space *mapping;
mapping = page_mapping(page);
if (mapping)
blk_run_backing_dev(mapping->backing_dev_info, page);
- return 0;
}
/*
/*
* Buffer-head allocation
*/
-static kmem_cache_t *bh_cachep;
+static struct kmem_cache *bh_cachep;
/*
* Once the number of bh's in the machine exceeds this level, we start
if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
return;
__get_cpu_var(bh_accounting).ratelimit = 0;
- for_each_cpu(i)
+ for_each_online_cpu(i)
tot += per_cpu(bh_accounting, i).nr;
buffer_heads_over_limit = (tot > max_buffer_heads);
}
-struct buffer_head *alloc_buffer_head(unsigned int __nocast gfp_flags)
+struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
{
struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags);
if (ret) {
- preempt_disable();
- __get_cpu_var(bh_accounting).nr++;
+ get_cpu_var(bh_accounting).nr++;
recalc_bh_state();
- preempt_enable();
+ put_cpu_var(bh_accounting);
}
return ret;
}
{
BUG_ON(!list_empty(&bh->b_assoc_buffers));
kmem_cache_free(bh_cachep, bh);
- preempt_disable();
- __get_cpu_var(bh_accounting).nr--;
+ get_cpu_var(bh_accounting).nr--;
recalc_bh_state();
- preempt_enable();
+ put_cpu_var(bh_accounting);
}
EXPORT_SYMBOL(free_buffer_head);
static void
-init_buffer_head(void *data, kmem_cache_t *cachep, unsigned long flags)
+init_buffer_head(void *data, struct kmem_cache *cachep, unsigned long flags)
{
if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
SLAB_CTOR_CONSTRUCTOR) {
}
}
-#ifdef CONFIG_HOTPLUG_CPU
static void buffer_exit_cpu(int cpu)
{
int i;
brelse(b->bhs[i]);
b->bhs[i] = NULL;
}
+ get_cpu_var(bh_accounting).nr += per_cpu(bh_accounting, cpu).nr;
+ per_cpu(bh_accounting, cpu).nr = 0;
+ put_cpu_var(bh_accounting);
}
static int buffer_cpu_notify(struct notifier_block *self,
buffer_exit_cpu((unsigned long)hcpu);
return NOTIFY_OK;
}
-#endif /* CONFIG_HOTPLUG_CPU */
void __init buffer_init(void)
{
int nrpages;
bh_cachep = kmem_cache_create("buffer_head",
- sizeof(struct buffer_head), 0,
- SLAB_RECLAIM_ACCOUNT|SLAB_PANIC, init_buffer_head, NULL);
+ sizeof(struct buffer_head), 0,
+ (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
+ SLAB_MEM_SPREAD),
+ init_buffer_head,
+ NULL);
/*
* Limit the bh occupancy to 10% of ZONE_NORMAL
EXPORT_SYMBOL(block_truncate_page);
EXPORT_SYMBOL(block_write_full_page);
EXPORT_SYMBOL(cont_prepare_write);
-EXPORT_SYMBOL(end_buffer_async_write);
EXPORT_SYMBOL(end_buffer_read_sync);
EXPORT_SYMBOL(end_buffer_write_sync);
EXPORT_SYMBOL(file_fsync);
EXPORT_SYMBOL(generic_block_bmap);
EXPORT_SYMBOL(generic_commit_write);
EXPORT_SYMBOL(generic_cont_expand);
+EXPORT_SYMBOL(generic_cont_expand_simple);
EXPORT_SYMBOL(init_buffer);
EXPORT_SYMBOL(invalidate_bdev);
EXPORT_SYMBOL(ll_rw_block);
Code Review / linux-2.6.git / blobdiff