NeilBrown <neilb@suse.de>
[linux-2.6.git] / mm / filemap.c
index 7c7addb..adbac10 100644 (file)
@@ -75,8 +75,8 @@ generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
  *  ->mmap_sem
  *    ->lock_page              (access_process_vm)
  *
- *  ->mmap_sem
- *    ->i_mutex                        (msync)
+ *  ->i_mutex                  (generic_file_buffered_write)
+ *    ->mmap_sem               (fault_in_pages_readable->do_page_fault)
  *
  *  ->i_mutex
  *    ->i_alloc_sem             (various)
@@ -120,6 +120,7 @@ void __remove_from_page_cache(struct page *page)
        page->mapping = NULL;
        mapping->nrpages--;
        __dec_zone_page_state(page, NR_FILE_PAGES);
+       BUG_ON(page_mapped(page));
 }
 
 void remove_from_page_cache(struct page *page)
@@ -327,7 +328,7 @@ EXPORT_SYMBOL(sync_page_range);
  * @pos:       beginning offset in pages to write
  * @count:     number of bytes to write
  *
- * Note: Holding i_mutex across sync_page_range_nolock is not a good idea
+ * Note: Holding i_mutex across sync_page_range_nolock() is not a good idea
  * as it forces O_SYNC writers to different parts of the same file
  * to be serialised right until io completion.
  */
@@ -467,25 +468,15 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
 }
 
 #ifdef CONFIG_NUMA
-struct page *page_cache_alloc(struct address_space *x)
+struct page *__page_cache_alloc(gfp_t gfp)
 {
        if (cpuset_do_page_mem_spread()) {
                int n = cpuset_mem_spread_node();
-               return alloc_pages_node(n, mapping_gfp_mask(x), 0);
+               return alloc_pages_node(n, gfp, 0);
        }
-       return alloc_pages(mapping_gfp_mask(x), 0);
+       return alloc_pages(gfp, 0);
 }
-EXPORT_SYMBOL(page_cache_alloc);
-
-struct page *page_cache_alloc_cold(struct address_space *x)
-{
-       if (cpuset_do_page_mem_spread()) {
-               int n = cpuset_mem_spread_node();
-               return alloc_pages_node(n, mapping_gfp_mask(x)|__GFP_COLD, 0);
-       }
-       return alloc_pages(mapping_gfp_mask(x)|__GFP_COLD, 0);
-}
-EXPORT_SYMBOL(page_cache_alloc_cold);
+EXPORT_SYMBOL(__page_cache_alloc);
 #endif
 
 static int __sleep_on_page_lock(void *word)
@@ -616,26 +607,6 @@ struct page * find_get_page(struct address_space *mapping, unsigned long offset)
 EXPORT_SYMBOL(find_get_page);
 
 /**
- * find_trylock_page - find and lock a page
- * @mapping: the address_space to search
- * @offset: the page index
- *
- * Same as find_get_page(), but trylock it instead of incrementing the count.
- */
-struct page *find_trylock_page(struct address_space *mapping, unsigned long offset)
-{
-       struct page *page;
-
-       read_lock_irq(&mapping->tree_lock);
-       page = radix_tree_lookup(&mapping->page_tree, offset);
-       if (page && TestSetPageLocked(page))
-               page = NULL;
-       read_unlock_irq(&mapping->tree_lock);
-       return page;
-}
-EXPORT_SYMBOL(find_trylock_page);
-
-/**
  * find_lock_page - locate, pin and lock a pagecache page
  * @mapping: the address_space to search
  * @offset: the page index
@@ -700,7 +671,8 @@ repeat:
        page = find_lock_page(mapping, index);
        if (!page) {
                if (!cached_page) {
-                       cached_page = alloc_page(gfp_mask);
+                       cached_page =
+                               __page_cache_alloc(gfp_mask);
                        if (!cached_page)
                                return NULL;
                }
@@ -780,6 +752,7 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
        read_unlock_irq(&mapping->tree_lock);
        return i;
 }
+EXPORT_SYMBOL(find_get_pages_contig);
 
 /**
  * find_get_pages_tag - find and return pages that match @tag
@@ -808,13 +781,14 @@ unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
        read_unlock_irq(&mapping->tree_lock);
        return ret;
 }
+EXPORT_SYMBOL(find_get_pages_tag);
 
 /**
  * grab_cache_page_nowait - returns locked page at given index in given cache
  * @mapping: target address_space
  * @index: the page index
  *
- * Same as grab_cache_page, but do not wait if the page is unavailable.
+ * Same as grab_cache_page(), but do not wait if the page is unavailable.
  * This is intended for speculative data generators, where the data can
  * be regenerated if the page couldn't be grabbed.  This routine should
  * be safe to call while holding the lock for another page.
@@ -826,7 +800,6 @@ struct page *
 grab_cache_page_nowait(struct address_space *mapping, unsigned long index)
 {
        struct page *page = find_get_page(mapping, index);
-       gfp_t gfp_mask;
 
        if (page) {
                if (!TestSetPageLocked(page))
@@ -834,9 +807,8 @@ grab_cache_page_nowait(struct address_space *mapping, unsigned long index)
                page_cache_release(page);
                return NULL;
        }
-       gfp_mask = mapping_gfp_mask(mapping) & ~__GFP_FS;
-       page = alloc_pages(gfp_mask, 0);
-       if (page && add_to_page_cache_lru(page, mapping, index, gfp_mask)) {
+       page = __page_cache_alloc(mapping_gfp_mask(mapping) & ~__GFP_FS);
+       if (page && add_to_page_cache_lru(page, mapping, index, GFP_KERNEL)) {
                page_cache_release(page);
                page = NULL;
        }
@@ -900,6 +872,7 @@ void do_generic_mapping_read(struct address_space *mapping,
        unsigned long last_index;
        unsigned long next_index;
        unsigned long prev_index;
+       unsigned int prev_offset;
        loff_t isize;
        struct page *cached_page;
        int error;
@@ -908,7 +881,8 @@ void do_generic_mapping_read(struct address_space *mapping,
        cached_page = NULL;
        index = *ppos >> PAGE_CACHE_SHIFT;
        next_index = index;
-       prev_index = ra.prev_page;
+       prev_index = ra.prev_index;
+       prev_offset = ra.prev_offset;
        last_index = (*ppos + desc->count + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
        offset = *ppos & ~PAGE_CACHE_MASK;
 
@@ -956,10 +930,10 @@ page_ok:
                        flush_dcache_page(page);
 
                /*
-                * When (part of) the same page is read multiple times
-                * in succession, only mark it as accessed the first time.
+                * When a sequential read accesses a page several times,
+                * only mark it as accessed the first time.
                 */
-               if (prev_index != index)
+               if (prev_index != index || offset != prev_offset)
                        mark_page_accessed(page);
                prev_index = index;
 
@@ -977,6 +951,8 @@ page_ok:
                offset += ret;
                index += offset >> PAGE_CACHE_SHIFT;
                offset &= ~PAGE_CACHE_MASK;
+               prev_offset = offset;
+               ra.prev_offset = offset;
 
                page_cache_release(page);
                if (ret == nr && desc->count)
@@ -1138,6 +1114,45 @@ success:
        return size;
 }
 
+/*
+ * Performs necessary checks before doing a write
+ * @iov:       io vector request
+ * @nr_segs:   number of segments in the iovec
+ * @count:     number of bytes to write
+ * @access_flags: type of access: %VERIFY_READ or %VERIFY_WRITE
+ *
+ * Adjust number of segments and amount of bytes to write (nr_segs should be
+ * properly initialized first). Returns appropriate error code that caller
+ * should return or zero in case that write should be allowed.
+ */
+int generic_segment_checks(const struct iovec *iov,
+                       unsigned long *nr_segs, size_t *count, int access_flags)
+{
+       unsigned long   seg;
+       size_t cnt = 0;
+       for (seg = 0; seg < *nr_segs; seg++) {
+               const struct iovec *iv = &iov[seg];
+
+               /*
+                * If any segment has a negative length, or the cumulative
+                * length ever wraps negative then return -EINVAL.
+                */
+               cnt += iv->iov_len;
+               if (unlikely((ssize_t)(cnt|iv->iov_len) < 0))
+                       return -EINVAL;
+               if (access_ok(access_flags, iv->iov_base, iv->iov_len))
+                       continue;
+               if (seg == 0)
+                       return -EFAULT;
+               *nr_segs = seg;
+               cnt -= iv->iov_len;     /* This segment is no good */
+               break;
+       }
+       *count = cnt;
+       return 0;
+}
+EXPORT_SYMBOL(generic_segment_checks);
+
 /**
  * generic_file_aio_read - generic filesystem read routine
  * @iocb:      kernel I/O control block
@@ -1159,24 +1174,9 @@ generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
        loff_t *ppos = &iocb->ki_pos;
 
        count = 0;
-       for (seg = 0; seg < nr_segs; seg++) {
-               const struct iovec *iv = &iov[seg];
-
-               /*
-                * If any segment has a negative length, or the cumulative
-                * length ever wraps negative then return -EINVAL.
-                */
-               count += iv->iov_len;
-               if (unlikely((ssize_t)(count|iv->iov_len) < 0))
-                       return -EINVAL;
-               if (access_ok(VERIFY_WRITE, iv->iov_base, iv->iov_len))
-                       continue;
-               if (seg == 0)
-                       return -EFAULT;
-               nr_segs = seg;
-               count -= iv->iov_len;   /* This segment is no good */
-               break;
-       }
+       retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
+       if (retval)
+               return retval;
 
        /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
        if (filp->f_flags & O_DIRECT) {
@@ -1193,8 +1193,6 @@ generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
                if (pos < size) {
                        retval = generic_file_direct_IO(READ, iocb,
                                                iov, pos, nr_segs);
-                       if (retval > 0 && !is_sync_kiocb(iocb))
-                               retval = -EIOCBQUEUED;
                        if (retval > 0)
                                *ppos = pos + retval;
                }
@@ -1221,6 +1219,8 @@ generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
                                retval = retval ?: desc.error;
                                break;
                        }
+                       if (desc.count > 0)
+                               break;
                }
        }
 out:
@@ -1248,26 +1248,6 @@ int file_send_actor(read_descriptor_t * desc, struct page *page, unsigned long o
        return written;
 }
 
-ssize_t generic_file_sendfile(struct file *in_file, loff_t *ppos,
-                        size_t count, read_actor_t actor, void *target)
-{
-       read_descriptor_t desc;
-
-       if (!count)
-               return 0;
-
-       desc.written = 0;
-       desc.count = count;
-       desc.arg.data = target;
-       desc.error = 0;
-
-       do_generic_file_read(in_file, ppos, &desc, actor);
-       if (desc.written)
-               return desc.written;
-       return desc.error;
-}
-EXPORT_SYMBOL(generic_file_sendfile);
-
 static ssize_t
 do_readahead(struct address_space *mapping, struct file *filp,
             unsigned long index, unsigned long nr)
@@ -1457,7 +1437,6 @@ no_cached_page:
         * effect.
         */
        error = page_cache_read(file, pgoff);
-       grab_swap_token();
 
        /*
         * The page we want has now been added to the page cache.
@@ -1481,30 +1460,6 @@ page_not_uptodate:
                majmin = VM_FAULT_MAJOR;
                count_vm_event(PGMAJFAULT);
        }
-       lock_page(page);
-
-       /* Did it get unhashed while we waited for it? */
-       if (!page->mapping) {
-               unlock_page(page);
-               page_cache_release(page);
-               goto retry_all;
-       }
-
-       /* Did somebody else get it up-to-date? */
-       if (PageUptodate(page)) {
-               unlock_page(page);
-               goto success;
-       }
-
-       error = mapping->a_ops->readpage(file, page);
-       if (!error) {
-               wait_on_page_locked(page);
-               if (PageUptodate(page))
-                       goto success;
-       } else if (error == AOP_TRUNCATED_PAGE) {
-               page_cache_release(page);
-               goto retry_find;
-       }
 
        /*
         * Umm, take care of errors if the page isn't up-to-date.
@@ -1761,7 +1716,7 @@ int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
 EXPORT_SYMBOL(generic_file_mmap);
 EXPORT_SYMBOL(generic_file_readonly_mmap);
 
-static inline struct page *__read_cache_page(struct address_space *mapping,
+static struct page *__read_cache_page(struct address_space *mapping,
                                unsigned long index,
                                int (*filler)(void *,struct page*),
                                void *data)
@@ -1798,17 +1753,11 @@ repeat:
        return page;
 }
 
-/**
- * read_cache_page - read into page cache, fill it if needed
- * @mapping:   the page's address_space
- * @index:     the page index
- * @filler:    function to perform the read
- * @data:      destination for read data
- *
- * Read into the page cache. If a page already exists,
- * and PageUptodate() is not set, try to fill the page.
+/*
+ * Same as read_cache_page, but don't wait for page to become unlocked
+ * after submitting it to the filler.
  */
-struct page *read_cache_page(struct address_space *mapping,
+struct page *read_cache_page_async(struct address_space *mapping,
                                unsigned long index,
                                int (*filler)(void *,struct page*),
                                void *data)
@@ -1819,8 +1768,7 @@ struct page *read_cache_page(struct address_space *mapping,
 retry:
        page = __read_cache_page(mapping, index, filler, data);
        if (IS_ERR(page))
-               goto out;
-       mark_page_accessed(page);
+               return page;
        if (PageUptodate(page))
                goto out;
 
@@ -1837,7 +1785,40 @@ retry:
        err = filler(data, page);
        if (err < 0) {
                page_cache_release(page);
-               page = ERR_PTR(err);
+               return ERR_PTR(err);
+       }
+out:
+       mark_page_accessed(page);
+       return page;
+}
+EXPORT_SYMBOL(read_cache_page_async);
+
+/**
+ * read_cache_page - read into page cache, fill it if needed
+ * @mapping:   the page's address_space
+ * @index:     the page index
+ * @filler:    function to perform the read
+ * @data:      destination for read data
+ *
+ * Read into the page cache. If a page already exists, and PageUptodate() is
+ * not set, try to fill the page then wait for it to become unlocked.
+ *
+ * If the page does not get brought uptodate, return -EIO.
+ */
+struct page *read_cache_page(struct address_space *mapping,
+                               unsigned long index,
+                               int (*filler)(void *,struct page*),
+                               void *data)
+{
+       struct page *page;
+
+       page = read_cache_page_async(mapping, index, filler, data);
+       if (IS_ERR(page))
+               goto out;
+       wait_on_page_locked(page);
+       if (!PageUptodate(page)) {
+               page_cache_release(page);
+               page = ERR_PTR(-EIO);
        }
  out:
        return page;
@@ -1905,6 +1886,7 @@ int should_remove_suid(struct dentry *dentry)
 
        return 0;
 }
+EXPORT_SYMBOL(should_remove_suid);
 
 int __remove_suid(struct dentry *dentry, int kill)
 {
@@ -2059,15 +2041,14 @@ generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
         * Sync the fs metadata but not the minor inode changes and
         * of course not the data as we did direct DMA for the IO.
         * i_mutex is held, which protects generic_osync_inode() from
-        * livelocking.
+        * livelocking.  AIO O_DIRECT ops attempt to sync metadata here.
         */
-       if (written >= 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
+       if ((written >= 0 || written == -EIOCBQUEUED) &&
+           ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
                int err = generic_osync_inode(inode, mapping, OSYNC_METADATA);
                if (err < 0)
                        written = err;
        }
-       if (written == count && !is_sync_kiocb(iocb))
-               written = -EIOCBQUEUED;
        return written;
 }
 EXPORT_SYMBOL(generic_file_direct_write);
@@ -2114,21 +2095,27 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
                /* Limit the size of the copy to the caller's write size */
                bytes = min(bytes, count);
 
-               /*
-                * Limit the size of the copy to that of the current segment,
-                * because fault_in_pages_readable() doesn't know how to walk
-                * segments.
+               /* We only need to worry about prefaulting when writes are from
+                * user-space.  NFSd uses vfs_writev with several non-aligned
+                * segments in the vector, and limiting to one segment a time is
+                * a noticeable performance for re-write
                 */
-               bytes = min(bytes, cur_iov->iov_len - iov_base);
-
-               /*
-                * Bring in the user page that we will copy from _first_.
-                * Otherwise there's a nasty deadlock on copying from the
-                * same page as we're writing to, without it being marked
-                * up-to-date.
-                */
-               fault_in_pages_readable(buf, bytes);
+               if (!segment_eq(get_fs(), KERNEL_DS)) {
+                       /*
+                        * Limit the size of the copy to that of the current
+                        * segment, because fault_in_pages_readable() doesn't
+                        * know how to walk segments.
+                        */
+                       bytes = min(bytes, cur_iov->iov_len - iov_base);
 
+                       /*
+                        * Bring in the user page that we will copy from
+                        * _first_.  Otherwise there's a nasty deadlock on
+                        * copying from the same page as we're writing to,
+                        * without it being marked up-to-date.
+                        */
+                       fault_in_pages_readable(buf, bytes);
+               }
                page = __grab_cache_page(mapping,index,&cached_page,&lru_pvec);
                if (!page) {
                        status = -ENOMEM;
@@ -2236,34 +2223,18 @@ __generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
                                unsigned long nr_segs, loff_t *ppos)
 {
        struct file *file = iocb->ki_filp;
-       const struct address_space * mapping = file->f_mapping;
+       struct address_space * mapping = file->f_mapping;
        size_t ocount;          /* original count */
        size_t count;           /* after file limit checks */
        struct inode    *inode = mapping->host;
-       unsigned long   seg;
        loff_t          pos;
        ssize_t         written;
        ssize_t         err;
 
        ocount = 0;
-       for (seg = 0; seg < nr_segs; seg++) {
-               const struct iovec *iv = &iov[seg];
-
-               /*
-                * If any segment has a negative length, or the cumulative
-                * length ever wraps negative then return -EINVAL.
-                */
-               ocount += iv->iov_len;
-               if (unlikely((ssize_t)(ocount|iv->iov_len) < 0))
-                       return -EINVAL;
-               if (access_ok(VERIFY_READ, iv->iov_base, iv->iov_len))
-                       continue;
-               if (seg == 0)
-                       return -EFAULT;
-               nr_segs = seg;
-               ocount -= iv->iov_len;  /* This segment is no good */
-               break;
-       }
+       err = generic_segment_checks(iov, &nr_segs, &ocount, VERIFY_READ);
+       if (err)
+               return err;
 
        count = ocount;
        pos = *ppos;
@@ -2281,7 +2252,7 @@ __generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
        if (count == 0)
                goto out;
 
-       err = remove_suid(file->f_dentry);
+       err = remove_suid(file->f_path.dentry);
        if (err)
                goto out;
 
@@ -2289,8 +2260,11 @@ __generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
 
        /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
        if (unlikely(file->f_flags & O_DIRECT)) {
-               written = generic_file_direct_write(iocb, iov,
-                               &nr_segs, pos, ppos, count, ocount);
+               loff_t endbyte;
+               ssize_t written_buffered;
+
+               written = generic_file_direct_write(iocb, iov, &nr_segs, pos,
+                                                       ppos, count, ocount);
                if (written < 0 || written == count)
                        goto out;
                /*
@@ -2299,10 +2273,46 @@ __generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
                 */
                pos += written;
                count -= written;
-       }
+               written_buffered = generic_file_buffered_write(iocb, iov,
+                                               nr_segs, pos, ppos, count,
+                                               written);
+               /*
+                * If generic_file_buffered_write() retuned a synchronous error
+                * then we want to return the number of bytes which were
+                * direct-written, or the error code if that was zero.  Note
+                * that this differs from normal direct-io semantics, which
+                * will return -EFOO even if some bytes were written.
+                */
+               if (written_buffered < 0) {
+                       err = written_buffered;
+                       goto out;
+               }
 
-       written = generic_file_buffered_write(iocb, iov, nr_segs,
-                       pos, ppos, count, written);
+               /*
+                * We need to ensure that the page cache pages are written to
+                * disk and invalidated to preserve the expected O_DIRECT
+                * semantics.
+                */
+               endbyte = pos + written_buffered - written - 1;
+               err = do_sync_mapping_range(file->f_mapping, pos, endbyte,
+                                           SYNC_FILE_RANGE_WAIT_BEFORE|
+                                           SYNC_FILE_RANGE_WRITE|
+                                           SYNC_FILE_RANGE_WAIT_AFTER);
+               if (err == 0) {
+                       written = written_buffered;
+                       invalidate_mapping_pages(mapping,
+                                                pos >> PAGE_CACHE_SHIFT,
+                                                endbyte >> PAGE_CACHE_SHIFT);
+               } else {
+                       /*
+                        * We don't know how much we wrote, so just return
+                        * the number of bytes which were direct-written
+                        */
+               }
+       } else {
+               written = generic_file_buffered_write(iocb, iov, nr_segs,
+                               pos, ppos, count, written);
+       }
 out:
        current->backing_dev_info = NULL;
        return written ? written : err;
@@ -2369,7 +2379,8 @@ generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
        struct file *file = iocb->ki_filp;
        struct address_space *mapping = file->f_mapping;
        ssize_t retval;
-       size_t write_len = 0;
+       size_t write_len;
+       pgoff_t end = 0; /* silence gcc */
 
        /*
         * If it's a write, unmap all mmappings of the file up-front.  This
@@ -2378,23 +2389,46 @@ generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
         */
        if (rw == WRITE) {
                write_len = iov_length(iov, nr_segs);
+               end = (offset + write_len - 1) >> PAGE_CACHE_SHIFT;
                if (mapping_mapped(mapping))
                        unmap_mapping_range(mapping, offset, write_len, 0);
        }
 
        retval = filemap_write_and_wait(mapping);
-       if (retval == 0) {
-               retval = mapping->a_ops->direct_IO(rw, iocb, iov,
-                                               offset, nr_segs);
-               if (rw == WRITE && mapping->nrpages) {
-                       pgoff_t end = (offset + write_len - 1)
-                                               >> PAGE_CACHE_SHIFT;
-                       int err = invalidate_inode_pages2_range(mapping,
+       if (retval)
+               goto out;
+
+       /*
+        * After a write we want buffered reads to be sure to go to disk to get
+        * the new data.  We invalidate clean cached page from the region we're
+        * about to write.  We do this *before* the write so that we can return
+        * -EIO without clobbering -EIOCBQUEUED from ->direct_IO().
+        */
+       if (rw == WRITE && mapping->nrpages) {
+               retval = invalidate_inode_pages2_range(mapping,
                                        offset >> PAGE_CACHE_SHIFT, end);
-                       if (err)
-                               retval = err;
-               }
+               if (retval)
+                       goto out;
+       }
+
+       retval = mapping->a_ops->direct_IO(rw, iocb, iov, offset, nr_segs);
+       if (retval)
+               goto out;
+
+       /*
+        * Finally, try again to invalidate clean pages which might have been
+        * faulted in by get_user_pages() if the source of the write was an
+        * mmap()ed region of the file we're writing.  That's a pretty crazy
+        * thing to do, so we don't support it 100%.  If this invalidation
+        * fails and we have -EIOCBQUEUED we ignore the failure.
+        */
+       if (rw == WRITE && mapping->nrpages) {
+               int err = invalidate_inode_pages2_range(mapping,
+                                             offset >> PAGE_CACHE_SHIFT, end);
+               if (err && retval >= 0)
+                       retval = err;
        }
+out:
        return retval;
 }