PM: Remove CONFIG_PM_OPS
[linux-2.6.git] / drivers / scsi / scsi_lib.c
index 681ed1b..fb2bb35 100644 (file)
@@ -8,6 +8,7 @@
  */
 
 #include <linux/bio.h>
+#include <linux/bitops.h>
 #include <linux/blkdev.h>
 #include <linux/completion.h>
 #include <linux/kernel.h>
 #define SG_MEMPOOL_NR          ARRAY_SIZE(scsi_sg_pools)
 #define SG_MEMPOOL_SIZE                2
 
-/*
- * The maximum number of SG segments that we will put inside a scatterlist
- * (unless chaining is used). Should ideally fit inside a single page, to
- * avoid a higher order allocation.
- */
-#define SCSI_MAX_SG_SEGMENTS   128
-
 struct scsi_host_sg_pool {
        size_t          size;
        char            *name;
@@ -48,23 +42,30 @@ struct scsi_host_sg_pool {
        mempool_t       *pool;
 };
 
-#define SP(x) { x, "sgpool-" #x }
+#define SP(x) { x, "sgpool-" __stringify(x) }
+#if (SCSI_MAX_SG_SEGMENTS < 32)
+#error SCSI_MAX_SG_SEGMENTS is too small (must be 32 or greater)
+#endif
 static struct scsi_host_sg_pool scsi_sg_pools[] = {
        SP(8),
        SP(16),
-#if (SCSI_MAX_SG_SEGMENTS > 16)
-       SP(32),
 #if (SCSI_MAX_SG_SEGMENTS > 32)
-       SP(64),
+       SP(32),
 #if (SCSI_MAX_SG_SEGMENTS > 64)
+       SP(64),
+#if (SCSI_MAX_SG_SEGMENTS > 128)
        SP(128),
+#if (SCSI_MAX_SG_SEGMENTS > 256)
+#error SCSI_MAX_SG_SEGMENTS is too large (256 MAX)
+#endif
 #endif
 #endif
 #endif
+       SP(SCSI_MAX_SG_SEGMENTS)
 };
 #undef SP
 
-static struct kmem_cache *scsi_bidi_sdb_cache;
+struct kmem_cache *scsi_sdb_cache;
 
 static void scsi_run_queue(struct request_queue *q);
 
@@ -84,35 +85,29 @@ static void scsi_unprep_request(struct request *req)
 {
        struct scsi_cmnd *cmd = req->special;
 
-       req->cmd_flags &= ~REQ_DONTPREP;
+       blk_unprep_request(req);
        req->special = NULL;
 
        scsi_put_command(cmd);
 }
 
-/*
- * Function:    scsi_queue_insert()
- *
- * Purpose:     Insert a command in the midlevel queue.
- *
- * Arguments:   cmd    - command that we are adding to queue.
- *              reason - why we are inserting command to queue.
- *
- * Lock status: Assumed that lock is not held upon entry.
- *
- * Returns:     Nothing.
- *
- * Notes:       We do this for one of two cases.  Either the host is busy
- *              and it cannot accept any more commands for the time being,
- *              or the device returned QUEUE_FULL and can accept no more
- *              commands.
- * Notes:       This could be called either from an interrupt context or a
- *              normal process context.
+/**
+ * __scsi_queue_insert - private queue insertion
+ * @cmd: The SCSI command being requeued
+ * @reason:  The reason for the requeue
+ * @unbusy: Whether the queue should be unbusied
+ *
+ * This is a private queue insertion.  The public interface
+ * scsi_queue_insert() always assumes the queue should be unbusied
+ * because it's always called before the completion.  This function is
+ * for a requeue after completion, which should only occur in this
+ * file.
  */
-int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
+static int __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy)
 {
        struct Scsi_Host *host = cmd->device->host;
        struct scsi_device *device = cmd->device;
+       struct scsi_target *starget = scsi_target(device);
        struct request_queue *q = device->request_queue;
        unsigned long flags;
 
@@ -132,16 +127,24 @@ int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
         * if a command is requeued with no other commands outstanding
         * either for the device or for the host.
         */
-       if (reason == SCSI_MLQUEUE_HOST_BUSY)
+       switch (reason) {
+       case SCSI_MLQUEUE_HOST_BUSY:
                host->host_blocked = host->max_host_blocked;
-       else if (reason == SCSI_MLQUEUE_DEVICE_BUSY)
+               break;
+       case SCSI_MLQUEUE_DEVICE_BUSY:
                device->device_blocked = device->max_device_blocked;
+               break;
+       case SCSI_MLQUEUE_TARGET_BUSY:
+               starget->target_blocked = starget->max_target_blocked;
+               break;
+       }
 
        /*
         * Decrement the counters, since these commands are no longer
         * active on the host/device.
         */
-       scsi_device_unbusy(device);
+       if (unbusy)
+               scsi_device_unbusy(device);
 
        /*
         * Requeue this command.  It will go before all other commands
@@ -163,6 +166,29 @@ int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
        return 0;
 }
 
+/*
+ * Function:    scsi_queue_insert()
+ *
+ * Purpose:     Insert a command in the midlevel queue.
+ *
+ * Arguments:   cmd    - command that we are adding to queue.
+ *              reason - why we are inserting command to queue.
+ *
+ * Lock status: Assumed that lock is not held upon entry.
+ *
+ * Returns:     Nothing.
+ *
+ * Notes:       We do this for one of two cases.  Either the host is busy
+ *              and it cannot accept any more commands for the time being,
+ *              or the device returned QUEUE_FULL and can accept no more
+ *              commands.
+ * Notes:       This could be called either from an interrupt context or a
+ *              normal process context.
+ */
+int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
+{
+       return __scsi_queue_insert(cmd, reason, 1);
+}
 /**
  * scsi_execute - insert request and wait for the result
  * @sdev:      scsi device
@@ -174,13 +200,15 @@ int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
  * @timeout:   request timeout in seconds
  * @retries:   number of times to retry request
  * @flags:     or into request flags;
+ * @resid:     optional residual length
  *
  * returns the req->errors value which is the scsi_cmnd result
  * field.
  */
 int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
                 int data_direction, void *buffer, unsigned bufflen,
-                unsigned char *sense, int timeout, int retries, int flags)
+                unsigned char *sense, int timeout, int retries, int flags,
+                int *resid)
 {
        struct request *req;
        int write = (data_direction == DMA_TO_DEVICE);
@@ -206,6 +234,17 @@ int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
         */
        blk_execute_rq(req->q, NULL, req, 1);
 
+       /*
+        * Some devices (USB mass-storage in particular) may transfer
+        * garbage data together with a residue indicating that the data
+        * is invalid.  Prevent the garbage from being misinterpreted
+        * and prevent security leaks by zeroing out the excess data.
+        */
+       if (unlikely(req->resid_len > 0 && req->resid_len <= bufflen))
+               memset(buffer + (bufflen - req->resid_len), 0, req->resid_len);
+
+       if (resid)
+               *resid = req->resid_len;
        ret = req->errors;
  out:
        blk_put_request(req);
@@ -217,7 +256,8 @@ EXPORT_SYMBOL(scsi_execute);
 
 int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd,
                     int data_direction, void *buffer, unsigned bufflen,
-                    struct scsi_sense_hdr *sshdr, int timeout, int retries)
+                    struct scsi_sense_hdr *sshdr, int timeout, int retries,
+                    int *resid)
 {
        char *sense = NULL;
        int result;
@@ -228,7 +268,7 @@ int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd,
                        return DRIVER_ERROR << 24;
        }
        result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen,
-                             sense, timeout, retries, 0);
+                             sense, timeout, retries, 0, resid);
        if (sshdr)
                scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr);
 
@@ -237,197 +277,6 @@ int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd,
 }
 EXPORT_SYMBOL(scsi_execute_req);
 
-struct scsi_io_context {
-       void *data;
-       void (*done)(void *data, char *sense, int result, int resid);
-       char sense[SCSI_SENSE_BUFFERSIZE];
-};
-
-static struct kmem_cache *scsi_io_context_cache;
-
-static void scsi_end_async(struct request *req, int uptodate)
-{
-       struct scsi_io_context *sioc = req->end_io_data;
-
-       if (sioc->done)
-               sioc->done(sioc->data, sioc->sense, req->errors, req->data_len);
-
-       kmem_cache_free(scsi_io_context_cache, sioc);
-       __blk_put_request(req->q, req);
-}
-
-static int scsi_merge_bio(struct request *rq, struct bio *bio)
-{
-       struct request_queue *q = rq->q;
-
-       bio->bi_flags &= ~(1 << BIO_SEG_VALID);
-       if (rq_data_dir(rq) == WRITE)
-               bio->bi_rw |= (1 << BIO_RW);
-       blk_queue_bounce(q, &bio);
-
-       return blk_rq_append_bio(q, rq, bio);
-}
-
-static void scsi_bi_endio(struct bio *bio, int error)
-{
-       bio_put(bio);
-}
-
-/**
- * scsi_req_map_sg - map a scatterlist into a request
- * @rq:                request to fill
- * @sgl:       scatterlist
- * @nsegs:     number of elements
- * @bufflen:   len of buffer
- * @gfp:       memory allocation flags
- *
- * scsi_req_map_sg maps a scatterlist into a request so that the
- * request can be sent to the block layer. We do not trust the scatterlist
- * sent to use, as some ULDs use that struct to only organize the pages.
- */
-static int scsi_req_map_sg(struct request *rq, struct scatterlist *sgl,
-                          int nsegs, unsigned bufflen, gfp_t gfp)
-{
-       struct request_queue *q = rq->q;
-       int nr_pages = (bufflen + sgl[0].offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
-       unsigned int data_len = bufflen, len, bytes, off;
-       struct scatterlist *sg;
-       struct page *page;
-       struct bio *bio = NULL;
-       int i, err, nr_vecs = 0;
-
-       for_each_sg(sgl, sg, nsegs, i) {
-               page = sg_page(sg);
-               off = sg->offset;
-               len = sg->length;
-               data_len += len;
-
-               while (len > 0 && data_len > 0) {
-                       /*
-                        * sg sends a scatterlist that is larger than
-                        * the data_len it wants transferred for certain
-                        * IO sizes
-                        */
-                       bytes = min_t(unsigned int, len, PAGE_SIZE - off);
-                       bytes = min(bytes, data_len);
-
-                       if (!bio) {
-                               nr_vecs = min_t(int, BIO_MAX_PAGES, nr_pages);
-                               nr_pages -= nr_vecs;
-
-                               bio = bio_alloc(gfp, nr_vecs);
-                               if (!bio) {
-                                       err = -ENOMEM;
-                                       goto free_bios;
-                               }
-                               bio->bi_end_io = scsi_bi_endio;
-                       }
-
-                       if (bio_add_pc_page(q, bio, page, bytes, off) !=
-                           bytes) {
-                               bio_put(bio);
-                               err = -EINVAL;
-                               goto free_bios;
-                       }
-
-                       if (bio->bi_vcnt >= nr_vecs) {
-                               err = scsi_merge_bio(rq, bio);
-                               if (err) {
-                                       bio_endio(bio, 0);
-                                       goto free_bios;
-                               }
-                               bio = NULL;
-                       }
-
-                       page++;
-                       len -= bytes;
-                       data_len -=bytes;
-                       off = 0;
-               }
-       }
-
-       rq->buffer = rq->data = NULL;
-       rq->data_len = bufflen;
-       return 0;
-
-free_bios:
-       while ((bio = rq->bio) != NULL) {
-               rq->bio = bio->bi_next;
-               /*
-                * call endio instead of bio_put incase it was bounced
-                */
-               bio_endio(bio, 0);
-       }
-
-       return err;
-}
-
-/**
- * scsi_execute_async - insert request
- * @sdev:      scsi device
- * @cmd:       scsi command
- * @cmd_len:   length of scsi cdb
- * @data_direction: DMA_TO_DEVICE, DMA_FROM_DEVICE, or DMA_NONE
- * @buffer:    data buffer (this can be a kernel buffer or scatterlist)
- * @bufflen:   len of buffer
- * @use_sg:    if buffer is a scatterlist this is the number of elements
- * @timeout:   request timeout in seconds
- * @retries:   number of times to retry request
- * @privdata:  data passed to done()
- * @done:      callback function when done
- * @gfp:       memory allocation flags
- */
-int scsi_execute_async(struct scsi_device *sdev, const unsigned char *cmd,
-                      int cmd_len, int data_direction, void *buffer, unsigned bufflen,
-                      int use_sg, int timeout, int retries, void *privdata,
-                      void (*done)(void *, char *, int, int), gfp_t gfp)
-{
-       struct request *req;
-       struct scsi_io_context *sioc;
-       int err = 0;
-       int write = (data_direction == DMA_TO_DEVICE);
-
-       sioc = kmem_cache_zalloc(scsi_io_context_cache, gfp);
-       if (!sioc)
-               return DRIVER_ERROR << 24;
-
-       req = blk_get_request(sdev->request_queue, write, gfp);
-       if (!req)
-               goto free_sense;
-       req->cmd_type = REQ_TYPE_BLOCK_PC;
-       req->cmd_flags |= REQ_QUIET;
-
-       if (use_sg)
-               err = scsi_req_map_sg(req, buffer, use_sg, bufflen, gfp);
-       else if (bufflen)
-               err = blk_rq_map_kern(req->q, req, buffer, bufflen, gfp);
-
-       if (err)
-               goto free_req;
-
-       req->cmd_len = cmd_len;
-       memset(req->cmd, 0, BLK_MAX_CDB); /* ATAPI hates garbage after CDB */
-       memcpy(req->cmd, cmd, req->cmd_len);
-       req->sense = sioc->sense;
-       req->sense_len = 0;
-       req->timeout = timeout;
-       req->retries = retries;
-       req->end_io_data = sioc;
-
-       sioc->data = privdata;
-       sioc->done = done;
-
-       blk_execute_rq_nowait(req->q, NULL, req, 1, scsi_end_async);
-       return 0;
-
-free_req:
-       blk_put_request(req);
-free_sense:
-       kmem_cache_free(scsi_io_context_cache, sioc);
-       return DRIVER_ERROR << 24;
-}
-EXPORT_SYMBOL_GPL(scsi_execute_async);
-
 /*
  * Function:    scsi_init_cmd_errh()
  *
@@ -445,16 +294,18 @@ static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)
        scsi_set_resid(cmd, 0);
        memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
        if (cmd->cmd_len == 0)
-               cmd->cmd_len = COMMAND_SIZE(cmd->cmnd[0]);
+               cmd->cmd_len = scsi_command_size(cmd->cmnd);
 }
 
 void scsi_device_unbusy(struct scsi_device *sdev)
 {
        struct Scsi_Host *shost = sdev->host;
+       struct scsi_target *starget = scsi_target(sdev);
        unsigned long flags;
 
        spin_lock_irqsave(shost->host_lock, flags);
        shost->host_busy--;
+       starget->target_busy--;
        if (unlikely(scsi_host_in_recovery(shost) &&
                     (shost->host_failed || shost->host_eh_scheduled)))
                scsi_eh_wakeup(shost);
@@ -510,6 +361,30 @@ static void scsi_single_lun_run(struct scsi_device *current_sdev)
        spin_unlock_irqrestore(shost->host_lock, flags);
 }
 
+static inline int scsi_device_is_busy(struct scsi_device *sdev)
+{
+       if (sdev->device_busy >= sdev->queue_depth || sdev->device_blocked)
+               return 1;
+
+       return 0;
+}
+
+static inline int scsi_target_is_busy(struct scsi_target *starget)
+{
+       return ((starget->can_queue > 0 &&
+                starget->target_busy >= starget->can_queue) ||
+                starget->target_blocked);
+}
+
+static inline int scsi_host_is_busy(struct Scsi_Host *shost)
+{
+       if ((shost->can_queue > 0 && shost->host_busy >= shost->can_queue) ||
+           shost->host_blocked || shost->host_self_blocked)
+               return 1;
+
+       return 0;
+}
+
 /*
  * Function:   scsi_run_queue()
  *
@@ -526,16 +401,18 @@ static void scsi_run_queue(struct request_queue *q)
 {
        struct scsi_device *sdev = q->queuedata;
        struct Scsi_Host *shost = sdev->host;
+       LIST_HEAD(starved_list);
        unsigned long flags;
 
        if (scsi_target(sdev)->single_lun)
                scsi_single_lun_run(sdev);
 
        spin_lock_irqsave(shost->host_lock, flags);
-       while (!list_empty(&shost->starved_list) &&
-              !shost->host_blocked && !shost->host_self_blocked &&
-               !((shost->can_queue > 0) &&
-                 (shost->host_busy >= shost->can_queue))) {
+       list_splice_init(&shost->starved_list, &starved_list);
+
+       while (!list_empty(&starved_list)) {
+               int flagset;
+
                /*
                 * As long as shost is accepting commands and we have
                 * starved queues, call blk_run_queue. scsi_request_fn
@@ -546,30 +423,35 @@ static void scsi_run_queue(struct request_queue *q)
                 * scsi_request_fn must get the host_lock before checking
                 * or modifying starved_list or starved_entry.
                 */
-               sdev = list_entry(shost->starved_list.next,
-                                         struct scsi_device, starved_entry);
+               if (scsi_host_is_busy(shost))
+                       break;
+
+               sdev = list_entry(starved_list.next,
+                                 struct scsi_device, starved_entry);
                list_del_init(&sdev->starved_entry);
-               spin_unlock_irqrestore(shost->host_lock, flags);
+               if (scsi_target_is_busy(scsi_target(sdev))) {
+                       list_move_tail(&sdev->starved_entry,
+                                      &shost->starved_list);
+                       continue;
+               }
 
+               spin_unlock(shost->host_lock);
 
-               if (test_bit(QUEUE_FLAG_REENTER, &q->queue_flags) &&
-                   !test_and_set_bit(QUEUE_FLAG_REENTER,
-                                     &sdev->request_queue->queue_flags)) {
-                       blk_run_queue(sdev->request_queue);
-                       clear_bit(QUEUE_FLAG_REENTER,
-                                 &sdev->request_queue->queue_flags);
-               } else
-                       blk_run_queue(sdev->request_queue);
+               spin_lock(sdev->request_queue->queue_lock);
+               flagset = test_bit(QUEUE_FLAG_REENTER, &q->queue_flags) &&
+                               !test_bit(QUEUE_FLAG_REENTER,
+                                       &sdev->request_queue->queue_flags);
+               if (flagset)
+                       queue_flag_set(QUEUE_FLAG_REENTER, sdev->request_queue);
+               __blk_run_queue(sdev->request_queue, false);
+               if (flagset)
+                       queue_flag_clear(QUEUE_FLAG_REENTER, sdev->request_queue);
+               spin_unlock(sdev->request_queue->queue_lock);
 
-               spin_lock_irqsave(shost->host_lock, flags);
-               if (unlikely(!list_empty(&sdev->starved_entry)))
-                       /*
-                        * sdev lost a race, and was put back on the
-                        * starved list. This is unlikely but without this
-                        * in theory we could loop forever.
-                        */
-                       break;
+               spin_lock(shost->host_lock);
        }
+       /* put any unprocessed entries back */
+       list_splice(&starved_list, &shost->starved_list);
        spin_unlock_irqrestore(shost->host_lock, flags);
 
        blk_run_queue(q);
@@ -598,8 +480,8 @@ static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd)
        struct request *req = cmd->request;
        unsigned long flags;
 
-       scsi_unprep_request(req);
        spin_lock_irqsave(q->queue_lock, flags);
+       scsi_unprep_request(req);
        blk_requeue_request(q, req);
        spin_unlock_irqrestore(q->queue_lock, flags);
 
@@ -629,6 +511,8 @@ void scsi_run_host_queues(struct Scsi_Host *shost)
                scsi_run_queue(sdev->request_queue);
 }
 
+static void __scsi_release_buffers(struct scsi_cmnd *, int);
+
 /*
  * Function:    scsi_end_request()
  *
@@ -662,14 +546,9 @@ static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int error,
         * to queue the remainder of them.
         */
        if (blk_end_request(req, error, bytes)) {
-               int leftover = (req->hard_nr_sectors << 9);
-
-               if (blk_pc_request(req))
-                       leftover = req->data_len;
-
                /* kill remainder if no retrys */
-               if (error && blk_noretry_request(req))
-                       blk_end_request(req, error, leftover);
+               if (error && scsi_noretry_cmd(cmd))
+                       blk_end_request_all(req, error);
                else {
                        if (requeue) {
                                /*
@@ -677,6 +556,7 @@ static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int error,
                                 * leftovers in the front of the
                                 * queue, and goose the queue again.
                                 */
+                               scsi_release_buffers(cmd);
                                scsi_requeue_command(q, cmd);
                                cmd = NULL;
                        }
@@ -688,46 +568,21 @@ static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int error,
         * This will goose the queue request function at the end, so we don't
         * need to worry about launching another command.
         */
+       __scsi_release_buffers(cmd, 0);
        scsi_next_command(cmd);
        return NULL;
 }
 
-/*
- * Like SCSI_MAX_SG_SEGMENTS, but for archs that have sg chaining. This limit
- * is totally arbitrary, a setting of 2048 will get you at least 8mb ios.
- */
-#define SCSI_MAX_SG_CHAIN_SEGMENTS     2048
-
 static inline unsigned int scsi_sgtable_index(unsigned short nents)
 {
        unsigned int index;
 
-       switch (nents) {
-       case 1 ... 8:
+       BUG_ON(nents > SCSI_MAX_SG_SEGMENTS);
+
+       if (nents <= 8)
                index = 0;
-               break;
-       case 9 ... 16:
-               index = 1;
-               break;
-#if (SCSI_MAX_SG_SEGMENTS > 16)
-       case 17 ... 32:
-               index = 2;
-               break;
-#if (SCSI_MAX_SG_SEGMENTS > 32)
-       case 33 ... 64:
-               index = 3;
-               break;
-#if (SCSI_MAX_SG_SEGMENTS > 64)
-       case 65 ... 128:
-               index = 4;
-               break;
-#endif
-#endif
-#endif
-       default:
-               printk(KERN_ERR "scsi: bad segment count=%d\n", nents);
-               BUG();
-       }
+       else
+               index = get_count_order(nents) - 3;
 
        return index;
 }
@@ -769,6 +624,26 @@ static void scsi_free_sgtable(struct scsi_data_buffer *sdb)
        __sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS, scsi_sg_free);
 }
 
+static void __scsi_release_buffers(struct scsi_cmnd *cmd, int do_bidi_check)
+{
+
+       if (cmd->sdb.table.nents)
+               scsi_free_sgtable(&cmd->sdb);
+
+       memset(&cmd->sdb, 0, sizeof(cmd->sdb));
+
+       if (do_bidi_check && scsi_bidi_cmnd(cmd)) {
+               struct scsi_data_buffer *bidi_sdb =
+                       cmd->request->next_rq->special;
+               scsi_free_sgtable(bidi_sdb);
+               kmem_cache_free(scsi_sdb_cache, bidi_sdb);
+               cmd->request->next_rq->special = NULL;
+       }
+
+       if (scsi_prot_sg_count(cmd))
+               scsi_free_sgtable(cmd->prot_sdb);
+}
+
 /*
  * Function:    scsi_release_buffers()
  *
@@ -788,42 +663,11 @@ static void scsi_free_sgtable(struct scsi_data_buffer *sdb)
  */
 void scsi_release_buffers(struct scsi_cmnd *cmd)
 {
-       if (cmd->sdb.table.nents)
-               scsi_free_sgtable(&cmd->sdb);
-
-       memset(&cmd->sdb, 0, sizeof(cmd->sdb));
-
-       if (scsi_bidi_cmnd(cmd)) {
-               struct scsi_data_buffer *bidi_sdb =
-                       cmd->request->next_rq->special;
-               scsi_free_sgtable(bidi_sdb);
-               kmem_cache_free(scsi_bidi_sdb_cache, bidi_sdb);
-               cmd->request->next_rq->special = NULL;
-       }
+       __scsi_release_buffers(cmd, 1);
 }
 EXPORT_SYMBOL(scsi_release_buffers);
 
 /*
- * Bidi commands Must be complete as a whole, both sides at once.
- * If part of the bytes were written and lld returned
- * scsi_in()->resid and/or scsi_out()->resid this information will be left
- * in req->data_len and req->next_rq->data_len. The upper-layer driver can
- * decide what to do with this information.
- */
-void scsi_end_bidi_request(struct scsi_cmnd *cmd)
-{
-       blk_end_bidi_request(cmd->request, 0, scsi_out(cmd)->resid,
-                                                       scsi_in(cmd)->resid);
-       scsi_release_buffers(cmd);
-
-       /*
-        * This will goose the queue request function at the end, so we don't
-        * need to worry about launching another command.
-        */
-       scsi_next_command(cmd);
-}
-
-/*
  * Function:    scsi_io_completion()
  *
  * Purpose:     Completion processing for block device I/O requests.
@@ -840,27 +684,37 @@ void scsi_end_bidi_request(struct scsi_cmnd *cmd)
  *              (the normal case for most drivers), we don't need
  *              the logic to deal with cleaning up afterwards.
  *
- *             We must do one of several things here:
+ *             We must call scsi_end_request().  This will finish off
+ *             the specified number of sectors.  If we are done, the
+ *             command block will be released and the queue function
+ *             will be goosed.  If we are not done then we have to
+ *             figure out what to do next:
+ *
+ *             a) We can call scsi_requeue_command().  The request
+ *                will be unprepared and put back on the queue.  Then
+ *                a new command will be created for it.  This should
+ *                be used if we made forward progress, or if we want
+ *                to switch from READ(10) to READ(6) for example.
  *
- *             a) Call scsi_end_request.  This will finish off the
- *                specified number of sectors.  If we are done, the
- *                command block will be released, and the queue
- *                function will be goosed.  If we are not done, then
- *                scsi_end_request will directly goose the queue.
+ *             b) We can call scsi_queue_insert().  The request will
+ *                be put back on the queue and retried using the same
+ *                command as before, possibly after a delay.
  *
- *             b) We can just use scsi_requeue_command() here.  This would
- *                be used if we just wanted to retry, for example.
+ *             c) We can call blk_end_request() with -EIO to fail
+ *                the remainder of the request.
  */
 void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
 {
        int result = cmd->result;
-       int this_count = scsi_bufflen(cmd);
        struct request_queue *q = cmd->device->request_queue;
        struct request *req = cmd->request;
-       int clear_errors = 1;
+       int error = 0;
        struct scsi_sense_hdr sshdr;
        int sense_valid = 0;
        int sense_deferred = 0;
+       enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY,
+             ACTION_DELAYED_RETRY} action;
+       char *description = NULL;
 
        if (result) {
                sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
@@ -868,10 +722,9 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
                        sense_deferred = scsi_sense_is_deferred(&sshdr);
        }
 
-       if (blk_pc_request(req)) { /* SG_IO ioctl from block level */
+       if (req->cmd_type == REQ_TYPE_BLOCK_PC) { /* SG_IO ioctl from block level */
                req->errors = result;
                if (result) {
-                       clear_errors = 0;
                        if (sense_valid && req->sense) {
                                /*
                                 * SG_IO wants current and deferred errors
@@ -883,40 +736,75 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
                                memcpy(req->sense, cmd->sense_buffer,  len);
                                req->sense_len = len;
                        }
+                       if (!sense_deferred)
+                               error = -EIO;
                }
+
+               req->resid_len = scsi_get_resid(cmd);
+
                if (scsi_bidi_cmnd(cmd)) {
-                       /* will also release_buffers */
-                       scsi_end_bidi_request(cmd);
+                       /*
+                        * Bidi commands Must be complete as a whole,
+                        * both sides at once.
+                        */
+                       req->next_rq->resid_len = scsi_in(cmd)->resid;
+
+                       scsi_release_buffers(cmd);
+                       blk_end_request_all(req, 0);
+
+                       scsi_next_command(cmd);
                        return;
                }
-               req->data_len = scsi_get_resid(cmd);
        }
 
-       BUG_ON(blk_bidi_rq(req)); /* bidi not support for !blk_pc_request yet */
-       scsi_release_buffers(cmd);
+       /* no bidi support for !REQ_TYPE_BLOCK_PC yet */
+       BUG_ON(blk_bidi_rq(req));
 
        /*
         * Next deal with any sectors which we were able to correctly
         * handle.
         */
-       SCSI_LOG_HLCOMPLETE(1, printk("%ld sectors total, "
+       SCSI_LOG_HLCOMPLETE(1, printk("%u sectors total, "
                                      "%d bytes done.\n",
-                                     req->nr_sectors, good_bytes));
+                                     blk_rq_sectors(req), good_bytes));
 
-       if (clear_errors)
-               req->errors = 0;
+       /*
+        * Recovered errors need reporting, but they're always treated
+        * as success, so fiddle the result code here.  For BLOCK_PC
+        * we already took a copy of the original into rq->errors which
+        * is what gets returned to the user
+        */
+       if (sense_valid && (sshdr.sense_key == RECOVERED_ERROR)) {
+               /* if ATA PASS-THROUGH INFORMATION AVAILABLE skip
+                * print since caller wants ATA registers. Only occurs on
+                * SCSI ATA PASS_THROUGH commands when CK_COND=1
+                */
+               if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d))
+                       ;
+               else if (!(req->cmd_flags & REQ_QUIET))
+                       scsi_print_sense("", cmd);
+               result = 0;
+               /* BLOCK_PC may have set error */
+               error = 0;
+       }
 
-       /* A number of bytes were successfully read.  If there
+       /*
+        * A number of bytes were successfully read.  If there
         * are leftovers and there is some kind of error
         * (result != 0), retry the rest.
         */
-       if (scsi_end_request(cmd, 0, good_bytes, result == 0) == NULL)
+       if (scsi_end_request(cmd, error, good_bytes, result == 0) == NULL)
                return;
 
-       /* good_bytes = 0, or (inclusive) there were leftovers and
-        * result = 0, so scsi_end_request couldn't retry.
-        */
-       if (sense_valid && !sense_deferred) {
+       error = -EIO;
+
+       if (host_byte(result) == DID_RESET) {
+               /* Third party bus reset or reset for error recovery
+                * reasons.  Just retry the command and see what
+                * happens.
+                */
+               action = ACTION_RETRY;
+       } else if (sense_valid && !sense_deferred) {
                switch (sshdr.sense_key) {
                case UNIT_ATTENTION:
                        if (cmd->device->removable) {
@@ -924,16 +812,15 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
                                 * and quietly refuse further access.
                                 */
                                cmd->device->changed = 1;
-                               scsi_end_request(cmd, -EIO, this_count, 1);
-                               return;
+                               description = "Media Changed";
+                               action = ACTION_FAIL;
                        } else {
                                /* Must have been a power glitch, or a
                                 * bus reset.  Could not have been a
                                 * media change, so we just retry the
-                                * request and see what happens.
+                                * command and see what happens.
                                 */
-                               scsi_requeue_command(q, cmd);
-                               return;
+                               action = ACTION_RETRY;
                        }
                        break;
                case ILLEGAL_REQUEST:
@@ -949,15 +836,21 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
                            sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&
                            (cmd->cmnd[0] == READ_10 ||
                             cmd->cmnd[0] == WRITE_10)) {
+                               /* This will issue a new 6-byte command. */
                                cmd->device->use_10_for_rw = 0;
-                               /* This will cause a retry with a
-                                * 6-byte command.
-                                */
-                               scsi_requeue_command(q, cmd);
-                               return;
-                       } else {
-                               scsi_end_request(cmd, -EIO, this_count, 1);
-                               return;
+                               action = ACTION_REPREP;
+                       } else if (sshdr.asc == 0x10) /* DIX */ {
+                               description = "Host Data Integrity Failure";
+                               action = ACTION_FAIL;
+                               error = -EILSEQ;
+                       } else
+                               action = ACTION_FAIL;
+                       break;
+               case ABORTED_COMMAND:
+                       action = ACTION_FAIL;
+                       if (sshdr.asc == 0x10) { /* DIF */
+                               description = "Target Data Integrity Failure";
+                               error = -EILSEQ;
                        }
                        break;
                case NOT_READY:
@@ -973,49 +866,67 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
                                case 0x07: /* operation in progress */
                                case 0x08: /* Long write in progress */
                                case 0x09: /* self test in progress */
-                                       scsi_requeue_command(q, cmd);
-                                       return;
+                               case 0x14: /* space allocation in progress */
+                                       action = ACTION_DELAYED_RETRY;
+                                       break;
                                default:
+                                       description = "Device not ready";
+                                       action = ACTION_FAIL;
                                        break;
                                }
+                       } else {
+                               description = "Device not ready";
+                               action = ACTION_FAIL;
                        }
-                       if (!(req->cmd_flags & REQ_QUIET))
-                               scsi_cmd_print_sense_hdr(cmd,
-                                                        "Device not ready",
-                                                        &sshdr);
-
-                       scsi_end_request(cmd, -EIO, this_count, 1);
-                       return;
+                       break;
                case VOLUME_OVERFLOW:
-                       if (!(req->cmd_flags & REQ_QUIET)) {
-                               scmd_printk(KERN_INFO, cmd,
-                                           "Volume overflow, CDB: ");
-                               __scsi_print_command(cmd->cmnd);
-                               scsi_print_sense("", cmd);
-                       }
                        /* See SSC3rXX or current. */
-                       scsi_end_request(cmd, -EIO, this_count, 1);
-                       return;
+                       action = ACTION_FAIL;
+                       break;
                default:
+                       description = "Unhandled sense code";
+                       action = ACTION_FAIL;
                        break;
                }
+       } else {
+               description = "Unhandled error code";
+               action = ACTION_FAIL;
        }
-       if (host_byte(result) == DID_RESET) {
-               /* Third party bus reset or reset for error recovery
-                * reasons.  Just retry the request and see what
-                * happens.
-                */
-               scsi_requeue_command(q, cmd);
-               return;
-       }
-       if (result) {
+
+       switch (action) {
+       case ACTION_FAIL:
+               /* Give up and fail the remainder of the request */
+               scsi_release_buffers(cmd);
                if (!(req->cmd_flags & REQ_QUIET)) {
+                       if (description)
+                               scmd_printk(KERN_INFO, cmd, "%s\n",
+                                           description);
                        scsi_print_result(cmd);
                        if (driver_byte(result) & DRIVER_SENSE)
                                scsi_print_sense("", cmd);
+                       scsi_print_command(cmd);
                }
+               if (blk_end_request_err(req, error))
+                       scsi_requeue_command(q, cmd);
+               else
+                       scsi_next_command(cmd);
+               break;
+       case ACTION_REPREP:
+               /* Unprep the request and put it back at the head of the queue.
+                * A new command will be prepared and issued.
+                */
+               scsi_release_buffers(cmd);
+               scsi_requeue_command(q, cmd);
+               break;
+       case ACTION_RETRY:
+               /* Retry the same command immediately */
+               __scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, 0);
+               break;
+       case ACTION_DELAYED_RETRY:
+               /* Retry the same command after a delay */
+               __scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, 0);
+               break;
        }
-       scsi_end_request(cmd, -EIO, this_count, !result);
 }
 
 static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb,
@@ -1032,10 +943,6 @@ static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb,
        }
 
        req->buffer = NULL;
-       if (blk_pc_request(req))
-               sdb->length = req->data_len;
-       else
-               sdb->length = req->nr_sectors << 9;
 
        /* 
         * Next, walk the list, and fill in the addresses and sizes of
@@ -1044,6 +951,7 @@ static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb,
        count = blk_rq_map_sg(req->q, req, sdb->table.sgl);
        BUG_ON(count > sdb->table.nents);
        sdb->table.nents = count;
+       sdb->length = blk_rq_bytes(req);
        return BLKPREP_OK;
 }
 
@@ -1060,34 +968,53 @@ static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb,
  */
 int scsi_init_io(struct scsi_cmnd *cmd, gfp_t gfp_mask)
 {
-       int error = scsi_init_sgtable(cmd->request, &cmd->sdb, gfp_mask);
+       struct request *rq = cmd->request;
+
+       int error = scsi_init_sgtable(rq, &cmd->sdb, gfp_mask);
        if (error)
                goto err_exit;
 
-       if (blk_bidi_rq(cmd->request)) {
+       if (blk_bidi_rq(rq)) {
                struct scsi_data_buffer *bidi_sdb = kmem_cache_zalloc(
-                       scsi_bidi_sdb_cache, GFP_ATOMIC);
+                       scsi_sdb_cache, GFP_ATOMIC);
                if (!bidi_sdb) {
                        error = BLKPREP_DEFER;
                        goto err_exit;
                }
 
-               cmd->request->next_rq->special = bidi_sdb;
-               error = scsi_init_sgtable(cmd->request->next_rq, bidi_sdb,
-                                                                   GFP_ATOMIC);
+               rq->next_rq->special = bidi_sdb;
+               error = scsi_init_sgtable(rq->next_rq, bidi_sdb, GFP_ATOMIC);
                if (error)
                        goto err_exit;
        }
 
+       if (blk_integrity_rq(rq)) {
+               struct scsi_data_buffer *prot_sdb = cmd->prot_sdb;
+               int ivecs, count;
+
+               BUG_ON(prot_sdb == NULL);
+               ivecs = blk_rq_count_integrity_sg(rq->q, rq->bio);
+
+               if (scsi_alloc_sgtable(prot_sdb, ivecs, gfp_mask)) {
+                       error = BLKPREP_DEFER;
+                       goto err_exit;
+               }
+
+               count = blk_rq_map_integrity_sg(rq->q, rq->bio,
+                                               prot_sdb->table.sgl);
+               BUG_ON(unlikely(count > ivecs));
+               BUG_ON(unlikely(count > queue_max_integrity_segments(rq->q)));
+
+               cmd->prot_sdb = prot_sdb;
+               cmd->prot_sdb->table.nents = count;
+       }
+
        return BLKPREP_OK ;
 
 err_exit:
        scsi_release_buffers(cmd);
-       if (error == BLKPREP_KILL)
-               scsi_put_command(cmd);
-       else /* BLKPREP_DEFER */
-               scsi_unprep_request(cmd->request);
-
+       cmd->request->special = NULL;
+       scsi_put_command(cmd);
        return error;
 }
 EXPORT_SYMBOL(scsi_init_io);
@@ -1110,6 +1037,8 @@ static struct scsi_cmnd *scsi_get_cmd_from_req(struct scsi_device *sdev,
        cmd->tag = req->tag;
        cmd->request = req;
 
+       cmd->cmnd = req->cmd;
+
        return cmd;
 }
 
@@ -1140,26 +1069,22 @@ int scsi_setup_blk_pc_cmnd(struct scsi_device *sdev, struct request *req)
                if (unlikely(ret))
                        return ret;
        } else {
-               BUG_ON(req->data_len);
-               BUG_ON(req->data);
+               BUG_ON(blk_rq_bytes(req));
 
                memset(&cmd->sdb, 0, sizeof(cmd->sdb));
                req->buffer = NULL;
        }
 
-       BUILD_BUG_ON(sizeof(req->cmd) > sizeof(cmd->cmnd));
-       memcpy(cmd->cmnd, req->cmd, sizeof(cmd->cmnd));
        cmd->cmd_len = req->cmd_len;
-       if (!req->data_len)
+       if (!blk_rq_bytes(req))
                cmd->sc_data_direction = DMA_NONE;
        else if (rq_data_dir(req) == WRITE)
                cmd->sc_data_direction = DMA_TO_DEVICE;
        else
                cmd->sc_data_direction = DMA_FROM_DEVICE;
        
-       cmd->transfersize = req->data_len;
+       cmd->transfersize = blk_rq_bytes(req);
        cmd->allowed = req->retries;
-       cmd->timeout_per_command = req->timeout;
        return BLKPREP_OK;
 }
 EXPORT_SYMBOL(scsi_setup_blk_pc_cmnd);
@@ -1176,6 +1101,14 @@ int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req)
 
        if (ret != BLKPREP_OK)
                return ret;
+
+       if (unlikely(sdev->scsi_dh_data && sdev->scsi_dh_data->scsi_dh
+                        && sdev->scsi_dh_data->scsi_dh->prep_fn)) {
+               ret = sdev->scsi_dh_data->scsi_dh->prep_fn(sdev, req);
+               if (ret != BLKPREP_OK)
+                       return ret;
+       }
+
        /*
         * Filesystem requests must transfer data.
         */
@@ -1185,6 +1118,7 @@ int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req)
        if (unlikely(!cmd))
                return BLKPREP_DEFER;
 
+       memset(cmd->cmnd, 0, BLK_MAX_CDB);
        return scsi_init_io(cmd, GFP_ATOMIC);
 }
 EXPORT_SYMBOL(scsi_setup_fs_cmnd);
@@ -1220,6 +1154,7 @@ int scsi_prep_state_check(struct scsi_device *sdev, struct request *req)
                        break;
                case SDEV_QUIESCE:
                case SDEV_BLOCK:
+               case SDEV_CREATED_BLOCK:
                        /*
                         * If the devices is blocked we defer normal commands.
                         */
@@ -1258,7 +1193,7 @@ int scsi_prep_return(struct request_queue *q, struct request *req, int ret)
                break;
        case BLKPREP_DEFER:
                /*
-                * If we defer, the elv_next_request() returns NULL, but the
+                * If we defer, the blk_peek_request() returns NULL, but the
                 * queue must be restarted, so we plug here if no returning
                 * command will automatically do that.
                 */
@@ -1282,6 +1217,7 @@ int scsi_prep_fn(struct request_queue *q, struct request *req)
                ret = scsi_setup_blk_pc_cmnd(sdev, req);
        return scsi_prep_return(q, req, ret);
 }
+EXPORT_SYMBOL(scsi_prep_fn);
 
 /*
  * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
@@ -1292,8 +1228,6 @@ int scsi_prep_fn(struct request_queue *q, struct request *req)
 static inline int scsi_dev_queue_ready(struct request_queue *q,
                                  struct scsi_device *sdev)
 {
-       if (sdev->device_busy >= sdev->queue_depth)
-               return 0;
        if (sdev->device_busy == 0 && sdev->device_blocked) {
                /*
                 * unblock after device_blocked iterates to zero
@@ -1307,12 +1241,55 @@ static inline int scsi_dev_queue_ready(struct request_queue *q,
                        return 0;
                }
        }
-       if (sdev->device_blocked)
+       if (scsi_device_is_busy(sdev))
                return 0;
 
        return 1;
 }
 
+
+/*
+ * scsi_target_queue_ready: checks if there we can send commands to target
+ * @sdev: scsi device on starget to check.
+ *
+ * Called with the host lock held.
+ */
+static inline int scsi_target_queue_ready(struct Scsi_Host *shost,
+                                          struct scsi_device *sdev)
+{
+       struct scsi_target *starget = scsi_target(sdev);
+
+       if (starget->single_lun) {
+               if (starget->starget_sdev_user &&
+                   starget->starget_sdev_user != sdev)
+                       return 0;
+               starget->starget_sdev_user = sdev;
+       }
+
+       if (starget->target_busy == 0 && starget->target_blocked) {
+               /*
+                * unblock after target_blocked iterates to zero
+                */
+               if (--starget->target_blocked == 0) {
+                       SCSI_LOG_MLQUEUE(3, starget_printk(KERN_INFO, starget,
+                                        "unblocking target at zero depth\n"));
+               } else
+                       return 0;
+       }
+
+       if (scsi_target_is_busy(starget)) {
+               if (list_empty(&sdev->starved_entry))
+                       list_add_tail(&sdev->starved_entry,
+                                     &shost->starved_list);
+               return 0;
+       }
+
+       /* We're OK to process the command, so we can't be starved */
+       if (!list_empty(&sdev->starved_entry))
+               list_del_init(&sdev->starved_entry);
+       return 1;
+}
+
 /*
  * scsi_host_queue_ready: if we can send requests to shost, return 1 else
  * return 0. We must end up running the queue again whenever 0 is
@@ -1335,12 +1312,10 @@ static inline int scsi_host_queue_ready(struct request_queue *q,
                                printk("scsi%d unblocking host at zero depth\n",
                                        shost->host_no));
                } else {
-                       blk_plug_device(q);
                        return 0;
                }
        }
-       if ((shost->can_queue > 0 && shost->host_busy >= shost->can_queue) ||
-           shost->host_blocked || shost->host_self_blocked) {
+       if (scsi_host_is_busy(shost)) {
                if (list_empty(&sdev->starved_entry))
                        list_add_tail(&sdev->starved_entry, &shost->starved_list);
                return 0;
@@ -1354,22 +1329,51 @@ static inline int scsi_host_queue_ready(struct request_queue *q,
 }
 
 /*
+ * Busy state exporting function for request stacking drivers.
+ *
+ * For efficiency, no lock is taken to check the busy state of
+ * shost/starget/sdev, since the returned value is not guaranteed and
+ * may be changed after request stacking drivers call the function,
+ * regardless of taking lock or not.
+ *
+ * When scsi can't dispatch I/Os anymore and needs to kill I/Os
+ * (e.g. !sdev), scsi needs to return 'not busy'.
+ * Otherwise, request stacking drivers may hold requests forever.
+ */
+static int scsi_lld_busy(struct request_queue *q)
+{
+       struct scsi_device *sdev = q->queuedata;
+       struct Scsi_Host *shost;
+       struct scsi_target *starget;
+
+       if (!sdev)
+               return 0;
+
+       shost = sdev->host;
+       starget = scsi_target(sdev);
+
+       if (scsi_host_in_recovery(shost) || scsi_host_is_busy(shost) ||
+           scsi_target_is_busy(starget) || scsi_device_is_busy(sdev))
+               return 1;
+
+       return 0;
+}
+
+/*
  * Kill a request for a dead device
  */
 static void scsi_kill_request(struct request *req, struct request_queue *q)
 {
        struct scsi_cmnd *cmd = req->special;
-       struct scsi_device *sdev = cmd->device;
-       struct Scsi_Host *shost = sdev->host;
-
-       blkdev_dequeue_request(req);
+       struct scsi_device *sdev;
+       struct scsi_target *starget;
+       struct Scsi_Host *shost;
 
-       if (unlikely(cmd == NULL)) {
-               printk(KERN_CRIT "impossible request in %s.\n",
-                                __FUNCTION__);
-               BUG();
-       }
+       blk_start_request(req);
 
+       sdev = cmd->device;
+       starget = scsi_target(sdev);
+       shost = sdev->host;
        scsi_init_cmd_errh(cmd);
        cmd->result = DID_NO_CONNECT << 16;
        atomic_inc(&cmd->device->iorequest_cnt);
@@ -1383,20 +1387,25 @@ static void scsi_kill_request(struct request *req, struct request_queue *q)
        spin_unlock(sdev->request_queue->queue_lock);
        spin_lock(shost->host_lock);
        shost->host_busy++;
+       starget->target_busy++;
        spin_unlock(shost->host_lock);
        spin_lock(sdev->request_queue->queue_lock);
 
-       __scsi_done(cmd);
+       blk_complete_request(req);
 }
 
 static void scsi_softirq_done(struct request *rq)
 {
-       struct scsi_cmnd *cmd = rq->completion_data;
-       unsigned long wait_for = (cmd->allowed + 1) * cmd->timeout_per_command;
+       struct scsi_cmnd *cmd = rq->special;
+       unsigned long wait_for = (cmd->allowed + 1) * rq->timeout;
        int disposition;
 
        INIT_LIST_HEAD(&cmd->eh_entry);
 
+       atomic_inc(&cmd->device->iodone_cnt);
+       if (cmd->result)
+               atomic_inc(&cmd->device->ioerr_cnt);
+
        disposition = scsi_decide_disposition(cmd);
        if (disposition != SUCCESS &&
            time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
@@ -1444,7 +1453,7 @@ static void scsi_request_fn(struct request_queue *q)
 
        if (!sdev) {
                printk("scsi: killing requests for dead queue\n");
-               while ((req = elv_next_request(q)) != NULL)
+               while ((req = blk_peek_request(q)) != NULL)
                        scsi_kill_request(req, q);
                return;
        }
@@ -1465,7 +1474,7 @@ static void scsi_request_fn(struct request_queue *q)
                 * that the request is fully prepared even if we cannot 
                 * accept it.
                 */
-               req = elv_next_request(q);
+               req = blk_peek_request(q);
                if (!req || !scsi_dev_queue_ready(q, sdev))
                        break;
 
@@ -1481,7 +1490,7 @@ static void scsi_request_fn(struct request_queue *q)
                 * Remove the request from the request list.
                 */
                if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
-                       blkdev_dequeue_request(req);
+                       blk_start_request(req);
                sdev->device_busy++;
 
                spin_unlock(q->queue_lock);
@@ -1490,20 +1499,34 @@ static void scsi_request_fn(struct request_queue *q)
                        printk(KERN_CRIT "impossible request in %s.\n"
                                         "please mail a stack trace to "
                                         "linux-scsi@vger.kernel.org\n",
-                                        __FUNCTION__);
+                                        __func__);
                        blk_dump_rq_flags(req, "foo");
                        BUG();
                }
                spin_lock(shost->host_lock);
 
-               if (!scsi_host_queue_ready(q, shost, sdev))
+               /*
+                * We hit this when the driver is using a host wide
+                * tag map. For device level tag maps the queue_depth check
+                * in the device ready fn would prevent us from trying
+                * to allocate a tag. Since the map is a shared host resource
+                * we add the dev to the starved list so it eventually gets
+                * a run when a tag is freed.
+                */
+               if (blk_queue_tagged(q) && !blk_rq_tagged(req)) {
+                       if (list_empty(&sdev->starved_entry))
+                               list_add_tail(&sdev->starved_entry,
+                                             &shost->starved_list);
                        goto not_ready;
-               if (scsi_target(sdev)->single_lun) {
-                       if (scsi_target(sdev)->starget_sdev_user &&
-                           scsi_target(sdev)->starget_sdev_user != sdev)
-                               goto not_ready;
-                       scsi_target(sdev)->starget_sdev_user = sdev;
                }
+
+               if (!scsi_target_queue_ready(shost, sdev))
+                       goto not_ready;
+
+               if (!scsi_host_queue_ready(q, shost, sdev))
+                       goto not_ready;
+
+               scsi_target(sdev)->target_busy++;
                shost->host_busy++;
 
                /*
@@ -1586,6 +1609,7 @@ struct request_queue *__scsi_alloc_queue(struct Scsi_Host *shost,
                                         request_fn_proc *request_fn)
 {
        struct request_queue *q;
+       struct device *dev = shost->shost_gendev.parent;
 
        q = blk_init_queue(request_fn, NULL);
        if (!q)
@@ -1594,28 +1618,26 @@ struct request_queue *__scsi_alloc_queue(struct Scsi_Host *shost,
        /*
         * this limit is imposed by hardware restrictions
         */
-       blk_queue_max_hw_segments(q, shost->sg_tablesize);
-
-       /*
-        * In the future, sg chaining support will be mandatory and this
-        * ifdef can then go away. Right now we don't have all archs
-        * converted, so better keep it safe.
-        */
-#ifdef ARCH_HAS_SG_CHAIN
-       if (shost->use_sg_chaining)
-               blk_queue_max_phys_segments(q, SCSI_MAX_SG_CHAIN_SEGMENTS);
-       else
-               blk_queue_max_phys_segments(q, SCSI_MAX_SG_SEGMENTS);
-#else
-       blk_queue_max_phys_segments(q, SCSI_MAX_SG_SEGMENTS);
-#endif
+       blk_queue_max_segments(q, min_t(unsigned short, shost->sg_tablesize,
+                                       SCSI_MAX_SG_CHAIN_SEGMENTS));
+
+       if (scsi_host_prot_dma(shost)) {
+               shost->sg_prot_tablesize =
+                       min_not_zero(shost->sg_prot_tablesize,
+                                    (unsigned short)SCSI_MAX_PROT_SG_SEGMENTS);
+               BUG_ON(shost->sg_prot_tablesize < shost->sg_tablesize);
+               blk_queue_max_integrity_segments(q, shost->sg_prot_tablesize);
+       }
 
-       blk_queue_max_sectors(q, shost->max_sectors);
+       blk_queue_max_hw_sectors(q, shost->max_sectors);
        blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
        blk_queue_segment_boundary(q, shost->dma_boundary);
+       dma_set_seg_boundary(dev, shost->dma_boundary);
+
+       blk_queue_max_segment_size(q, dma_get_max_seg_size(dev));
 
        if (!shost->use_clustering)
-               clear_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags);
+               q->limits.cluster = 0;
 
        /*
         * set a reasonable default alignment on word boundaries: the
@@ -1638,6 +1660,8 @@ struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
 
        blk_queue_prep_rq(q, scsi_prep_fn);
        blk_queue_softirq_done(q, scsi_softirq_done);
+       blk_queue_rq_timed_out(q, scsi_times_out);
+       blk_queue_lld_busy(q, scsi_lld_busy);
        return q;
 }
 
@@ -1699,19 +1723,11 @@ int __init scsi_init_queue(void)
 {
        int i;
 
-       scsi_io_context_cache = kmem_cache_create("scsi_io_context",
-                                       sizeof(struct scsi_io_context),
-                                       0, 0, NULL);
-       if (!scsi_io_context_cache) {
-               printk(KERN_ERR "SCSI: can't init scsi io context cache\n");
-               return -ENOMEM;
-       }
-
-       scsi_bidi_sdb_cache = kmem_cache_create("scsi_bidi_sdb",
-                                       sizeof(struct scsi_data_buffer),
-                                       0, 0, NULL);
-       if (!scsi_bidi_sdb_cache) {
-               printk(KERN_ERR "SCSI: can't init scsi bidi sdb cache\n");
+       scsi_sdb_cache = kmem_cache_create("scsi_data_buffer",
+                                          sizeof(struct scsi_data_buffer),
+                                          0, 0, NULL);
+       if (!scsi_sdb_cache) {
+               printk(KERN_ERR "SCSI: can't init scsi sdb cache\n");
                return -ENOMEM;
        }
 
@@ -1724,6 +1740,7 @@ int __init scsi_init_queue(void)
                if (!sgp->slab) {
                        printk(KERN_ERR "SCSI: can't init sg slab %s\n",
                                        sgp->name);
+                       goto cleanup_sdb;
                }
 
                sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
@@ -1731,17 +1748,30 @@ int __init scsi_init_queue(void)
                if (!sgp->pool) {
                        printk(KERN_ERR "SCSI: can't init sg mempool %s\n",
                                        sgp->name);
+                       goto cleanup_sdb;
                }
        }
 
        return 0;
+
+cleanup_sdb:
+       for (i = 0; i < SG_MEMPOOL_NR; i++) {
+               struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
+               if (sgp->pool)
+                       mempool_destroy(sgp->pool);
+               if (sgp->slab)
+                       kmem_cache_destroy(sgp->slab);
+       }
+       kmem_cache_destroy(scsi_sdb_cache);
+
+       return -ENOMEM;
 }
 
 void scsi_exit_queue(void)
 {
        int i;
 
-       kmem_cache_destroy(scsi_io_context_cache);
+       kmem_cache_destroy(scsi_sdb_cache);
 
        for (i = 0; i < SG_MEMPOOL_NR; i++) {
                struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
@@ -1821,7 +1851,7 @@ scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage,
        }
 
        ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len,
-                              sshdr, timeout, retries);
+                              sshdr, timeout, retries, NULL);
        kfree(real_buffer);
        return ret;
 }
@@ -1886,7 +1916,7 @@ scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
        memset(buffer, 0, len);
 
        result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len,
-                                 sshdr, timeout, retries);
+                                 sshdr, timeout, retries, NULL);
 
        /* This code looks awful: what it's doing is making sure an
         * ILLEGAL REQUEST sense return identifies the actual command
@@ -1947,8 +1977,7 @@ EXPORT_SYMBOL(scsi_mode_sense);
  *             in.
  *
  *     Returns zero if unsuccessful or an error if TUR failed.  For
- *     removable media, a return of NOT_READY or UNIT_ATTENTION is
- *     translated to success, with the ->changed flag updated.
+ *     removable media, UNIT_ATTENTION sets ->changed flag.
  **/
 int
 scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries,
@@ -1968,24 +1997,13 @@ scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries,
        /* try to eat the UNIT_ATTENTION if there are enough retries */
        do {
                result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, sshdr,
-                                         timeout, retries);
-       } while ((driver_byte(result) & DRIVER_SENSE) &&
-                sshdr && sshdr->sense_key == UNIT_ATTENTION &&
-                --retries);
-
-       if (!sshdr)
-               /* could not allocate sense buffer, so can't process it */
-               return result;
-
-       if ((driver_byte(result) & DRIVER_SENSE) && sdev->removable) {
-
-               if ((scsi_sense_valid(sshdr)) &&
-                   ((sshdr->sense_key == UNIT_ATTENTION) ||
-                    (sshdr->sense_key == NOT_READY))) {
+                                         timeout, retries, NULL);
+               if (sdev->removable && scsi_sense_valid(sshdr) &&
+                   sshdr->sense_key == UNIT_ATTENTION)
                        sdev->changed = 1;
-                       result = 0;
-               }
-       }
+       } while (scsi_sense_valid(sshdr) &&
+                sshdr->sense_key == UNIT_ATTENTION && --retries);
+
        if (!sshdr_external)
                kfree(sshdr);
        return result;
@@ -2010,10 +2028,13 @@ scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
 
        switch (state) {
        case SDEV_CREATED:
-               /* There are no legal states that come back to
-                * created.  This is the manually initialised start
-                * state */
-               goto illegal;
+               switch (oldstate) {
+               case SDEV_CREATED_BLOCK:
+                       break;
+               default:
+                       goto illegal;
+               }
+               break;
                        
        case SDEV_RUNNING:
                switch (oldstate) {
@@ -2051,8 +2072,17 @@ scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
 
        case SDEV_BLOCK:
                switch (oldstate) {
-               case SDEV_CREATED:
                case SDEV_RUNNING:
+               case SDEV_CREATED_BLOCK:
+                       break;
+               default:
+                       goto illegal;
+               }
+               break;
+
+       case SDEV_CREATED_BLOCK:
+               switch (oldstate) {
+               case SDEV_CREATED:
                        break;
                default:
                        goto illegal;
@@ -2172,10 +2202,15 @@ void sdev_evt_send(struct scsi_device *sdev, struct scsi_event *evt)
 {
        unsigned long flags;
 
+#if 0
+       /* FIXME: currently this check eliminates all media change events
+        * for polled devices.  Need to update to discriminate between AN
+        * and polled events */
        if (!test_bit(evt->evt_type, sdev->supported_events)) {
                kfree(evt);
                return;
        }
+#endif
 
        spin_lock_irqsave(&sdev->list_lock, flags);
        list_add_tail(&evt->node, &sdev->event_list);
@@ -2335,8 +2370,12 @@ scsi_internal_device_block(struct scsi_device *sdev)
        int err = 0;
 
        err = scsi_device_set_state(sdev, SDEV_BLOCK);
-       if (err)
-               return err;
+       if (err) {
+               err = scsi_device_set_state(sdev, SDEV_CREATED_BLOCK);
+
+               if (err)
+                       return err;
+       }
 
        /* 
         * The device has transitioned to SDEV_BLOCK.  Stop the
@@ -2371,16 +2410,19 @@ int
 scsi_internal_device_unblock(struct scsi_device *sdev)
 {
        struct request_queue *q = sdev->request_queue; 
-       int err;
        unsigned long flags;
        
        /* 
         * Try to transition the scsi device to SDEV_RUNNING
         * and goose the device queue if successful.  
         */
-       err = scsi_device_set_state(sdev, SDEV_RUNNING);
-       if (err)
-               return err;
+       if (sdev->sdev_state == SDEV_BLOCK)
+               sdev->sdev_state = SDEV_RUNNING;
+       else if (sdev->sdev_state == SDEV_CREATED_BLOCK)
+               sdev->sdev_state = SDEV_CREATED;
+       else if (sdev->sdev_state != SDEV_CANCEL &&
+                sdev->sdev_state != SDEV_OFFLINE)
+               return -EINVAL;
 
        spin_lock_irqsave(q->queue_lock, flags);
        blk_start_queue(q);
@@ -2471,7 +2513,7 @@ void *scsi_kmap_atomic_sg(struct scatterlist *sgl, int sg_count,
        if (unlikely(i == sg_count)) {
                printk(KERN_ERR "%s: Bytes in sg: %zu, requested offset %zu, "
                        "elements %d\n",
-                      __FUNCTION__, sg_len, *offset, sg_count);
+                      __func__, sg_len, *offset, sg_count);
                WARN_ON(1);
                return NULL;
        }