PM: Remove CONFIG_PM_OPS
[linux-2.6.git] / drivers / scsi / scsi_lib.c
index aa17e71..fb2bb35 100644 (file)
@@ -8,6 +8,7 @@
  */
 
 #include <linux/bio.h>
+#include <linux/bitops.h>
 #include <linux/blkdev.h>
 #include <linux/completion.h>
 #include <linux/kernel.h>
 #define SG_MEMPOOL_NR          ARRAY_SIZE(scsi_sg_pools)
 #define SG_MEMPOOL_SIZE                2
 
-/*
- * The maximum number of SG segments that we will put inside a scatterlist
- * (unless chaining is used). Should ideally fit inside a single page, to
- * avoid a higher order allocation.
- */
-#define SCSI_MAX_SG_SEGMENTS   128
-
 struct scsi_host_sg_pool {
        size_t          size;
        char            *name;
@@ -48,22 +42,31 @@ struct scsi_host_sg_pool {
        mempool_t       *pool;
 };
 
-#define SP(x) { x, "sgpool-" #x }
+#define SP(x) { x, "sgpool-" __stringify(x) }
+#if (SCSI_MAX_SG_SEGMENTS < 32)
+#error SCSI_MAX_SG_SEGMENTS is too small (must be 32 or greater)
+#endif
 static struct scsi_host_sg_pool scsi_sg_pools[] = {
        SP(8),
        SP(16),
-#if (SCSI_MAX_SG_SEGMENTS > 16)
-       SP(32),
 #if (SCSI_MAX_SG_SEGMENTS > 32)
-       SP(64),
+       SP(32),
 #if (SCSI_MAX_SG_SEGMENTS > 64)
+       SP(64),
+#if (SCSI_MAX_SG_SEGMENTS > 128)
        SP(128),
+#if (SCSI_MAX_SG_SEGMENTS > 256)
+#error SCSI_MAX_SG_SEGMENTS is too large (256 MAX)
 #endif
 #endif
 #endif
+#endif
+       SP(SCSI_MAX_SG_SEGMENTS)
 };
 #undef SP
 
+struct kmem_cache *scsi_sdb_cache;
+
 static void scsi_run_queue(struct request_queue *q);
 
 /*
@@ -82,35 +85,29 @@ static void scsi_unprep_request(struct request *req)
 {
        struct scsi_cmnd *cmd = req->special;
 
-       req->cmd_flags &= ~REQ_DONTPREP;
+       blk_unprep_request(req);
        req->special = NULL;
 
        scsi_put_command(cmd);
 }
 
-/*
- * Function:    scsi_queue_insert()
- *
- * Purpose:     Insert a command in the midlevel queue.
- *
- * Arguments:   cmd    - command that we are adding to queue.
- *              reason - why we are inserting command to queue.
- *
- * Lock status: Assumed that lock is not held upon entry.
- *
- * Returns:     Nothing.
- *
- * Notes:       We do this for one of two cases.  Either the host is busy
- *              and it cannot accept any more commands for the time being,
- *              or the device returned QUEUE_FULL and can accept no more
- *              commands.
- * Notes:       This could be called either from an interrupt context or a
- *              normal process context.
+/**
+ * __scsi_queue_insert - private queue insertion
+ * @cmd: The SCSI command being requeued
+ * @reason:  The reason for the requeue
+ * @unbusy: Whether the queue should be unbusied
+ *
+ * This is a private queue insertion.  The public interface
+ * scsi_queue_insert() always assumes the queue should be unbusied
+ * because it's always called before the completion.  This function is
+ * for a requeue after completion, which should only occur in this
+ * file.
  */
-int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
+static int __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy)
 {
        struct Scsi_Host *host = cmd->device->host;
        struct scsi_device *device = cmd->device;
+       struct scsi_target *starget = scsi_target(device);
        struct request_queue *q = device->request_queue;
        unsigned long flags;
 
@@ -130,16 +127,24 @@ int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
         * if a command is requeued with no other commands outstanding
         * either for the device or for the host.
         */
-       if (reason == SCSI_MLQUEUE_HOST_BUSY)
+       switch (reason) {
+       case SCSI_MLQUEUE_HOST_BUSY:
                host->host_blocked = host->max_host_blocked;
-       else if (reason == SCSI_MLQUEUE_DEVICE_BUSY)
+               break;
+       case SCSI_MLQUEUE_DEVICE_BUSY:
                device->device_blocked = device->max_device_blocked;
+               break;
+       case SCSI_MLQUEUE_TARGET_BUSY:
+               starget->target_blocked = starget->max_target_blocked;
+               break;
+       }
 
        /*
         * Decrement the counters, since these commands are no longer
         * active on the host/device.
         */
-       scsi_device_unbusy(device);
+       if (unbusy)
+               scsi_device_unbusy(device);
 
        /*
         * Requeue this command.  It will go before all other commands
@@ -161,6 +166,29 @@ int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
        return 0;
 }
 
+/*
+ * Function:    scsi_queue_insert()
+ *
+ * Purpose:     Insert a command in the midlevel queue.
+ *
+ * Arguments:   cmd    - command that we are adding to queue.
+ *              reason - why we are inserting command to queue.
+ *
+ * Lock status: Assumed that lock is not held upon entry.
+ *
+ * Returns:     Nothing.
+ *
+ * Notes:       We do this for one of two cases.  Either the host is busy
+ *              and it cannot accept any more commands for the time being,
+ *              or the device returned QUEUE_FULL and can accept no more
+ *              commands.
+ * Notes:       This could be called either from an interrupt context or a
+ *              normal process context.
+ */
+int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
+{
+       return __scsi_queue_insert(cmd, reason, 1);
+}
 /**
  * scsi_execute - insert request and wait for the result
  * @sdev:      scsi device
@@ -172,13 +200,15 @@ int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
  * @timeout:   request timeout in seconds
  * @retries:   number of times to retry request
  * @flags:     or into request flags;
+ * @resid:     optional residual length
  *
  * returns the req->errors value which is the scsi_cmnd result
  * field.
  */
 int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
                 int data_direction, void *buffer, unsigned bufflen,
-                unsigned char *sense, int timeout, int retries, int flags)
+                unsigned char *sense, int timeout, int retries, int flags,
+                int *resid)
 {
        struct request *req;
        int write = (data_direction == DMA_TO_DEVICE);
@@ -204,6 +234,17 @@ int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
         */
        blk_execute_rq(req->q, NULL, req, 1);
 
+       /*
+        * Some devices (USB mass-storage in particular) may transfer
+        * garbage data together with a residue indicating that the data
+        * is invalid.  Prevent the garbage from being misinterpreted
+        * and prevent security leaks by zeroing out the excess data.
+        */
+       if (unlikely(req->resid_len > 0 && req->resid_len <= bufflen))
+               memset(buffer + (bufflen - req->resid_len), 0, req->resid_len);
+
+       if (resid)
+               *resid = req->resid_len;
        ret = req->errors;
  out:
        blk_put_request(req);
@@ -215,7 +256,8 @@ EXPORT_SYMBOL(scsi_execute);
 
 int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd,
                     int data_direction, void *buffer, unsigned bufflen,
-                    struct scsi_sense_hdr *sshdr, int timeout, int retries)
+                    struct scsi_sense_hdr *sshdr, int timeout, int retries,
+                    int *resid)
 {
        char *sense = NULL;
        int result;
@@ -226,7 +268,7 @@ int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd,
                        return DRIVER_ERROR << 24;
        }
        result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen,
-                             sense, timeout, retries, 0);
+                             sense, timeout, retries, 0, resid);
        if (sshdr)
                scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr);
 
@@ -235,197 +277,6 @@ int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd,
 }
 EXPORT_SYMBOL(scsi_execute_req);
 
-struct scsi_io_context {
-       void *data;
-       void (*done)(void *data, char *sense, int result, int resid);
-       char sense[SCSI_SENSE_BUFFERSIZE];
-};
-
-static struct kmem_cache *scsi_io_context_cache;
-
-static void scsi_end_async(struct request *req, int uptodate)
-{
-       struct scsi_io_context *sioc = req->end_io_data;
-
-       if (sioc->done)
-               sioc->done(sioc->data, sioc->sense, req->errors, req->data_len);
-
-       kmem_cache_free(scsi_io_context_cache, sioc);
-       __blk_put_request(req->q, req);
-}
-
-static int scsi_merge_bio(struct request *rq, struct bio *bio)
-{
-       struct request_queue *q = rq->q;
-
-       bio->bi_flags &= ~(1 << BIO_SEG_VALID);
-       if (rq_data_dir(rq) == WRITE)
-               bio->bi_rw |= (1 << BIO_RW);
-       blk_queue_bounce(q, &bio);
-
-       return blk_rq_append_bio(q, rq, bio);
-}
-
-static void scsi_bi_endio(struct bio *bio, int error)
-{
-       bio_put(bio);
-}
-
-/**
- * scsi_req_map_sg - map a scatterlist into a request
- * @rq:                request to fill
- * @sgl:       scatterlist
- * @nsegs:     number of elements
- * @bufflen:   len of buffer
- * @gfp:       memory allocation flags
- *
- * scsi_req_map_sg maps a scatterlist into a request so that the
- * request can be sent to the block layer. We do not trust the scatterlist
- * sent to use, as some ULDs use that struct to only organize the pages.
- */
-static int scsi_req_map_sg(struct request *rq, struct scatterlist *sgl,
-                          int nsegs, unsigned bufflen, gfp_t gfp)
-{
-       struct request_queue *q = rq->q;
-       int nr_pages = (bufflen + sgl[0].offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
-       unsigned int data_len = bufflen, len, bytes, off;
-       struct scatterlist *sg;
-       struct page *page;
-       struct bio *bio = NULL;
-       int i, err, nr_vecs = 0;
-
-       for_each_sg(sgl, sg, nsegs, i) {
-               page = sg_page(sg);
-               off = sg->offset;
-               len = sg->length;
-               data_len += len;
-
-               while (len > 0 && data_len > 0) {
-                       /*
-                        * sg sends a scatterlist that is larger than
-                        * the data_len it wants transferred for certain
-                        * IO sizes
-                        */
-                       bytes = min_t(unsigned int, len, PAGE_SIZE - off);
-                       bytes = min(bytes, data_len);
-
-                       if (!bio) {
-                               nr_vecs = min_t(int, BIO_MAX_PAGES, nr_pages);
-                               nr_pages -= nr_vecs;
-
-                               bio = bio_alloc(gfp, nr_vecs);
-                               if (!bio) {
-                                       err = -ENOMEM;
-                                       goto free_bios;
-                               }
-                               bio->bi_end_io = scsi_bi_endio;
-                       }
-
-                       if (bio_add_pc_page(q, bio, page, bytes, off) !=
-                           bytes) {
-                               bio_put(bio);
-                               err = -EINVAL;
-                               goto free_bios;
-                       }
-
-                       if (bio->bi_vcnt >= nr_vecs) {
-                               err = scsi_merge_bio(rq, bio);
-                               if (err) {
-                                       bio_endio(bio, 0);
-                                       goto free_bios;
-                               }
-                               bio = NULL;
-                       }
-
-                       page++;
-                       len -= bytes;
-                       data_len -=bytes;
-                       off = 0;
-               }
-       }
-
-       rq->buffer = rq->data = NULL;
-       rq->data_len = bufflen;
-       return 0;
-
-free_bios:
-       while ((bio = rq->bio) != NULL) {
-               rq->bio = bio->bi_next;
-               /*
-                * call endio instead of bio_put incase it was bounced
-                */
-               bio_endio(bio, 0);
-       }
-
-       return err;
-}
-
-/**
- * scsi_execute_async - insert request
- * @sdev:      scsi device
- * @cmd:       scsi command
- * @cmd_len:   length of scsi cdb
- * @data_direction: DMA_TO_DEVICE, DMA_FROM_DEVICE, or DMA_NONE
- * @buffer:    data buffer (this can be a kernel buffer or scatterlist)
- * @bufflen:   len of buffer
- * @use_sg:    if buffer is a scatterlist this is the number of elements
- * @timeout:   request timeout in seconds
- * @retries:   number of times to retry request
- * @privdata:  data passed to done()
- * @done:      callback function when done
- * @gfp:       memory allocation flags
- */
-int scsi_execute_async(struct scsi_device *sdev, const unsigned char *cmd,
-                      int cmd_len, int data_direction, void *buffer, unsigned bufflen,
-                      int use_sg, int timeout, int retries, void *privdata,
-                      void (*done)(void *, char *, int, int), gfp_t gfp)
-{
-       struct request *req;
-       struct scsi_io_context *sioc;
-       int err = 0;
-       int write = (data_direction == DMA_TO_DEVICE);
-
-       sioc = kmem_cache_zalloc(scsi_io_context_cache, gfp);
-       if (!sioc)
-               return DRIVER_ERROR << 24;
-
-       req = blk_get_request(sdev->request_queue, write, gfp);
-       if (!req)
-               goto free_sense;
-       req->cmd_type = REQ_TYPE_BLOCK_PC;
-       req->cmd_flags |= REQ_QUIET;
-
-       if (use_sg)
-               err = scsi_req_map_sg(req, buffer, use_sg, bufflen, gfp);
-       else if (bufflen)
-               err = blk_rq_map_kern(req->q, req, buffer, bufflen, gfp);
-
-       if (err)
-               goto free_req;
-
-       req->cmd_len = cmd_len;
-       memset(req->cmd, 0, BLK_MAX_CDB); /* ATAPI hates garbage after CDB */
-       memcpy(req->cmd, cmd, req->cmd_len);
-       req->sense = sioc->sense;
-       req->sense_len = 0;
-       req->timeout = timeout;
-       req->retries = retries;
-       req->end_io_data = sioc;
-
-       sioc->data = privdata;
-       sioc->done = done;
-
-       blk_execute_rq_nowait(req->q, NULL, req, 1, scsi_end_async);
-       return 0;
-
-free_req:
-       blk_put_request(req);
-free_sense:
-       kmem_cache_free(scsi_io_context_cache, sioc);
-       return DRIVER_ERROR << 24;
-}
-EXPORT_SYMBOL_GPL(scsi_execute_async);
-
 /*
  * Function:    scsi_init_cmd_errh()
  *
@@ -440,19 +291,21 @@ EXPORT_SYMBOL_GPL(scsi_execute_async);
 static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)
 {
        cmd->serial_number = 0;
-       cmd->resid = 0;
-       memset(cmd->sense_buffer, 0, sizeof cmd->sense_buffer);
+       scsi_set_resid(cmd, 0);
+       memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
        if (cmd->cmd_len == 0)
-               cmd->cmd_len = COMMAND_SIZE(cmd->cmnd[0]);
+               cmd->cmd_len = scsi_command_size(cmd->cmnd);
 }
 
 void scsi_device_unbusy(struct scsi_device *sdev)
 {
        struct Scsi_Host *shost = sdev->host;
+       struct scsi_target *starget = scsi_target(sdev);
        unsigned long flags;
 
        spin_lock_irqsave(shost->host_lock, flags);
        shost->host_busy--;
+       starget->target_busy--;
        if (unlikely(scsi_host_in_recovery(shost) &&
                     (shost->host_failed || shost->host_eh_scheduled)))
                scsi_eh_wakeup(shost);
@@ -508,6 +361,30 @@ static void scsi_single_lun_run(struct scsi_device *current_sdev)
        spin_unlock_irqrestore(shost->host_lock, flags);
 }
 
+static inline int scsi_device_is_busy(struct scsi_device *sdev)
+{
+       if (sdev->device_busy >= sdev->queue_depth || sdev->device_blocked)
+               return 1;
+
+       return 0;
+}
+
+static inline int scsi_target_is_busy(struct scsi_target *starget)
+{
+       return ((starget->can_queue > 0 &&
+                starget->target_busy >= starget->can_queue) ||
+                starget->target_blocked);
+}
+
+static inline int scsi_host_is_busy(struct Scsi_Host *shost)
+{
+       if ((shost->can_queue > 0 && shost->host_busy >= shost->can_queue) ||
+           shost->host_blocked || shost->host_self_blocked)
+               return 1;
+
+       return 0;
+}
+
 /*
  * Function:   scsi_run_queue()
  *
@@ -524,16 +401,18 @@ static void scsi_run_queue(struct request_queue *q)
 {
        struct scsi_device *sdev = q->queuedata;
        struct Scsi_Host *shost = sdev->host;
+       LIST_HEAD(starved_list);
        unsigned long flags;
 
        if (scsi_target(sdev)->single_lun)
                scsi_single_lun_run(sdev);
 
        spin_lock_irqsave(shost->host_lock, flags);
-       while (!list_empty(&shost->starved_list) &&
-              !shost->host_blocked && !shost->host_self_blocked &&
-               !((shost->can_queue > 0) &&
-                 (shost->host_busy >= shost->can_queue))) {
+       list_splice_init(&shost->starved_list, &starved_list);
+
+       while (!list_empty(&starved_list)) {
+               int flagset;
+
                /*
                 * As long as shost is accepting commands and we have
                 * starved queues, call blk_run_queue. scsi_request_fn
@@ -544,30 +423,35 @@ static void scsi_run_queue(struct request_queue *q)
                 * scsi_request_fn must get the host_lock before checking
                 * or modifying starved_list or starved_entry.
                 */
-               sdev = list_entry(shost->starved_list.next,
-                                         struct scsi_device, starved_entry);
+               if (scsi_host_is_busy(shost))
+                       break;
+
+               sdev = list_entry(starved_list.next,
+                                 struct scsi_device, starved_entry);
                list_del_init(&sdev->starved_entry);
-               spin_unlock_irqrestore(shost->host_lock, flags);
+               if (scsi_target_is_busy(scsi_target(sdev))) {
+                       list_move_tail(&sdev->starved_entry,
+                                      &shost->starved_list);
+                       continue;
+               }
 
+               spin_unlock(shost->host_lock);
 
-               if (test_bit(QUEUE_FLAG_REENTER, &q->queue_flags) &&
-                   !test_and_set_bit(QUEUE_FLAG_REENTER,
-                                     &sdev->request_queue->queue_flags)) {
-                       blk_run_queue(sdev->request_queue);
-                       clear_bit(QUEUE_FLAG_REENTER,
-                                 &sdev->request_queue->queue_flags);
-               } else
-                       blk_run_queue(sdev->request_queue);
+               spin_lock(sdev->request_queue->queue_lock);
+               flagset = test_bit(QUEUE_FLAG_REENTER, &q->queue_flags) &&
+                               !test_bit(QUEUE_FLAG_REENTER,
+                                       &sdev->request_queue->queue_flags);
+               if (flagset)
+                       queue_flag_set(QUEUE_FLAG_REENTER, sdev->request_queue);
+               __blk_run_queue(sdev->request_queue, false);
+               if (flagset)
+                       queue_flag_clear(QUEUE_FLAG_REENTER, sdev->request_queue);
+               spin_unlock(sdev->request_queue->queue_lock);
 
-               spin_lock_irqsave(shost->host_lock, flags);
-               if (unlikely(!list_empty(&sdev->starved_entry)))
-                       /*
-                        * sdev lost a race, and was put back on the
-                        * starved list. This is unlikely but without this
-                        * in theory we could loop forever.
-                        */
-                       break;
+               spin_lock(shost->host_lock);
        }
+       /* put any unprocessed entries back */
+       list_splice(&starved_list, &shost->starved_list);
        spin_unlock_irqrestore(shost->host_lock, flags);
 
        blk_run_queue(q);
@@ -596,8 +480,8 @@ static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd)
        struct request *req = cmd->request;
        unsigned long flags;
 
-       scsi_unprep_request(req);
        spin_lock_irqsave(q->queue_lock, flags);
+       scsi_unprep_request(req);
        blk_requeue_request(q, req);
        spin_unlock_irqrestore(q->queue_lock, flags);
 
@@ -627,6 +511,8 @@ void scsi_run_host_queues(struct Scsi_Host *shost)
                scsi_run_queue(sdev->request_queue);
 }
 
+static void __scsi_release_buffers(struct scsi_cmnd *, int);
+
 /*
  * Function:    scsi_end_request()
  *
@@ -634,7 +520,7 @@ void scsi_run_host_queues(struct Scsi_Host *shost)
  *             of upper level post-processing and scsi_io_completion).
  *
  * Arguments:   cmd     - command that is complete.
- *              uptodate - 1 if I/O indicates success, <= 0 for I/O error.
+ *              error    - 0 if I/O indicates success, < 0 for I/O error.
  *              bytes    - number of bytes of completed I/O
  *             requeue  - indicates whether we should requeue leftovers.
  *
@@ -649,26 +535,20 @@ void scsi_run_host_queues(struct Scsi_Host *shost)
  *             at some point during this call.
  * Notes:      If cmd was requeued, upon return it will be a stale pointer.
  */
-static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int uptodate,
+static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int error,
                                          int bytes, int requeue)
 {
        struct request_queue *q = cmd->device->request_queue;
        struct request *req = cmd->request;
-       unsigned long flags;
 
        /*
         * If there are blocks left over at the end, set up the command
         * to queue the remainder of them.
         */
-       if (end_that_request_chunk(req, uptodate, bytes)) {
-               int leftover = (req->hard_nr_sectors << 9);
-
-               if (blk_pc_request(req))
-                       leftover = req->data_len;
-
+       if (blk_end_request(req, error, bytes)) {
                /* kill remainder if no retrys */
-               if (!uptodate && blk_noretry_request(req))
-                       end_that_request_chunk(req, 0, leftover);
+               if (error && scsi_noretry_cmd(cmd))
+                       blk_end_request_all(req, error);
                else {
                        if (requeue) {
                                /*
@@ -676,6 +556,7 @@ static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int uptodate,
                                 * leftovers in the front of the
                                 * queue, and goose the queue again.
                                 */
+                               scsi_release_buffers(cmd);
                                scsi_requeue_command(q, cmd);
                                cmd = NULL;
                        }
@@ -683,198 +564,86 @@ static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int uptodate,
                }
        }
 
-       add_disk_randomness(req->rq_disk);
-
-       spin_lock_irqsave(q->queue_lock, flags);
-       if (blk_rq_tagged(req))
-               blk_queue_end_tag(q, req);
-       end_that_request_last(req, uptodate);
-       spin_unlock_irqrestore(q->queue_lock, flags);
-
        /*
         * This will goose the queue request function at the end, so we don't
         * need to worry about launching another command.
         */
+       __scsi_release_buffers(cmd, 0);
        scsi_next_command(cmd);
        return NULL;
 }
 
-/*
- * Like SCSI_MAX_SG_SEGMENTS, but for archs that have sg chaining. This limit
- * is totally arbitrary, a setting of 2048 will get you at least 8mb ios.
- */
-#define SCSI_MAX_SG_CHAIN_SEGMENTS     2048
-
 static inline unsigned int scsi_sgtable_index(unsigned short nents)
 {
        unsigned int index;
 
-       switch (nents) {
-       case 1 ... 8:
+       BUG_ON(nents > SCSI_MAX_SG_SEGMENTS);
+
+       if (nents <= 8)
                index = 0;
-               break;
-       case 9 ... 16:
-               index = 1;
-               break;
-#if (SCSI_MAX_SG_SEGMENTS > 16)
-       case 17 ... 32:
-               index = 2;
-               break;
-#if (SCSI_MAX_SG_SEGMENTS > 32)
-       case 33 ... 64:
-               index = 3;
-               break;
-#if (SCSI_MAX_SG_SEGMENTS > 64)
-       case 65 ... 128:
-               index = 4;
-               break;
-#endif
-#endif
-#endif
-       default:
-               printk(KERN_ERR "scsi: bad segment count=%d\n", nents);
-               BUG();
-       }
+       else
+               index = get_count_order(nents) - 3;
 
        return index;
 }
 
-struct scatterlist *scsi_alloc_sgtable(struct scsi_cmnd *cmd, gfp_t gfp_mask)
+static void scsi_sg_free(struct scatterlist *sgl, unsigned int nents)
 {
        struct scsi_host_sg_pool *sgp;
-       struct scatterlist *sgl, *prev, *ret;
-       unsigned int index;
-       int this, left;
 
-       BUG_ON(!cmd->use_sg);
-
-       left = cmd->use_sg;
-       ret = prev = NULL;
-       do {
-               this = left;
-               if (this > SCSI_MAX_SG_SEGMENTS) {
-                       this = SCSI_MAX_SG_SEGMENTS - 1;
-                       index = SG_MEMPOOL_NR - 1;
-               } else
-                       index = scsi_sgtable_index(this);
-
-               left -= this;
-
-               sgp = scsi_sg_pools + index;
-
-               sgl = mempool_alloc(sgp->pool, gfp_mask);
-               if (unlikely(!sgl))
-                       goto enomem;
+       sgp = scsi_sg_pools + scsi_sgtable_index(nents);
+       mempool_free(sgl, sgp->pool);
+}
 
-               sg_init_table(sgl, sgp->size);
+static struct scatterlist *scsi_sg_alloc(unsigned int nents, gfp_t gfp_mask)
+{
+       struct scsi_host_sg_pool *sgp;
 
-               /*
-                * first loop through, set initial index and return value
-                */
-               if (!ret)
-                       ret = sgl;
+       sgp = scsi_sg_pools + scsi_sgtable_index(nents);
+       return mempool_alloc(sgp->pool, gfp_mask);
+}
 
-               /*
-                * chain previous sglist, if any. we know the previous
-                * sglist must be the biggest one, or we would not have
-                * ended up doing another loop.
-                */
-               if (prev)
-                       sg_chain(prev, SCSI_MAX_SG_SEGMENTS, sgl);
+static int scsi_alloc_sgtable(struct scsi_data_buffer *sdb, int nents,
+                             gfp_t gfp_mask)
+{
+       int ret;
 
-               /*
-                * if we have nothing left, mark the last segment as
-                * end-of-list
-                */
-               if (!left)
-                       sg_mark_end(&sgl[this - 1]);
+       BUG_ON(!nents);
 
-               /*
-                * don't allow subsequent mempool allocs to sleep, it would
-                * violate the mempool principle.
-                */
-               gfp_mask &= ~__GFP_WAIT;
-               gfp_mask |= __GFP_HIGH;
-               prev = sgl;
-       } while (left);
+       ret = __sg_alloc_table(&sdb->table, nents, SCSI_MAX_SG_SEGMENTS,
+                              gfp_mask, scsi_sg_alloc);
+       if (unlikely(ret))
+               __sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS,
+                               scsi_sg_free);
 
-       /*
-        * ->use_sg may get modified after dma mapping has potentially
-        * shrunk the number of segments, so keep a copy of it for free.
-        */
-       cmd->__use_sg = cmd->use_sg;
        return ret;
-enomem:
-       if (ret) {
-               /*
-                * Free entries chained off ret. Since we were trying to
-                * allocate another sglist, we know that all entries are of
-                * the max size.
-                */
-               sgp = scsi_sg_pools + SG_MEMPOOL_NR - 1;
-               prev = ret;
-               ret = &ret[SCSI_MAX_SG_SEGMENTS - 1];
-
-               while ((sgl = sg_chain_ptr(ret)) != NULL) {
-                       ret = &sgl[SCSI_MAX_SG_SEGMENTS - 1];
-                       mempool_free(sgl, sgp->pool);
-               }
-
-               mempool_free(prev, sgp->pool);
-       }
-       return NULL;
 }
 
-EXPORT_SYMBOL(scsi_alloc_sgtable);
-
-void scsi_free_sgtable(struct scsi_cmnd *cmd)
+static void scsi_free_sgtable(struct scsi_data_buffer *sdb)
 {
-       struct scatterlist *sgl = cmd->request_buffer;
-       struct scsi_host_sg_pool *sgp;
-
-       /*
-        * if this is the biggest size sglist, check if we have
-        * chained parts we need to free
-        */
-       if (cmd->__use_sg > SCSI_MAX_SG_SEGMENTS) {
-               unsigned short this, left;
-               struct scatterlist *next;
-               unsigned int index;
-
-               left = cmd->__use_sg - (SCSI_MAX_SG_SEGMENTS - 1);
-               next = sg_chain_ptr(&sgl[SCSI_MAX_SG_SEGMENTS - 1]);
-               while (left && next) {
-                       sgl = next;
-                       this = left;
-                       if (this > SCSI_MAX_SG_SEGMENTS) {
-                               this = SCSI_MAX_SG_SEGMENTS - 1;
-                               index = SG_MEMPOOL_NR - 1;
-                       } else
-                               index = scsi_sgtable_index(this);
-
-                       left -= this;
+       __sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS, scsi_sg_free);
+}
 
-                       sgp = scsi_sg_pools + index;
+static void __scsi_release_buffers(struct scsi_cmnd *cmd, int do_bidi_check)
+{
 
-                       if (left)
-                               next = sg_chain_ptr(&sgl[sgp->size - 1]);
+       if (cmd->sdb.table.nents)
+               scsi_free_sgtable(&cmd->sdb);
 
-                       mempool_free(sgl, sgp->pool);
-               }
+       memset(&cmd->sdb, 0, sizeof(cmd->sdb));
 
-               /*
-                * Restore original, will be freed below
-                */
-               sgl = cmd->request_buffer;
-               sgp = scsi_sg_pools + SG_MEMPOOL_NR - 1;
-       } else
-               sgp = scsi_sg_pools + scsi_sgtable_index(cmd->__use_sg);
+       if (do_bidi_check && scsi_bidi_cmnd(cmd)) {
+               struct scsi_data_buffer *bidi_sdb =
+                       cmd->request->next_rq->special;
+               scsi_free_sgtable(bidi_sdb);
+               kmem_cache_free(scsi_sdb_cache, bidi_sdb);
+               cmd->request->next_rq->special = NULL;
+       }
 
-       mempool_free(sgl, sgp->pool);
+       if (scsi_prot_sg_count(cmd))
+               scsi_free_sgtable(cmd->prot_sdb);
 }
 
-EXPORT_SYMBOL(scsi_free_sgtable);
-
 /*
  * Function:    scsi_release_buffers()
  *
@@ -892,18 +661,11 @@ EXPORT_SYMBOL(scsi_free_sgtable);
  *             the scatter-gather table, and potentially any bounce
  *             buffers.
  */
-static void scsi_release_buffers(struct scsi_cmnd *cmd)
+void scsi_release_buffers(struct scsi_cmnd *cmd)
 {
-       if (cmd->use_sg)
-               scsi_free_sgtable(cmd);
-
-       /*
-        * Zero these out.  They now point to freed memory, and it is
-        * dangerous to hang onto the pointers.
-        */
-       cmd->request_buffer = NULL;
-       cmd->request_bufflen = 0;
+       __scsi_release_buffers(cmd, 1);
 }
+EXPORT_SYMBOL(scsi_release_buffers);
 
 /*
  * Function:    scsi_io_completion()
@@ -922,29 +684,37 @@ static void scsi_release_buffers(struct scsi_cmnd *cmd)
  *              (the normal case for most drivers), we don't need
  *              the logic to deal with cleaning up afterwards.
  *
- *             We must do one of several things here:
+ *             We must call scsi_end_request().  This will finish off
+ *             the specified number of sectors.  If we are done, the
+ *             command block will be released and the queue function
+ *             will be goosed.  If we are not done then we have to
+ *             figure out what to do next:
+ *
+ *             a) We can call scsi_requeue_command().  The request
+ *                will be unprepared and put back on the queue.  Then
+ *                a new command will be created for it.  This should
+ *                be used if we made forward progress, or if we want
+ *                to switch from READ(10) to READ(6) for example.
  *
- *             a) Call scsi_end_request.  This will finish off the
- *                specified number of sectors.  If we are done, the
- *                command block will be released, and the queue
- *                function will be goosed.  If we are not done, then
- *                scsi_end_request will directly goose the queue.
+ *             b) We can call scsi_queue_insert().  The request will
+ *                be put back on the queue and retried using the same
+ *                command as before, possibly after a delay.
  *
- *             b) We can just use scsi_requeue_command() here.  This would
- *                be used if we just wanted to retry, for example.
+ *             c) We can call blk_end_request() with -EIO to fail
+ *                the remainder of the request.
  */
 void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
 {
        int result = cmd->result;
-       int this_count = cmd->request_bufflen;
        struct request_queue *q = cmd->device->request_queue;
        struct request *req = cmd->request;
-       int clear_errors = 1;
+       int error = 0;
        struct scsi_sense_hdr sshdr;
        int sense_valid = 0;
        int sense_deferred = 0;
-
-       scsi_release_buffers(cmd);
+       enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY,
+             ACTION_DELAYED_RETRY} action;
+       char *description = NULL;
 
        if (result) {
                sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
@@ -952,10 +722,9 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
                        sense_deferred = scsi_sense_is_deferred(&sshdr);
        }
 
-       if (blk_pc_request(req)) { /* SG_IO ioctl from block level */
+       if (req->cmd_type == REQ_TYPE_BLOCK_PC) { /* SG_IO ioctl from block level */
                req->errors = result;
                if (result) {
-                       clear_errors = 0;
                        if (sense_valid && req->sense) {
                                /*
                                 * SG_IO wants current and deferred errors
@@ -967,33 +736,75 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
                                memcpy(req->sense, cmd->sense_buffer,  len);
                                req->sense_len = len;
                        }
+                       if (!sense_deferred)
+                               error = -EIO;
+               }
+
+               req->resid_len = scsi_get_resid(cmd);
+
+               if (scsi_bidi_cmnd(cmd)) {
+                       /*
+                        * Bidi commands Must be complete as a whole,
+                        * both sides at once.
+                        */
+                       req->next_rq->resid_len = scsi_in(cmd)->resid;
+
+                       scsi_release_buffers(cmd);
+                       blk_end_request_all(req, 0);
+
+                       scsi_next_command(cmd);
+                       return;
                }
-               req->data_len = cmd->resid;
        }
 
+       /* no bidi support for !REQ_TYPE_BLOCK_PC yet */
+       BUG_ON(blk_bidi_rq(req));
+
        /*
         * Next deal with any sectors which we were able to correctly
         * handle.
         */
-       SCSI_LOG_HLCOMPLETE(1, printk("%ld sectors total, "
+       SCSI_LOG_HLCOMPLETE(1, printk("%u sectors total, "
                                      "%d bytes done.\n",
-                                     req->nr_sectors, good_bytes));
-       SCSI_LOG_HLCOMPLETE(1, printk("use_sg is %d\n", cmd->use_sg));
+                                     blk_rq_sectors(req), good_bytes));
 
-       if (clear_errors)
-               req->errors = 0;
+       /*
+        * Recovered errors need reporting, but they're always treated
+        * as success, so fiddle the result code here.  For BLOCK_PC
+        * we already took a copy of the original into rq->errors which
+        * is what gets returned to the user
+        */
+       if (sense_valid && (sshdr.sense_key == RECOVERED_ERROR)) {
+               /* if ATA PASS-THROUGH INFORMATION AVAILABLE skip
+                * print since caller wants ATA registers. Only occurs on
+                * SCSI ATA PASS_THROUGH commands when CK_COND=1
+                */
+               if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d))
+                       ;
+               else if (!(req->cmd_flags & REQ_QUIET))
+                       scsi_print_sense("", cmd);
+               result = 0;
+               /* BLOCK_PC may have set error */
+               error = 0;
+       }
 
-       /* A number of bytes were successfully read.  If there
+       /*
+        * A number of bytes were successfully read.  If there
         * are leftovers and there is some kind of error
         * (result != 0), retry the rest.
         */
-       if (scsi_end_request(cmd, 1, good_bytes, result == 0) == NULL)
+       if (scsi_end_request(cmd, error, good_bytes, result == 0) == NULL)
                return;
 
-       /* good_bytes = 0, or (inclusive) there were leftovers and
-        * result = 0, so scsi_end_request couldn't retry.
-        */
-       if (sense_valid && !sense_deferred) {
+       error = -EIO;
+
+       if (host_byte(result) == DID_RESET) {
+               /* Third party bus reset or reset for error recovery
+                * reasons.  Just retry the command and see what
+                * happens.
+                */
+               action = ACTION_RETRY;
+       } else if (sense_valid && !sense_deferred) {
                switch (sshdr.sense_key) {
                case UNIT_ATTENTION:
                        if (cmd->device->removable) {
@@ -1001,16 +812,15 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
                                 * and quietly refuse further access.
                                 */
                                cmd->device->changed = 1;
-                               scsi_end_request(cmd, 0, this_count, 1);
-                               return;
+                               description = "Media Changed";
+                               action = ACTION_FAIL;
                        } else {
                                /* Must have been a power glitch, or a
                                 * bus reset.  Could not have been a
                                 * media change, so we just retry the
-                                * request and see what happens.
+                                * command and see what happens.
                                 */
-                               scsi_requeue_command(q, cmd);
-                               return;
+                               action = ACTION_RETRY;
                        }
                        break;
                case ILLEGAL_REQUEST:
@@ -1026,15 +836,21 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
                            sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&
                            (cmd->cmnd[0] == READ_10 ||
                             cmd->cmnd[0] == WRITE_10)) {
+                               /* This will issue a new 6-byte command. */
                                cmd->device->use_10_for_rw = 0;
-                               /* This will cause a retry with a
-                                * 6-byte command.
-                                */
-                               scsi_requeue_command(q, cmd);
-                               return;
-                       } else {
-                               scsi_end_request(cmd, 0, this_count, 1);
-                               return;
+                               action = ACTION_REPREP;
+                       } else if (sshdr.asc == 0x10) /* DIX */ {
+                               description = "Host Data Integrity Failure";
+                               action = ACTION_FAIL;
+                               error = -EILSEQ;
+                       } else
+                               action = ACTION_FAIL;
+                       break;
+               case ABORTED_COMMAND:
+                       action = ACTION_FAIL;
+                       if (sshdr.asc == 0x10) { /* DIF */
+                               description = "Target Data Integrity Failure";
+                               error = -EILSEQ;
                        }
                        break;
                case NOT_READY:
@@ -1050,49 +866,93 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
                                case 0x07: /* operation in progress */
                                case 0x08: /* Long write in progress */
                                case 0x09: /* self test in progress */
-                                       scsi_requeue_command(q, cmd);
-                                       return;
+                               case 0x14: /* space allocation in progress */
+                                       action = ACTION_DELAYED_RETRY;
+                                       break;
                                default:
+                                       description = "Device not ready";
+                                       action = ACTION_FAIL;
                                        break;
                                }
+                       } else {
+                               description = "Device not ready";
+                               action = ACTION_FAIL;
                        }
-                       if (!(req->cmd_flags & REQ_QUIET))
-                               scsi_cmd_print_sense_hdr(cmd,
-                                                        "Device not ready",
-                                                        &sshdr);
-
-                       scsi_end_request(cmd, 0, this_count, 1);
-                       return;
+                       break;
                case VOLUME_OVERFLOW:
-                       if (!(req->cmd_flags & REQ_QUIET)) {
-                               scmd_printk(KERN_INFO, cmd,
-                                           "Volume overflow, CDB: ");
-                               __scsi_print_command(cmd->cmnd);
-                               scsi_print_sense("", cmd);
-                       }
                        /* See SSC3rXX or current. */
-                       scsi_end_request(cmd, 0, this_count, 1);
-                       return;
+                       action = ACTION_FAIL;
+                       break;
                default:
+                       description = "Unhandled sense code";
+                       action = ACTION_FAIL;
                        break;
                }
+       } else {
+               description = "Unhandled error code";
+               action = ACTION_FAIL;
        }
-       if (host_byte(result) == DID_RESET) {
-               /* Third party bus reset or reset for error recovery
-                * reasons.  Just retry the request and see what
-                * happens.
-                */
-               scsi_requeue_command(q, cmd);
-               return;
-       }
-       if (result) {
+
+       switch (action) {
+       case ACTION_FAIL:
+               /* Give up and fail the remainder of the request */
+               scsi_release_buffers(cmd);
                if (!(req->cmd_flags & REQ_QUIET)) {
+                       if (description)
+                               scmd_printk(KERN_INFO, cmd, "%s\n",
+                                           description);
                        scsi_print_result(cmd);
                        if (driver_byte(result) & DRIVER_SENSE)
                                scsi_print_sense("", cmd);
+                       scsi_print_command(cmd);
                }
+               if (blk_end_request_err(req, error))
+                       scsi_requeue_command(q, cmd);
+               else
+                       scsi_next_command(cmd);
+               break;
+       case ACTION_REPREP:
+               /* Unprep the request and put it back at the head of the queue.
+                * A new command will be prepared and issued.
+                */
+               scsi_release_buffers(cmd);
+               scsi_requeue_command(q, cmd);
+               break;
+       case ACTION_RETRY:
+               /* Retry the same command immediately */
+               __scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, 0);
+               break;
+       case ACTION_DELAYED_RETRY:
+               /* Retry the same command after a delay */
+               __scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, 0);
+               break;
+       }
+}
+
+static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb,
+                            gfp_t gfp_mask)
+{
+       int count;
+
+       /*
+        * If sg table allocation fails, requeue request later.
+        */
+       if (unlikely(scsi_alloc_sgtable(sdb, req->nr_phys_segments,
+                                       gfp_mask))) {
+               return BLKPREP_DEFER;
        }
-       scsi_end_request(cmd, 0, this_count, !result);
+
+       req->buffer = NULL;
+
+       /* 
+        * Next, walk the list, and fill in the addresses and sizes of
+        * each segment.
+        */
+       count = blk_rq_map_sg(req->q, req, sdb->table.sgl);
+       BUG_ON(count > sdb->table.nents);
+       sdb->table.nents = count;
+       sdb->length = blk_rq_bytes(req);
+       return BLKPREP_OK;
 }
 
 /*
@@ -1104,43 +964,60 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
  *
  * Returns:     0 on success
  *             BLKPREP_DEFER if the failure is retryable
+ *             BLKPREP_KILL if the failure is fatal
  */
-static int scsi_init_io(struct scsi_cmnd *cmd)
+int scsi_init_io(struct scsi_cmnd *cmd, gfp_t gfp_mask)
 {
-       struct request     *req = cmd->request;
-       int                count;
+       struct request *rq = cmd->request;
 
-       /*
-        * We used to not use scatter-gather for single segment request,
-        * but now we do (it makes highmem I/O easier to support without
-        * kmapping pages)
-        */
-       cmd->use_sg = req->nr_phys_segments;
+       int error = scsi_init_sgtable(rq, &cmd->sdb, gfp_mask);
+       if (error)
+               goto err_exit;
 
-       /*
-        * If sg table allocation fails, requeue request later.
-        */
-       cmd->request_buffer = scsi_alloc_sgtable(cmd, GFP_ATOMIC);
-       if (unlikely(!cmd->request_buffer)) {
-               scsi_unprep_request(req);
-               return BLKPREP_DEFER;
+       if (blk_bidi_rq(rq)) {
+               struct scsi_data_buffer *bidi_sdb = kmem_cache_zalloc(
+                       scsi_sdb_cache, GFP_ATOMIC);
+               if (!bidi_sdb) {
+                       error = BLKPREP_DEFER;
+                       goto err_exit;
+               }
+
+               rq->next_rq->special = bidi_sdb;
+               error = scsi_init_sgtable(rq->next_rq, bidi_sdb, GFP_ATOMIC);
+               if (error)
+                       goto err_exit;
        }
 
-       req->buffer = NULL;
-       if (blk_pc_request(req))
-               cmd->request_bufflen = req->data_len;
-       else
-               cmd->request_bufflen = req->nr_sectors << 9;
+       if (blk_integrity_rq(rq)) {
+               struct scsi_data_buffer *prot_sdb = cmd->prot_sdb;
+               int ivecs, count;
 
-       /* 
-        * Next, walk the list, and fill in the addresses and sizes of
-        * each segment.
-        */
-       count = blk_rq_map_sg(req->q, req, cmd->request_buffer);
-       BUG_ON(count > cmd->use_sg);
-       cmd->use_sg = count;
-       return BLKPREP_OK;
+               BUG_ON(prot_sdb == NULL);
+               ivecs = blk_rq_count_integrity_sg(rq->q, rq->bio);
+
+               if (scsi_alloc_sgtable(prot_sdb, ivecs, gfp_mask)) {
+                       error = BLKPREP_DEFER;
+                       goto err_exit;
+               }
+
+               count = blk_rq_map_integrity_sg(rq->q, rq->bio,
+                                               prot_sdb->table.sgl);
+               BUG_ON(unlikely(count > ivecs));
+               BUG_ON(unlikely(count > queue_max_integrity_segments(rq->q)));
+
+               cmd->prot_sdb = prot_sdb;
+               cmd->prot_sdb->table.nents = count;
+       }
+
+       return BLKPREP_OK ;
+
+err_exit:
+       scsi_release_buffers(cmd);
+       cmd->request->special = NULL;
+       scsi_put_command(cmd);
+       return error;
 }
+EXPORT_SYMBOL(scsi_init_io);
 
 static struct scsi_cmnd *scsi_get_cmd_from_req(struct scsi_device *sdev,
                struct request *req)
@@ -1160,6 +1037,8 @@ static struct scsi_cmnd *scsi_get_cmd_from_req(struct scsi_device *sdev,
        cmd->tag = req->tag;
        cmd->request = req;
 
+       cmd->cmnd = req->cmd;
+
        return cmd;
 }
 
@@ -1186,32 +1065,26 @@ int scsi_setup_blk_pc_cmnd(struct scsi_device *sdev, struct request *req)
 
                BUG_ON(!req->nr_phys_segments);
 
-               ret = scsi_init_io(cmd);
+               ret = scsi_init_io(cmd, GFP_ATOMIC);
                if (unlikely(ret))
                        return ret;
        } else {
-               BUG_ON(req->data_len);
-               BUG_ON(req->data);
+               BUG_ON(blk_rq_bytes(req));
 
-               cmd->request_bufflen = 0;
-               cmd->request_buffer = NULL;
-               cmd->use_sg = 0;
+               memset(&cmd->sdb, 0, sizeof(cmd->sdb));
                req->buffer = NULL;
        }
 
-       BUILD_BUG_ON(sizeof(req->cmd) > sizeof(cmd->cmnd));
-       memcpy(cmd->cmnd, req->cmd, sizeof(cmd->cmnd));
        cmd->cmd_len = req->cmd_len;
-       if (!req->data_len)
+       if (!blk_rq_bytes(req))
                cmd->sc_data_direction = DMA_NONE;
        else if (rq_data_dir(req) == WRITE)
                cmd->sc_data_direction = DMA_TO_DEVICE;
        else
                cmd->sc_data_direction = DMA_FROM_DEVICE;
        
-       cmd->transfersize = req->data_len;
+       cmd->transfersize = blk_rq_bytes(req);
        cmd->allowed = req->retries;
-       cmd->timeout_per_command = req->timeout;
        return BLKPREP_OK;
 }
 EXPORT_SYMBOL(scsi_setup_blk_pc_cmnd);
@@ -1228,6 +1101,14 @@ int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req)
 
        if (ret != BLKPREP_OK)
                return ret;
+
+       if (unlikely(sdev->scsi_dh_data && sdev->scsi_dh_data->scsi_dh
+                        && sdev->scsi_dh_data->scsi_dh->prep_fn)) {
+               ret = sdev->scsi_dh_data->scsi_dh->prep_fn(sdev, req);
+               if (ret != BLKPREP_OK)
+                       return ret;
+       }
+
        /*
         * Filesystem requests must transfer data.
         */
@@ -1237,7 +1118,8 @@ int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req)
        if (unlikely(!cmd))
                return BLKPREP_DEFER;
 
-       return scsi_init_io(cmd);
+       memset(cmd->cmnd, 0, BLK_MAX_CDB);
+       return scsi_init_io(cmd, GFP_ATOMIC);
 }
 EXPORT_SYMBOL(scsi_setup_fs_cmnd);
 
@@ -1272,6 +1154,7 @@ int scsi_prep_state_check(struct scsi_device *sdev, struct request *req)
                        break;
                case SDEV_QUIESCE:
                case SDEV_BLOCK:
+               case SDEV_CREATED_BLOCK:
                        /*
                         * If the devices is blocked we defer normal commands.
                         */
@@ -1310,7 +1193,7 @@ int scsi_prep_return(struct request_queue *q, struct request *req, int ret)
                break;
        case BLKPREP_DEFER:
                /*
-                * If we defer, the elv_next_request() returns NULL, but the
+                * If we defer, the blk_peek_request() returns NULL, but the
                 * queue must be restarted, so we plug here if no returning
                 * command will automatically do that.
                 */
@@ -1334,6 +1217,7 @@ int scsi_prep_fn(struct request_queue *q, struct request *req)
                ret = scsi_setup_blk_pc_cmnd(sdev, req);
        return scsi_prep_return(q, req, ret);
 }
+EXPORT_SYMBOL(scsi_prep_fn);
 
 /*
  * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
@@ -1344,8 +1228,6 @@ int scsi_prep_fn(struct request_queue *q, struct request *req)
 static inline int scsi_dev_queue_ready(struct request_queue *q,
                                  struct scsi_device *sdev)
 {
-       if (sdev->device_busy >= sdev->queue_depth)
-               return 0;
        if (sdev->device_busy == 0 && sdev->device_blocked) {
                /*
                 * unblock after device_blocked iterates to zero
@@ -1359,9 +1241,52 @@ static inline int scsi_dev_queue_ready(struct request_queue *q,
                        return 0;
                }
        }
-       if (sdev->device_blocked)
+       if (scsi_device_is_busy(sdev))
+               return 0;
+
+       return 1;
+}
+
+
+/*
+ * scsi_target_queue_ready: checks if there we can send commands to target
+ * @sdev: scsi device on starget to check.
+ *
+ * Called with the host lock held.
+ */
+static inline int scsi_target_queue_ready(struct Scsi_Host *shost,
+                                          struct scsi_device *sdev)
+{
+       struct scsi_target *starget = scsi_target(sdev);
+
+       if (starget->single_lun) {
+               if (starget->starget_sdev_user &&
+                   starget->starget_sdev_user != sdev)
+                       return 0;
+               starget->starget_sdev_user = sdev;
+       }
+
+       if (starget->target_busy == 0 && starget->target_blocked) {
+               /*
+                * unblock after target_blocked iterates to zero
+                */
+               if (--starget->target_blocked == 0) {
+                       SCSI_LOG_MLQUEUE(3, starget_printk(KERN_INFO, starget,
+                                        "unblocking target at zero depth\n"));
+               } else
+                       return 0;
+       }
+
+       if (scsi_target_is_busy(starget)) {
+               if (list_empty(&sdev->starved_entry))
+                       list_add_tail(&sdev->starved_entry,
+                                     &shost->starved_list);
                return 0;
+       }
 
+       /* We're OK to process the command, so we can't be starved */
+       if (!list_empty(&sdev->starved_entry))
+               list_del_init(&sdev->starved_entry);
        return 1;
 }
 
@@ -1387,12 +1312,10 @@ static inline int scsi_host_queue_ready(struct request_queue *q,
                                printk("scsi%d unblocking host at zero depth\n",
                                        shost->host_no));
                } else {
-                       blk_plug_device(q);
                        return 0;
                }
        }
-       if ((shost->can_queue > 0 && shost->host_busy >= shost->can_queue) ||
-           shost->host_blocked || shost->host_self_blocked) {
+       if (scsi_host_is_busy(shost)) {
                if (list_empty(&sdev->starved_entry))
                        list_add_tail(&sdev->starved_entry, &shost->starved_list);
                return 0;
@@ -1406,22 +1329,51 @@ static inline int scsi_host_queue_ready(struct request_queue *q,
 }
 
 /*
+ * Busy state exporting function for request stacking drivers.
+ *
+ * For efficiency, no lock is taken to check the busy state of
+ * shost/starget/sdev, since the returned value is not guaranteed and
+ * may be changed after request stacking drivers call the function,
+ * regardless of taking lock or not.
+ *
+ * When scsi can't dispatch I/Os anymore and needs to kill I/Os
+ * (e.g. !sdev), scsi needs to return 'not busy'.
+ * Otherwise, request stacking drivers may hold requests forever.
+ */
+static int scsi_lld_busy(struct request_queue *q)
+{
+       struct scsi_device *sdev = q->queuedata;
+       struct Scsi_Host *shost;
+       struct scsi_target *starget;
+
+       if (!sdev)
+               return 0;
+
+       shost = sdev->host;
+       starget = scsi_target(sdev);
+
+       if (scsi_host_in_recovery(shost) || scsi_host_is_busy(shost) ||
+           scsi_target_is_busy(starget) || scsi_device_is_busy(sdev))
+               return 1;
+
+       return 0;
+}
+
+/*
  * Kill a request for a dead device
  */
 static void scsi_kill_request(struct request *req, struct request_queue *q)
 {
        struct scsi_cmnd *cmd = req->special;
-       struct scsi_device *sdev = cmd->device;
-       struct Scsi_Host *shost = sdev->host;
-
-       blkdev_dequeue_request(req);
+       struct scsi_device *sdev;
+       struct scsi_target *starget;
+       struct Scsi_Host *shost;
 
-       if (unlikely(cmd == NULL)) {
-               printk(KERN_CRIT "impossible request in %s.\n",
-                                __FUNCTION__);
-               BUG();
-       }
+       blk_start_request(req);
 
+       sdev = cmd->device;
+       starget = scsi_target(sdev);
+       shost = sdev->host;
        scsi_init_cmd_errh(cmd);
        cmd->result = DID_NO_CONNECT << 16;
        atomic_inc(&cmd->device->iorequest_cnt);
@@ -1435,20 +1387,25 @@ static void scsi_kill_request(struct request *req, struct request_queue *q)
        spin_unlock(sdev->request_queue->queue_lock);
        spin_lock(shost->host_lock);
        shost->host_busy++;
+       starget->target_busy++;
        spin_unlock(shost->host_lock);
        spin_lock(sdev->request_queue->queue_lock);
 
-       __scsi_done(cmd);
+       blk_complete_request(req);
 }
 
 static void scsi_softirq_done(struct request *rq)
 {
-       struct scsi_cmnd *cmd = rq->completion_data;
-       unsigned long wait_for = (cmd->allowed + 1) * cmd->timeout_per_command;
+       struct scsi_cmnd *cmd = rq->special;
+       unsigned long wait_for = (cmd->allowed + 1) * rq->timeout;
        int disposition;
 
        INIT_LIST_HEAD(&cmd->eh_entry);
 
+       atomic_inc(&cmd->device->iodone_cnt);
+       if (cmd->result)
+               atomic_inc(&cmd->device->ioerr_cnt);
+
        disposition = scsi_decide_disposition(cmd);
        if (disposition != SUCCESS &&
            time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
@@ -1496,7 +1453,7 @@ static void scsi_request_fn(struct request_queue *q)
 
        if (!sdev) {
                printk("scsi: killing requests for dead queue\n");
-               while ((req = elv_next_request(q)) != NULL)
+               while ((req = blk_peek_request(q)) != NULL)
                        scsi_kill_request(req, q);
                return;
        }
@@ -1517,7 +1474,7 @@ static void scsi_request_fn(struct request_queue *q)
                 * that the request is fully prepared even if we cannot 
                 * accept it.
                 */
-               req = elv_next_request(q);
+               req = blk_peek_request(q);
                if (!req || !scsi_dev_queue_ready(q, sdev))
                        break;
 
@@ -1533,7 +1490,7 @@ static void scsi_request_fn(struct request_queue *q)
                 * Remove the request from the request list.
                 */
                if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
-                       blkdev_dequeue_request(req);
+                       blk_start_request(req);
                sdev->device_busy++;
 
                spin_unlock(q->queue_lock);
@@ -1542,20 +1499,34 @@ static void scsi_request_fn(struct request_queue *q)
                        printk(KERN_CRIT "impossible request in %s.\n"
                                         "please mail a stack trace to "
                                         "linux-scsi@vger.kernel.org\n",
-                                        __FUNCTION__);
+                                        __func__);
                        blk_dump_rq_flags(req, "foo");
                        BUG();
                }
                spin_lock(shost->host_lock);
 
-               if (!scsi_host_queue_ready(q, shost, sdev))
+               /*
+                * We hit this when the driver is using a host wide
+                * tag map. For device level tag maps the queue_depth check
+                * in the device ready fn would prevent us from trying
+                * to allocate a tag. Since the map is a shared host resource
+                * we add the dev to the starved list so it eventually gets
+                * a run when a tag is freed.
+                */
+               if (blk_queue_tagged(q) && !blk_rq_tagged(req)) {
+                       if (list_empty(&sdev->starved_entry))
+                               list_add_tail(&sdev->starved_entry,
+                                             &shost->starved_list);
                        goto not_ready;
-               if (scsi_target(sdev)->single_lun) {
-                       if (scsi_target(sdev)->starget_sdev_user &&
-                           scsi_target(sdev)->starget_sdev_user != sdev)
-                               goto not_ready;
-                       scsi_target(sdev)->starget_sdev_user = sdev;
                }
+
+               if (!scsi_target_queue_ready(shost, sdev))
+                       goto not_ready;
+
+               if (!scsi_host_queue_ready(q, shost, sdev))
+                       goto not_ready;
+
+               scsi_target(sdev)->target_busy++;
                shost->host_busy++;
 
                /*
@@ -1638,6 +1609,7 @@ struct request_queue *__scsi_alloc_queue(struct Scsi_Host *shost,
                                         request_fn_proc *request_fn)
 {
        struct request_queue *q;
+       struct device *dev = shost->shost_gendev.parent;
 
        q = blk_init_queue(request_fn, NULL);
        if (!q)
@@ -1646,28 +1618,34 @@ struct request_queue *__scsi_alloc_queue(struct Scsi_Host *shost,
        /*
         * this limit is imposed by hardware restrictions
         */
-       blk_queue_max_hw_segments(q, shost->sg_tablesize);
-
-       /*
-        * In the future, sg chaining support will be mandatory and this
-        * ifdef can then go away. Right now we don't have all archs
-        * converted, so better keep it safe.
-        */
-#ifdef ARCH_HAS_SG_CHAIN
-       if (shost->use_sg_chaining)
-               blk_queue_max_phys_segments(q, SCSI_MAX_SG_CHAIN_SEGMENTS);
-       else
-               blk_queue_max_phys_segments(q, SCSI_MAX_SG_SEGMENTS);
-#else
-       blk_queue_max_phys_segments(q, SCSI_MAX_SG_SEGMENTS);
-#endif
+       blk_queue_max_segments(q, min_t(unsigned short, shost->sg_tablesize,
+                                       SCSI_MAX_SG_CHAIN_SEGMENTS));
+
+       if (scsi_host_prot_dma(shost)) {
+               shost->sg_prot_tablesize =
+                       min_not_zero(shost->sg_prot_tablesize,
+                                    (unsigned short)SCSI_MAX_PROT_SG_SEGMENTS);
+               BUG_ON(shost->sg_prot_tablesize < shost->sg_tablesize);
+               blk_queue_max_integrity_segments(q, shost->sg_prot_tablesize);
+       }
 
-       blk_queue_max_sectors(q, shost->max_sectors);
+       blk_queue_max_hw_sectors(q, shost->max_sectors);
        blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
        blk_queue_segment_boundary(q, shost->dma_boundary);
+       dma_set_seg_boundary(dev, shost->dma_boundary);
+
+       blk_queue_max_segment_size(q, dma_get_max_seg_size(dev));
 
        if (!shost->use_clustering)
-               clear_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags);
+               q->limits.cluster = 0;
+
+       /*
+        * set a reasonable default alignment on word boundaries: the
+        * host and device may alter it using
+        * blk_queue_update_dma_alignment() later.
+        */
+       blk_queue_dma_alignment(q, 0x03);
+
        return q;
 }
 EXPORT_SYMBOL(__scsi_alloc_queue);
@@ -1682,6 +1660,8 @@ struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
 
        blk_queue_prep_rq(q, scsi_prep_fn);
        blk_queue_softirq_done(q, scsi_softirq_done);
+       blk_queue_rq_timed_out(q, scsi_times_out);
+       blk_queue_lld_busy(q, scsi_lld_busy);
        return q;
 }
 
@@ -1743,11 +1723,11 @@ int __init scsi_init_queue(void)
 {
        int i;
 
-       scsi_io_context_cache = kmem_cache_create("scsi_io_context",
-                                       sizeof(struct scsi_io_context),
-                                       0, 0, NULL);
-       if (!scsi_io_context_cache) {
-               printk(KERN_ERR "SCSI: can't init scsi io context cache\n");
+       scsi_sdb_cache = kmem_cache_create("scsi_data_buffer",
+                                          sizeof(struct scsi_data_buffer),
+                                          0, 0, NULL);
+       if (!scsi_sdb_cache) {
+               printk(KERN_ERR "SCSI: can't init scsi sdb cache\n");
                return -ENOMEM;
        }
 
@@ -1760,6 +1740,7 @@ int __init scsi_init_queue(void)
                if (!sgp->slab) {
                        printk(KERN_ERR "SCSI: can't init sg slab %s\n",
                                        sgp->name);
+                       goto cleanup_sdb;
                }
 
                sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
@@ -1767,17 +1748,30 @@ int __init scsi_init_queue(void)
                if (!sgp->pool) {
                        printk(KERN_ERR "SCSI: can't init sg mempool %s\n",
                                        sgp->name);
+                       goto cleanup_sdb;
                }
        }
 
        return 0;
+
+cleanup_sdb:
+       for (i = 0; i < SG_MEMPOOL_NR; i++) {
+               struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
+               if (sgp->pool)
+                       mempool_destroy(sgp->pool);
+               if (sgp->slab)
+                       kmem_cache_destroy(sgp->slab);
+       }
+       kmem_cache_destroy(scsi_sdb_cache);
+
+       return -ENOMEM;
 }
 
 void scsi_exit_queue(void)
 {
        int i;
 
-       kmem_cache_destroy(scsi_io_context_cache);
+       kmem_cache_destroy(scsi_sdb_cache);
 
        for (i = 0; i < SG_MEMPOOL_NR; i++) {
                struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
@@ -1857,7 +1851,7 @@ scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage,
        }
 
        ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len,
-                              sshdr, timeout, retries);
+                              sshdr, timeout, retries, NULL);
        kfree(real_buffer);
        return ret;
 }
@@ -1922,7 +1916,7 @@ scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
        memset(buffer, 0, len);
 
        result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len,
-                                 sshdr, timeout, retries);
+                                 sshdr, timeout, retries, NULL);
 
        /* This code looks awful: what it's doing is making sure an
         * ILLEGAL REQUEST sense return identifies the actual command
@@ -1973,27 +1967,45 @@ scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
 }
 EXPORT_SYMBOL(scsi_mode_sense);
 
+/**
+ *     scsi_test_unit_ready - test if unit is ready
+ *     @sdev:  scsi device to change the state of.
+ *     @timeout: command timeout
+ *     @retries: number of retries before failing
+ *     @sshdr_external: Optional pointer to struct scsi_sense_hdr for
+ *             returning sense. Make sure that this is cleared before passing
+ *             in.
+ *
+ *     Returns zero if unsuccessful or an error if TUR failed.  For
+ *     removable media, UNIT_ATTENTION sets ->changed flag.
+ **/
 int
-scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries)
+scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries,
+                    struct scsi_sense_hdr *sshdr_external)
 {
        char cmd[] = {
                TEST_UNIT_READY, 0, 0, 0, 0, 0,
        };
-       struct scsi_sense_hdr sshdr;
+       struct scsi_sense_hdr *sshdr;
        int result;
-       
-       result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, &sshdr,
-                                 timeout, retries);
 
-       if ((driver_byte(result) & DRIVER_SENSE) && sdev->removable) {
+       if (!sshdr_external)
+               sshdr = kzalloc(sizeof(*sshdr), GFP_KERNEL);
+       else
+               sshdr = sshdr_external;
 
-               if ((scsi_sense_valid(&sshdr)) &&
-                   ((sshdr.sense_key == UNIT_ATTENTION) ||
-                    (sshdr.sense_key == NOT_READY))) {
+       /* try to eat the UNIT_ATTENTION if there are enough retries */
+       do {
+               result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, sshdr,
+                                         timeout, retries, NULL);
+               if (sdev->removable && scsi_sense_valid(sshdr) &&
+                   sshdr->sense_key == UNIT_ATTENTION)
                        sdev->changed = 1;
-                       result = 0;
-               }
-       }
+       } while (scsi_sense_valid(sshdr) &&
+                sshdr->sense_key == UNIT_ATTENTION && --retries);
+
+       if (!sshdr_external)
+               kfree(sshdr);
        return result;
 }
 EXPORT_SYMBOL(scsi_test_unit_ready);
@@ -2016,10 +2028,13 @@ scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
 
        switch (state) {
        case SDEV_CREATED:
-               /* There are no legal states that come back to
-                * created.  This is the manually initialised start
-                * state */
-               goto illegal;
+               switch (oldstate) {
+               case SDEV_CREATED_BLOCK:
+                       break;
+               default:
+                       goto illegal;
+               }
+               break;
                        
        case SDEV_RUNNING:
                switch (oldstate) {
@@ -2057,8 +2072,17 @@ scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
 
        case SDEV_BLOCK:
                switch (oldstate) {
-               case SDEV_CREATED:
                case SDEV_RUNNING:
+               case SDEV_CREATED_BLOCK:
+                       break;
+               default:
+                       goto illegal;
+               }
+               break;
+
+       case SDEV_CREATED_BLOCK:
+               switch (oldstate) {
+               case SDEV_CREATED:
                        break;
                default:
                        goto illegal;
@@ -2178,10 +2202,15 @@ void sdev_evt_send(struct scsi_device *sdev, struct scsi_event *evt)
 {
        unsigned long flags;
 
+#if 0
+       /* FIXME: currently this check eliminates all media change events
+        * for polled devices.  Need to update to discriminate between AN
+        * and polled events */
        if (!test_bit(evt->evt_type, sdev->supported_events)) {
                kfree(evt);
                return;
        }
+#endif
 
        spin_lock_irqsave(&sdev->list_lock, flags);
        list_add_tail(&evt->node, &sdev->event_list);
@@ -2341,8 +2370,12 @@ scsi_internal_device_block(struct scsi_device *sdev)
        int err = 0;
 
        err = scsi_device_set_state(sdev, SDEV_BLOCK);
-       if (err)
-               return err;
+       if (err) {
+               err = scsi_device_set_state(sdev, SDEV_CREATED_BLOCK);
+
+               if (err)
+                       return err;
+       }
 
        /* 
         * The device has transitioned to SDEV_BLOCK.  Stop the
@@ -2377,16 +2410,19 @@ int
 scsi_internal_device_unblock(struct scsi_device *sdev)
 {
        struct request_queue *q = sdev->request_queue; 
-       int err;
        unsigned long flags;
        
        /* 
         * Try to transition the scsi device to SDEV_RUNNING
         * and goose the device queue if successful.  
         */
-       err = scsi_device_set_state(sdev, SDEV_RUNNING);
-       if (err)
-               return err;
+       if (sdev->sdev_state == SDEV_BLOCK)
+               sdev->sdev_state = SDEV_RUNNING;
+       else if (sdev->sdev_state == SDEV_CREATED_BLOCK)
+               sdev->sdev_state = SDEV_CREATED;
+       else if (sdev->sdev_state != SDEV_CANCEL &&
+                sdev->sdev_state != SDEV_OFFLINE)
+               return -EINVAL;
 
        spin_lock_irqsave(q->queue_lock, flags);
        blk_start_queue(q);
@@ -2477,7 +2513,7 @@ void *scsi_kmap_atomic_sg(struct scatterlist *sgl, int sg_count,
        if (unlikely(i == sg_count)) {
                printk(KERN_ERR "%s: Bytes in sg: %zu, requested offset %zu, "
                        "elements %d\n",
-                      __FUNCTION__, sg_len, *offset, sg_count);
+                      __func__, sg_len, *offset, sg_count);
                WARN_ON(1);
                return NULL;
        }