[SCSI] remove REQ_SPECIAL in scsi_init_io()
[linux-2.6.git] / drivers / scsi / scsi_lib.c
1 /*
2  *  scsi_lib.c Copyright (C) 1999 Eric Youngdale
3  *
4  *  SCSI queueing library.
5  *      Initial versions: Eric Youngdale (eric@andante.org).
6  *                        Based upon conversations with large numbers
7  *                        of people at Linux Expo.
8  */
9
10 #include <linux/bio.h>
11 #include <linux/blkdev.h>
12 #include <linux/completion.h>
13 #include <linux/kernel.h>
14 #include <linux/mempool.h>
15 #include <linux/slab.h>
16 #include <linux/init.h>
17 #include <linux/pci.h>
18 #include <linux/delay.h>
19
20 #include <scsi/scsi.h>
21 #include <scsi/scsi_dbg.h>
22 #include <scsi/scsi_device.h>
23 #include <scsi/scsi_driver.h>
24 #include <scsi/scsi_eh.h>
25 #include <scsi/scsi_host.h>
26 #include <scsi/scsi_request.h>
27
28 #include "scsi_priv.h"
29 #include "scsi_logging.h"
30
31
32 #define SG_MEMPOOL_NR           (sizeof(scsi_sg_pools)/sizeof(struct scsi_host_sg_pool))
33 #define SG_MEMPOOL_SIZE         32
34
35 struct scsi_host_sg_pool {
36         size_t          size;
37         char            *name; 
38         kmem_cache_t    *slab;
39         mempool_t       *pool;
40 };
41
42 #if (SCSI_MAX_PHYS_SEGMENTS < 32)
43 #error SCSI_MAX_PHYS_SEGMENTS is too small
44 #endif
45
46 #define SP(x) { x, "sgpool-" #x } 
47 struct scsi_host_sg_pool scsi_sg_pools[] = { 
48         SP(8),
49         SP(16),
50         SP(32),
51 #if (SCSI_MAX_PHYS_SEGMENTS > 32)
52         SP(64),
53 #if (SCSI_MAX_PHYS_SEGMENTS > 64)
54         SP(128),
55 #if (SCSI_MAX_PHYS_SEGMENTS > 128)
56         SP(256),
57 #if (SCSI_MAX_PHYS_SEGMENTS > 256)
58 #error SCSI_MAX_PHYS_SEGMENTS is too large
59 #endif
60 #endif
61 #endif
62 #endif
63 };      
64 #undef SP
65
66
67 /*
68  * Function:    scsi_insert_special_req()
69  *
70  * Purpose:     Insert pre-formed request into request queue.
71  *
72  * Arguments:   sreq    - request that is ready to be queued.
73  *              at_head - boolean.  True if we should insert at head
74  *                        of queue, false if we should insert at tail.
75  *
76  * Lock status: Assumed that lock is not held upon entry.
77  *
78  * Returns:     Nothing
79  *
80  * Notes:       This function is called from character device and from
81  *              ioctl types of functions where the caller knows exactly
82  *              what SCSI command needs to be issued.   The idea is that
83  *              we merely inject the command into the queue (at the head
84  *              for now), and then call the queue request function to actually
85  *              process it.
86  */
87 int scsi_insert_special_req(struct scsi_request *sreq, int at_head)
88 {
89         /*
90          * Because users of this function are apt to reuse requests with no
91          * modification, we have to sanitise the request flags here
92          */
93         sreq->sr_request->flags &= ~REQ_DONTPREP;
94         blk_insert_request(sreq->sr_device->request_queue, sreq->sr_request,
95                            at_head, sreq, 0);
96         return 0;
97 }
98
99 /*
100  * Function:    scsi_queue_insert()
101  *
102  * Purpose:     Insert a command in the midlevel queue.
103  *
104  * Arguments:   cmd    - command that we are adding to queue.
105  *              reason - why we are inserting command to queue.
106  *
107  * Lock status: Assumed that lock is not held upon entry.
108  *
109  * Returns:     Nothing.
110  *
111  * Notes:       We do this for one of two cases.  Either the host is busy
112  *              and it cannot accept any more commands for the time being,
113  *              or the device returned QUEUE_FULL and can accept no more
114  *              commands.
115  * Notes:       This could be called either from an interrupt context or a
116  *              normal process context.
117  */
118 int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
119 {
120         struct Scsi_Host *host = cmd->device->host;
121         struct scsi_device *device = cmd->device;
122
123         SCSI_LOG_MLQUEUE(1,
124                  printk("Inserting command %p into mlqueue\n", cmd));
125
126         /*
127          * We are inserting the command into the ml queue.  First, we
128          * cancel the timer, so it doesn't time out.
129          */
130         scsi_delete_timer(cmd);
131
132         /*
133          * Next, set the appropriate busy bit for the device/host.
134          *
135          * If the host/device isn't busy, assume that something actually
136          * completed, and that we should be able to queue a command now.
137          *
138          * Note that the prior mid-layer assumption that any host could
139          * always queue at least one command is now broken.  The mid-layer
140          * will implement a user specifiable stall (see
141          * scsi_host.max_host_blocked and scsi_device.max_device_blocked)
142          * if a command is requeued with no other commands outstanding
143          * either for the device or for the host.
144          */
145         if (reason == SCSI_MLQUEUE_HOST_BUSY)
146                 host->host_blocked = host->max_host_blocked;
147         else if (reason == SCSI_MLQUEUE_DEVICE_BUSY)
148                 device->device_blocked = device->max_device_blocked;
149
150         /*
151          * Register the fact that we own the thing for now.
152          */
153         cmd->state = SCSI_STATE_MLQUEUE;
154         cmd->owner = SCSI_OWNER_MIDLEVEL;
155
156         /*
157          * Decrement the counters, since these commands are no longer
158          * active on the host/device.
159          */
160         scsi_device_unbusy(device);
161
162         /*
163          * Insert this command at the head of the queue for it's device.
164          * It will go before all other commands that are already in the queue.
165          *
166          * NOTE: there is magic here about the way the queue is plugged if
167          * we have no outstanding commands.
168          * 
169          * Although this *doesn't* plug the queue, it does call the request
170          * function.  The SCSI request function detects the blocked condition
171          * and plugs the queue appropriately.
172          */
173         blk_insert_request(device->request_queue, cmd->request, 1, cmd, 1);
174         return 0;
175 }
176
177 /*
178  * Function:    scsi_do_req
179  *
180  * Purpose:     Queue a SCSI request
181  *
182  * Arguments:   sreq      - command descriptor.
183  *              cmnd      - actual SCSI command to be performed.
184  *              buffer    - data buffer.
185  *              bufflen   - size of data buffer.
186  *              done      - completion function to be run.
187  *              timeout   - how long to let it run before timeout.
188  *              retries   - number of retries we allow.
189  *
190  * Lock status: No locks held upon entry.
191  *
192  * Returns:     Nothing.
193  *
194  * Notes:       This function is only used for queueing requests for things
195  *              like ioctls and character device requests - this is because
196  *              we essentially just inject a request into the queue for the
197  *              device.
198  *
199  *              In order to support the scsi_device_quiesce function, we
200  *              now inject requests on the *head* of the device queue
201  *              rather than the tail.
202  */
203 void scsi_do_req(struct scsi_request *sreq, const void *cmnd,
204                  void *buffer, unsigned bufflen,
205                  void (*done)(struct scsi_cmnd *),
206                  int timeout, int retries)
207 {
208         /*
209          * If the upper level driver is reusing these things, then
210          * we should release the low-level block now.  Another one will
211          * be allocated later when this request is getting queued.
212          */
213         __scsi_release_request(sreq);
214
215         /*
216          * Our own function scsi_done (which marks the host as not busy,
217          * disables the timeout counter, etc) will be called by us or by the
218          * scsi_hosts[host].queuecommand() function needs to also call
219          * the completion function for the high level driver.
220          */
221         memcpy(sreq->sr_cmnd, cmnd, sizeof(sreq->sr_cmnd));
222         sreq->sr_bufflen = bufflen;
223         sreq->sr_buffer = buffer;
224         sreq->sr_allowed = retries;
225         sreq->sr_done = done;
226         sreq->sr_timeout_per_command = timeout;
227
228         if (sreq->sr_cmd_len == 0)
229                 sreq->sr_cmd_len = COMMAND_SIZE(sreq->sr_cmnd[0]);
230
231         /*
232          * head injection *required* here otherwise quiesce won't work
233          */
234         scsi_insert_special_req(sreq, 1);
235 }
236 EXPORT_SYMBOL(scsi_do_req);
237
238 static void scsi_wait_done(struct scsi_cmnd *cmd)
239 {
240         struct request *req = cmd->request;
241         struct request_queue *q = cmd->device->request_queue;
242         unsigned long flags;
243
244         req->rq_status = RQ_SCSI_DONE;  /* Busy, but indicate request done */
245
246         spin_lock_irqsave(q->queue_lock, flags);
247         if (blk_rq_tagged(req))
248                 blk_queue_end_tag(q, req);
249         spin_unlock_irqrestore(q->queue_lock, flags);
250
251         if (req->waiting)
252                 complete(req->waiting);
253 }
254
255 /* This is the end routine we get to if a command was never attached
256  * to the request.  Simply complete the request without changing
257  * rq_status; this will cause a DRIVER_ERROR. */
258 static void scsi_wait_req_end_io(struct request *req)
259 {
260         BUG_ON(!req->waiting);
261
262         complete(req->waiting);
263 }
264
265 void scsi_wait_req(struct scsi_request *sreq, const void *cmnd, void *buffer,
266                    unsigned bufflen, int timeout, int retries)
267 {
268         DECLARE_COMPLETION(wait);
269         
270         sreq->sr_request->waiting = &wait;
271         sreq->sr_request->rq_status = RQ_SCSI_BUSY;
272         sreq->sr_request->end_io = scsi_wait_req_end_io;
273         scsi_do_req(sreq, cmnd, buffer, bufflen, scsi_wait_done,
274                         timeout, retries);
275         wait_for_completion(&wait);
276         sreq->sr_request->waiting = NULL;
277         if (sreq->sr_request->rq_status != RQ_SCSI_DONE)
278                 sreq->sr_result |= (DRIVER_ERROR << 24);
279
280         __scsi_release_request(sreq);
281 }
282 EXPORT_SYMBOL(scsi_wait_req);
283
284 /*
285  * Function:    scsi_init_cmd_errh()
286  *
287  * Purpose:     Initialize cmd fields related to error handling.
288  *
289  * Arguments:   cmd     - command that is ready to be queued.
290  *
291  * Returns:     Nothing
292  *
293  * Notes:       This function has the job of initializing a number of
294  *              fields related to error handling.   Typically this will
295  *              be called once for each command, as required.
296  */
297 static int scsi_init_cmd_errh(struct scsi_cmnd *cmd)
298 {
299         cmd->owner = SCSI_OWNER_MIDLEVEL;
300         cmd->serial_number = 0;
301         cmd->abort_reason = 0;
302
303         memset(cmd->sense_buffer, 0, sizeof cmd->sense_buffer);
304
305         if (cmd->cmd_len == 0)
306                 cmd->cmd_len = COMMAND_SIZE(cmd->cmnd[0]);
307
308         /*
309          * We need saved copies of a number of fields - this is because
310          * error handling may need to overwrite these with different values
311          * to run different commands, and once error handling is complete,
312          * we will need to restore these values prior to running the actual
313          * command.
314          */
315         cmd->old_use_sg = cmd->use_sg;
316         cmd->old_cmd_len = cmd->cmd_len;
317         cmd->sc_old_data_direction = cmd->sc_data_direction;
318         cmd->old_underflow = cmd->underflow;
319         memcpy(cmd->data_cmnd, cmd->cmnd, sizeof(cmd->cmnd));
320         cmd->buffer = cmd->request_buffer;
321         cmd->bufflen = cmd->request_bufflen;
322         cmd->abort_reason = 0;
323
324         return 1;
325 }
326
327 /*
328  * Function:   scsi_setup_cmd_retry()
329  *
330  * Purpose:    Restore the command state for a retry
331  *
332  * Arguments:  cmd      - command to be restored
333  *
334  * Returns:    Nothing
335  *
336  * Notes:      Immediately prior to retrying a command, we need
337  *             to restore certain fields that we saved above.
338  */
339 void scsi_setup_cmd_retry(struct scsi_cmnd *cmd)
340 {
341         memcpy(cmd->cmnd, cmd->data_cmnd, sizeof(cmd->data_cmnd));
342         cmd->request_buffer = cmd->buffer;
343         cmd->request_bufflen = cmd->bufflen;
344         cmd->use_sg = cmd->old_use_sg;
345         cmd->cmd_len = cmd->old_cmd_len;
346         cmd->sc_data_direction = cmd->sc_old_data_direction;
347         cmd->underflow = cmd->old_underflow;
348 }
349
350 void scsi_device_unbusy(struct scsi_device *sdev)
351 {
352         struct Scsi_Host *shost = sdev->host;
353         unsigned long flags;
354
355         spin_lock_irqsave(shost->host_lock, flags);
356         shost->host_busy--;
357         if (unlikely(test_bit(SHOST_RECOVERY, &shost->shost_state) &&
358                      shost->host_failed))
359                 scsi_eh_wakeup(shost);
360         spin_unlock(shost->host_lock);
361         spin_lock(sdev->request_queue->queue_lock);
362         sdev->device_busy--;
363         spin_unlock_irqrestore(sdev->request_queue->queue_lock, flags);
364 }
365
366 /*
367  * Called for single_lun devices on IO completion. Clear starget_sdev_user,
368  * and call blk_run_queue for all the scsi_devices on the target -
369  * including current_sdev first.
370  *
371  * Called with *no* scsi locks held.
372  */
373 static void scsi_single_lun_run(struct scsi_device *current_sdev)
374 {
375         struct Scsi_Host *shost = current_sdev->host;
376         struct scsi_device *sdev, *tmp;
377         struct scsi_target *starget = scsi_target(current_sdev);
378         unsigned long flags;
379
380         spin_lock_irqsave(shost->host_lock, flags);
381         starget->starget_sdev_user = NULL;
382         spin_unlock_irqrestore(shost->host_lock, flags);
383
384         /*
385          * Call blk_run_queue for all LUNs on the target, starting with
386          * current_sdev. We race with others (to set starget_sdev_user),
387          * but in most cases, we will be first. Ideally, each LU on the
388          * target would get some limited time or requests on the target.
389          */
390         blk_run_queue(current_sdev->request_queue);
391
392         spin_lock_irqsave(shost->host_lock, flags);
393         if (starget->starget_sdev_user)
394                 goto out;
395         list_for_each_entry_safe(sdev, tmp, &starget->devices,
396                         same_target_siblings) {
397                 if (sdev == current_sdev)
398                         continue;
399                 if (scsi_device_get(sdev))
400                         continue;
401
402                 spin_unlock_irqrestore(shost->host_lock, flags);
403                 blk_run_queue(sdev->request_queue);
404                 spin_lock_irqsave(shost->host_lock, flags);
405         
406                 scsi_device_put(sdev);
407         }
408  out:
409         spin_unlock_irqrestore(shost->host_lock, flags);
410 }
411
412 /*
413  * Function:    scsi_run_queue()
414  *
415  * Purpose:     Select a proper request queue to serve next
416  *
417  * Arguments:   q       - last request's queue
418  *
419  * Returns:     Nothing
420  *
421  * Notes:       The previous command was completely finished, start
422  *              a new one if possible.
423  */
424 static void scsi_run_queue(struct request_queue *q)
425 {
426         struct scsi_device *sdev = q->queuedata;
427         struct Scsi_Host *shost = sdev->host;
428         unsigned long flags;
429
430         if (sdev->single_lun)
431                 scsi_single_lun_run(sdev);
432
433         spin_lock_irqsave(shost->host_lock, flags);
434         while (!list_empty(&shost->starved_list) &&
435                !shost->host_blocked && !shost->host_self_blocked &&
436                 !((shost->can_queue > 0) &&
437                   (shost->host_busy >= shost->can_queue))) {
438                 /*
439                  * As long as shost is accepting commands and we have
440                  * starved queues, call blk_run_queue. scsi_request_fn
441                  * drops the queue_lock and can add us back to the
442                  * starved_list.
443                  *
444                  * host_lock protects the starved_list and starved_entry.
445                  * scsi_request_fn must get the host_lock before checking
446                  * or modifying starved_list or starved_entry.
447                  */
448                 sdev = list_entry(shost->starved_list.next,
449                                           struct scsi_device, starved_entry);
450                 list_del_init(&sdev->starved_entry);
451                 spin_unlock_irqrestore(shost->host_lock, flags);
452
453                 blk_run_queue(sdev->request_queue);
454
455                 spin_lock_irqsave(shost->host_lock, flags);
456                 if (unlikely(!list_empty(&sdev->starved_entry)))
457                         /*
458                          * sdev lost a race, and was put back on the
459                          * starved list. This is unlikely but without this
460                          * in theory we could loop forever.
461                          */
462                         break;
463         }
464         spin_unlock_irqrestore(shost->host_lock, flags);
465
466         blk_run_queue(q);
467 }
468
469 /*
470  * Function:    scsi_requeue_command()
471  *
472  * Purpose:     Handle post-processing of completed commands.
473  *
474  * Arguments:   q       - queue to operate on
475  *              cmd     - command that may need to be requeued.
476  *
477  * Returns:     Nothing
478  *
479  * Notes:       After command completion, there may be blocks left
480  *              over which weren't finished by the previous command
481  *              this can be for a number of reasons - the main one is
482  *              I/O errors in the middle of the request, in which case
483  *              we need to request the blocks that come after the bad
484  *              sector.
485  */
486 static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd)
487 {
488         cmd->request->flags &= ~REQ_DONTPREP;
489         blk_insert_request(q, cmd->request, 1, cmd, 1);
490
491         scsi_run_queue(q);
492 }
493
494 void scsi_next_command(struct scsi_cmnd *cmd)
495 {
496         struct request_queue *q = cmd->device->request_queue;
497
498         scsi_put_command(cmd);
499         scsi_run_queue(q);
500 }
501
502 void scsi_run_host_queues(struct Scsi_Host *shost)
503 {
504         struct scsi_device *sdev;
505
506         shost_for_each_device(sdev, shost)
507                 scsi_run_queue(sdev->request_queue);
508 }
509
510 /*
511  * Function:    scsi_end_request()
512  *
513  * Purpose:     Post-processing of completed commands (usually invoked at end
514  *              of upper level post-processing and scsi_io_completion).
515  *
516  * Arguments:   cmd      - command that is complete.
517  *              uptodate - 1 if I/O indicates success, <= 0 for I/O error.
518  *              bytes    - number of bytes of completed I/O
519  *              requeue  - indicates whether we should requeue leftovers.
520  *
521  * Lock status: Assumed that lock is not held upon entry.
522  *
523  * Returns:     cmd if requeue done or required, NULL otherwise
524  *
525  * Notes:       This is called for block device requests in order to
526  *              mark some number of sectors as complete.
527  * 
528  *              We are guaranteeing that the request queue will be goosed
529  *              at some point during this call.
530  */
531 static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int uptodate,
532                                           int bytes, int requeue)
533 {
534         request_queue_t *q = cmd->device->request_queue;
535         struct request *req = cmd->request;
536         unsigned long flags;
537
538         /*
539          * If there are blocks left over at the end, set up the command
540          * to queue the remainder of them.
541          */
542         if (end_that_request_chunk(req, uptodate, bytes)) {
543                 int leftover = (req->hard_nr_sectors << 9);
544
545                 if (blk_pc_request(req))
546                         leftover = req->data_len;
547
548                 /* kill remainder if no retrys */
549                 if (!uptodate && blk_noretry_request(req))
550                         end_that_request_chunk(req, 0, leftover);
551                 else {
552                         if (requeue)
553                                 /*
554                                  * Bleah.  Leftovers again.  Stick the
555                                  * leftovers in the front of the
556                                  * queue, and goose the queue again.
557                                  */
558                                 scsi_requeue_command(q, cmd);
559
560                         return cmd;
561                 }
562         }
563
564         add_disk_randomness(req->rq_disk);
565
566         spin_lock_irqsave(q->queue_lock, flags);
567         if (blk_rq_tagged(req))
568                 blk_queue_end_tag(q, req);
569         end_that_request_last(req);
570         spin_unlock_irqrestore(q->queue_lock, flags);
571
572         /*
573          * This will goose the queue request function at the end, so we don't
574          * need to worry about launching another command.
575          */
576         scsi_next_command(cmd);
577         return NULL;
578 }
579
580 static struct scatterlist *scsi_alloc_sgtable(struct scsi_cmnd *cmd, int gfp_mask)
581 {
582         struct scsi_host_sg_pool *sgp;
583         struct scatterlist *sgl;
584
585         BUG_ON(!cmd->use_sg);
586
587         switch (cmd->use_sg) {
588         case 1 ... 8:
589                 cmd->sglist_len = 0;
590                 break;
591         case 9 ... 16:
592                 cmd->sglist_len = 1;
593                 break;
594         case 17 ... 32:
595                 cmd->sglist_len = 2;
596                 break;
597 #if (SCSI_MAX_PHYS_SEGMENTS > 32)
598         case 33 ... 64:
599                 cmd->sglist_len = 3;
600                 break;
601 #if (SCSI_MAX_PHYS_SEGMENTS > 64)
602         case 65 ... 128:
603                 cmd->sglist_len = 4;
604                 break;
605 #if (SCSI_MAX_PHYS_SEGMENTS  > 128)
606         case 129 ... 256:
607                 cmd->sglist_len = 5;
608                 break;
609 #endif
610 #endif
611 #endif
612         default:
613                 return NULL;
614         }
615
616         sgp = scsi_sg_pools + cmd->sglist_len;
617         sgl = mempool_alloc(sgp->pool, gfp_mask);
618         if (sgl)
619                 memset(sgl, 0, sgp->size);
620         return sgl;
621 }
622
623 static void scsi_free_sgtable(struct scatterlist *sgl, int index)
624 {
625         struct scsi_host_sg_pool *sgp;
626
627         BUG_ON(index > SG_MEMPOOL_NR);
628
629         sgp = scsi_sg_pools + index;
630         mempool_free(sgl, sgp->pool);
631 }
632
633 /*
634  * Function:    scsi_release_buffers()
635  *
636  * Purpose:     Completion processing for block device I/O requests.
637  *
638  * Arguments:   cmd     - command that we are bailing.
639  *
640  * Lock status: Assumed that no lock is held upon entry.
641  *
642  * Returns:     Nothing
643  *
644  * Notes:       In the event that an upper level driver rejects a
645  *              command, we must release resources allocated during
646  *              the __init_io() function.  Primarily this would involve
647  *              the scatter-gather table, and potentially any bounce
648  *              buffers.
649  */
650 static void scsi_release_buffers(struct scsi_cmnd *cmd)
651 {
652         struct request *req = cmd->request;
653
654         /*
655          * Free up any indirection buffers we allocated for DMA purposes. 
656          */
657         if (cmd->use_sg)
658                 scsi_free_sgtable(cmd->request_buffer, cmd->sglist_len);
659         else if (cmd->request_buffer != req->buffer)
660                 kfree(cmd->request_buffer);
661
662         /*
663          * Zero these out.  They now point to freed memory, and it is
664          * dangerous to hang onto the pointers.
665          */
666         cmd->buffer  = NULL;
667         cmd->bufflen = 0;
668         cmd->request_buffer = NULL;
669         cmd->request_bufflen = 0;
670 }
671
672 /*
673  * Function:    scsi_io_completion()
674  *
675  * Purpose:     Completion processing for block device I/O requests.
676  *
677  * Arguments:   cmd   - command that is finished.
678  *
679  * Lock status: Assumed that no lock is held upon entry.
680  *
681  * Returns:     Nothing
682  *
683  * Notes:       This function is matched in terms of capabilities to
684  *              the function that created the scatter-gather list.
685  *              In other words, if there are no bounce buffers
686  *              (the normal case for most drivers), we don't need
687  *              the logic to deal with cleaning up afterwards.
688  *
689  *              We must do one of several things here:
690  *
691  *              a) Call scsi_end_request.  This will finish off the
692  *                 specified number of sectors.  If we are done, the
693  *                 command block will be released, and the queue
694  *                 function will be goosed.  If we are not done, then
695  *                 scsi_end_request will directly goose the queue.
696  *
697  *              b) We can just use scsi_requeue_command() here.  This would
698  *                 be used if we just wanted to retry, for example.
699  */
700 void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes,
701                         unsigned int block_bytes)
702 {
703         int result = cmd->result;
704         int this_count = cmd->bufflen;
705         request_queue_t *q = cmd->device->request_queue;
706         struct request *req = cmd->request;
707         int clear_errors = 1;
708         struct scsi_sense_hdr sshdr;
709         int sense_valid = 0;
710         int sense_deferred = 0;
711
712         if (blk_complete_barrier_rq(q, req, good_bytes >> 9))
713                 return;
714
715         /*
716          * Free up any indirection buffers we allocated for DMA purposes. 
717          * For the case of a READ, we need to copy the data out of the
718          * bounce buffer and into the real buffer.
719          */
720         if (cmd->use_sg)
721                 scsi_free_sgtable(cmd->buffer, cmd->sglist_len);
722         else if (cmd->buffer != req->buffer) {
723                 if (rq_data_dir(req) == READ) {
724                         unsigned long flags;
725                         char *to = bio_kmap_irq(req->bio, &flags);
726                         memcpy(to, cmd->buffer, cmd->bufflen);
727                         bio_kunmap_irq(to, &flags);
728                 }
729                 kfree(cmd->buffer);
730         }
731
732         if (result) {
733                 sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
734                 if (sense_valid)
735                         sense_deferred = scsi_sense_is_deferred(&sshdr);
736         }
737         if (blk_pc_request(req)) { /* SG_IO ioctl from block level */
738                 req->errors = result;
739                 if (result) {
740                         clear_errors = 0;
741                         if (sense_valid && req->sense) {
742                                 /*
743                                  * SG_IO wants current and deferred errors
744                                  */
745                                 int len = 8 + cmd->sense_buffer[7];
746
747                                 if (len > SCSI_SENSE_BUFFERSIZE)
748                                         len = SCSI_SENSE_BUFFERSIZE;
749                                 memcpy(req->sense, cmd->sense_buffer,  len);
750                                 req->sense_len = len;
751                         }
752                 } else
753                         req->data_len = cmd->resid;
754         }
755
756         /*
757          * Zero these out.  They now point to freed memory, and it is
758          * dangerous to hang onto the pointers.
759          */
760         cmd->buffer  = NULL;
761         cmd->bufflen = 0;
762         cmd->request_buffer = NULL;
763         cmd->request_bufflen = 0;
764
765         /*
766          * Next deal with any sectors which we were able to correctly
767          * handle.
768          */
769         if (good_bytes >= 0) {
770                 SCSI_LOG_HLCOMPLETE(1, printk("%ld sectors total, %d bytes done.\n",
771                                               req->nr_sectors, good_bytes));
772                 SCSI_LOG_HLCOMPLETE(1, printk("use_sg is %d\n", cmd->use_sg));
773
774                 if (clear_errors)
775                         req->errors = 0;
776                 /*
777                  * If multiple sectors are requested in one buffer, then
778                  * they will have been finished off by the first command.
779                  * If not, then we have a multi-buffer command.
780                  *
781                  * If block_bytes != 0, it means we had a medium error
782                  * of some sort, and that we want to mark some number of
783                  * sectors as not uptodate.  Thus we want to inhibit
784                  * requeueing right here - we will requeue down below
785                  * when we handle the bad sectors.
786                  */
787                 cmd = scsi_end_request(cmd, 1, good_bytes, result == 0);
788
789                 /*
790                  * If the command completed without error, then either finish off the
791                  * rest of the command, or start a new one.
792                  */
793                 if (result == 0 || cmd == NULL ) {
794                         return;
795                 }
796         }
797         /*
798          * Now, if we were good little boys and girls, Santa left us a request
799          * sense buffer.  We can extract information from this, so we
800          * can choose a block to remap, etc.
801          */
802         if (sense_valid && !sense_deferred) {
803                 switch (sshdr.sense_key) {
804                 case UNIT_ATTENTION:
805                         if (cmd->device->removable) {
806                                 /* detected disc change.  set a bit 
807                                  * and quietly refuse further access.
808                                  */
809                                 cmd->device->changed = 1;
810                                 cmd = scsi_end_request(cmd, 0,
811                                                 this_count, 1);
812                                 return;
813                         } else {
814                                 /*
815                                 * Must have been a power glitch, or a
816                                 * bus reset.  Could not have been a
817                                 * media change, so we just retry the
818                                 * request and see what happens.  
819                                 */
820                                 scsi_requeue_command(q, cmd);
821                                 return;
822                         }
823                         break;
824                 case ILLEGAL_REQUEST:
825                         /*
826                         * If we had an ILLEGAL REQUEST returned, then we may
827                         * have performed an unsupported command.  The only
828                         * thing this should be would be a ten byte read where
829                         * only a six byte read was supported.  Also, on a
830                         * system where READ CAPACITY failed, we may have read
831                         * past the end of the disk.
832                         */
833                         if (cmd->device->use_10_for_rw &&
834                             (cmd->cmnd[0] == READ_10 ||
835                              cmd->cmnd[0] == WRITE_10)) {
836                                 cmd->device->use_10_for_rw = 0;
837                                 /*
838                                  * This will cause a retry with a 6-byte
839                                  * command.
840                                  */
841                                 scsi_requeue_command(q, cmd);
842                                 result = 0;
843                         } else {
844                                 cmd = scsi_end_request(cmd, 0, this_count, 1);
845                                 return;
846                         }
847                         break;
848                 case NOT_READY:
849                         /*
850                          * If the device is in the process of becoming ready,
851                          * retry.
852                          */
853                         if (sshdr.asc == 0x04 && sshdr.ascq == 0x01) {
854                                 scsi_requeue_command(q, cmd);
855                                 return;
856                         }
857                         printk(KERN_INFO "Device %s not ready.\n",
858                                req->rq_disk ? req->rq_disk->disk_name : "");
859                         cmd = scsi_end_request(cmd, 0, this_count, 1);
860                         return;
861                 case VOLUME_OVERFLOW:
862                         printk(KERN_INFO "Volume overflow <%d %d %d %d> CDB: ",
863                                cmd->device->host->host_no,
864                                (int)cmd->device->channel,
865                                (int)cmd->device->id, (int)cmd->device->lun);
866                         __scsi_print_command(cmd->data_cmnd);
867                         scsi_print_sense("", cmd);
868                         cmd = scsi_end_request(cmd, 0, block_bytes, 1);
869                         return;
870                 default:
871                         break;
872                 }
873         }                       /* driver byte != 0 */
874         if (host_byte(result) == DID_RESET) {
875                 /*
876                  * Third party bus reset or reset for error
877                  * recovery reasons.  Just retry the request
878                  * and see what happens.  
879                  */
880                 scsi_requeue_command(q, cmd);
881                 return;
882         }
883         if (result) {
884                 printk(KERN_INFO "SCSI error : <%d %d %d %d> return code "
885                        "= 0x%x\n", cmd->device->host->host_no,
886                        cmd->device->channel,
887                        cmd->device->id,
888                        cmd->device->lun, result);
889
890                 if (driver_byte(result) & DRIVER_SENSE)
891                         scsi_print_sense("", cmd);
892                 /*
893                  * Mark a single buffer as not uptodate.  Queue the remainder.
894                  * We sometimes get this cruft in the event that a medium error
895                  * isn't properly reported.
896                  */
897                 block_bytes = req->hard_cur_sectors << 9;
898                 if (!block_bytes)
899                         block_bytes = req->data_len;
900                 cmd = scsi_end_request(cmd, 0, block_bytes, 1);
901         }
902 }
903 EXPORT_SYMBOL(scsi_io_completion);
904
905 /*
906  * Function:    scsi_init_io()
907  *
908  * Purpose:     SCSI I/O initialize function.
909  *
910  * Arguments:   cmd   - Command descriptor we wish to initialize
911  *
912  * Returns:     0 on success
913  *              BLKPREP_DEFER if the failure is retryable
914  *              BLKPREP_KILL if the failure is fatal
915  */
916 static int scsi_init_io(struct scsi_cmnd *cmd)
917 {
918         struct request     *req = cmd->request;
919         struct scatterlist *sgpnt;
920         int                count;
921
922         /*
923          * if this is a rq->data based REQ_BLOCK_PC, setup for a non-sg xfer
924          */
925         if ((req->flags & REQ_BLOCK_PC) && !req->bio) {
926                 cmd->request_bufflen = req->data_len;
927                 cmd->request_buffer = req->data;
928                 req->buffer = req->data;
929                 cmd->use_sg = 0;
930                 return 0;
931         }
932
933         /*
934          * we used to not use scatter-gather for single segment request,
935          * but now we do (it makes highmem I/O easier to support without
936          * kmapping pages)
937          */
938         cmd->use_sg = req->nr_phys_segments;
939
940         /*
941          * if sg table allocation fails, requeue request later.
942          */
943         sgpnt = scsi_alloc_sgtable(cmd, GFP_ATOMIC);
944         if (unlikely(!sgpnt))
945                 return BLKPREP_DEFER;
946
947         cmd->request_buffer = (char *) sgpnt;
948         cmd->request_bufflen = req->nr_sectors << 9;
949         if (blk_pc_request(req))
950                 cmd->request_bufflen = req->data_len;
951         req->buffer = NULL;
952
953         /* 
954          * Next, walk the list, and fill in the addresses and sizes of
955          * each segment.
956          */
957         count = blk_rq_map_sg(req->q, req, cmd->request_buffer);
958
959         /*
960          * mapped well, send it off
961          */
962         if (likely(count <= cmd->use_sg)) {
963                 cmd->use_sg = count;
964                 return 0;
965         }
966
967         printk(KERN_ERR "Incorrect number of segments after building list\n");
968         printk(KERN_ERR "counted %d, received %d\n", count, cmd->use_sg);
969         printk(KERN_ERR "req nr_sec %lu, cur_nr_sec %u\n", req->nr_sectors,
970                         req->current_nr_sectors);
971
972         /* release the command and kill it */
973         scsi_release_buffers(cmd);
974         scsi_put_command(cmd);
975         return BLKPREP_KILL;
976 }
977
978 static int scsi_prepare_flush_fn(request_queue_t *q, struct request *rq)
979 {
980         struct scsi_device *sdev = q->queuedata;
981         struct scsi_driver *drv;
982
983         if (sdev->sdev_state == SDEV_RUNNING) {
984                 drv = *(struct scsi_driver **) rq->rq_disk->private_data;
985
986                 if (drv->prepare_flush)
987                         return drv->prepare_flush(q, rq);
988         }
989
990         return 0;
991 }
992
993 static void scsi_end_flush_fn(request_queue_t *q, struct request *rq)
994 {
995         struct scsi_device *sdev = q->queuedata;
996         struct request *flush_rq = rq->end_io_data;
997         struct scsi_driver *drv;
998
999         if (flush_rq->errors) {
1000                 printk("scsi: barrier error, disabling flush support\n");
1001                 blk_queue_ordered(q, QUEUE_ORDERED_NONE);
1002         }
1003
1004         if (sdev->sdev_state == SDEV_RUNNING) {
1005                 drv = *(struct scsi_driver **) rq->rq_disk->private_data;
1006                 drv->end_flush(q, rq);
1007         }
1008 }
1009
1010 static int scsi_issue_flush_fn(request_queue_t *q, struct gendisk *disk,
1011                                sector_t *error_sector)
1012 {
1013         struct scsi_device *sdev = q->queuedata;
1014         struct scsi_driver *drv;
1015
1016         if (sdev->sdev_state != SDEV_RUNNING)
1017                 return -ENXIO;
1018
1019         drv = *(struct scsi_driver **) disk->private_data;
1020         if (drv->issue_flush)
1021                 return drv->issue_flush(&sdev->sdev_gendev, error_sector);
1022
1023         return -EOPNOTSUPP;
1024 }
1025
1026 static int scsi_prep_fn(struct request_queue *q, struct request *req)
1027 {
1028         struct scsi_device *sdev = q->queuedata;
1029         struct scsi_cmnd *cmd;
1030         int specials_only = 0;
1031
1032         /*
1033          * Just check to see if the device is online.  If it isn't, we
1034          * refuse to process any commands.  The device must be brought
1035          * online before trying any recovery commands
1036          */
1037         if (unlikely(!scsi_device_online(sdev))) {
1038                 printk(KERN_ERR "scsi%d (%d:%d): rejecting I/O to offline device\n",
1039                        sdev->host->host_no, sdev->id, sdev->lun);
1040                 return BLKPREP_KILL;
1041         }
1042         if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
1043                 /* OK, we're not in a running state don't prep
1044                  * user commands */
1045                 if (sdev->sdev_state == SDEV_DEL) {
1046                         /* Device is fully deleted, no commands
1047                          * at all allowed down */
1048                         printk(KERN_ERR "scsi%d (%d:%d): rejecting I/O to dead device\n",
1049                                sdev->host->host_no, sdev->id, sdev->lun);
1050                         return BLKPREP_KILL;
1051                 }
1052                 /* OK, we only allow special commands (i.e. not
1053                  * user initiated ones */
1054                 specials_only = sdev->sdev_state;
1055         }
1056
1057         /*
1058          * Find the actual device driver associated with this command.
1059          * The SPECIAL requests are things like character device or
1060          * ioctls, which did not originate from ll_rw_blk.  Note that
1061          * the special field is also used to indicate the cmd for
1062          * the remainder of a partially fulfilled request that can 
1063          * come up when there is a medium error.  We have to treat
1064          * these two cases differently.  We differentiate by looking
1065          * at request->cmd, as this tells us the real story.
1066          */
1067         if (req->flags & REQ_SPECIAL) {
1068                 struct scsi_request *sreq = req->special;
1069
1070                 if (sreq->sr_magic == SCSI_REQ_MAGIC) {
1071                         cmd = scsi_get_command(sreq->sr_device, GFP_ATOMIC);
1072                         if (unlikely(!cmd))
1073                                 goto defer;
1074                         scsi_init_cmd_from_req(cmd, sreq);
1075                 } else
1076                         cmd = req->special;
1077         } else if (req->flags & (REQ_CMD | REQ_BLOCK_PC)) {
1078
1079                 if(unlikely(specials_only)) {
1080                         if(specials_only == SDEV_QUIESCE ||
1081                                         specials_only == SDEV_BLOCK)
1082                                 return BLKPREP_DEFER;
1083                         
1084                         printk(KERN_ERR "scsi%d (%d:%d): rejecting I/O to device being removed\n",
1085                                sdev->host->host_no, sdev->id, sdev->lun);
1086                         return BLKPREP_KILL;
1087                 }
1088                         
1089                         
1090                 /*
1091                  * Now try and find a command block that we can use.
1092                  */
1093                 if (!req->special) {
1094                         cmd = scsi_get_command(sdev, GFP_ATOMIC);
1095                         if (unlikely(!cmd))
1096                                 goto defer;
1097                 } else
1098                         cmd = req->special;
1099                 
1100                 /* pull a tag out of the request if we have one */
1101                 cmd->tag = req->tag;
1102         } else {
1103                 blk_dump_rq_flags(req, "SCSI bad req");
1104                 return BLKPREP_KILL;
1105         }
1106         
1107         /* note the overloading of req->special.  When the tag
1108          * is active it always means cmd.  If the tag goes
1109          * back for re-queueing, it may be reset */
1110         req->special = cmd;
1111         cmd->request = req;
1112         
1113         /*
1114          * FIXME: drop the lock here because the functions below
1115          * expect to be called without the queue lock held.  Also,
1116          * previously, we dequeued the request before dropping the
1117          * lock.  We hope REQ_STARTED prevents anything untoward from
1118          * happening now.
1119          */
1120         if (req->flags & (REQ_CMD | REQ_BLOCK_PC)) {
1121                 struct scsi_driver *drv;
1122                 int ret;
1123
1124                 /*
1125                  * This will do a couple of things:
1126                  *  1) Fill in the actual SCSI command.
1127                  *  2) Fill in any other upper-level specific fields
1128                  * (timeout).
1129                  *
1130                  * If this returns 0, it means that the request failed
1131                  * (reading past end of disk, reading offline device,
1132                  * etc).   This won't actually talk to the device, but
1133                  * some kinds of consistency checking may cause the     
1134                  * request to be rejected immediately.
1135                  */
1136
1137                 /* 
1138                  * This sets up the scatter-gather table (allocating if
1139                  * required).
1140                  */
1141                 ret = scsi_init_io(cmd);
1142                 if (ret)        /* BLKPREP_KILL return also releases the command */
1143                         return ret;
1144                 
1145                 /*
1146                  * Initialize the actual SCSI command for this request.
1147                  */
1148                 drv = *(struct scsi_driver **)req->rq_disk->private_data;
1149                 if (unlikely(!drv->init_command(cmd))) {
1150                         scsi_release_buffers(cmd);
1151                         scsi_put_command(cmd);
1152                         return BLKPREP_KILL;
1153                 }
1154         }
1155
1156         /*
1157          * The request is now prepped, no need to come back here
1158          */
1159         req->flags |= REQ_DONTPREP;
1160         return BLKPREP_OK;
1161
1162  defer:
1163         /* If we defer, the elv_next_request() returns NULL, but the
1164          * queue must be restarted, so we plug here if no returning
1165          * command will automatically do that. */
1166         if (sdev->device_busy == 0)
1167                 blk_plug_device(q);
1168         return BLKPREP_DEFER;
1169 }
1170
1171 /*
1172  * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
1173  * return 0.
1174  *
1175  * Called with the queue_lock held.
1176  */
1177 static inline int scsi_dev_queue_ready(struct request_queue *q,
1178                                   struct scsi_device *sdev)
1179 {
1180         if (sdev->device_busy >= sdev->queue_depth)
1181                 return 0;
1182         if (sdev->device_busy == 0 && sdev->device_blocked) {
1183                 /*
1184                  * unblock after device_blocked iterates to zero
1185                  */
1186                 if (--sdev->device_blocked == 0) {
1187                         SCSI_LOG_MLQUEUE(3,
1188                                 printk("scsi%d (%d:%d) unblocking device at"
1189                                        " zero depth\n", sdev->host->host_no,
1190                                        sdev->id, sdev->lun));
1191                 } else {
1192                         blk_plug_device(q);
1193                         return 0;
1194                 }
1195         }
1196         if (sdev->device_blocked)
1197                 return 0;
1198
1199         return 1;
1200 }
1201
1202 /*
1203  * scsi_host_queue_ready: if we can send requests to shost, return 1 else
1204  * return 0. We must end up running the queue again whenever 0 is
1205  * returned, else IO can hang.
1206  *
1207  * Called with host_lock held.
1208  */
1209 static inline int scsi_host_queue_ready(struct request_queue *q,
1210                                    struct Scsi_Host *shost,
1211                                    struct scsi_device *sdev)
1212 {
1213         if (test_bit(SHOST_RECOVERY, &shost->shost_state))
1214                 return 0;
1215         if (shost->host_busy == 0 && shost->host_blocked) {
1216                 /*
1217                  * unblock after host_blocked iterates to zero
1218                  */
1219                 if (--shost->host_blocked == 0) {
1220                         SCSI_LOG_MLQUEUE(3,
1221                                 printk("scsi%d unblocking host at zero depth\n",
1222                                         shost->host_no));
1223                 } else {
1224                         blk_plug_device(q);
1225                         return 0;
1226                 }
1227         }
1228         if ((shost->can_queue > 0 && shost->host_busy >= shost->can_queue) ||
1229             shost->host_blocked || shost->host_self_blocked) {
1230                 if (list_empty(&sdev->starved_entry))
1231                         list_add_tail(&sdev->starved_entry, &shost->starved_list);
1232                 return 0;
1233         }
1234
1235         /* We're OK to process the command, so we can't be starved */
1236         if (!list_empty(&sdev->starved_entry))
1237                 list_del_init(&sdev->starved_entry);
1238
1239         return 1;
1240 }
1241
1242 /*
1243  * Kill requests for a dead device
1244  */
1245 static void scsi_kill_requests(request_queue_t *q)
1246 {
1247         struct request *req;
1248
1249         while ((req = elv_next_request(q)) != NULL) {
1250                 blkdev_dequeue_request(req);
1251                 req->flags |= REQ_QUIET;
1252                 while (end_that_request_first(req, 0, req->nr_sectors))
1253                         ;
1254                 end_that_request_last(req);
1255         }
1256 }
1257
1258 /*
1259  * Function:    scsi_request_fn()
1260  *
1261  * Purpose:     Main strategy routine for SCSI.
1262  *
1263  * Arguments:   q       - Pointer to actual queue.
1264  *
1265  * Returns:     Nothing
1266  *
1267  * Lock status: IO request lock assumed to be held when called.
1268  */
1269 static void scsi_request_fn(struct request_queue *q)
1270 {
1271         struct scsi_device *sdev = q->queuedata;
1272         struct Scsi_Host *shost;
1273         struct scsi_cmnd *cmd;
1274         struct request *req;
1275
1276         if (!sdev) {
1277                 printk("scsi: killing requests for dead queue\n");
1278                 scsi_kill_requests(q);
1279                 return;
1280         }
1281
1282         if(!get_device(&sdev->sdev_gendev))
1283                 /* We must be tearing the block queue down already */
1284                 return;
1285
1286         /*
1287          * To start with, we keep looping until the queue is empty, or until
1288          * the host is no longer able to accept any more requests.
1289          */
1290         shost = sdev->host;
1291         while (!blk_queue_plugged(q)) {
1292                 int rtn;
1293                 /*
1294                  * get next queueable request.  We do this early to make sure
1295                  * that the request is fully prepared even if we cannot 
1296                  * accept it.
1297                  */
1298                 req = elv_next_request(q);
1299                 if (!req || !scsi_dev_queue_ready(q, sdev))
1300                         break;
1301
1302                 if (unlikely(!scsi_device_online(sdev))) {
1303                         printk(KERN_ERR "scsi%d (%d:%d): rejecting I/O to offline device\n",
1304                                sdev->host->host_no, sdev->id, sdev->lun);
1305                         blkdev_dequeue_request(req);
1306                         req->flags |= REQ_QUIET;
1307                         while (end_that_request_first(req, 0, req->nr_sectors))
1308                                 ;
1309                         end_that_request_last(req);
1310                         continue;
1311                 }
1312
1313
1314                 /*
1315                  * Remove the request from the request list.
1316                  */
1317                 if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
1318                         blkdev_dequeue_request(req);
1319                 sdev->device_busy++;
1320
1321                 spin_unlock(q->queue_lock);
1322                 spin_lock(shost->host_lock);
1323
1324                 if (!scsi_host_queue_ready(q, shost, sdev))
1325                         goto not_ready;
1326                 if (sdev->single_lun) {
1327                         if (scsi_target(sdev)->starget_sdev_user &&
1328                             scsi_target(sdev)->starget_sdev_user != sdev)
1329                                 goto not_ready;
1330                         scsi_target(sdev)->starget_sdev_user = sdev;
1331                 }
1332                 shost->host_busy++;
1333
1334                 /*
1335                  * XXX(hch): This is rather suboptimal, scsi_dispatch_cmd will
1336                  *              take the lock again.
1337                  */
1338                 spin_unlock_irq(shost->host_lock);
1339
1340                 cmd = req->special;
1341                 if (unlikely(cmd == NULL)) {
1342                         printk(KERN_CRIT "impossible request in %s.\n"
1343                                          "please mail a stack trace to "
1344                                          "linux-scsi@vger.kernel.org",
1345                                          __FUNCTION__);
1346                         BUG();
1347                 }
1348
1349                 /*
1350                  * Finally, initialize any error handling parameters, and set up
1351                  * the timers for timeouts.
1352                  */
1353                 scsi_init_cmd_errh(cmd);
1354
1355                 /*
1356                  * Dispatch the command to the low-level driver.
1357                  */
1358                 rtn = scsi_dispatch_cmd(cmd);
1359                 spin_lock_irq(q->queue_lock);
1360                 if(rtn) {
1361                         /* we're refusing the command; because of
1362                          * the way locks get dropped, we need to 
1363                          * check here if plugging is required */
1364                         if(sdev->device_busy == 0)
1365                                 blk_plug_device(q);
1366
1367                         break;
1368                 }
1369         }
1370
1371         goto out;
1372
1373  not_ready:
1374         spin_unlock_irq(shost->host_lock);
1375
1376         /*
1377          * lock q, handle tag, requeue req, and decrement device_busy. We
1378          * must return with queue_lock held.
1379          *
1380          * Decrementing device_busy without checking it is OK, as all such
1381          * cases (host limits or settings) should run the queue at some
1382          * later time.
1383          */
1384         spin_lock_irq(q->queue_lock);
1385         blk_requeue_request(q, req);
1386         sdev->device_busy--;
1387         if(sdev->device_busy == 0)
1388                 blk_plug_device(q);
1389  out:
1390         /* must be careful here...if we trigger the ->remove() function
1391          * we cannot be holding the q lock */
1392         spin_unlock_irq(q->queue_lock);
1393         put_device(&sdev->sdev_gendev);
1394         spin_lock_irq(q->queue_lock);
1395 }
1396
1397 u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
1398 {
1399         struct device *host_dev;
1400         u64 bounce_limit = 0xffffffff;
1401
1402         if (shost->unchecked_isa_dma)
1403                 return BLK_BOUNCE_ISA;
1404         /*
1405          * Platforms with virtual-DMA translation
1406          * hardware have no practical limit.
1407          */
1408         if (!PCI_DMA_BUS_IS_PHYS)
1409                 return BLK_BOUNCE_ANY;
1410
1411         host_dev = scsi_get_device(shost);
1412         if (host_dev && host_dev->dma_mask)
1413                 bounce_limit = *host_dev->dma_mask;
1414
1415         return bounce_limit;
1416 }
1417 EXPORT_SYMBOL(scsi_calculate_bounce_limit);
1418
1419 struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
1420 {
1421         struct Scsi_Host *shost = sdev->host;
1422         struct request_queue *q;
1423
1424         q = blk_init_queue(scsi_request_fn, NULL);
1425         if (!q)
1426                 return NULL;
1427
1428         blk_queue_prep_rq(q, scsi_prep_fn);
1429
1430         blk_queue_max_hw_segments(q, shost->sg_tablesize);
1431         blk_queue_max_phys_segments(q, SCSI_MAX_PHYS_SEGMENTS);
1432         blk_queue_max_sectors(q, shost->max_sectors);
1433         blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
1434         blk_queue_segment_boundary(q, shost->dma_boundary);
1435         blk_queue_issue_flush_fn(q, scsi_issue_flush_fn);
1436
1437         /*
1438          * ordered tags are superior to flush ordering
1439          */
1440         if (shost->ordered_tag)
1441                 blk_queue_ordered(q, QUEUE_ORDERED_TAG);
1442         else if (shost->ordered_flush) {
1443                 blk_queue_ordered(q, QUEUE_ORDERED_FLUSH);
1444                 q->prepare_flush_fn = scsi_prepare_flush_fn;
1445                 q->end_flush_fn = scsi_end_flush_fn;
1446         }
1447
1448         if (!shost->use_clustering)
1449                 clear_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags);
1450         return q;
1451 }
1452
1453 void scsi_free_queue(struct request_queue *q)
1454 {
1455         blk_cleanup_queue(q);
1456 }
1457
1458 /*
1459  * Function:    scsi_block_requests()
1460  *
1461  * Purpose:     Utility function used by low-level drivers to prevent further
1462  *              commands from being queued to the device.
1463  *
1464  * Arguments:   shost       - Host in question
1465  *
1466  * Returns:     Nothing
1467  *
1468  * Lock status: No locks are assumed held.
1469  *
1470  * Notes:       There is no timer nor any other means by which the requests
1471  *              get unblocked other than the low-level driver calling
1472  *              scsi_unblock_requests().
1473  */
1474 void scsi_block_requests(struct Scsi_Host *shost)
1475 {
1476         shost->host_self_blocked = 1;
1477 }
1478 EXPORT_SYMBOL(scsi_block_requests);
1479
1480 /*
1481  * Function:    scsi_unblock_requests()
1482  *
1483  * Purpose:     Utility function used by low-level drivers to allow further
1484  *              commands from being queued to the device.
1485  *
1486  * Arguments:   shost       - Host in question
1487  *
1488  * Returns:     Nothing
1489  *
1490  * Lock status: No locks are assumed held.
1491  *
1492  * Notes:       There is no timer nor any other means by which the requests
1493  *              get unblocked other than the low-level driver calling
1494  *              scsi_unblock_requests().
1495  *
1496  *              This is done as an API function so that changes to the
1497  *              internals of the scsi mid-layer won't require wholesale
1498  *              changes to drivers that use this feature.
1499  */
1500 void scsi_unblock_requests(struct Scsi_Host *shost)
1501 {
1502         shost->host_self_blocked = 0;
1503         scsi_run_host_queues(shost);
1504 }
1505 EXPORT_SYMBOL(scsi_unblock_requests);
1506
1507 int __init scsi_init_queue(void)
1508 {
1509         int i;
1510
1511         for (i = 0; i < SG_MEMPOOL_NR; i++) {
1512                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1513                 int size = sgp->size * sizeof(struct scatterlist);
1514
1515                 sgp->slab = kmem_cache_create(sgp->name, size, 0,
1516                                 SLAB_HWCACHE_ALIGN, NULL, NULL);
1517                 if (!sgp->slab) {
1518                         printk(KERN_ERR "SCSI: can't init sg slab %s\n",
1519                                         sgp->name);
1520                 }
1521
1522                 sgp->pool = mempool_create(SG_MEMPOOL_SIZE,
1523                                 mempool_alloc_slab, mempool_free_slab,
1524                                 sgp->slab);
1525                 if (!sgp->pool) {
1526                         printk(KERN_ERR "SCSI: can't init sg mempool %s\n",
1527                                         sgp->name);
1528                 }
1529         }
1530
1531         return 0;
1532 }
1533
1534 void scsi_exit_queue(void)
1535 {
1536         int i;
1537
1538         for (i = 0; i < SG_MEMPOOL_NR; i++) {
1539                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1540                 mempool_destroy(sgp->pool);
1541                 kmem_cache_destroy(sgp->slab);
1542         }
1543 }
1544 /**
1545  *      __scsi_mode_sense - issue a mode sense, falling back from 10 to 
1546  *              six bytes if necessary.
1547  *      @sreq:  SCSI request to fill in with the MODE_SENSE
1548  *      @dbd:   set if mode sense will allow block descriptors to be returned
1549  *      @modepage: mode page being requested
1550  *      @buffer: request buffer (may not be smaller than eight bytes)
1551  *      @len:   length of request buffer.
1552  *      @timeout: command timeout
1553  *      @retries: number of retries before failing
1554  *      @data: returns a structure abstracting the mode header data
1555  *
1556  *      Returns zero if unsuccessful, or the header offset (either 4
1557  *      or 8 depending on whether a six or ten byte command was
1558  *      issued) if successful.
1559  **/
1560 int
1561 __scsi_mode_sense(struct scsi_request *sreq, int dbd, int modepage,
1562                   unsigned char *buffer, int len, int timeout, int retries,
1563                   struct scsi_mode_data *data) {
1564         unsigned char cmd[12];
1565         int use_10_for_ms;
1566         int header_length;
1567
1568         memset(data, 0, sizeof(*data));
1569         memset(&cmd[0], 0, 12);
1570         cmd[1] = dbd & 0x18;    /* allows DBD and LLBA bits */
1571         cmd[2] = modepage;
1572
1573  retry:
1574         use_10_for_ms = sreq->sr_device->use_10_for_ms;
1575
1576         if (use_10_for_ms) {
1577                 if (len < 8)
1578                         len = 8;
1579
1580                 cmd[0] = MODE_SENSE_10;
1581                 cmd[8] = len;
1582                 header_length = 8;
1583         } else {
1584                 if (len < 4)
1585                         len = 4;
1586
1587                 cmd[0] = MODE_SENSE;
1588                 cmd[4] = len;
1589                 header_length = 4;
1590         }
1591
1592         sreq->sr_cmd_len = 0;
1593         memset(sreq->sr_sense_buffer, 0, sizeof(sreq->sr_sense_buffer));
1594         sreq->sr_data_direction = DMA_FROM_DEVICE;
1595
1596         memset(buffer, 0, len);
1597
1598         scsi_wait_req(sreq, cmd, buffer, len, timeout, retries);
1599
1600         /* This code looks awful: what it's doing is making sure an
1601          * ILLEGAL REQUEST sense return identifies the actual command
1602          * byte as the problem.  MODE_SENSE commands can return
1603          * ILLEGAL REQUEST if the code page isn't supported */
1604
1605         if (use_10_for_ms && !scsi_status_is_good(sreq->sr_result) &&
1606             (driver_byte(sreq->sr_result) & DRIVER_SENSE)) {
1607                 struct scsi_sense_hdr sshdr;
1608
1609                 if (scsi_request_normalize_sense(sreq, &sshdr)) {
1610                         if ((sshdr.sense_key == ILLEGAL_REQUEST) &&
1611                             (sshdr.asc == 0x20) && (sshdr.ascq == 0)) {
1612                                 /* 
1613                                  * Invalid command operation code
1614                                  */
1615                                 sreq->sr_device->use_10_for_ms = 0;
1616                                 goto retry;
1617                         }
1618                 }
1619         }
1620
1621         if(scsi_status_is_good(sreq->sr_result)) {
1622                 data->header_length = header_length;
1623                 if(use_10_for_ms) {
1624                         data->length = buffer[0]*256 + buffer[1] + 2;
1625                         data->medium_type = buffer[2];
1626                         data->device_specific = buffer[3];
1627                         data->longlba = buffer[4] & 0x01;
1628                         data->block_descriptor_length = buffer[6]*256
1629                                 + buffer[7];
1630                 } else {
1631                         data->length = buffer[0] + 1;
1632                         data->medium_type = buffer[1];
1633                         data->device_specific = buffer[2];
1634                         data->block_descriptor_length = buffer[3];
1635                 }
1636         }
1637
1638         return sreq->sr_result;
1639 }
1640 EXPORT_SYMBOL(__scsi_mode_sense);
1641
1642 /**
1643  *      scsi_mode_sense - issue a mode sense, falling back from 10 to 
1644  *              six bytes if necessary.
1645  *      @sdev:  scsi device to send command to.
1646  *      @dbd:   set if mode sense will disable block descriptors in the return
1647  *      @modepage: mode page being requested
1648  *      @buffer: request buffer (may not be smaller than eight bytes)
1649  *      @len:   length of request buffer.
1650  *      @timeout: command timeout
1651  *      @retries: number of retries before failing
1652  *
1653  *      Returns zero if unsuccessful, or the header offset (either 4
1654  *      or 8 depending on whether a six or ten byte command was
1655  *      issued) if successful.
1656  **/
1657 int
1658 scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
1659                 unsigned char *buffer, int len, int timeout, int retries,
1660                 struct scsi_mode_data *data)
1661 {
1662         struct scsi_request *sreq = scsi_allocate_request(sdev, GFP_KERNEL);
1663         int ret;
1664
1665         if (!sreq)
1666                 return -1;
1667
1668         ret = __scsi_mode_sense(sreq, dbd, modepage, buffer, len,
1669                                 timeout, retries, data);
1670
1671         scsi_release_request(sreq);
1672
1673         return ret;
1674 }
1675 EXPORT_SYMBOL(scsi_mode_sense);
1676
1677 int
1678 scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries)
1679 {
1680         struct scsi_request *sreq;
1681         char cmd[] = {
1682                 TEST_UNIT_READY, 0, 0, 0, 0, 0,
1683         };
1684         int result;
1685         
1686         sreq = scsi_allocate_request(sdev, GFP_KERNEL);
1687         if (!sreq)
1688                 return -ENOMEM;
1689
1690         sreq->sr_data_direction = DMA_NONE;
1691         scsi_wait_req(sreq, cmd, NULL, 0, timeout, retries);
1692
1693         if ((driver_byte(sreq->sr_result) & DRIVER_SENSE) && sdev->removable) {
1694                 struct scsi_sense_hdr sshdr;
1695
1696                 if ((scsi_request_normalize_sense(sreq, &sshdr)) &&
1697                     ((sshdr.sense_key == UNIT_ATTENTION) ||
1698                      (sshdr.sense_key == NOT_READY))) {
1699                         sdev->changed = 1;
1700                         sreq->sr_result = 0;
1701                 }
1702         }
1703         result = sreq->sr_result;
1704         scsi_release_request(sreq);
1705         return result;
1706 }
1707 EXPORT_SYMBOL(scsi_test_unit_ready);
1708
1709 /**
1710  *      scsi_device_set_state - Take the given device through the device
1711  *              state model.
1712  *      @sdev:  scsi device to change the state of.
1713  *      @state: state to change to.
1714  *
1715  *      Returns zero if unsuccessful or an error if the requested 
1716  *      transition is illegal.
1717  **/
1718 int
1719 scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
1720 {
1721         enum scsi_device_state oldstate = sdev->sdev_state;
1722
1723         if (state == oldstate)
1724                 return 0;
1725
1726         switch (state) {
1727         case SDEV_CREATED:
1728                 /* There are no legal states that come back to
1729                  * created.  This is the manually initialised start
1730                  * state */
1731                 goto illegal;
1732                         
1733         case SDEV_RUNNING:
1734                 switch (oldstate) {
1735                 case SDEV_CREATED:
1736                 case SDEV_OFFLINE:
1737                 case SDEV_QUIESCE:
1738                 case SDEV_BLOCK:
1739                         break;
1740                 default:
1741                         goto illegal;
1742                 }
1743                 break;
1744
1745         case SDEV_QUIESCE:
1746                 switch (oldstate) {
1747                 case SDEV_RUNNING:
1748                 case SDEV_OFFLINE:
1749                         break;
1750                 default:
1751                         goto illegal;
1752                 }
1753                 break;
1754
1755         case SDEV_OFFLINE:
1756                 switch (oldstate) {
1757                 case SDEV_CREATED:
1758                 case SDEV_RUNNING:
1759                 case SDEV_QUIESCE:
1760                 case SDEV_BLOCK:
1761                         break;
1762                 default:
1763                         goto illegal;
1764                 }
1765                 break;
1766
1767         case SDEV_BLOCK:
1768                 switch (oldstate) {
1769                 case SDEV_CREATED:
1770                 case SDEV_RUNNING:
1771                         break;
1772                 default:
1773                         goto illegal;
1774                 }
1775                 break;
1776
1777         case SDEV_CANCEL:
1778                 switch (oldstate) {
1779                 case SDEV_CREATED:
1780                 case SDEV_RUNNING:
1781                 case SDEV_OFFLINE:
1782                 case SDEV_BLOCK:
1783                         break;
1784                 default:
1785                         goto illegal;
1786                 }
1787                 break;
1788
1789         case SDEV_DEL:
1790                 switch (oldstate) {
1791                 case SDEV_CANCEL:
1792                         break;
1793                 default:
1794                         goto illegal;
1795                 }
1796                 break;
1797
1798         }
1799         sdev->sdev_state = state;
1800         return 0;
1801
1802  illegal:
1803         SCSI_LOG_ERROR_RECOVERY(1, 
1804                                 dev_printk(KERN_ERR, &sdev->sdev_gendev,
1805                                            "Illegal state transition %s->%s\n",
1806                                            scsi_device_state_name(oldstate),
1807                                            scsi_device_state_name(state))
1808                                 );
1809         return -EINVAL;
1810 }
1811 EXPORT_SYMBOL(scsi_device_set_state);
1812
1813 /**
1814  *      scsi_device_quiesce - Block user issued commands.
1815  *      @sdev:  scsi device to quiesce.
1816  *
1817  *      This works by trying to transition to the SDEV_QUIESCE state
1818  *      (which must be a legal transition).  When the device is in this
1819  *      state, only special requests will be accepted, all others will
1820  *      be deferred.  Since special requests may also be requeued requests,
1821  *      a successful return doesn't guarantee the device will be 
1822  *      totally quiescent.
1823  *
1824  *      Must be called with user context, may sleep.
1825  *
1826  *      Returns zero if unsuccessful or an error if not.
1827  **/
1828 int
1829 scsi_device_quiesce(struct scsi_device *sdev)
1830 {
1831         int err = scsi_device_set_state(sdev, SDEV_QUIESCE);
1832         if (err)
1833                 return err;
1834
1835         scsi_run_queue(sdev->request_queue);
1836         while (sdev->device_busy) {
1837                 msleep_interruptible(200);
1838                 scsi_run_queue(sdev->request_queue);
1839         }
1840         return 0;
1841 }
1842 EXPORT_SYMBOL(scsi_device_quiesce);
1843
1844 /**
1845  *      scsi_device_resume - Restart user issued commands to a quiesced device.
1846  *      @sdev:  scsi device to resume.
1847  *
1848  *      Moves the device from quiesced back to running and restarts the
1849  *      queues.
1850  *
1851  *      Must be called with user context, may sleep.
1852  **/
1853 void
1854 scsi_device_resume(struct scsi_device *sdev)
1855 {
1856         if(scsi_device_set_state(sdev, SDEV_RUNNING))
1857                 return;
1858         scsi_run_queue(sdev->request_queue);
1859 }
1860 EXPORT_SYMBOL(scsi_device_resume);
1861
1862 static void
1863 device_quiesce_fn(struct scsi_device *sdev, void *data)
1864 {
1865         scsi_device_quiesce(sdev);
1866 }
1867
1868 void
1869 scsi_target_quiesce(struct scsi_target *starget)
1870 {
1871         starget_for_each_device(starget, NULL, device_quiesce_fn);
1872 }
1873 EXPORT_SYMBOL(scsi_target_quiesce);
1874
1875 static void
1876 device_resume_fn(struct scsi_device *sdev, void *data)
1877 {
1878         scsi_device_resume(sdev);
1879 }
1880
1881 void
1882 scsi_target_resume(struct scsi_target *starget)
1883 {
1884         starget_for_each_device(starget, NULL, device_resume_fn);
1885 }
1886 EXPORT_SYMBOL(scsi_target_resume);
1887
1888 /**
1889  * scsi_internal_device_block - internal function to put a device
1890  *                              temporarily into the SDEV_BLOCK state
1891  * @sdev:       device to block
1892  *
1893  * Block request made by scsi lld's to temporarily stop all
1894  * scsi commands on the specified device.  Called from interrupt
1895  * or normal process context.
1896  *
1897  * Returns zero if successful or error if not
1898  *
1899  * Notes:       
1900  *      This routine transitions the device to the SDEV_BLOCK state
1901  *      (which must be a legal transition).  When the device is in this
1902  *      state, all commands are deferred until the scsi lld reenables
1903  *      the device with scsi_device_unblock or device_block_tmo fires.
1904  *      This routine assumes the host_lock is held on entry.
1905  **/
1906 int
1907 scsi_internal_device_block(struct scsi_device *sdev)
1908 {
1909         request_queue_t *q = sdev->request_queue;
1910         unsigned long flags;
1911         int err = 0;
1912
1913         err = scsi_device_set_state(sdev, SDEV_BLOCK);
1914         if (err)
1915                 return err;
1916
1917         /* 
1918          * The device has transitioned to SDEV_BLOCK.  Stop the
1919          * block layer from calling the midlayer with this device's
1920          * request queue. 
1921          */
1922         spin_lock_irqsave(q->queue_lock, flags);
1923         blk_stop_queue(q);
1924         spin_unlock_irqrestore(q->queue_lock, flags);
1925
1926         return 0;
1927 }
1928 EXPORT_SYMBOL_GPL(scsi_internal_device_block);
1929  
1930 /**
1931  * scsi_internal_device_unblock - resume a device after a block request
1932  * @sdev:       device to resume
1933  *
1934  * Called by scsi lld's or the midlayer to restart the device queue
1935  * for the previously suspended scsi device.  Called from interrupt or
1936  * normal process context.
1937  *
1938  * Returns zero if successful or error if not.
1939  *
1940  * Notes:       
1941  *      This routine transitions the device to the SDEV_RUNNING state
1942  *      (which must be a legal transition) allowing the midlayer to
1943  *      goose the queue for this device.  This routine assumes the 
1944  *      host_lock is held upon entry.
1945  **/
1946 int
1947 scsi_internal_device_unblock(struct scsi_device *sdev)
1948 {
1949         request_queue_t *q = sdev->request_queue; 
1950         int err;
1951         unsigned long flags;
1952         
1953         /* 
1954          * Try to transition the scsi device to SDEV_RUNNING
1955          * and goose the device queue if successful.  
1956          */
1957         err = scsi_device_set_state(sdev, SDEV_RUNNING);
1958         if (err)
1959                 return err;
1960
1961         spin_lock_irqsave(q->queue_lock, flags);
1962         blk_start_queue(q);
1963         spin_unlock_irqrestore(q->queue_lock, flags);
1964
1965         return 0;
1966 }
1967 EXPORT_SYMBOL_GPL(scsi_internal_device_unblock);
1968
1969 static void
1970 device_block(struct scsi_device *sdev, void *data)
1971 {
1972         scsi_internal_device_block(sdev);
1973 }
1974
1975 static int
1976 target_block(struct device *dev, void *data)
1977 {
1978         if (scsi_is_target_device(dev))
1979                 starget_for_each_device(to_scsi_target(dev), NULL,
1980                                         device_block);
1981         return 0;
1982 }
1983
1984 void
1985 scsi_target_block(struct device *dev)
1986 {
1987         if (scsi_is_target_device(dev))
1988                 starget_for_each_device(to_scsi_target(dev), NULL,
1989                                         device_block);
1990         else
1991                 device_for_each_child(dev, NULL, target_block);
1992 }
1993 EXPORT_SYMBOL_GPL(scsi_target_block);
1994
1995 static void
1996 device_unblock(struct scsi_device *sdev, void *data)
1997 {
1998         scsi_internal_device_unblock(sdev);
1999 }
2000
2001 static int
2002 target_unblock(struct device *dev, void *data)
2003 {
2004         if (scsi_is_target_device(dev))
2005                 starget_for_each_device(to_scsi_target(dev), NULL,
2006                                         device_unblock);
2007         return 0;
2008 }
2009
2010 void
2011 scsi_target_unblock(struct device *dev)
2012 {
2013         if (scsi_is_target_device(dev))
2014                 starget_for_each_device(to_scsi_target(dev), NULL,
2015                                         device_unblock);
2016         else
2017                 device_for_each_child(dev, NULL, target_unblock);
2018 }
2019 EXPORT_SYMBOL_GPL(scsi_target_unblock);