Merge branch 'core-iommu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[linux-2.6.git] / drivers / scsi / scsi_lib.c
1 /*
2  *  scsi_lib.c Copyright (C) 1999 Eric Youngdale
3  *
4  *  SCSI queueing library.
5  *      Initial versions: Eric Youngdale (eric@andante.org).
6  *                        Based upon conversations with large numbers
7  *                        of people at Linux Expo.
8  */
9
10 #include <linux/bio.h>
11 #include <linux/bitops.h>
12 #include <linux/blkdev.h>
13 #include <linux/completion.h>
14 #include <linux/kernel.h>
15 #include <linux/mempool.h>
16 #include <linux/slab.h>
17 #include <linux/init.h>
18 #include <linux/pci.h>
19 #include <linux/delay.h>
20 #include <linux/hardirq.h>
21 #include <linux/scatterlist.h>
22
23 #include <scsi/scsi.h>
24 #include <scsi/scsi_cmnd.h>
25 #include <scsi/scsi_dbg.h>
26 #include <scsi/scsi_device.h>
27 #include <scsi/scsi_driver.h>
28 #include <scsi/scsi_eh.h>
29 #include <scsi/scsi_host.h>
30
31 #include "scsi_priv.h"
32 #include "scsi_logging.h"
33
34
35 #define SG_MEMPOOL_NR           ARRAY_SIZE(scsi_sg_pools)
36 #define SG_MEMPOOL_SIZE         2
37
38 struct scsi_host_sg_pool {
39         size_t          size;
40         char            *name;
41         struct kmem_cache       *slab;
42         mempool_t       *pool;
43 };
44
45 #define SP(x) { x, "sgpool-" __stringify(x) }
46 #if (SCSI_MAX_SG_SEGMENTS < 32)
47 #error SCSI_MAX_SG_SEGMENTS is too small (must be 32 or greater)
48 #endif
49 static struct scsi_host_sg_pool scsi_sg_pools[] = {
50         SP(8),
51         SP(16),
52 #if (SCSI_MAX_SG_SEGMENTS > 32)
53         SP(32),
54 #if (SCSI_MAX_SG_SEGMENTS > 64)
55         SP(64),
56 #if (SCSI_MAX_SG_SEGMENTS > 128)
57         SP(128),
58 #if (SCSI_MAX_SG_SEGMENTS > 256)
59 #error SCSI_MAX_SG_SEGMENTS is too large (256 MAX)
60 #endif
61 #endif
62 #endif
63 #endif
64         SP(SCSI_MAX_SG_SEGMENTS)
65 };
66 #undef SP
67
68 struct kmem_cache *scsi_sdb_cache;
69
70 /*
71  * When to reinvoke queueing after a resource shortage. It's 3 msecs to
72  * not change behaviour from the previous unplug mechanism, experimentation
73  * may prove this needs changing.
74  */
75 #define SCSI_QUEUE_DELAY        3
76
77 /*
78  * Function:    scsi_unprep_request()
79  *
80  * Purpose:     Remove all preparation done for a request, including its
81  *              associated scsi_cmnd, so that it can be requeued.
82  *
83  * Arguments:   req     - request to unprepare
84  *
85  * Lock status: Assumed that no locks are held upon entry.
86  *
87  * Returns:     Nothing.
88  */
89 static void scsi_unprep_request(struct request *req)
90 {
91         struct scsi_cmnd *cmd = req->special;
92
93         blk_unprep_request(req);
94         req->special = NULL;
95
96         scsi_put_command(cmd);
97 }
98
99 /**
100  * __scsi_queue_insert - private queue insertion
101  * @cmd: The SCSI command being requeued
102  * @reason:  The reason for the requeue
103  * @unbusy: Whether the queue should be unbusied
104  *
105  * This is a private queue insertion.  The public interface
106  * scsi_queue_insert() always assumes the queue should be unbusied
107  * because it's always called before the completion.  This function is
108  * for a requeue after completion, which should only occur in this
109  * file.
110  */
111 static int __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy)
112 {
113         struct Scsi_Host *host = cmd->device->host;
114         struct scsi_device *device = cmd->device;
115         struct scsi_target *starget = scsi_target(device);
116         struct request_queue *q = device->request_queue;
117         unsigned long flags;
118
119         SCSI_LOG_MLQUEUE(1,
120                  printk("Inserting command %p into mlqueue\n", cmd));
121
122         /*
123          * Set the appropriate busy bit for the device/host.
124          *
125          * If the host/device isn't busy, assume that something actually
126          * completed, and that we should be able to queue a command now.
127          *
128          * Note that the prior mid-layer assumption that any host could
129          * always queue at least one command is now broken.  The mid-layer
130          * will implement a user specifiable stall (see
131          * scsi_host.max_host_blocked and scsi_device.max_device_blocked)
132          * if a command is requeued with no other commands outstanding
133          * either for the device or for the host.
134          */
135         switch (reason) {
136         case SCSI_MLQUEUE_HOST_BUSY:
137                 host->host_blocked = host->max_host_blocked;
138                 break;
139         case SCSI_MLQUEUE_DEVICE_BUSY:
140                 device->device_blocked = device->max_device_blocked;
141                 break;
142         case SCSI_MLQUEUE_TARGET_BUSY:
143                 starget->target_blocked = starget->max_target_blocked;
144                 break;
145         }
146
147         /*
148          * Decrement the counters, since these commands are no longer
149          * active on the host/device.
150          */
151         if (unbusy)
152                 scsi_device_unbusy(device);
153
154         /*
155          * Requeue this command.  It will go before all other commands
156          * that are already in the queue.
157          */
158         spin_lock_irqsave(q->queue_lock, flags);
159         blk_requeue_request(q, cmd->request);
160         spin_unlock_irqrestore(q->queue_lock, flags);
161
162         kblockd_schedule_work(q, &device->requeue_work);
163
164         return 0;
165 }
166
167 /*
168  * Function:    scsi_queue_insert()
169  *
170  * Purpose:     Insert a command in the midlevel queue.
171  *
172  * Arguments:   cmd    - command that we are adding to queue.
173  *              reason - why we are inserting command to queue.
174  *
175  * Lock status: Assumed that lock is not held upon entry.
176  *
177  * Returns:     Nothing.
178  *
179  * Notes:       We do this for one of two cases.  Either the host is busy
180  *              and it cannot accept any more commands for the time being,
181  *              or the device returned QUEUE_FULL and can accept no more
182  *              commands.
183  * Notes:       This could be called either from an interrupt context or a
184  *              normal process context.
185  */
186 int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
187 {
188         return __scsi_queue_insert(cmd, reason, 1);
189 }
190 /**
191  * scsi_execute - insert request and wait for the result
192  * @sdev:       scsi device
193  * @cmd:        scsi command
194  * @data_direction: data direction
195  * @buffer:     data buffer
196  * @bufflen:    len of buffer
197  * @sense:      optional sense buffer
198  * @timeout:    request timeout in seconds
199  * @retries:    number of times to retry request
200  * @flags:      or into request flags;
201  * @resid:      optional residual length
202  *
203  * returns the req->errors value which is the scsi_cmnd result
204  * field.
205  */
206 int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
207                  int data_direction, void *buffer, unsigned bufflen,
208                  unsigned char *sense, int timeout, int retries, int flags,
209                  int *resid)
210 {
211         struct request *req;
212         int write = (data_direction == DMA_TO_DEVICE);
213         int ret = DRIVER_ERROR << 24;
214
215         req = blk_get_request(sdev->request_queue, write, __GFP_WAIT);
216
217         if (bufflen &&  blk_rq_map_kern(sdev->request_queue, req,
218                                         buffer, bufflen, __GFP_WAIT))
219                 goto out;
220
221         req->cmd_len = COMMAND_SIZE(cmd[0]);
222         memcpy(req->cmd, cmd, req->cmd_len);
223         req->sense = sense;
224         req->sense_len = 0;
225         req->retries = retries;
226         req->timeout = timeout;
227         req->cmd_type = REQ_TYPE_BLOCK_PC;
228         req->cmd_flags |= flags | REQ_QUIET | REQ_PREEMPT;
229
230         /*
231          * head injection *required* here otherwise quiesce won't work
232          */
233         blk_execute_rq(req->q, NULL, req, 1);
234
235         /*
236          * Some devices (USB mass-storage in particular) may transfer
237          * garbage data together with a residue indicating that the data
238          * is invalid.  Prevent the garbage from being misinterpreted
239          * and prevent security leaks by zeroing out the excess data.
240          */
241         if (unlikely(req->resid_len > 0 && req->resid_len <= bufflen))
242                 memset(buffer + (bufflen - req->resid_len), 0, req->resid_len);
243
244         if (resid)
245                 *resid = req->resid_len;
246         ret = req->errors;
247  out:
248         blk_put_request(req);
249
250         return ret;
251 }
252 EXPORT_SYMBOL(scsi_execute);
253
254
255 int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd,
256                      int data_direction, void *buffer, unsigned bufflen,
257                      struct scsi_sense_hdr *sshdr, int timeout, int retries,
258                      int *resid)
259 {
260         char *sense = NULL;
261         int result;
262         
263         if (sshdr) {
264                 sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO);
265                 if (!sense)
266                         return DRIVER_ERROR << 24;
267         }
268         result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen,
269                               sense, timeout, retries, 0, resid);
270         if (sshdr)
271                 scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr);
272
273         kfree(sense);
274         return result;
275 }
276 EXPORT_SYMBOL(scsi_execute_req);
277
278 /*
279  * Function:    scsi_init_cmd_errh()
280  *
281  * Purpose:     Initialize cmd fields related to error handling.
282  *
283  * Arguments:   cmd     - command that is ready to be queued.
284  *
285  * Notes:       This function has the job of initializing a number of
286  *              fields related to error handling.   Typically this will
287  *              be called once for each command, as required.
288  */
289 static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)
290 {
291         cmd->serial_number = 0;
292         scsi_set_resid(cmd, 0);
293         memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
294         if (cmd->cmd_len == 0)
295                 cmd->cmd_len = scsi_command_size(cmd->cmnd);
296 }
297
298 void scsi_device_unbusy(struct scsi_device *sdev)
299 {
300         struct Scsi_Host *shost = sdev->host;
301         struct scsi_target *starget = scsi_target(sdev);
302         unsigned long flags;
303
304         spin_lock_irqsave(shost->host_lock, flags);
305         shost->host_busy--;
306         starget->target_busy--;
307         if (unlikely(scsi_host_in_recovery(shost) &&
308                      (shost->host_failed || shost->host_eh_scheduled)))
309                 scsi_eh_wakeup(shost);
310         spin_unlock(shost->host_lock);
311         spin_lock(sdev->request_queue->queue_lock);
312         sdev->device_busy--;
313         spin_unlock_irqrestore(sdev->request_queue->queue_lock, flags);
314 }
315
316 /*
317  * Called for single_lun devices on IO completion. Clear starget_sdev_user,
318  * and call blk_run_queue for all the scsi_devices on the target -
319  * including current_sdev first.
320  *
321  * Called with *no* scsi locks held.
322  */
323 static void scsi_single_lun_run(struct scsi_device *current_sdev)
324 {
325         struct Scsi_Host *shost = current_sdev->host;
326         struct scsi_device *sdev, *tmp;
327         struct scsi_target *starget = scsi_target(current_sdev);
328         unsigned long flags;
329
330         spin_lock_irqsave(shost->host_lock, flags);
331         starget->starget_sdev_user = NULL;
332         spin_unlock_irqrestore(shost->host_lock, flags);
333
334         /*
335          * Call blk_run_queue for all LUNs on the target, starting with
336          * current_sdev. We race with others (to set starget_sdev_user),
337          * but in most cases, we will be first. Ideally, each LU on the
338          * target would get some limited time or requests on the target.
339          */
340         blk_run_queue(current_sdev->request_queue);
341
342         spin_lock_irqsave(shost->host_lock, flags);
343         if (starget->starget_sdev_user)
344                 goto out;
345         list_for_each_entry_safe(sdev, tmp, &starget->devices,
346                         same_target_siblings) {
347                 if (sdev == current_sdev)
348                         continue;
349                 if (scsi_device_get(sdev))
350                         continue;
351
352                 spin_unlock_irqrestore(shost->host_lock, flags);
353                 blk_run_queue(sdev->request_queue);
354                 spin_lock_irqsave(shost->host_lock, flags);
355         
356                 scsi_device_put(sdev);
357         }
358  out:
359         spin_unlock_irqrestore(shost->host_lock, flags);
360 }
361
362 static inline int scsi_device_is_busy(struct scsi_device *sdev)
363 {
364         if (sdev->device_busy >= sdev->queue_depth || sdev->device_blocked)
365                 return 1;
366
367         return 0;
368 }
369
370 static inline int scsi_target_is_busy(struct scsi_target *starget)
371 {
372         return ((starget->can_queue > 0 &&
373                  starget->target_busy >= starget->can_queue) ||
374                  starget->target_blocked);
375 }
376
377 static inline int scsi_host_is_busy(struct Scsi_Host *shost)
378 {
379         if ((shost->can_queue > 0 && shost->host_busy >= shost->can_queue) ||
380             shost->host_blocked || shost->host_self_blocked)
381                 return 1;
382
383         return 0;
384 }
385
386 /*
387  * Function:    scsi_run_queue()
388  *
389  * Purpose:     Select a proper request queue to serve next
390  *
391  * Arguments:   q       - last request's queue
392  *
393  * Returns:     Nothing
394  *
395  * Notes:       The previous command was completely finished, start
396  *              a new one if possible.
397  */
398 static void scsi_run_queue(struct request_queue *q)
399 {
400         struct scsi_device *sdev = q->queuedata;
401         struct Scsi_Host *shost;
402         LIST_HEAD(starved_list);
403         unsigned long flags;
404
405         /* if the device is dead, sdev will be NULL, so no queue to run */
406         if (!sdev)
407                 return;
408
409         shost = sdev->host;
410         if (scsi_target(sdev)->single_lun)
411                 scsi_single_lun_run(sdev);
412
413         spin_lock_irqsave(shost->host_lock, flags);
414         list_splice_init(&shost->starved_list, &starved_list);
415
416         while (!list_empty(&starved_list)) {
417                 /*
418                  * As long as shost is accepting commands and we have
419                  * starved queues, call blk_run_queue. scsi_request_fn
420                  * drops the queue_lock and can add us back to the
421                  * starved_list.
422                  *
423                  * host_lock protects the starved_list and starved_entry.
424                  * scsi_request_fn must get the host_lock before checking
425                  * or modifying starved_list or starved_entry.
426                  */
427                 if (scsi_host_is_busy(shost))
428                         break;
429
430                 sdev = list_entry(starved_list.next,
431                                   struct scsi_device, starved_entry);
432                 list_del_init(&sdev->starved_entry);
433                 if (scsi_target_is_busy(scsi_target(sdev))) {
434                         list_move_tail(&sdev->starved_entry,
435                                        &shost->starved_list);
436                         continue;
437                 }
438
439                 spin_unlock(shost->host_lock);
440                 spin_lock(sdev->request_queue->queue_lock);
441                 __blk_run_queue(sdev->request_queue);
442                 spin_unlock(sdev->request_queue->queue_lock);
443                 spin_lock(shost->host_lock);
444         }
445         /* put any unprocessed entries back */
446         list_splice(&starved_list, &shost->starved_list);
447         spin_unlock_irqrestore(shost->host_lock, flags);
448
449         blk_run_queue(q);
450 }
451
452 void scsi_requeue_run_queue(struct work_struct *work)
453 {
454         struct scsi_device *sdev;
455         struct request_queue *q;
456
457         sdev = container_of(work, struct scsi_device, requeue_work);
458         q = sdev->request_queue;
459         scsi_run_queue(q);
460 }
461
462 /*
463  * Function:    scsi_requeue_command()
464  *
465  * Purpose:     Handle post-processing of completed commands.
466  *
467  * Arguments:   q       - queue to operate on
468  *              cmd     - command that may need to be requeued.
469  *
470  * Returns:     Nothing
471  *
472  * Notes:       After command completion, there may be blocks left
473  *              over which weren't finished by the previous command
474  *              this can be for a number of reasons - the main one is
475  *              I/O errors in the middle of the request, in which case
476  *              we need to request the blocks that come after the bad
477  *              sector.
478  * Notes:       Upon return, cmd is a stale pointer.
479  */
480 static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd)
481 {
482         struct request *req = cmd->request;
483         unsigned long flags;
484
485         spin_lock_irqsave(q->queue_lock, flags);
486         scsi_unprep_request(req);
487         blk_requeue_request(q, req);
488         spin_unlock_irqrestore(q->queue_lock, flags);
489
490         scsi_run_queue(q);
491 }
492
493 void scsi_next_command(struct scsi_cmnd *cmd)
494 {
495         struct scsi_device *sdev = cmd->device;
496         struct request_queue *q = sdev->request_queue;
497
498         /* need to hold a reference on the device before we let go of the cmd */
499         get_device(&sdev->sdev_gendev);
500
501         scsi_put_command(cmd);
502         scsi_run_queue(q);
503
504         /* ok to remove device now */
505         put_device(&sdev->sdev_gendev);
506 }
507
508 void scsi_run_host_queues(struct Scsi_Host *shost)
509 {
510         struct scsi_device *sdev;
511
512         shost_for_each_device(sdev, shost)
513                 scsi_run_queue(sdev->request_queue);
514 }
515
516 static void __scsi_release_buffers(struct scsi_cmnd *, int);
517
518 /*
519  * Function:    scsi_end_request()
520  *
521  * Purpose:     Post-processing of completed commands (usually invoked at end
522  *              of upper level post-processing and scsi_io_completion).
523  *
524  * Arguments:   cmd      - command that is complete.
525  *              error    - 0 if I/O indicates success, < 0 for I/O error.
526  *              bytes    - number of bytes of completed I/O
527  *              requeue  - indicates whether we should requeue leftovers.
528  *
529  * Lock status: Assumed that lock is not held upon entry.
530  *
531  * Returns:     cmd if requeue required, NULL otherwise.
532  *
533  * Notes:       This is called for block device requests in order to
534  *              mark some number of sectors as complete.
535  * 
536  *              We are guaranteeing that the request queue will be goosed
537  *              at some point during this call.
538  * Notes:       If cmd was requeued, upon return it will be a stale pointer.
539  */
540 static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int error,
541                                           int bytes, int requeue)
542 {
543         struct request_queue *q = cmd->device->request_queue;
544         struct request *req = cmd->request;
545
546         /*
547          * If there are blocks left over at the end, set up the command
548          * to queue the remainder of them.
549          */
550         if (blk_end_request(req, error, bytes)) {
551                 /* kill remainder if no retrys */
552                 if (error && scsi_noretry_cmd(cmd))
553                         blk_end_request_all(req, error);
554                 else {
555                         if (requeue) {
556                                 /*
557                                  * Bleah.  Leftovers again.  Stick the
558                                  * leftovers in the front of the
559                                  * queue, and goose the queue again.
560                                  */
561                                 scsi_release_buffers(cmd);
562                                 scsi_requeue_command(q, cmd);
563                                 cmd = NULL;
564                         }
565                         return cmd;
566                 }
567         }
568
569         /*
570          * This will goose the queue request function at the end, so we don't
571          * need to worry about launching another command.
572          */
573         __scsi_release_buffers(cmd, 0);
574         scsi_next_command(cmd);
575         return NULL;
576 }
577
578 static inline unsigned int scsi_sgtable_index(unsigned short nents)
579 {
580         unsigned int index;
581
582         BUG_ON(nents > SCSI_MAX_SG_SEGMENTS);
583
584         if (nents <= 8)
585                 index = 0;
586         else
587                 index = get_count_order(nents) - 3;
588
589         return index;
590 }
591
592 static void scsi_sg_free(struct scatterlist *sgl, unsigned int nents)
593 {
594         struct scsi_host_sg_pool *sgp;
595
596         sgp = scsi_sg_pools + scsi_sgtable_index(nents);
597         mempool_free(sgl, sgp->pool);
598 }
599
600 static struct scatterlist *scsi_sg_alloc(unsigned int nents, gfp_t gfp_mask)
601 {
602         struct scsi_host_sg_pool *sgp;
603
604         sgp = scsi_sg_pools + scsi_sgtable_index(nents);
605         return mempool_alloc(sgp->pool, gfp_mask);
606 }
607
608 static int scsi_alloc_sgtable(struct scsi_data_buffer *sdb, int nents,
609                               gfp_t gfp_mask)
610 {
611         int ret;
612
613         BUG_ON(!nents);
614
615         ret = __sg_alloc_table(&sdb->table, nents, SCSI_MAX_SG_SEGMENTS,
616                                gfp_mask, scsi_sg_alloc);
617         if (unlikely(ret))
618                 __sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS,
619                                 scsi_sg_free);
620
621         return ret;
622 }
623
624 static void scsi_free_sgtable(struct scsi_data_buffer *sdb)
625 {
626         __sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS, scsi_sg_free);
627 }
628
629 static void __scsi_release_buffers(struct scsi_cmnd *cmd, int do_bidi_check)
630 {
631
632         if (cmd->sdb.table.nents)
633                 scsi_free_sgtable(&cmd->sdb);
634
635         memset(&cmd->sdb, 0, sizeof(cmd->sdb));
636
637         if (do_bidi_check && scsi_bidi_cmnd(cmd)) {
638                 struct scsi_data_buffer *bidi_sdb =
639                         cmd->request->next_rq->special;
640                 scsi_free_sgtable(bidi_sdb);
641                 kmem_cache_free(scsi_sdb_cache, bidi_sdb);
642                 cmd->request->next_rq->special = NULL;
643         }
644
645         if (scsi_prot_sg_count(cmd))
646                 scsi_free_sgtable(cmd->prot_sdb);
647 }
648
649 /*
650  * Function:    scsi_release_buffers()
651  *
652  * Purpose:     Completion processing for block device I/O requests.
653  *
654  * Arguments:   cmd     - command that we are bailing.
655  *
656  * Lock status: Assumed that no lock is held upon entry.
657  *
658  * Returns:     Nothing
659  *
660  * Notes:       In the event that an upper level driver rejects a
661  *              command, we must release resources allocated during
662  *              the __init_io() function.  Primarily this would involve
663  *              the scatter-gather table, and potentially any bounce
664  *              buffers.
665  */
666 void scsi_release_buffers(struct scsi_cmnd *cmd)
667 {
668         __scsi_release_buffers(cmd, 1);
669 }
670 EXPORT_SYMBOL(scsi_release_buffers);
671
672 static int __scsi_error_from_host_byte(struct scsi_cmnd *cmd, int result)
673 {
674         int error = 0;
675
676         switch(host_byte(result)) {
677         case DID_TRANSPORT_FAILFAST:
678                 error = -ENOLINK;
679                 break;
680         case DID_TARGET_FAILURE:
681                 cmd->result |= (DID_OK << 16);
682                 error = -EREMOTEIO;
683                 break;
684         case DID_NEXUS_FAILURE:
685                 cmd->result |= (DID_OK << 16);
686                 error = -EBADE;
687                 break;
688         default:
689                 error = -EIO;
690                 break;
691         }
692
693         return error;
694 }
695
696 /*
697  * Function:    scsi_io_completion()
698  *
699  * Purpose:     Completion processing for block device I/O requests.
700  *
701  * Arguments:   cmd   - command that is finished.
702  *
703  * Lock status: Assumed that no lock is held upon entry.
704  *
705  * Returns:     Nothing
706  *
707  * Notes:       This function is matched in terms of capabilities to
708  *              the function that created the scatter-gather list.
709  *              In other words, if there are no bounce buffers
710  *              (the normal case for most drivers), we don't need
711  *              the logic to deal with cleaning up afterwards.
712  *
713  *              We must call scsi_end_request().  This will finish off
714  *              the specified number of sectors.  If we are done, the
715  *              command block will be released and the queue function
716  *              will be goosed.  If we are not done then we have to
717  *              figure out what to do next:
718  *
719  *              a) We can call scsi_requeue_command().  The request
720  *                 will be unprepared and put back on the queue.  Then
721  *                 a new command will be created for it.  This should
722  *                 be used if we made forward progress, or if we want
723  *                 to switch from READ(10) to READ(6) for example.
724  *
725  *              b) We can call scsi_queue_insert().  The request will
726  *                 be put back on the queue and retried using the same
727  *                 command as before, possibly after a delay.
728  *
729  *              c) We can call blk_end_request() with -EIO to fail
730  *                 the remainder of the request.
731  */
732 void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
733 {
734         int result = cmd->result;
735         struct request_queue *q = cmd->device->request_queue;
736         struct request *req = cmd->request;
737         int error = 0;
738         struct scsi_sense_hdr sshdr;
739         int sense_valid = 0;
740         int sense_deferred = 0;
741         enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY,
742               ACTION_DELAYED_RETRY} action;
743         char *description = NULL;
744
745         if (result) {
746                 sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
747                 if (sense_valid)
748                         sense_deferred = scsi_sense_is_deferred(&sshdr);
749         }
750
751         if (req->cmd_type == REQ_TYPE_BLOCK_PC) { /* SG_IO ioctl from block level */
752                 req->errors = result;
753                 if (result) {
754                         if (sense_valid && req->sense) {
755                                 /*
756                                  * SG_IO wants current and deferred errors
757                                  */
758                                 int len = 8 + cmd->sense_buffer[7];
759
760                                 if (len > SCSI_SENSE_BUFFERSIZE)
761                                         len = SCSI_SENSE_BUFFERSIZE;
762                                 memcpy(req->sense, cmd->sense_buffer,  len);
763                                 req->sense_len = len;
764                         }
765                         if (!sense_deferred)
766                                 error = __scsi_error_from_host_byte(cmd, result);
767                 }
768
769                 req->resid_len = scsi_get_resid(cmd);
770
771                 if (scsi_bidi_cmnd(cmd)) {
772                         /*
773                          * Bidi commands Must be complete as a whole,
774                          * both sides at once.
775                          */
776                         req->next_rq->resid_len = scsi_in(cmd)->resid;
777
778                         scsi_release_buffers(cmd);
779                         blk_end_request_all(req, 0);
780
781                         scsi_next_command(cmd);
782                         return;
783                 }
784         }
785
786         /* no bidi support for !REQ_TYPE_BLOCK_PC yet */
787         BUG_ON(blk_bidi_rq(req));
788
789         /*
790          * Next deal with any sectors which we were able to correctly
791          * handle.
792          */
793         SCSI_LOG_HLCOMPLETE(1, printk("%u sectors total, "
794                                       "%d bytes done.\n",
795                                       blk_rq_sectors(req), good_bytes));
796
797         /*
798          * Recovered errors need reporting, but they're always treated
799          * as success, so fiddle the result code here.  For BLOCK_PC
800          * we already took a copy of the original into rq->errors which
801          * is what gets returned to the user
802          */
803         if (sense_valid && (sshdr.sense_key == RECOVERED_ERROR)) {
804                 /* if ATA PASS-THROUGH INFORMATION AVAILABLE skip
805                  * print since caller wants ATA registers. Only occurs on
806                  * SCSI ATA PASS_THROUGH commands when CK_COND=1
807                  */
808                 if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d))
809                         ;
810                 else if (!(req->cmd_flags & REQ_QUIET))
811                         scsi_print_sense("", cmd);
812                 result = 0;
813                 /* BLOCK_PC may have set error */
814                 error = 0;
815         }
816
817         /*
818          * A number of bytes were successfully read.  If there
819          * are leftovers and there is some kind of error
820          * (result != 0), retry the rest.
821          */
822         if (scsi_end_request(cmd, error, good_bytes, result == 0) == NULL)
823                 return;
824
825         error = __scsi_error_from_host_byte(cmd, result);
826
827         if (host_byte(result) == DID_RESET) {
828                 /* Third party bus reset or reset for error recovery
829                  * reasons.  Just retry the command and see what
830                  * happens.
831                  */
832                 action = ACTION_RETRY;
833         } else if (sense_valid && !sense_deferred) {
834                 switch (sshdr.sense_key) {
835                 case UNIT_ATTENTION:
836                         if (cmd->device->removable) {
837                                 /* Detected disc change.  Set a bit
838                                  * and quietly refuse further access.
839                                  */
840                                 cmd->device->changed = 1;
841                                 description = "Media Changed";
842                                 action = ACTION_FAIL;
843                         } else {
844                                 /* Must have been a power glitch, or a
845                                  * bus reset.  Could not have been a
846                                  * media change, so we just retry the
847                                  * command and see what happens.
848                                  */
849                                 action = ACTION_RETRY;
850                         }
851                         break;
852                 case ILLEGAL_REQUEST:
853                         /* If we had an ILLEGAL REQUEST returned, then
854                          * we may have performed an unsupported
855                          * command.  The only thing this should be
856                          * would be a ten byte read where only a six
857                          * byte read was supported.  Also, on a system
858                          * where READ CAPACITY failed, we may have
859                          * read past the end of the disk.
860                          */
861                         if ((cmd->device->use_10_for_rw &&
862                             sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&
863                             (cmd->cmnd[0] == READ_10 ||
864                              cmd->cmnd[0] == WRITE_10)) {
865                                 /* This will issue a new 6-byte command. */
866                                 cmd->device->use_10_for_rw = 0;
867                                 action = ACTION_REPREP;
868                         } else if (sshdr.asc == 0x10) /* DIX */ {
869                                 description = "Host Data Integrity Failure";
870                                 action = ACTION_FAIL;
871                                 error = -EILSEQ;
872                         /* INVALID COMMAND OPCODE or INVALID FIELD IN CDB */
873                         } else if ((sshdr.asc == 0x20 || sshdr.asc == 0x24) &&
874                                    (cmd->cmnd[0] == UNMAP ||
875                                     cmd->cmnd[0] == WRITE_SAME_16 ||
876                                     cmd->cmnd[0] == WRITE_SAME)) {
877                                 description = "Discard failure";
878                                 action = ACTION_FAIL;
879                         } else
880                                 action = ACTION_FAIL;
881                         break;
882                 case ABORTED_COMMAND:
883                         action = ACTION_FAIL;
884                         if (sshdr.asc == 0x10) { /* DIF */
885                                 description = "Target Data Integrity Failure";
886                                 error = -EILSEQ;
887                         }
888                         break;
889                 case NOT_READY:
890                         /* If the device is in the process of becoming
891                          * ready, or has a temporary blockage, retry.
892                          */
893                         if (sshdr.asc == 0x04) {
894                                 switch (sshdr.ascq) {
895                                 case 0x01: /* becoming ready */
896                                 case 0x04: /* format in progress */
897                                 case 0x05: /* rebuild in progress */
898                                 case 0x06: /* recalculation in progress */
899                                 case 0x07: /* operation in progress */
900                                 case 0x08: /* Long write in progress */
901                                 case 0x09: /* self test in progress */
902                                 case 0x14: /* space allocation in progress */
903                                         action = ACTION_DELAYED_RETRY;
904                                         break;
905                                 default:
906                                         description = "Device not ready";
907                                         action = ACTION_FAIL;
908                                         break;
909                                 }
910                         } else {
911                                 description = "Device not ready";
912                                 action = ACTION_FAIL;
913                         }
914                         break;
915                 case VOLUME_OVERFLOW:
916                         /* See SSC3rXX or current. */
917                         action = ACTION_FAIL;
918                         break;
919                 default:
920                         description = "Unhandled sense code";
921                         action = ACTION_FAIL;
922                         break;
923                 }
924         } else {
925                 description = "Unhandled error code";
926                 action = ACTION_FAIL;
927         }
928
929         switch (action) {
930         case ACTION_FAIL:
931                 /* Give up and fail the remainder of the request */
932                 scsi_release_buffers(cmd);
933                 if (!(req->cmd_flags & REQ_QUIET)) {
934                         if (description)
935                                 scmd_printk(KERN_INFO, cmd, "%s\n",
936                                             description);
937                         scsi_print_result(cmd);
938                         if (driver_byte(result) & DRIVER_SENSE)
939                                 scsi_print_sense("", cmd);
940                         scsi_print_command(cmd);
941                 }
942                 if (blk_end_request_err(req, error))
943                         scsi_requeue_command(q, cmd);
944                 else
945                         scsi_next_command(cmd);
946                 break;
947         case ACTION_REPREP:
948                 /* Unprep the request and put it back at the head of the queue.
949                  * A new command will be prepared and issued.
950                  */
951                 scsi_release_buffers(cmd);
952                 scsi_requeue_command(q, cmd);
953                 break;
954         case ACTION_RETRY:
955                 /* Retry the same command immediately */
956                 __scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, 0);
957                 break;
958         case ACTION_DELAYED_RETRY:
959                 /* Retry the same command after a delay */
960                 __scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, 0);
961                 break;
962         }
963 }
964
965 static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb,
966                              gfp_t gfp_mask)
967 {
968         int count;
969
970         /*
971          * If sg table allocation fails, requeue request later.
972          */
973         if (unlikely(scsi_alloc_sgtable(sdb, req->nr_phys_segments,
974                                         gfp_mask))) {
975                 return BLKPREP_DEFER;
976         }
977
978         req->buffer = NULL;
979
980         /* 
981          * Next, walk the list, and fill in the addresses and sizes of
982          * each segment.
983          */
984         count = blk_rq_map_sg(req->q, req, sdb->table.sgl);
985         BUG_ON(count > sdb->table.nents);
986         sdb->table.nents = count;
987         sdb->length = blk_rq_bytes(req);
988         return BLKPREP_OK;
989 }
990
991 /*
992  * Function:    scsi_init_io()
993  *
994  * Purpose:     SCSI I/O initialize function.
995  *
996  * Arguments:   cmd   - Command descriptor we wish to initialize
997  *
998  * Returns:     0 on success
999  *              BLKPREP_DEFER if the failure is retryable
1000  *              BLKPREP_KILL if the failure is fatal
1001  */
1002 int scsi_init_io(struct scsi_cmnd *cmd, gfp_t gfp_mask)
1003 {
1004         struct request *rq = cmd->request;
1005
1006         int error = scsi_init_sgtable(rq, &cmd->sdb, gfp_mask);
1007         if (error)
1008                 goto err_exit;
1009
1010         if (blk_bidi_rq(rq)) {
1011                 struct scsi_data_buffer *bidi_sdb = kmem_cache_zalloc(
1012                         scsi_sdb_cache, GFP_ATOMIC);
1013                 if (!bidi_sdb) {
1014                         error = BLKPREP_DEFER;
1015                         goto err_exit;
1016                 }
1017
1018                 rq->next_rq->special = bidi_sdb;
1019                 error = scsi_init_sgtable(rq->next_rq, bidi_sdb, GFP_ATOMIC);
1020                 if (error)
1021                         goto err_exit;
1022         }
1023
1024         if (blk_integrity_rq(rq)) {
1025                 struct scsi_data_buffer *prot_sdb = cmd->prot_sdb;
1026                 int ivecs, count;
1027
1028                 BUG_ON(prot_sdb == NULL);
1029                 ivecs = blk_rq_count_integrity_sg(rq->q, rq->bio);
1030
1031                 if (scsi_alloc_sgtable(prot_sdb, ivecs, gfp_mask)) {
1032                         error = BLKPREP_DEFER;
1033                         goto err_exit;
1034                 }
1035
1036                 count = blk_rq_map_integrity_sg(rq->q, rq->bio,
1037                                                 prot_sdb->table.sgl);
1038                 BUG_ON(unlikely(count > ivecs));
1039                 BUG_ON(unlikely(count > queue_max_integrity_segments(rq->q)));
1040
1041                 cmd->prot_sdb = prot_sdb;
1042                 cmd->prot_sdb->table.nents = count;
1043         }
1044
1045         return BLKPREP_OK ;
1046
1047 err_exit:
1048         scsi_release_buffers(cmd);
1049         cmd->request->special = NULL;
1050         scsi_put_command(cmd);
1051         return error;
1052 }
1053 EXPORT_SYMBOL(scsi_init_io);
1054
1055 static struct scsi_cmnd *scsi_get_cmd_from_req(struct scsi_device *sdev,
1056                 struct request *req)
1057 {
1058         struct scsi_cmnd *cmd;
1059
1060         if (!req->special) {
1061                 cmd = scsi_get_command(sdev, GFP_ATOMIC);
1062                 if (unlikely(!cmd))
1063                         return NULL;
1064                 req->special = cmd;
1065         } else {
1066                 cmd = req->special;
1067         }
1068
1069         /* pull a tag out of the request if we have one */
1070         cmd->tag = req->tag;
1071         cmd->request = req;
1072
1073         cmd->cmnd = req->cmd;
1074         cmd->prot_op = SCSI_PROT_NORMAL;
1075
1076         return cmd;
1077 }
1078
1079 int scsi_setup_blk_pc_cmnd(struct scsi_device *sdev, struct request *req)
1080 {
1081         struct scsi_cmnd *cmd;
1082         int ret = scsi_prep_state_check(sdev, req);
1083
1084         if (ret != BLKPREP_OK)
1085                 return ret;
1086
1087         cmd = scsi_get_cmd_from_req(sdev, req);
1088         if (unlikely(!cmd))
1089                 return BLKPREP_DEFER;
1090
1091         /*
1092          * BLOCK_PC requests may transfer data, in which case they must
1093          * a bio attached to them.  Or they might contain a SCSI command
1094          * that does not transfer data, in which case they may optionally
1095          * submit a request without an attached bio.
1096          */
1097         if (req->bio) {
1098                 int ret;
1099
1100                 BUG_ON(!req->nr_phys_segments);
1101
1102                 ret = scsi_init_io(cmd, GFP_ATOMIC);
1103                 if (unlikely(ret))
1104                         return ret;
1105         } else {
1106                 BUG_ON(blk_rq_bytes(req));
1107
1108                 memset(&cmd->sdb, 0, sizeof(cmd->sdb));
1109                 req->buffer = NULL;
1110         }
1111
1112         cmd->cmd_len = req->cmd_len;
1113         if (!blk_rq_bytes(req))
1114                 cmd->sc_data_direction = DMA_NONE;
1115         else if (rq_data_dir(req) == WRITE)
1116                 cmd->sc_data_direction = DMA_TO_DEVICE;
1117         else
1118                 cmd->sc_data_direction = DMA_FROM_DEVICE;
1119         
1120         cmd->transfersize = blk_rq_bytes(req);
1121         cmd->allowed = req->retries;
1122         return BLKPREP_OK;
1123 }
1124 EXPORT_SYMBOL(scsi_setup_blk_pc_cmnd);
1125
1126 /*
1127  * Setup a REQ_TYPE_FS command.  These are simple read/write request
1128  * from filesystems that still need to be translated to SCSI CDBs from
1129  * the ULD.
1130  */
1131 int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req)
1132 {
1133         struct scsi_cmnd *cmd;
1134         int ret = scsi_prep_state_check(sdev, req);
1135
1136         if (ret != BLKPREP_OK)
1137                 return ret;
1138
1139         if (unlikely(sdev->scsi_dh_data && sdev->scsi_dh_data->scsi_dh
1140                          && sdev->scsi_dh_data->scsi_dh->prep_fn)) {
1141                 ret = sdev->scsi_dh_data->scsi_dh->prep_fn(sdev, req);
1142                 if (ret != BLKPREP_OK)
1143                         return ret;
1144         }
1145
1146         /*
1147          * Filesystem requests must transfer data.
1148          */
1149         BUG_ON(!req->nr_phys_segments);
1150
1151         cmd = scsi_get_cmd_from_req(sdev, req);
1152         if (unlikely(!cmd))
1153                 return BLKPREP_DEFER;
1154
1155         memset(cmd->cmnd, 0, BLK_MAX_CDB);
1156         return scsi_init_io(cmd, GFP_ATOMIC);
1157 }
1158 EXPORT_SYMBOL(scsi_setup_fs_cmnd);
1159
1160 int scsi_prep_state_check(struct scsi_device *sdev, struct request *req)
1161 {
1162         int ret = BLKPREP_OK;
1163
1164         /*
1165          * If the device is not in running state we will reject some
1166          * or all commands.
1167          */
1168         if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
1169                 switch (sdev->sdev_state) {
1170                 case SDEV_OFFLINE:
1171                         /*
1172                          * If the device is offline we refuse to process any
1173                          * commands.  The device must be brought online
1174                          * before trying any recovery commands.
1175                          */
1176                         sdev_printk(KERN_ERR, sdev,
1177                                     "rejecting I/O to offline device\n");
1178                         ret = BLKPREP_KILL;
1179                         break;
1180                 case SDEV_DEL:
1181                         /*
1182                          * If the device is fully deleted, we refuse to
1183                          * process any commands as well.
1184                          */
1185                         sdev_printk(KERN_ERR, sdev,
1186                                     "rejecting I/O to dead device\n");
1187                         ret = BLKPREP_KILL;
1188                         break;
1189                 case SDEV_QUIESCE:
1190                 case SDEV_BLOCK:
1191                 case SDEV_CREATED_BLOCK:
1192                         /*
1193                          * If the devices is blocked we defer normal commands.
1194                          */
1195                         if (!(req->cmd_flags & REQ_PREEMPT))
1196                                 ret = BLKPREP_DEFER;
1197                         break;
1198                 default:
1199                         /*
1200                          * For any other not fully online state we only allow
1201                          * special commands.  In particular any user initiated
1202                          * command is not allowed.
1203                          */
1204                         if (!(req->cmd_flags & REQ_PREEMPT))
1205                                 ret = BLKPREP_KILL;
1206                         break;
1207                 }
1208         }
1209         return ret;
1210 }
1211 EXPORT_SYMBOL(scsi_prep_state_check);
1212
1213 int scsi_prep_return(struct request_queue *q, struct request *req, int ret)
1214 {
1215         struct scsi_device *sdev = q->queuedata;
1216
1217         switch (ret) {
1218         case BLKPREP_KILL:
1219                 req->errors = DID_NO_CONNECT << 16;
1220                 /* release the command and kill it */
1221                 if (req->special) {
1222                         struct scsi_cmnd *cmd = req->special;
1223                         scsi_release_buffers(cmd);
1224                         scsi_put_command(cmd);
1225                         req->special = NULL;
1226                 }
1227                 break;
1228         case BLKPREP_DEFER:
1229                 /*
1230                  * If we defer, the blk_peek_request() returns NULL, but the
1231                  * queue must be restarted, so we schedule a callback to happen
1232                  * shortly.
1233                  */
1234                 if (sdev->device_busy == 0)
1235                         blk_delay_queue(q, SCSI_QUEUE_DELAY);
1236                 break;
1237         default:
1238                 req->cmd_flags |= REQ_DONTPREP;
1239         }
1240
1241         return ret;
1242 }
1243 EXPORT_SYMBOL(scsi_prep_return);
1244
1245 int scsi_prep_fn(struct request_queue *q, struct request *req)
1246 {
1247         struct scsi_device *sdev = q->queuedata;
1248         int ret = BLKPREP_KILL;
1249
1250         if (req->cmd_type == REQ_TYPE_BLOCK_PC)
1251                 ret = scsi_setup_blk_pc_cmnd(sdev, req);
1252         return scsi_prep_return(q, req, ret);
1253 }
1254 EXPORT_SYMBOL(scsi_prep_fn);
1255
1256 /*
1257  * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
1258  * return 0.
1259  *
1260  * Called with the queue_lock held.
1261  */
1262 static inline int scsi_dev_queue_ready(struct request_queue *q,
1263                                   struct scsi_device *sdev)
1264 {
1265         if (sdev->device_busy == 0 && sdev->device_blocked) {
1266                 /*
1267                  * unblock after device_blocked iterates to zero
1268                  */
1269                 if (--sdev->device_blocked == 0) {
1270                         SCSI_LOG_MLQUEUE(3,
1271                                    sdev_printk(KERN_INFO, sdev,
1272                                    "unblocking device at zero depth\n"));
1273                 } else {
1274                         blk_delay_queue(q, SCSI_QUEUE_DELAY);
1275                         return 0;
1276                 }
1277         }
1278         if (scsi_device_is_busy(sdev))
1279                 return 0;
1280
1281         return 1;
1282 }
1283
1284
1285 /*
1286  * scsi_target_queue_ready: checks if there we can send commands to target
1287  * @sdev: scsi device on starget to check.
1288  *
1289  * Called with the host lock held.
1290  */
1291 static inline int scsi_target_queue_ready(struct Scsi_Host *shost,
1292                                            struct scsi_device *sdev)
1293 {
1294         struct scsi_target *starget = scsi_target(sdev);
1295
1296         if (starget->single_lun) {
1297                 if (starget->starget_sdev_user &&
1298                     starget->starget_sdev_user != sdev)
1299                         return 0;
1300                 starget->starget_sdev_user = sdev;
1301         }
1302
1303         if (starget->target_busy == 0 && starget->target_blocked) {
1304                 /*
1305                  * unblock after target_blocked iterates to zero
1306                  */
1307                 if (--starget->target_blocked == 0) {
1308                         SCSI_LOG_MLQUEUE(3, starget_printk(KERN_INFO, starget,
1309                                          "unblocking target at zero depth\n"));
1310                 } else
1311                         return 0;
1312         }
1313
1314         if (scsi_target_is_busy(starget)) {
1315                 if (list_empty(&sdev->starved_entry))
1316                         list_add_tail(&sdev->starved_entry,
1317                                       &shost->starved_list);
1318                 return 0;
1319         }
1320
1321         /* We're OK to process the command, so we can't be starved */
1322         if (!list_empty(&sdev->starved_entry))
1323                 list_del_init(&sdev->starved_entry);
1324         return 1;
1325 }
1326
1327 /*
1328  * scsi_host_queue_ready: if we can send requests to shost, return 1 else
1329  * return 0. We must end up running the queue again whenever 0 is
1330  * returned, else IO can hang.
1331  *
1332  * Called with host_lock held.
1333  */
1334 static inline int scsi_host_queue_ready(struct request_queue *q,
1335                                    struct Scsi_Host *shost,
1336                                    struct scsi_device *sdev)
1337 {
1338         if (scsi_host_in_recovery(shost))
1339                 return 0;
1340         if (shost->host_busy == 0 && shost->host_blocked) {
1341                 /*
1342                  * unblock after host_blocked iterates to zero
1343                  */
1344                 if (--shost->host_blocked == 0) {
1345                         SCSI_LOG_MLQUEUE(3,
1346                                 printk("scsi%d unblocking host at zero depth\n",
1347                                         shost->host_no));
1348                 } else {
1349                         return 0;
1350                 }
1351         }
1352         if (scsi_host_is_busy(shost)) {
1353                 if (list_empty(&sdev->starved_entry))
1354                         list_add_tail(&sdev->starved_entry, &shost->starved_list);
1355                 return 0;
1356         }
1357
1358         /* We're OK to process the command, so we can't be starved */
1359         if (!list_empty(&sdev->starved_entry))
1360                 list_del_init(&sdev->starved_entry);
1361
1362         return 1;
1363 }
1364
1365 /*
1366  * Busy state exporting function for request stacking drivers.
1367  *
1368  * For efficiency, no lock is taken to check the busy state of
1369  * shost/starget/sdev, since the returned value is not guaranteed and
1370  * may be changed after request stacking drivers call the function,
1371  * regardless of taking lock or not.
1372  *
1373  * When scsi can't dispatch I/Os anymore and needs to kill I/Os
1374  * (e.g. !sdev), scsi needs to return 'not busy'.
1375  * Otherwise, request stacking drivers may hold requests forever.
1376  */
1377 static int scsi_lld_busy(struct request_queue *q)
1378 {
1379         struct scsi_device *sdev = q->queuedata;
1380         struct Scsi_Host *shost;
1381         struct scsi_target *starget;
1382
1383         if (!sdev)
1384                 return 0;
1385
1386         shost = sdev->host;
1387         starget = scsi_target(sdev);
1388
1389         if (scsi_host_in_recovery(shost) || scsi_host_is_busy(shost) ||
1390             scsi_target_is_busy(starget) || scsi_device_is_busy(sdev))
1391                 return 1;
1392
1393         return 0;
1394 }
1395
1396 /*
1397  * Kill a request for a dead device
1398  */
1399 static void scsi_kill_request(struct request *req, struct request_queue *q)
1400 {
1401         struct scsi_cmnd *cmd = req->special;
1402         struct scsi_device *sdev;
1403         struct scsi_target *starget;
1404         struct Scsi_Host *shost;
1405
1406         blk_start_request(req);
1407
1408         sdev = cmd->device;
1409         starget = scsi_target(sdev);
1410         shost = sdev->host;
1411         scsi_init_cmd_errh(cmd);
1412         cmd->result = DID_NO_CONNECT << 16;
1413         atomic_inc(&cmd->device->iorequest_cnt);
1414
1415         /*
1416          * SCSI request completion path will do scsi_device_unbusy(),
1417          * bump busy counts.  To bump the counters, we need to dance
1418          * with the locks as normal issue path does.
1419          */
1420         sdev->device_busy++;
1421         spin_unlock(sdev->request_queue->queue_lock);
1422         spin_lock(shost->host_lock);
1423         shost->host_busy++;
1424         starget->target_busy++;
1425         spin_unlock(shost->host_lock);
1426         spin_lock(sdev->request_queue->queue_lock);
1427
1428         blk_complete_request(req);
1429 }
1430
1431 static void scsi_softirq_done(struct request *rq)
1432 {
1433         struct scsi_cmnd *cmd = rq->special;
1434         unsigned long wait_for = (cmd->allowed + 1) * rq->timeout;
1435         int disposition;
1436
1437         INIT_LIST_HEAD(&cmd->eh_entry);
1438
1439         atomic_inc(&cmd->device->iodone_cnt);
1440         if (cmd->result)
1441                 atomic_inc(&cmd->device->ioerr_cnt);
1442
1443         disposition = scsi_decide_disposition(cmd);
1444         if (disposition != SUCCESS &&
1445             time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
1446                 sdev_printk(KERN_ERR, cmd->device,
1447                             "timing out command, waited %lus\n",
1448                             wait_for/HZ);
1449                 disposition = SUCCESS;
1450         }
1451                         
1452         scsi_log_completion(cmd, disposition);
1453
1454         switch (disposition) {
1455                 case SUCCESS:
1456                         scsi_finish_command(cmd);
1457                         break;
1458                 case NEEDS_RETRY:
1459                         scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);
1460                         break;
1461                 case ADD_TO_MLQUEUE:
1462                         scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
1463                         break;
1464                 default:
1465                         if (!scsi_eh_scmd_add(cmd, 0))
1466                                 scsi_finish_command(cmd);
1467         }
1468 }
1469
1470 /*
1471  * Function:    scsi_request_fn()
1472  *
1473  * Purpose:     Main strategy routine for SCSI.
1474  *
1475  * Arguments:   q       - Pointer to actual queue.
1476  *
1477  * Returns:     Nothing
1478  *
1479  * Lock status: IO request lock assumed to be held when called.
1480  */
1481 static void scsi_request_fn(struct request_queue *q)
1482 {
1483         struct scsi_device *sdev = q->queuedata;
1484         struct Scsi_Host *shost;
1485         struct scsi_cmnd *cmd;
1486         struct request *req;
1487
1488         if (!sdev) {
1489                 printk("scsi: killing requests for dead queue\n");
1490                 while ((req = blk_peek_request(q)) != NULL)
1491                         scsi_kill_request(req, q);
1492                 return;
1493         }
1494
1495         if(!get_device(&sdev->sdev_gendev))
1496                 /* We must be tearing the block queue down already */
1497                 return;
1498
1499         /*
1500          * To start with, we keep looping until the queue is empty, or until
1501          * the host is no longer able to accept any more requests.
1502          */
1503         shost = sdev->host;
1504         for (;;) {
1505                 int rtn;
1506                 /*
1507                  * get next queueable request.  We do this early to make sure
1508                  * that the request is fully prepared even if we cannot 
1509                  * accept it.
1510                  */
1511                 req = blk_peek_request(q);
1512                 if (!req || !scsi_dev_queue_ready(q, sdev))
1513                         break;
1514
1515                 if (unlikely(!scsi_device_online(sdev))) {
1516                         sdev_printk(KERN_ERR, sdev,
1517                                     "rejecting I/O to offline device\n");
1518                         scsi_kill_request(req, q);
1519                         continue;
1520                 }
1521
1522
1523                 /*
1524                  * Remove the request from the request list.
1525                  */
1526                 if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
1527                         blk_start_request(req);
1528                 sdev->device_busy++;
1529
1530                 spin_unlock(q->queue_lock);
1531                 cmd = req->special;
1532                 if (unlikely(cmd == NULL)) {
1533                         printk(KERN_CRIT "impossible request in %s.\n"
1534                                          "please mail a stack trace to "
1535                                          "linux-scsi@vger.kernel.org\n",
1536                                          __func__);
1537                         blk_dump_rq_flags(req, "foo");
1538                         BUG();
1539                 }
1540                 spin_lock(shost->host_lock);
1541
1542                 /*
1543                  * We hit this when the driver is using a host wide
1544                  * tag map. For device level tag maps the queue_depth check
1545                  * in the device ready fn would prevent us from trying
1546                  * to allocate a tag. Since the map is a shared host resource
1547                  * we add the dev to the starved list so it eventually gets
1548                  * a run when a tag is freed.
1549                  */
1550                 if (blk_queue_tagged(q) && !blk_rq_tagged(req)) {
1551                         if (list_empty(&sdev->starved_entry))
1552                                 list_add_tail(&sdev->starved_entry,
1553                                               &shost->starved_list);
1554                         goto not_ready;
1555                 }
1556
1557                 if (!scsi_target_queue_ready(shost, sdev))
1558                         goto not_ready;
1559
1560                 if (!scsi_host_queue_ready(q, shost, sdev))
1561                         goto not_ready;
1562
1563                 scsi_target(sdev)->target_busy++;
1564                 shost->host_busy++;
1565
1566                 /*
1567                  * XXX(hch): This is rather suboptimal, scsi_dispatch_cmd will
1568                  *              take the lock again.
1569                  */
1570                 spin_unlock_irq(shost->host_lock);
1571
1572                 /*
1573                  * Finally, initialize any error handling parameters, and set up
1574                  * the timers for timeouts.
1575                  */
1576                 scsi_init_cmd_errh(cmd);
1577
1578                 /*
1579                  * Dispatch the command to the low-level driver.
1580                  */
1581                 rtn = scsi_dispatch_cmd(cmd);
1582                 spin_lock_irq(q->queue_lock);
1583                 if (rtn)
1584                         goto out_delay;
1585         }
1586
1587         goto out;
1588
1589  not_ready:
1590         spin_unlock_irq(shost->host_lock);
1591
1592         /*
1593          * lock q, handle tag, requeue req, and decrement device_busy. We
1594          * must return with queue_lock held.
1595          *
1596          * Decrementing device_busy without checking it is OK, as all such
1597          * cases (host limits or settings) should run the queue at some
1598          * later time.
1599          */
1600         spin_lock_irq(q->queue_lock);
1601         blk_requeue_request(q, req);
1602         sdev->device_busy--;
1603 out_delay:
1604         if (sdev->device_busy == 0)
1605                 blk_delay_queue(q, SCSI_QUEUE_DELAY);
1606 out:
1607         /* must be careful here...if we trigger the ->remove() function
1608          * we cannot be holding the q lock */
1609         spin_unlock_irq(q->queue_lock);
1610         put_device(&sdev->sdev_gendev);
1611         spin_lock_irq(q->queue_lock);
1612 }
1613
1614 u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
1615 {
1616         struct device *host_dev;
1617         u64 bounce_limit = 0xffffffff;
1618
1619         if (shost->unchecked_isa_dma)
1620                 return BLK_BOUNCE_ISA;
1621         /*
1622          * Platforms with virtual-DMA translation
1623          * hardware have no practical limit.
1624          */
1625         if (!PCI_DMA_BUS_IS_PHYS)
1626                 return BLK_BOUNCE_ANY;
1627
1628         host_dev = scsi_get_device(shost);
1629         if (host_dev && host_dev->dma_mask)
1630                 bounce_limit = *host_dev->dma_mask;
1631
1632         return bounce_limit;
1633 }
1634 EXPORT_SYMBOL(scsi_calculate_bounce_limit);
1635
1636 struct request_queue *__scsi_alloc_queue(struct Scsi_Host *shost,
1637                                          request_fn_proc *request_fn)
1638 {
1639         struct request_queue *q;
1640         struct device *dev = shost->shost_gendev.parent;
1641
1642         q = blk_init_queue(request_fn, NULL);
1643         if (!q)
1644                 return NULL;
1645
1646         /*
1647          * this limit is imposed by hardware restrictions
1648          */
1649         blk_queue_max_segments(q, min_t(unsigned short, shost->sg_tablesize,
1650                                         SCSI_MAX_SG_CHAIN_SEGMENTS));
1651
1652         if (scsi_host_prot_dma(shost)) {
1653                 shost->sg_prot_tablesize =
1654                         min_not_zero(shost->sg_prot_tablesize,
1655                                      (unsigned short)SCSI_MAX_PROT_SG_SEGMENTS);
1656                 BUG_ON(shost->sg_prot_tablesize < shost->sg_tablesize);
1657                 blk_queue_max_integrity_segments(q, shost->sg_prot_tablesize);
1658         }
1659
1660         blk_queue_max_hw_sectors(q, shost->max_sectors);
1661         blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
1662         blk_queue_segment_boundary(q, shost->dma_boundary);
1663         dma_set_seg_boundary(dev, shost->dma_boundary);
1664
1665         blk_queue_max_segment_size(q, dma_get_max_seg_size(dev));
1666
1667         if (!shost->use_clustering)
1668                 q->limits.cluster = 0;
1669
1670         /*
1671          * set a reasonable default alignment on word boundaries: the
1672          * host and device may alter it using
1673          * blk_queue_update_dma_alignment() later.
1674          */
1675         blk_queue_dma_alignment(q, 0x03);
1676
1677         return q;
1678 }
1679 EXPORT_SYMBOL(__scsi_alloc_queue);
1680
1681 struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
1682 {
1683         struct request_queue *q;
1684
1685         q = __scsi_alloc_queue(sdev->host, scsi_request_fn);
1686         if (!q)
1687                 return NULL;
1688
1689         blk_queue_prep_rq(q, scsi_prep_fn);
1690         blk_queue_softirq_done(q, scsi_softirq_done);
1691         blk_queue_rq_timed_out(q, scsi_times_out);
1692         blk_queue_lld_busy(q, scsi_lld_busy);
1693         return q;
1694 }
1695
1696 void scsi_free_queue(struct request_queue *q)
1697 {
1698         blk_cleanup_queue(q);
1699 }
1700
1701 /*
1702  * Function:    scsi_block_requests()
1703  *
1704  * Purpose:     Utility function used by low-level drivers to prevent further
1705  *              commands from being queued to the device.
1706  *
1707  * Arguments:   shost       - Host in question
1708  *
1709  * Returns:     Nothing
1710  *
1711  * Lock status: No locks are assumed held.
1712  *
1713  * Notes:       There is no timer nor any other means by which the requests
1714  *              get unblocked other than the low-level driver calling
1715  *              scsi_unblock_requests().
1716  */
1717 void scsi_block_requests(struct Scsi_Host *shost)
1718 {
1719         shost->host_self_blocked = 1;
1720 }
1721 EXPORT_SYMBOL(scsi_block_requests);
1722
1723 /*
1724  * Function:    scsi_unblock_requests()
1725  *
1726  * Purpose:     Utility function used by low-level drivers to allow further
1727  *              commands from being queued to the device.
1728  *
1729  * Arguments:   shost       - Host in question
1730  *
1731  * Returns:     Nothing
1732  *
1733  * Lock status: No locks are assumed held.
1734  *
1735  * Notes:       There is no timer nor any other means by which the requests
1736  *              get unblocked other than the low-level driver calling
1737  *              scsi_unblock_requests().
1738  *
1739  *              This is done as an API function so that changes to the
1740  *              internals of the scsi mid-layer won't require wholesale
1741  *              changes to drivers that use this feature.
1742  */
1743 void scsi_unblock_requests(struct Scsi_Host *shost)
1744 {
1745         shost->host_self_blocked = 0;
1746         scsi_run_host_queues(shost);
1747 }
1748 EXPORT_SYMBOL(scsi_unblock_requests);
1749
1750 int __init scsi_init_queue(void)
1751 {
1752         int i;
1753
1754         scsi_sdb_cache = kmem_cache_create("scsi_data_buffer",
1755                                            sizeof(struct scsi_data_buffer),
1756                                            0, 0, NULL);
1757         if (!scsi_sdb_cache) {
1758                 printk(KERN_ERR "SCSI: can't init scsi sdb cache\n");
1759                 return -ENOMEM;
1760         }
1761
1762         for (i = 0; i < SG_MEMPOOL_NR; i++) {
1763                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1764                 int size = sgp->size * sizeof(struct scatterlist);
1765
1766                 sgp->slab = kmem_cache_create(sgp->name, size, 0,
1767                                 SLAB_HWCACHE_ALIGN, NULL);
1768                 if (!sgp->slab) {
1769                         printk(KERN_ERR "SCSI: can't init sg slab %s\n",
1770                                         sgp->name);
1771                         goto cleanup_sdb;
1772                 }
1773
1774                 sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
1775                                                      sgp->slab);
1776                 if (!sgp->pool) {
1777                         printk(KERN_ERR "SCSI: can't init sg mempool %s\n",
1778                                         sgp->name);
1779                         goto cleanup_sdb;
1780                 }
1781         }
1782
1783         return 0;
1784
1785 cleanup_sdb:
1786         for (i = 0; i < SG_MEMPOOL_NR; i++) {
1787                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1788                 if (sgp->pool)
1789                         mempool_destroy(sgp->pool);
1790                 if (sgp->slab)
1791                         kmem_cache_destroy(sgp->slab);
1792         }
1793         kmem_cache_destroy(scsi_sdb_cache);
1794
1795         return -ENOMEM;
1796 }
1797
1798 void scsi_exit_queue(void)
1799 {
1800         int i;
1801
1802         kmem_cache_destroy(scsi_sdb_cache);
1803
1804         for (i = 0; i < SG_MEMPOOL_NR; i++) {
1805                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1806                 mempool_destroy(sgp->pool);
1807                 kmem_cache_destroy(sgp->slab);
1808         }
1809 }
1810
1811 /**
1812  *      scsi_mode_select - issue a mode select
1813  *      @sdev:  SCSI device to be queried
1814  *      @pf:    Page format bit (1 == standard, 0 == vendor specific)
1815  *      @sp:    Save page bit (0 == don't save, 1 == save)
1816  *      @modepage: mode page being requested
1817  *      @buffer: request buffer (may not be smaller than eight bytes)
1818  *      @len:   length of request buffer.
1819  *      @timeout: command timeout
1820  *      @retries: number of retries before failing
1821  *      @data: returns a structure abstracting the mode header data
1822  *      @sshdr: place to put sense data (or NULL if no sense to be collected).
1823  *              must be SCSI_SENSE_BUFFERSIZE big.
1824  *
1825  *      Returns zero if successful; negative error number or scsi
1826  *      status on error
1827  *
1828  */
1829 int
1830 scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage,
1831                  unsigned char *buffer, int len, int timeout, int retries,
1832                  struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
1833 {
1834         unsigned char cmd[10];
1835         unsigned char *real_buffer;
1836         int ret;
1837
1838         memset(cmd, 0, sizeof(cmd));
1839         cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0);
1840
1841         if (sdev->use_10_for_ms) {
1842                 if (len > 65535)
1843                         return -EINVAL;
1844                 real_buffer = kmalloc(8 + len, GFP_KERNEL);
1845                 if (!real_buffer)
1846                         return -ENOMEM;
1847                 memcpy(real_buffer + 8, buffer, len);
1848                 len += 8;
1849                 real_buffer[0] = 0;
1850                 real_buffer[1] = 0;
1851                 real_buffer[2] = data->medium_type;
1852                 real_buffer[3] = data->device_specific;
1853                 real_buffer[4] = data->longlba ? 0x01 : 0;
1854                 real_buffer[5] = 0;
1855                 real_buffer[6] = data->block_descriptor_length >> 8;
1856                 real_buffer[7] = data->block_descriptor_length;
1857
1858                 cmd[0] = MODE_SELECT_10;
1859                 cmd[7] = len >> 8;
1860                 cmd[8] = len;
1861         } else {
1862                 if (len > 255 || data->block_descriptor_length > 255 ||
1863                     data->longlba)
1864                         return -EINVAL;
1865
1866                 real_buffer = kmalloc(4 + len, GFP_KERNEL);
1867                 if (!real_buffer)
1868                         return -ENOMEM;
1869                 memcpy(real_buffer + 4, buffer, len);
1870                 len += 4;
1871                 real_buffer[0] = 0;
1872                 real_buffer[1] = data->medium_type;
1873                 real_buffer[2] = data->device_specific;
1874                 real_buffer[3] = data->block_descriptor_length;
1875                 
1876
1877                 cmd[0] = MODE_SELECT;
1878                 cmd[4] = len;
1879         }
1880
1881         ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len,
1882                                sshdr, timeout, retries, NULL);
1883         kfree(real_buffer);
1884         return ret;
1885 }
1886 EXPORT_SYMBOL_GPL(scsi_mode_select);
1887
1888 /**
1889  *      scsi_mode_sense - issue a mode sense, falling back from 10 to six bytes if necessary.
1890  *      @sdev:  SCSI device to be queried
1891  *      @dbd:   set if mode sense will allow block descriptors to be returned
1892  *      @modepage: mode page being requested
1893  *      @buffer: request buffer (may not be smaller than eight bytes)
1894  *      @len:   length of request buffer.
1895  *      @timeout: command timeout
1896  *      @retries: number of retries before failing
1897  *      @data: returns a structure abstracting the mode header data
1898  *      @sshdr: place to put sense data (or NULL if no sense to be collected).
1899  *              must be SCSI_SENSE_BUFFERSIZE big.
1900  *
1901  *      Returns zero if unsuccessful, or the header offset (either 4
1902  *      or 8 depending on whether a six or ten byte command was
1903  *      issued) if successful.
1904  */
1905 int
1906 scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
1907                   unsigned char *buffer, int len, int timeout, int retries,
1908                   struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
1909 {
1910         unsigned char cmd[12];
1911         int use_10_for_ms;
1912         int header_length;
1913         int result;
1914         struct scsi_sense_hdr my_sshdr;
1915
1916         memset(data, 0, sizeof(*data));
1917         memset(&cmd[0], 0, 12);
1918         cmd[1] = dbd & 0x18;    /* allows DBD and LLBA bits */
1919         cmd[2] = modepage;
1920
1921         /* caller might not be interested in sense, but we need it */
1922         if (!sshdr)
1923                 sshdr = &my_sshdr;
1924
1925  retry:
1926         use_10_for_ms = sdev->use_10_for_ms;
1927
1928         if (use_10_for_ms) {
1929                 if (len < 8)
1930                         len = 8;
1931
1932                 cmd[0] = MODE_SENSE_10;
1933                 cmd[8] = len;
1934                 header_length = 8;
1935         } else {
1936                 if (len < 4)
1937                         len = 4;
1938
1939                 cmd[0] = MODE_SENSE;
1940                 cmd[4] = len;
1941                 header_length = 4;
1942         }
1943
1944         memset(buffer, 0, len);
1945
1946         result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len,
1947                                   sshdr, timeout, retries, NULL);
1948
1949         /* This code looks awful: what it's doing is making sure an
1950          * ILLEGAL REQUEST sense return identifies the actual command
1951          * byte as the problem.  MODE_SENSE commands can return
1952          * ILLEGAL REQUEST if the code page isn't supported */
1953
1954         if (use_10_for_ms && !scsi_status_is_good(result) &&
1955             (driver_byte(result) & DRIVER_SENSE)) {
1956                 if (scsi_sense_valid(sshdr)) {
1957                         if ((sshdr->sense_key == ILLEGAL_REQUEST) &&
1958                             (sshdr->asc == 0x20) && (sshdr->ascq == 0)) {
1959                                 /* 
1960                                  * Invalid command operation code
1961                                  */
1962                                 sdev->use_10_for_ms = 0;
1963                                 goto retry;
1964                         }
1965                 }
1966         }
1967
1968         if(scsi_status_is_good(result)) {
1969                 if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b &&
1970                              (modepage == 6 || modepage == 8))) {
1971                         /* Initio breakage? */
1972                         header_length = 0;
1973                         data->length = 13;
1974                         data->medium_type = 0;
1975                         data->device_specific = 0;
1976                         data->longlba = 0;
1977                         data->block_descriptor_length = 0;
1978                 } else if(use_10_for_ms) {
1979                         data->length = buffer[0]*256 + buffer[1] + 2;
1980                         data->medium_type = buffer[2];
1981                         data->device_specific = buffer[3];
1982                         data->longlba = buffer[4] & 0x01;
1983                         data->block_descriptor_length = buffer[6]*256
1984                                 + buffer[7];
1985                 } else {
1986                         data->length = buffer[0] + 1;
1987                         data->medium_type = buffer[1];
1988                         data->device_specific = buffer[2];
1989                         data->block_descriptor_length = buffer[3];
1990                 }
1991                 data->header_length = header_length;
1992         }
1993
1994         return result;
1995 }
1996 EXPORT_SYMBOL(scsi_mode_sense);
1997
1998 /**
1999  *      scsi_test_unit_ready - test if unit is ready
2000  *      @sdev:  scsi device to change the state of.
2001  *      @timeout: command timeout
2002  *      @retries: number of retries before failing
2003  *      @sshdr_external: Optional pointer to struct scsi_sense_hdr for
2004  *              returning sense. Make sure that this is cleared before passing
2005  *              in.
2006  *
2007  *      Returns zero if unsuccessful or an error if TUR failed.  For
2008  *      removable media, UNIT_ATTENTION sets ->changed flag.
2009  **/
2010 int
2011 scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries,
2012                      struct scsi_sense_hdr *sshdr_external)
2013 {
2014         char cmd[] = {
2015                 TEST_UNIT_READY, 0, 0, 0, 0, 0,
2016         };
2017         struct scsi_sense_hdr *sshdr;
2018         int result;
2019
2020         if (!sshdr_external)
2021                 sshdr = kzalloc(sizeof(*sshdr), GFP_KERNEL);
2022         else
2023                 sshdr = sshdr_external;
2024
2025         /* try to eat the UNIT_ATTENTION if there are enough retries */
2026         do {
2027                 result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, sshdr,
2028                                           timeout, retries, NULL);
2029                 if (sdev->removable && scsi_sense_valid(sshdr) &&
2030                     sshdr->sense_key == UNIT_ATTENTION)
2031                         sdev->changed = 1;
2032         } while (scsi_sense_valid(sshdr) &&
2033                  sshdr->sense_key == UNIT_ATTENTION && --retries);
2034
2035         if (!sshdr_external)
2036                 kfree(sshdr);
2037         return result;
2038 }
2039 EXPORT_SYMBOL(scsi_test_unit_ready);
2040
2041 /**
2042  *      scsi_device_set_state - Take the given device through the device state model.
2043  *      @sdev:  scsi device to change the state of.
2044  *      @state: state to change to.
2045  *
2046  *      Returns zero if unsuccessful or an error if the requested 
2047  *      transition is illegal.
2048  */
2049 int
2050 scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
2051 {
2052         enum scsi_device_state oldstate = sdev->sdev_state;
2053
2054         if (state == oldstate)
2055                 return 0;
2056
2057         switch (state) {
2058         case SDEV_CREATED:
2059                 switch (oldstate) {
2060                 case SDEV_CREATED_BLOCK:
2061                         break;
2062                 default:
2063                         goto illegal;
2064                 }
2065                 break;
2066                         
2067         case SDEV_RUNNING:
2068                 switch (oldstate) {
2069                 case SDEV_CREATED:
2070                 case SDEV_OFFLINE:
2071                 case SDEV_QUIESCE:
2072                 case SDEV_BLOCK:
2073                         break;
2074                 default:
2075                         goto illegal;
2076                 }
2077                 break;
2078
2079         case SDEV_QUIESCE:
2080                 switch (oldstate) {
2081                 case SDEV_RUNNING:
2082                 case SDEV_OFFLINE:
2083                         break;
2084                 default:
2085                         goto illegal;
2086                 }
2087                 break;
2088
2089         case SDEV_OFFLINE:
2090                 switch (oldstate) {
2091                 case SDEV_CREATED:
2092                 case SDEV_RUNNING:
2093                 case SDEV_QUIESCE:
2094                 case SDEV_BLOCK:
2095                         break;
2096                 default:
2097                         goto illegal;
2098                 }
2099                 break;
2100
2101         case SDEV_BLOCK:
2102                 switch (oldstate) {
2103                 case SDEV_RUNNING:
2104                 case SDEV_CREATED_BLOCK:
2105                         break;
2106                 default:
2107                         goto illegal;
2108                 }
2109                 break;
2110
2111         case SDEV_CREATED_BLOCK:
2112                 switch (oldstate) {
2113                 case SDEV_CREATED:
2114                         break;
2115                 default:
2116                         goto illegal;
2117                 }
2118                 break;
2119
2120         case SDEV_CANCEL:
2121                 switch (oldstate) {
2122                 case SDEV_CREATED:
2123                 case SDEV_RUNNING:
2124                 case SDEV_QUIESCE:
2125                 case SDEV_OFFLINE:
2126                 case SDEV_BLOCK:
2127                         break;
2128                 default:
2129                         goto illegal;
2130                 }
2131                 break;
2132
2133         case SDEV_DEL:
2134                 switch (oldstate) {
2135                 case SDEV_CREATED:
2136                 case SDEV_RUNNING:
2137                 case SDEV_OFFLINE:
2138                 case SDEV_CANCEL:
2139                         break;
2140                 default:
2141                         goto illegal;
2142                 }
2143                 break;
2144
2145         }
2146         sdev->sdev_state = state;
2147         return 0;
2148
2149  illegal:
2150         SCSI_LOG_ERROR_RECOVERY(1, 
2151                                 sdev_printk(KERN_ERR, sdev,
2152                                             "Illegal state transition %s->%s\n",
2153                                             scsi_device_state_name(oldstate),
2154                                             scsi_device_state_name(state))
2155                                 );
2156         return -EINVAL;
2157 }
2158 EXPORT_SYMBOL(scsi_device_set_state);
2159
2160 /**
2161  *      sdev_evt_emit - emit a single SCSI device uevent
2162  *      @sdev: associated SCSI device
2163  *      @evt: event to emit
2164  *
2165  *      Send a single uevent (scsi_event) to the associated scsi_device.
2166  */
2167 static void scsi_evt_emit(struct scsi_device *sdev, struct scsi_event *evt)
2168 {
2169         int idx = 0;
2170         char *envp[3];
2171
2172         switch (evt->evt_type) {
2173         case SDEV_EVT_MEDIA_CHANGE:
2174                 envp[idx++] = "SDEV_MEDIA_CHANGE=1";
2175                 break;
2176
2177         default:
2178                 /* do nothing */
2179                 break;
2180         }
2181
2182         envp[idx++] = NULL;
2183
2184         kobject_uevent_env(&sdev->sdev_gendev.kobj, KOBJ_CHANGE, envp);
2185 }
2186
2187 /**
2188  *      sdev_evt_thread - send a uevent for each scsi event
2189  *      @work: work struct for scsi_device
2190  *
2191  *      Dispatch queued events to their associated scsi_device kobjects
2192  *      as uevents.
2193  */
2194 void scsi_evt_thread(struct work_struct *work)
2195 {
2196         struct scsi_device *sdev;
2197         LIST_HEAD(event_list);
2198
2199         sdev = container_of(work, struct scsi_device, event_work);
2200
2201         while (1) {
2202                 struct scsi_event *evt;
2203                 struct list_head *this, *tmp;
2204                 unsigned long flags;
2205
2206                 spin_lock_irqsave(&sdev->list_lock, flags);
2207                 list_splice_init(&sdev->event_list, &event_list);
2208                 spin_unlock_irqrestore(&sdev->list_lock, flags);
2209
2210                 if (list_empty(&event_list))
2211                         break;
2212
2213                 list_for_each_safe(this, tmp, &event_list) {
2214                         evt = list_entry(this, struct scsi_event, node);
2215                         list_del(&evt->node);
2216                         scsi_evt_emit(sdev, evt);
2217                         kfree(evt);
2218                 }
2219         }
2220 }
2221
2222 /**
2223  *      sdev_evt_send - send asserted event to uevent thread
2224  *      @sdev: scsi_device event occurred on
2225  *      @evt: event to send
2226  *
2227  *      Assert scsi device event asynchronously.
2228  */
2229 void sdev_evt_send(struct scsi_device *sdev, struct scsi_event *evt)
2230 {
2231         unsigned long flags;
2232
2233 #if 0
2234         /* FIXME: currently this check eliminates all media change events
2235          * for polled devices.  Need to update to discriminate between AN
2236          * and polled events */
2237         if (!test_bit(evt->evt_type, sdev->supported_events)) {
2238                 kfree(evt);
2239                 return;
2240         }
2241 #endif
2242
2243         spin_lock_irqsave(&sdev->list_lock, flags);
2244         list_add_tail(&evt->node, &sdev->event_list);
2245         schedule_work(&sdev->event_work);
2246         spin_unlock_irqrestore(&sdev->list_lock, flags);
2247 }
2248 EXPORT_SYMBOL_GPL(sdev_evt_send);
2249
2250 /**
2251  *      sdev_evt_alloc - allocate a new scsi event
2252  *      @evt_type: type of event to allocate
2253  *      @gfpflags: GFP flags for allocation
2254  *
2255  *      Allocates and returns a new scsi_event.
2256  */
2257 struct scsi_event *sdev_evt_alloc(enum scsi_device_event evt_type,
2258                                   gfp_t gfpflags)
2259 {
2260         struct scsi_event *evt = kzalloc(sizeof(struct scsi_event), gfpflags);
2261         if (!evt)
2262                 return NULL;
2263
2264         evt->evt_type = evt_type;
2265         INIT_LIST_HEAD(&evt->node);
2266
2267         /* evt_type-specific initialization, if any */
2268         switch (evt_type) {
2269         case SDEV_EVT_MEDIA_CHANGE:
2270         default:
2271                 /* do nothing */
2272                 break;
2273         }
2274
2275         return evt;
2276 }
2277 EXPORT_SYMBOL_GPL(sdev_evt_alloc);
2278
2279 /**
2280  *      sdev_evt_send_simple - send asserted event to uevent thread
2281  *      @sdev: scsi_device event occurred on
2282  *      @evt_type: type of event to send
2283  *      @gfpflags: GFP flags for allocation
2284  *
2285  *      Assert scsi device event asynchronously, given an event type.
2286  */
2287 void sdev_evt_send_simple(struct scsi_device *sdev,
2288                           enum scsi_device_event evt_type, gfp_t gfpflags)
2289 {
2290         struct scsi_event *evt = sdev_evt_alloc(evt_type, gfpflags);
2291         if (!evt) {
2292                 sdev_printk(KERN_ERR, sdev, "event %d eaten due to OOM\n",
2293                             evt_type);
2294                 return;
2295         }
2296
2297         sdev_evt_send(sdev, evt);
2298 }
2299 EXPORT_SYMBOL_GPL(sdev_evt_send_simple);
2300
2301 /**
2302  *      scsi_device_quiesce - Block user issued commands.
2303  *      @sdev:  scsi device to quiesce.
2304  *
2305  *      This works by trying to transition to the SDEV_QUIESCE state
2306  *      (which must be a legal transition).  When the device is in this
2307  *      state, only special requests will be accepted, all others will
2308  *      be deferred.  Since special requests may also be requeued requests,
2309  *      a successful return doesn't guarantee the device will be 
2310  *      totally quiescent.
2311  *
2312  *      Must be called with user context, may sleep.
2313  *
2314  *      Returns zero if unsuccessful or an error if not.
2315  */
2316 int
2317 scsi_device_quiesce(struct scsi_device *sdev)
2318 {
2319         int err = scsi_device_set_state(sdev, SDEV_QUIESCE);
2320         if (err)
2321                 return err;
2322
2323         scsi_run_queue(sdev->request_queue);
2324         while (sdev->device_busy) {
2325                 msleep_interruptible(200);
2326                 scsi_run_queue(sdev->request_queue);
2327         }
2328         return 0;
2329 }
2330 EXPORT_SYMBOL(scsi_device_quiesce);
2331
2332 /**
2333  *      scsi_device_resume - Restart user issued commands to a quiesced device.
2334  *      @sdev:  scsi device to resume.
2335  *
2336  *      Moves the device from quiesced back to running and restarts the
2337  *      queues.
2338  *
2339  *      Must be called with user context, may sleep.
2340  */
2341 void
2342 scsi_device_resume(struct scsi_device *sdev)
2343 {
2344         if(scsi_device_set_state(sdev, SDEV_RUNNING))
2345                 return;
2346         scsi_run_queue(sdev->request_queue);
2347 }
2348 EXPORT_SYMBOL(scsi_device_resume);
2349
2350 static void
2351 device_quiesce_fn(struct scsi_device *sdev, void *data)
2352 {
2353         scsi_device_quiesce(sdev);
2354 }
2355
2356 void
2357 scsi_target_quiesce(struct scsi_target *starget)
2358 {
2359         starget_for_each_device(starget, NULL, device_quiesce_fn);
2360 }
2361 EXPORT_SYMBOL(scsi_target_quiesce);
2362
2363 static void
2364 device_resume_fn(struct scsi_device *sdev, void *data)
2365 {
2366         scsi_device_resume(sdev);
2367 }
2368
2369 void
2370 scsi_target_resume(struct scsi_target *starget)
2371 {
2372         starget_for_each_device(starget, NULL, device_resume_fn);
2373 }
2374 EXPORT_SYMBOL(scsi_target_resume);
2375
2376 /**
2377  * scsi_internal_device_block - internal function to put a device temporarily into the SDEV_BLOCK state
2378  * @sdev:       device to block
2379  *
2380  * Block request made by scsi lld's to temporarily stop all
2381  * scsi commands on the specified device.  Called from interrupt
2382  * or normal process context.
2383  *
2384  * Returns zero if successful or error if not
2385  *
2386  * Notes:       
2387  *      This routine transitions the device to the SDEV_BLOCK state
2388  *      (which must be a legal transition).  When the device is in this
2389  *      state, all commands are deferred until the scsi lld reenables
2390  *      the device with scsi_device_unblock or device_block_tmo fires.
2391  *      This routine assumes the host_lock is held on entry.
2392  */
2393 int
2394 scsi_internal_device_block(struct scsi_device *sdev)
2395 {
2396         struct request_queue *q = sdev->request_queue;
2397         unsigned long flags;
2398         int err = 0;
2399
2400         err = scsi_device_set_state(sdev, SDEV_BLOCK);
2401         if (err) {
2402                 err = scsi_device_set_state(sdev, SDEV_CREATED_BLOCK);
2403
2404                 if (err)
2405                         return err;
2406         }
2407
2408         /* 
2409          * The device has transitioned to SDEV_BLOCK.  Stop the
2410          * block layer from calling the midlayer with this device's
2411          * request queue. 
2412          */
2413         spin_lock_irqsave(q->queue_lock, flags);
2414         blk_stop_queue(q);
2415         spin_unlock_irqrestore(q->queue_lock, flags);
2416
2417         return 0;
2418 }
2419 EXPORT_SYMBOL_GPL(scsi_internal_device_block);
2420  
2421 /**
2422  * scsi_internal_device_unblock - resume a device after a block request
2423  * @sdev:       device to resume
2424  *
2425  * Called by scsi lld's or the midlayer to restart the device queue
2426  * for the previously suspended scsi device.  Called from interrupt or
2427  * normal process context.
2428  *
2429  * Returns zero if successful or error if not.
2430  *
2431  * Notes:       
2432  *      This routine transitions the device to the SDEV_RUNNING state
2433  *      (which must be a legal transition) allowing the midlayer to
2434  *      goose the queue for this device.  This routine assumes the 
2435  *      host_lock is held upon entry.
2436  */
2437 int
2438 scsi_internal_device_unblock(struct scsi_device *sdev)
2439 {
2440         struct request_queue *q = sdev->request_queue; 
2441         unsigned long flags;
2442         
2443         /* 
2444          * Try to transition the scsi device to SDEV_RUNNING
2445          * and goose the device queue if successful.  
2446          */
2447         if (sdev->sdev_state == SDEV_BLOCK)
2448                 sdev->sdev_state = SDEV_RUNNING;
2449         else if (sdev->sdev_state == SDEV_CREATED_BLOCK)
2450                 sdev->sdev_state = SDEV_CREATED;
2451         else if (sdev->sdev_state != SDEV_CANCEL &&
2452                  sdev->sdev_state != SDEV_OFFLINE)
2453                 return -EINVAL;
2454
2455         spin_lock_irqsave(q->queue_lock, flags);
2456         blk_start_queue(q);
2457         spin_unlock_irqrestore(q->queue_lock, flags);
2458
2459         return 0;
2460 }
2461 EXPORT_SYMBOL_GPL(scsi_internal_device_unblock);
2462
2463 static void
2464 device_block(struct scsi_device *sdev, void *data)
2465 {
2466         scsi_internal_device_block(sdev);
2467 }
2468
2469 static int
2470 target_block(struct device *dev, void *data)
2471 {
2472         if (scsi_is_target_device(dev))
2473                 starget_for_each_device(to_scsi_target(dev), NULL,
2474                                         device_block);
2475         return 0;
2476 }
2477
2478 void
2479 scsi_target_block(struct device *dev)
2480 {
2481         if (scsi_is_target_device(dev))
2482                 starget_for_each_device(to_scsi_target(dev), NULL,
2483                                         device_block);
2484         else
2485                 device_for_each_child(dev, NULL, target_block);
2486 }
2487 EXPORT_SYMBOL_GPL(scsi_target_block);
2488
2489 static void
2490 device_unblock(struct scsi_device *sdev, void *data)
2491 {
2492         scsi_internal_device_unblock(sdev);
2493 }
2494
2495 static int
2496 target_unblock(struct device *dev, void *data)
2497 {
2498         if (scsi_is_target_device(dev))
2499                 starget_for_each_device(to_scsi_target(dev), NULL,
2500                                         device_unblock);
2501         return 0;
2502 }
2503
2504 void
2505 scsi_target_unblock(struct device *dev)
2506 {
2507         if (scsi_is_target_device(dev))
2508                 starget_for_each_device(to_scsi_target(dev), NULL,
2509                                         device_unblock);
2510         else
2511                 device_for_each_child(dev, NULL, target_unblock);
2512 }
2513 EXPORT_SYMBOL_GPL(scsi_target_unblock);
2514
2515 /**
2516  * scsi_kmap_atomic_sg - find and atomically map an sg-elemnt
2517  * @sgl:        scatter-gather list
2518  * @sg_count:   number of segments in sg
2519  * @offset:     offset in bytes into sg, on return offset into the mapped area
2520  * @len:        bytes to map, on return number of bytes mapped
2521  *
2522  * Returns virtual address of the start of the mapped page
2523  */
2524 void *scsi_kmap_atomic_sg(struct scatterlist *sgl, int sg_count,
2525                           size_t *offset, size_t *len)
2526 {
2527         int i;
2528         size_t sg_len = 0, len_complete = 0;
2529         struct scatterlist *sg;
2530         struct page *page;
2531
2532         WARN_ON(!irqs_disabled());
2533
2534         for_each_sg(sgl, sg, sg_count, i) {
2535                 len_complete = sg_len; /* Complete sg-entries */
2536                 sg_len += sg->length;
2537                 if (sg_len > *offset)
2538                         break;
2539         }
2540
2541         if (unlikely(i == sg_count)) {
2542                 printk(KERN_ERR "%s: Bytes in sg: %zu, requested offset %zu, "
2543                         "elements %d\n",
2544                        __func__, sg_len, *offset, sg_count);
2545                 WARN_ON(1);
2546                 return NULL;
2547         }
2548
2549         /* Offset starting from the beginning of first page in this sg-entry */
2550         *offset = *offset - len_complete + sg->offset;
2551
2552         /* Assumption: contiguous pages can be accessed as "page + i" */
2553         page = nth_page(sg_page(sg), (*offset >> PAGE_SHIFT));
2554         *offset &= ~PAGE_MASK;
2555
2556         /* Bytes in this sg-entry from *offset to the end of the page */
2557         sg_len = PAGE_SIZE - *offset;
2558         if (*len > sg_len)
2559                 *len = sg_len;
2560
2561         return kmap_atomic(page, KM_BIO_SRC_IRQ);
2562 }
2563 EXPORT_SYMBOL(scsi_kmap_atomic_sg);
2564
2565 /**
2566  * scsi_kunmap_atomic_sg - atomically unmap a virtual address, previously mapped with scsi_kmap_atomic_sg
2567  * @virt:       virtual address to be unmapped
2568  */
2569 void scsi_kunmap_atomic_sg(void *virt)
2570 {
2571         kunmap_atomic(virt, KM_BIO_SRC_IRQ);
2572 }
2573 EXPORT_SYMBOL(scsi_kunmap_atomic_sg);