d6743b959a72b151f1d6ade6d70b6d3b70c44d92
[linux-3.10.git] / drivers / scsi / scsi_lib.c
1 /*
2  *  scsi_lib.c Copyright (C) 1999 Eric Youngdale
3  *
4  *  SCSI queueing library.
5  *      Initial versions: Eric Youngdale (eric@andante.org).
6  *                        Based upon conversations with large numbers
7  *                        of people at Linux Expo.
8  */
9
10 #include <linux/bio.h>
11 #include <linux/blkdev.h>
12 #include <linux/completion.h>
13 #include <linux/kernel.h>
14 #include <linux/mempool.h>
15 #include <linux/slab.h>
16 #include <linux/init.h>
17 #include <linux/pci.h>
18 #include <linux/delay.h>
19 #include <linux/hardirq.h>
20
21 #include <scsi/scsi.h>
22 #include <scsi/scsi_cmnd.h>
23 #include <scsi/scsi_dbg.h>
24 #include <scsi/scsi_device.h>
25 #include <scsi/scsi_driver.h>
26 #include <scsi/scsi_eh.h>
27 #include <scsi/scsi_host.h>
28
29 #include "scsi_priv.h"
30 #include "scsi_logging.h"
31
32
33 #define SG_MEMPOOL_NR           ARRAY_SIZE(scsi_sg_pools)
34 #define SG_MEMPOOL_SIZE         32
35
36 struct scsi_host_sg_pool {
37         size_t          size;
38         char            *name; 
39         kmem_cache_t    *slab;
40         mempool_t       *pool;
41 };
42
43 #if (SCSI_MAX_PHYS_SEGMENTS < 32)
44 #error SCSI_MAX_PHYS_SEGMENTS is too small
45 #endif
46
47 #define SP(x) { x, "sgpool-" #x } 
48 static struct scsi_host_sg_pool scsi_sg_pools[] = {
49         SP(8),
50         SP(16),
51         SP(32),
52 #if (SCSI_MAX_PHYS_SEGMENTS > 32)
53         SP(64),
54 #if (SCSI_MAX_PHYS_SEGMENTS > 64)
55         SP(128),
56 #if (SCSI_MAX_PHYS_SEGMENTS > 128)
57         SP(256),
58 #if (SCSI_MAX_PHYS_SEGMENTS > 256)
59 #error SCSI_MAX_PHYS_SEGMENTS is too large
60 #endif
61 #endif
62 #endif
63 #endif
64 };      
65 #undef SP
66
67 static void scsi_run_queue(struct request_queue *q);
68
69 /*
70  * Function:    scsi_unprep_request()
71  *
72  * Purpose:     Remove all preparation done for a request, including its
73  *              associated scsi_cmnd, so that it can be requeued.
74  *
75  * Arguments:   req     - request to unprepare
76  *
77  * Lock status: Assumed that no locks are held upon entry.
78  *
79  * Returns:     Nothing.
80  */
81 static void scsi_unprep_request(struct request *req)
82 {
83         struct scsi_cmnd *cmd = req->special;
84
85         req->flags &= ~REQ_DONTPREP;
86         req->special = NULL;
87
88         scsi_put_command(cmd);
89 }
90
91 /*
92  * Function:    scsi_queue_insert()
93  *
94  * Purpose:     Insert a command in the midlevel queue.
95  *
96  * Arguments:   cmd    - command that we are adding to queue.
97  *              reason - why we are inserting command to queue.
98  *
99  * Lock status: Assumed that lock is not held upon entry.
100  *
101  * Returns:     Nothing.
102  *
103  * Notes:       We do this for one of two cases.  Either the host is busy
104  *              and it cannot accept any more commands for the time being,
105  *              or the device returned QUEUE_FULL and can accept no more
106  *              commands.
107  * Notes:       This could be called either from an interrupt context or a
108  *              normal process context.
109  */
110 int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
111 {
112         struct Scsi_Host *host = cmd->device->host;
113         struct scsi_device *device = cmd->device;
114         struct request_queue *q = device->request_queue;
115         unsigned long flags;
116
117         SCSI_LOG_MLQUEUE(1,
118                  printk("Inserting command %p into mlqueue\n", cmd));
119
120         /*
121          * Set the appropriate busy bit for the device/host.
122          *
123          * If the host/device isn't busy, assume that something actually
124          * completed, and that we should be able to queue a command now.
125          *
126          * Note that the prior mid-layer assumption that any host could
127          * always queue at least one command is now broken.  The mid-layer
128          * will implement a user specifiable stall (see
129          * scsi_host.max_host_blocked and scsi_device.max_device_blocked)
130          * if a command is requeued with no other commands outstanding
131          * either for the device or for the host.
132          */
133         if (reason == SCSI_MLQUEUE_HOST_BUSY)
134                 host->host_blocked = host->max_host_blocked;
135         else if (reason == SCSI_MLQUEUE_DEVICE_BUSY)
136                 device->device_blocked = device->max_device_blocked;
137
138         /*
139          * Decrement the counters, since these commands are no longer
140          * active on the host/device.
141          */
142         scsi_device_unbusy(device);
143
144         /*
145          * Requeue this command.  It will go before all other commands
146          * that are already in the queue.
147          *
148          * NOTE: there is magic here about the way the queue is plugged if
149          * we have no outstanding commands.
150          * 
151          * Although we *don't* plug the queue, we call the request
152          * function.  The SCSI request function detects the blocked condition
153          * and plugs the queue appropriately.
154          */
155         spin_lock_irqsave(q->queue_lock, flags);
156         blk_requeue_request(q, cmd->request);
157         spin_unlock_irqrestore(q->queue_lock, flags);
158
159         scsi_run_queue(q);
160
161         return 0;
162 }
163
164 /**
165  * scsi_execute - insert request and wait for the result
166  * @sdev:       scsi device
167  * @cmd:        scsi command
168  * @data_direction: data direction
169  * @buffer:     data buffer
170  * @bufflen:    len of buffer
171  * @sense:      optional sense buffer
172  * @timeout:    request timeout in seconds
173  * @retries:    number of times to retry request
174  * @flags:      or into request flags;
175  *
176  * returns the req->errors value which is the the scsi_cmnd result
177  * field.
178  **/
179 int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
180                  int data_direction, void *buffer, unsigned bufflen,
181                  unsigned char *sense, int timeout, int retries, int flags)
182 {
183         struct request *req;
184         int write = (data_direction == DMA_TO_DEVICE);
185         int ret = DRIVER_ERROR << 24;
186
187         req = blk_get_request(sdev->request_queue, write, __GFP_WAIT);
188
189         if (bufflen &&  blk_rq_map_kern(sdev->request_queue, req,
190                                         buffer, bufflen, __GFP_WAIT))
191                 goto out;
192
193         req->cmd_len = COMMAND_SIZE(cmd[0]);
194         memcpy(req->cmd, cmd, req->cmd_len);
195         req->sense = sense;
196         req->sense_len = 0;
197         req->retries = retries;
198         req->timeout = timeout;
199         req->flags |= flags | REQ_BLOCK_PC | REQ_SPECIAL | REQ_QUIET;
200
201         /*
202          * head injection *required* here otherwise quiesce won't work
203          */
204         blk_execute_rq(req->q, NULL, req, 1);
205
206         ret = req->errors;
207  out:
208         blk_put_request(req);
209
210         return ret;
211 }
212 EXPORT_SYMBOL(scsi_execute);
213
214
215 int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd,
216                      int data_direction, void *buffer, unsigned bufflen,
217                      struct scsi_sense_hdr *sshdr, int timeout, int retries)
218 {
219         char *sense = NULL;
220         int result;
221         
222         if (sshdr) {
223                 sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO);
224                 if (!sense)
225                         return DRIVER_ERROR << 24;
226         }
227         result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen,
228                               sense, timeout, retries, 0);
229         if (sshdr)
230                 scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr);
231
232         kfree(sense);
233         return result;
234 }
235 EXPORT_SYMBOL(scsi_execute_req);
236
237 struct scsi_io_context {
238         void *data;
239         void (*done)(void *data, char *sense, int result, int resid);
240         char sense[SCSI_SENSE_BUFFERSIZE];
241 };
242
243 static kmem_cache_t *scsi_io_context_cache;
244
245 static void scsi_end_async(struct request *req, int uptodate)
246 {
247         struct scsi_io_context *sioc = req->end_io_data;
248
249         if (sioc->done)
250                 sioc->done(sioc->data, sioc->sense, req->errors, req->data_len);
251
252         kmem_cache_free(scsi_io_context_cache, sioc);
253         __blk_put_request(req->q, req);
254 }
255
256 static int scsi_merge_bio(struct request *rq, struct bio *bio)
257 {
258         struct request_queue *q = rq->q;
259
260         bio->bi_flags &= ~(1 << BIO_SEG_VALID);
261         if (rq_data_dir(rq) == WRITE)
262                 bio->bi_rw |= (1 << BIO_RW);
263         blk_queue_bounce(q, &bio);
264
265         if (!rq->bio)
266                 blk_rq_bio_prep(q, rq, bio);
267         else if (!q->back_merge_fn(q, rq, bio))
268                 return -EINVAL;
269         else {
270                 rq->biotail->bi_next = bio;
271                 rq->biotail = bio;
272                 rq->hard_nr_sectors += bio_sectors(bio);
273                 rq->nr_sectors = rq->hard_nr_sectors;
274         }
275
276         return 0;
277 }
278
279 static int scsi_bi_endio(struct bio *bio, unsigned int bytes_done, int error)
280 {
281         if (bio->bi_size)
282                 return 1;
283
284         bio_put(bio);
285         return 0;
286 }
287
288 /**
289  * scsi_req_map_sg - map a scatterlist into a request
290  * @rq:         request to fill
291  * @sg:         scatterlist
292  * @nsegs:      number of elements
293  * @bufflen:    len of buffer
294  * @gfp:        memory allocation flags
295  *
296  * scsi_req_map_sg maps a scatterlist into a request so that the
297  * request can be sent to the block layer. We do not trust the scatterlist
298  * sent to use, as some ULDs use that struct to only organize the pages.
299  */
300 static int scsi_req_map_sg(struct request *rq, struct scatterlist *sgl,
301                            int nsegs, unsigned bufflen, gfp_t gfp)
302 {
303         struct request_queue *q = rq->q;
304         int nr_pages = (bufflen + sgl[0].offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
305         unsigned int data_len = 0, len, bytes, off;
306         struct page *page;
307         struct bio *bio = NULL;
308         int i, err, nr_vecs = 0;
309
310         for (i = 0; i < nsegs; i++) {
311                 page = sgl[i].page;
312                 off = sgl[i].offset;
313                 len = sgl[i].length;
314                 data_len += len;
315
316                 while (len > 0) {
317                         bytes = min_t(unsigned int, len, PAGE_SIZE - off);
318
319                         if (!bio) {
320                                 nr_vecs = min_t(int, BIO_MAX_PAGES, nr_pages);
321                                 nr_pages -= nr_vecs;
322
323                                 bio = bio_alloc(gfp, nr_vecs);
324                                 if (!bio) {
325                                         err = -ENOMEM;
326                                         goto free_bios;
327                                 }
328                                 bio->bi_end_io = scsi_bi_endio;
329                         }
330
331                         if (bio_add_pc_page(q, bio, page, bytes, off) !=
332                             bytes) {
333                                 bio_put(bio);
334                                 err = -EINVAL;
335                                 goto free_bios;
336                         }
337
338                         if (bio->bi_vcnt >= nr_vecs) {
339                                 err = scsi_merge_bio(rq, bio);
340                                 if (err) {
341                                         bio_endio(bio, bio->bi_size, 0);
342                                         goto free_bios;
343                                 }
344                                 bio = NULL;
345                         }
346
347                         page++;
348                         len -= bytes;
349                         off = 0;
350                 }
351         }
352
353         rq->buffer = rq->data = NULL;
354         rq->data_len = data_len;
355         return 0;
356
357 free_bios:
358         while ((bio = rq->bio) != NULL) {
359                 rq->bio = bio->bi_next;
360                 /*
361                  * call endio instead of bio_put incase it was bounced
362                  */
363                 bio_endio(bio, bio->bi_size, 0);
364         }
365
366         return err;
367 }
368
369 /**
370  * scsi_execute_async - insert request
371  * @sdev:       scsi device
372  * @cmd:        scsi command
373  * @cmd_len:    length of scsi cdb
374  * @data_direction: data direction
375  * @buffer:     data buffer (this can be a kernel buffer or scatterlist)
376  * @bufflen:    len of buffer
377  * @use_sg:     if buffer is a scatterlist this is the number of elements
378  * @timeout:    request timeout in seconds
379  * @retries:    number of times to retry request
380  * @flags:      or into request flags
381  **/
382 int scsi_execute_async(struct scsi_device *sdev, const unsigned char *cmd,
383                        int cmd_len, int data_direction, void *buffer, unsigned bufflen,
384                        int use_sg, int timeout, int retries, void *privdata,
385                        void (*done)(void *, char *, int, int), gfp_t gfp)
386 {
387         struct request *req;
388         struct scsi_io_context *sioc;
389         int err = 0;
390         int write = (data_direction == DMA_TO_DEVICE);
391
392         sioc = kmem_cache_alloc(scsi_io_context_cache, gfp);
393         if (!sioc)
394                 return DRIVER_ERROR << 24;
395         memset(sioc, 0, sizeof(*sioc));
396
397         req = blk_get_request(sdev->request_queue, write, gfp);
398         if (!req)
399                 goto free_sense;
400         req->flags |= REQ_BLOCK_PC | REQ_QUIET;
401
402         if (use_sg)
403                 err = scsi_req_map_sg(req, buffer, use_sg, bufflen, gfp);
404         else if (bufflen)
405                 err = blk_rq_map_kern(req->q, req, buffer, bufflen, gfp);
406
407         if (err)
408                 goto free_req;
409
410         req->cmd_len = cmd_len;
411         memcpy(req->cmd, cmd, req->cmd_len);
412         req->sense = sioc->sense;
413         req->sense_len = 0;
414         req->timeout = timeout;
415         req->retries = retries;
416         req->end_io_data = sioc;
417
418         sioc->data = privdata;
419         sioc->done = done;
420
421         blk_execute_rq_nowait(req->q, NULL, req, 1, scsi_end_async);
422         return 0;
423
424 free_req:
425         blk_put_request(req);
426 free_sense:
427         kfree(sioc);
428         return DRIVER_ERROR << 24;
429 }
430 EXPORT_SYMBOL_GPL(scsi_execute_async);
431
432 /*
433  * Function:    scsi_init_cmd_errh()
434  *
435  * Purpose:     Initialize cmd fields related to error handling.
436  *
437  * Arguments:   cmd     - command that is ready to be queued.
438  *
439  * Notes:       This function has the job of initializing a number of
440  *              fields related to error handling.   Typically this will
441  *              be called once for each command, as required.
442  */
443 static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)
444 {
445         cmd->serial_number = 0;
446         memset(cmd->sense_buffer, 0, sizeof cmd->sense_buffer);
447         if (cmd->cmd_len == 0)
448                 cmd->cmd_len = COMMAND_SIZE(cmd->cmnd[0]);
449 }
450
451 void scsi_device_unbusy(struct scsi_device *sdev)
452 {
453         struct Scsi_Host *shost = sdev->host;
454         unsigned long flags;
455
456         spin_lock_irqsave(shost->host_lock, flags);
457         shost->host_busy--;
458         if (unlikely(scsi_host_in_recovery(shost) &&
459                      (shost->host_failed || shost->host_eh_scheduled)))
460                 scsi_eh_wakeup(shost);
461         spin_unlock(shost->host_lock);
462         spin_lock(sdev->request_queue->queue_lock);
463         sdev->device_busy--;
464         spin_unlock_irqrestore(sdev->request_queue->queue_lock, flags);
465 }
466
467 /*
468  * Called for single_lun devices on IO completion. Clear starget_sdev_user,
469  * and call blk_run_queue for all the scsi_devices on the target -
470  * including current_sdev first.
471  *
472  * Called with *no* scsi locks held.
473  */
474 static void scsi_single_lun_run(struct scsi_device *current_sdev)
475 {
476         struct Scsi_Host *shost = current_sdev->host;
477         struct scsi_device *sdev, *tmp;
478         struct scsi_target *starget = scsi_target(current_sdev);
479         unsigned long flags;
480
481         spin_lock_irqsave(shost->host_lock, flags);
482         starget->starget_sdev_user = NULL;
483         spin_unlock_irqrestore(shost->host_lock, flags);
484
485         /*
486          * Call blk_run_queue for all LUNs on the target, starting with
487          * current_sdev. We race with others (to set starget_sdev_user),
488          * but in most cases, we will be first. Ideally, each LU on the
489          * target would get some limited time or requests on the target.
490          */
491         blk_run_queue(current_sdev->request_queue);
492
493         spin_lock_irqsave(shost->host_lock, flags);
494         if (starget->starget_sdev_user)
495                 goto out;
496         list_for_each_entry_safe(sdev, tmp, &starget->devices,
497                         same_target_siblings) {
498                 if (sdev == current_sdev)
499                         continue;
500                 if (scsi_device_get(sdev))
501                         continue;
502
503                 spin_unlock_irqrestore(shost->host_lock, flags);
504                 blk_run_queue(sdev->request_queue);
505                 spin_lock_irqsave(shost->host_lock, flags);
506         
507                 scsi_device_put(sdev);
508         }
509  out:
510         spin_unlock_irqrestore(shost->host_lock, flags);
511 }
512
513 /*
514  * Function:    scsi_run_queue()
515  *
516  * Purpose:     Select a proper request queue to serve next
517  *
518  * Arguments:   q       - last request's queue
519  *
520  * Returns:     Nothing
521  *
522  * Notes:       The previous command was completely finished, start
523  *              a new one if possible.
524  */
525 static void scsi_run_queue(struct request_queue *q)
526 {
527         struct scsi_device *sdev = q->queuedata;
528         struct Scsi_Host *shost = sdev->host;
529         unsigned long flags;
530
531         if (sdev->single_lun)
532                 scsi_single_lun_run(sdev);
533
534         spin_lock_irqsave(shost->host_lock, flags);
535         while (!list_empty(&shost->starved_list) &&
536                !shost->host_blocked && !shost->host_self_blocked &&
537                 !((shost->can_queue > 0) &&
538                   (shost->host_busy >= shost->can_queue))) {
539                 /*
540                  * As long as shost is accepting commands and we have
541                  * starved queues, call blk_run_queue. scsi_request_fn
542                  * drops the queue_lock and can add us back to the
543                  * starved_list.
544                  *
545                  * host_lock protects the starved_list and starved_entry.
546                  * scsi_request_fn must get the host_lock before checking
547                  * or modifying starved_list or starved_entry.
548                  */
549                 sdev = list_entry(shost->starved_list.next,
550                                           struct scsi_device, starved_entry);
551                 list_del_init(&sdev->starved_entry);
552                 spin_unlock_irqrestore(shost->host_lock, flags);
553
554
555                 if (test_bit(QUEUE_FLAG_REENTER, &q->queue_flags) &&
556                     !test_and_set_bit(QUEUE_FLAG_REENTER,
557                                       &sdev->request_queue->queue_flags)) {
558                         blk_run_queue(sdev->request_queue);
559                         clear_bit(QUEUE_FLAG_REENTER,
560                                   &sdev->request_queue->queue_flags);
561                 } else
562                         blk_run_queue(sdev->request_queue);
563
564                 spin_lock_irqsave(shost->host_lock, flags);
565                 if (unlikely(!list_empty(&sdev->starved_entry)))
566                         /*
567                          * sdev lost a race, and was put back on the
568                          * starved list. This is unlikely but without this
569                          * in theory we could loop forever.
570                          */
571                         break;
572         }
573         spin_unlock_irqrestore(shost->host_lock, flags);
574
575         blk_run_queue(q);
576 }
577
578 /*
579  * Function:    scsi_requeue_command()
580  *
581  * Purpose:     Handle post-processing of completed commands.
582  *
583  * Arguments:   q       - queue to operate on
584  *              cmd     - command that may need to be requeued.
585  *
586  * Returns:     Nothing
587  *
588  * Notes:       After command completion, there may be blocks left
589  *              over which weren't finished by the previous command
590  *              this can be for a number of reasons - the main one is
591  *              I/O errors in the middle of the request, in which case
592  *              we need to request the blocks that come after the bad
593  *              sector.
594  * Notes:       Upon return, cmd is a stale pointer.
595  */
596 static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd)
597 {
598         struct request *req = cmd->request;
599         unsigned long flags;
600
601         scsi_unprep_request(req);
602         spin_lock_irqsave(q->queue_lock, flags);
603         blk_requeue_request(q, req);
604         spin_unlock_irqrestore(q->queue_lock, flags);
605
606         scsi_run_queue(q);
607 }
608
609 void scsi_next_command(struct scsi_cmnd *cmd)
610 {
611         struct scsi_device *sdev = cmd->device;
612         struct request_queue *q = sdev->request_queue;
613
614         /* need to hold a reference on the device before we let go of the cmd */
615         get_device(&sdev->sdev_gendev);
616
617         scsi_put_command(cmd);
618         scsi_run_queue(q);
619
620         /* ok to remove device now */
621         put_device(&sdev->sdev_gendev);
622 }
623
624 void scsi_run_host_queues(struct Scsi_Host *shost)
625 {
626         struct scsi_device *sdev;
627
628         shost_for_each_device(sdev, shost)
629                 scsi_run_queue(sdev->request_queue);
630 }
631
632 /*
633  * Function:    scsi_end_request()
634  *
635  * Purpose:     Post-processing of completed commands (usually invoked at end
636  *              of upper level post-processing and scsi_io_completion).
637  *
638  * Arguments:   cmd      - command that is complete.
639  *              uptodate - 1 if I/O indicates success, <= 0 for I/O error.
640  *              bytes    - number of bytes of completed I/O
641  *              requeue  - indicates whether we should requeue leftovers.
642  *
643  * Lock status: Assumed that lock is not held upon entry.
644  *
645  * Returns:     cmd if requeue required, NULL otherwise.
646  *
647  * Notes:       This is called for block device requests in order to
648  *              mark some number of sectors as complete.
649  * 
650  *              We are guaranteeing that the request queue will be goosed
651  *              at some point during this call.
652  * Notes:       If cmd was requeued, upon return it will be a stale pointer.
653  */
654 static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int uptodate,
655                                           int bytes, int requeue)
656 {
657         request_queue_t *q = cmd->device->request_queue;
658         struct request *req = cmd->request;
659         unsigned long flags;
660
661         /*
662          * If there are blocks left over at the end, set up the command
663          * to queue the remainder of them.
664          */
665         if (end_that_request_chunk(req, uptodate, bytes)) {
666                 int leftover = (req->hard_nr_sectors << 9);
667
668                 if (blk_pc_request(req))
669                         leftover = req->data_len;
670
671                 /* kill remainder if no retrys */
672                 if (!uptodate && blk_noretry_request(req))
673                         end_that_request_chunk(req, 0, leftover);
674                 else {
675                         if (requeue) {
676                                 /*
677                                  * Bleah.  Leftovers again.  Stick the
678                                  * leftovers in the front of the
679                                  * queue, and goose the queue again.
680                                  */
681                                 scsi_requeue_command(q, cmd);
682                                 cmd = NULL;
683                         }
684                         return cmd;
685                 }
686         }
687
688         add_disk_randomness(req->rq_disk);
689
690         spin_lock_irqsave(q->queue_lock, flags);
691         if (blk_rq_tagged(req))
692                 blk_queue_end_tag(q, req);
693         end_that_request_last(req, uptodate);
694         spin_unlock_irqrestore(q->queue_lock, flags);
695
696         /*
697          * This will goose the queue request function at the end, so we don't
698          * need to worry about launching another command.
699          */
700         scsi_next_command(cmd);
701         return NULL;
702 }
703
704 static struct scatterlist *scsi_alloc_sgtable(struct scsi_cmnd *cmd, gfp_t gfp_mask)
705 {
706         struct scsi_host_sg_pool *sgp;
707         struct scatterlist *sgl;
708
709         BUG_ON(!cmd->use_sg);
710
711         switch (cmd->use_sg) {
712         case 1 ... 8:
713                 cmd->sglist_len = 0;
714                 break;
715         case 9 ... 16:
716                 cmd->sglist_len = 1;
717                 break;
718         case 17 ... 32:
719                 cmd->sglist_len = 2;
720                 break;
721 #if (SCSI_MAX_PHYS_SEGMENTS > 32)
722         case 33 ... 64:
723                 cmd->sglist_len = 3;
724                 break;
725 #if (SCSI_MAX_PHYS_SEGMENTS > 64)
726         case 65 ... 128:
727                 cmd->sglist_len = 4;
728                 break;
729 #if (SCSI_MAX_PHYS_SEGMENTS  > 128)
730         case 129 ... 256:
731                 cmd->sglist_len = 5;
732                 break;
733 #endif
734 #endif
735 #endif
736         default:
737                 return NULL;
738         }
739
740         sgp = scsi_sg_pools + cmd->sglist_len;
741         sgl = mempool_alloc(sgp->pool, gfp_mask);
742         return sgl;
743 }
744
745 static void scsi_free_sgtable(struct scatterlist *sgl, int index)
746 {
747         struct scsi_host_sg_pool *sgp;
748
749         BUG_ON(index >= SG_MEMPOOL_NR);
750
751         sgp = scsi_sg_pools + index;
752         mempool_free(sgl, sgp->pool);
753 }
754
755 /*
756  * Function:    scsi_release_buffers()
757  *
758  * Purpose:     Completion processing for block device I/O requests.
759  *
760  * Arguments:   cmd     - command that we are bailing.
761  *
762  * Lock status: Assumed that no lock is held upon entry.
763  *
764  * Returns:     Nothing
765  *
766  * Notes:       In the event that an upper level driver rejects a
767  *              command, we must release resources allocated during
768  *              the __init_io() function.  Primarily this would involve
769  *              the scatter-gather table, and potentially any bounce
770  *              buffers.
771  */
772 static void scsi_release_buffers(struct scsi_cmnd *cmd)
773 {
774         if (cmd->use_sg)
775                 scsi_free_sgtable(cmd->request_buffer, cmd->sglist_len);
776
777         /*
778          * Zero these out.  They now point to freed memory, and it is
779          * dangerous to hang onto the pointers.
780          */
781         cmd->request_buffer = NULL;
782         cmd->request_bufflen = 0;
783 }
784
785 /*
786  * Function:    scsi_io_completion()
787  *
788  * Purpose:     Completion processing for block device I/O requests.
789  *
790  * Arguments:   cmd   - command that is finished.
791  *
792  * Lock status: Assumed that no lock is held upon entry.
793  *
794  * Returns:     Nothing
795  *
796  * Notes:       This function is matched in terms of capabilities to
797  *              the function that created the scatter-gather list.
798  *              In other words, if there are no bounce buffers
799  *              (the normal case for most drivers), we don't need
800  *              the logic to deal with cleaning up afterwards.
801  *
802  *              We must do one of several things here:
803  *
804  *              a) Call scsi_end_request.  This will finish off the
805  *                 specified number of sectors.  If we are done, the
806  *                 command block will be released, and the queue
807  *                 function will be goosed.  If we are not done, then
808  *                 scsi_end_request will directly goose the queue.
809  *
810  *              b) We can just use scsi_requeue_command() here.  This would
811  *                 be used if we just wanted to retry, for example.
812  */
813 void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
814 {
815         int result = cmd->result;
816         int this_count = cmd->request_bufflen;
817         request_queue_t *q = cmd->device->request_queue;
818         struct request *req = cmd->request;
819         int clear_errors = 1;
820         struct scsi_sense_hdr sshdr;
821         int sense_valid = 0;
822         int sense_deferred = 0;
823
824         scsi_release_buffers(cmd);
825
826         if (result) {
827                 sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
828                 if (sense_valid)
829                         sense_deferred = scsi_sense_is_deferred(&sshdr);
830         }
831
832         if (blk_pc_request(req)) { /* SG_IO ioctl from block level */
833                 req->errors = result;
834                 if (result) {
835                         clear_errors = 0;
836                         if (sense_valid && req->sense) {
837                                 /*
838                                  * SG_IO wants current and deferred errors
839                                  */
840                                 int len = 8 + cmd->sense_buffer[7];
841
842                                 if (len > SCSI_SENSE_BUFFERSIZE)
843                                         len = SCSI_SENSE_BUFFERSIZE;
844                                 memcpy(req->sense, cmd->sense_buffer,  len);
845                                 req->sense_len = len;
846                         }
847                 } else
848                         req->data_len = cmd->resid;
849         }
850
851         /*
852          * Next deal with any sectors which we were able to correctly
853          * handle.
854          */
855         SCSI_LOG_HLCOMPLETE(1, printk("%ld sectors total, "
856                                       "%d bytes done.\n",
857                                       req->nr_sectors, good_bytes));
858         SCSI_LOG_HLCOMPLETE(1, printk("use_sg is %d\n", cmd->use_sg));
859
860         if (clear_errors)
861                 req->errors = 0;
862
863         /* A number of bytes were successfully read.  If there
864          * are leftovers and there is some kind of error
865          * (result != 0), retry the rest.
866          */
867         if (scsi_end_request(cmd, 1, good_bytes, result == 0) == NULL)
868                 return;
869
870         /* good_bytes = 0, or (inclusive) there were leftovers and
871          * result = 0, so scsi_end_request couldn't retry.
872          */
873         if (sense_valid && !sense_deferred) {
874                 switch (sshdr.sense_key) {
875                 case UNIT_ATTENTION:
876                         if (cmd->device->removable) {
877                                 /* Detected disc change.  Set a bit
878                                  * and quietly refuse further access.
879                                  */
880                                 cmd->device->changed = 1;
881                                 scsi_end_request(cmd, 0, this_count, 1);
882                                 return;
883                         } else {
884                                 /* Must have been a power glitch, or a
885                                  * bus reset.  Could not have been a
886                                  * media change, so we just retry the
887                                  * request and see what happens.
888                                  */
889                                 scsi_requeue_command(q, cmd);
890                                 return;
891                         }
892                         break;
893                 case ILLEGAL_REQUEST:
894                         /* If we had an ILLEGAL REQUEST returned, then
895                          * we may have performed an unsupported
896                          * command.  The only thing this should be
897                          * would be a ten byte read where only a six
898                          * byte read was supported.  Also, on a system
899                          * where READ CAPACITY failed, we may have
900                          * read past the end of the disk.
901                          */
902                         if ((cmd->device->use_10_for_rw &&
903                             sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&
904                             (cmd->cmnd[0] == READ_10 ||
905                              cmd->cmnd[0] == WRITE_10)) {
906                                 cmd->device->use_10_for_rw = 0;
907                                 /* This will cause a retry with a
908                                  * 6-byte command.
909                                  */
910                                 scsi_requeue_command(q, cmd);
911                                 return;
912                         } else {
913                                 scsi_end_request(cmd, 0, this_count, 1);
914                                 return;
915                         }
916                         break;
917                 case NOT_READY:
918                         /* If the device is in the process of becoming
919                          * ready, or has a temporary blockage, retry.
920                          */
921                         if (sshdr.asc == 0x04) {
922                                 switch (sshdr.ascq) {
923                                 case 0x01: /* becoming ready */
924                                 case 0x04: /* format in progress */
925                                 case 0x05: /* rebuild in progress */
926                                 case 0x06: /* recalculation in progress */
927                                 case 0x07: /* operation in progress */
928                                 case 0x08: /* Long write in progress */
929                                 case 0x09: /* self test in progress */
930                                         scsi_requeue_command(q, cmd);
931                                         return;
932                                 default:
933                                         break;
934                                 }
935                         }
936                         if (!(req->flags & REQ_QUIET)) {
937                                 scmd_printk(KERN_INFO, cmd,
938                                             "Device not ready: ");
939                                 scsi_print_sense_hdr("", &sshdr);
940                         }
941                         scsi_end_request(cmd, 0, this_count, 1);
942                         return;
943                 case VOLUME_OVERFLOW:
944                         if (!(req->flags & REQ_QUIET)) {
945                                 scmd_printk(KERN_INFO, cmd,
946                                             "Volume overflow, CDB: ");
947                                 __scsi_print_command(cmd->cmnd);
948                                 scsi_print_sense("", cmd);
949                         }
950                         /* See SSC3rXX or current. */
951                         scsi_end_request(cmd, 0, this_count, 1);
952                         return;
953                 default:
954                         break;
955                 }
956         }
957         if (host_byte(result) == DID_RESET) {
958                 /* Third party bus reset or reset for error recovery
959                  * reasons.  Just retry the request and see what
960                  * happens.
961                  */
962                 scsi_requeue_command(q, cmd);
963                 return;
964         }
965         if (result) {
966                 if (!(req->flags & REQ_QUIET)) {
967                         scmd_printk(KERN_INFO, cmd,
968                                     "SCSI error: return code = 0x%08x\n",
969                                     result);
970                         if (driver_byte(result) & DRIVER_SENSE)
971                                 scsi_print_sense("", cmd);
972                 }
973         }
974         scsi_end_request(cmd, 0, this_count, !result);
975 }
976 EXPORT_SYMBOL(scsi_io_completion);
977
978 /*
979  * Function:    scsi_init_io()
980  *
981  * Purpose:     SCSI I/O initialize function.
982  *
983  * Arguments:   cmd   - Command descriptor we wish to initialize
984  *
985  * Returns:     0 on success
986  *              BLKPREP_DEFER if the failure is retryable
987  *              BLKPREP_KILL if the failure is fatal
988  */
989 static int scsi_init_io(struct scsi_cmnd *cmd)
990 {
991         struct request     *req = cmd->request;
992         struct scatterlist *sgpnt;
993         int                count;
994
995         /*
996          * if this is a rq->data based REQ_BLOCK_PC, setup for a non-sg xfer
997          */
998         if ((req->flags & REQ_BLOCK_PC) && !req->bio) {
999                 cmd->request_bufflen = req->data_len;
1000                 cmd->request_buffer = req->data;
1001                 req->buffer = req->data;
1002                 cmd->use_sg = 0;
1003                 return 0;
1004         }
1005
1006         /*
1007          * we used to not use scatter-gather for single segment request,
1008          * but now we do (it makes highmem I/O easier to support without
1009          * kmapping pages)
1010          */
1011         cmd->use_sg = req->nr_phys_segments;
1012
1013         /*
1014          * if sg table allocation fails, requeue request later.
1015          */
1016         sgpnt = scsi_alloc_sgtable(cmd, GFP_ATOMIC);
1017         if (unlikely(!sgpnt)) {
1018                 scsi_unprep_request(req);
1019                 return BLKPREP_DEFER;
1020         }
1021
1022         cmd->request_buffer = (char *) sgpnt;
1023         cmd->request_bufflen = req->nr_sectors << 9;
1024         if (blk_pc_request(req))
1025                 cmd->request_bufflen = req->data_len;
1026         req->buffer = NULL;
1027
1028         /* 
1029          * Next, walk the list, and fill in the addresses and sizes of
1030          * each segment.
1031          */
1032         count = blk_rq_map_sg(req->q, req, cmd->request_buffer);
1033
1034         /*
1035          * mapped well, send it off
1036          */
1037         if (likely(count <= cmd->use_sg)) {
1038                 cmd->use_sg = count;
1039                 return 0;
1040         }
1041
1042         printk(KERN_ERR "Incorrect number of segments after building list\n");
1043         printk(KERN_ERR "counted %d, received %d\n", count, cmd->use_sg);
1044         printk(KERN_ERR "req nr_sec %lu, cur_nr_sec %u\n", req->nr_sectors,
1045                         req->current_nr_sectors);
1046
1047         /* release the command and kill it */
1048         scsi_release_buffers(cmd);
1049         scsi_put_command(cmd);
1050         return BLKPREP_KILL;
1051 }
1052
1053 static int scsi_issue_flush_fn(request_queue_t *q, struct gendisk *disk,
1054                                sector_t *error_sector)
1055 {
1056         struct scsi_device *sdev = q->queuedata;
1057         struct scsi_driver *drv;
1058
1059         if (sdev->sdev_state != SDEV_RUNNING)
1060                 return -ENXIO;
1061
1062         drv = *(struct scsi_driver **) disk->private_data;
1063         if (drv->issue_flush)
1064                 return drv->issue_flush(&sdev->sdev_gendev, error_sector);
1065
1066         return -EOPNOTSUPP;
1067 }
1068
1069 static void scsi_blk_pc_done(struct scsi_cmnd *cmd)
1070 {
1071         BUG_ON(!blk_pc_request(cmd->request));
1072         /*
1073          * This will complete the whole command with uptodate=1 so
1074          * as far as the block layer is concerned the command completed
1075          * successfully. Since this is a REQ_BLOCK_PC command the
1076          * caller should check the request's errors value
1077          */
1078         scsi_io_completion(cmd, cmd->request_bufflen);
1079 }
1080
1081 static void scsi_setup_blk_pc_cmnd(struct scsi_cmnd *cmd)
1082 {
1083         struct request *req = cmd->request;
1084
1085         BUG_ON(sizeof(req->cmd) > sizeof(cmd->cmnd));
1086         memcpy(cmd->cmnd, req->cmd, sizeof(cmd->cmnd));
1087         cmd->cmd_len = req->cmd_len;
1088         if (!req->data_len)
1089                 cmd->sc_data_direction = DMA_NONE;
1090         else if (rq_data_dir(req) == WRITE)
1091                 cmd->sc_data_direction = DMA_TO_DEVICE;
1092         else
1093                 cmd->sc_data_direction = DMA_FROM_DEVICE;
1094         
1095         cmd->transfersize = req->data_len;
1096         cmd->allowed = req->retries;
1097         cmd->timeout_per_command = req->timeout;
1098         cmd->done = scsi_blk_pc_done;
1099 }
1100
1101 static int scsi_prep_fn(struct request_queue *q, struct request *req)
1102 {
1103         struct scsi_device *sdev = q->queuedata;
1104         struct scsi_cmnd *cmd;
1105         int specials_only = 0;
1106
1107         /*
1108          * Just check to see if the device is online.  If it isn't, we
1109          * refuse to process any commands.  The device must be brought
1110          * online before trying any recovery commands
1111          */
1112         if (unlikely(!scsi_device_online(sdev))) {
1113                 sdev_printk(KERN_ERR, sdev,
1114                             "rejecting I/O to offline device\n");
1115                 goto kill;
1116         }
1117         if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
1118                 /* OK, we're not in a running state don't prep
1119                  * user commands */
1120                 if (sdev->sdev_state == SDEV_DEL) {
1121                         /* Device is fully deleted, no commands
1122                          * at all allowed down */
1123                         sdev_printk(KERN_ERR, sdev,
1124                                     "rejecting I/O to dead device\n");
1125                         goto kill;
1126                 }
1127                 /* OK, we only allow special commands (i.e. not
1128                  * user initiated ones */
1129                 specials_only = sdev->sdev_state;
1130         }
1131
1132         /*
1133          * Find the actual device driver associated with this command.
1134          * The SPECIAL requests are things like character device or
1135          * ioctls, which did not originate from ll_rw_blk.  Note that
1136          * the special field is also used to indicate the cmd for
1137          * the remainder of a partially fulfilled request that can 
1138          * come up when there is a medium error.  We have to treat
1139          * these two cases differently.  We differentiate by looking
1140          * at request->cmd, as this tells us the real story.
1141          */
1142         if (req->flags & REQ_SPECIAL && req->special) {
1143                 cmd = req->special;
1144         } else if (req->flags & (REQ_CMD | REQ_BLOCK_PC)) {
1145
1146                 if(unlikely(specials_only) && !(req->flags & REQ_SPECIAL)) {
1147                         if(specials_only == SDEV_QUIESCE ||
1148                                         specials_only == SDEV_BLOCK)
1149                                 goto defer;
1150                         
1151                         sdev_printk(KERN_ERR, sdev,
1152                                     "rejecting I/O to device being removed\n");
1153                         goto kill;
1154                 }
1155                         
1156                         
1157                 /*
1158                  * Now try and find a command block that we can use.
1159                  */
1160                 if (!req->special) {
1161                         cmd = scsi_get_command(sdev, GFP_ATOMIC);
1162                         if (unlikely(!cmd))
1163                                 goto defer;
1164                 } else
1165                         cmd = req->special;
1166                 
1167                 /* pull a tag out of the request if we have one */
1168                 cmd->tag = req->tag;
1169         } else {
1170                 blk_dump_rq_flags(req, "SCSI bad req");
1171                 goto kill;
1172         }
1173         
1174         /* note the overloading of req->special.  When the tag
1175          * is active it always means cmd.  If the tag goes
1176          * back for re-queueing, it may be reset */
1177         req->special = cmd;
1178         cmd->request = req;
1179         
1180         /*
1181          * FIXME: drop the lock here because the functions below
1182          * expect to be called without the queue lock held.  Also,
1183          * previously, we dequeued the request before dropping the
1184          * lock.  We hope REQ_STARTED prevents anything untoward from
1185          * happening now.
1186          */
1187         if (req->flags & (REQ_CMD | REQ_BLOCK_PC)) {
1188                 int ret;
1189
1190                 /*
1191                  * This will do a couple of things:
1192                  *  1) Fill in the actual SCSI command.
1193                  *  2) Fill in any other upper-level specific fields
1194                  * (timeout).
1195                  *
1196                  * If this returns 0, it means that the request failed
1197                  * (reading past end of disk, reading offline device,
1198                  * etc).   This won't actually talk to the device, but
1199                  * some kinds of consistency checking may cause the     
1200                  * request to be rejected immediately.
1201                  */
1202
1203                 /* 
1204                  * This sets up the scatter-gather table (allocating if
1205                  * required).
1206                  */
1207                 ret = scsi_init_io(cmd);
1208                 switch(ret) {
1209                         /* For BLKPREP_KILL/DEFER the cmd was released */
1210                 case BLKPREP_KILL:
1211                         goto kill;
1212                 case BLKPREP_DEFER:
1213                         goto defer;
1214                 }
1215                 
1216                 /*
1217                  * Initialize the actual SCSI command for this request.
1218                  */
1219                 if (req->flags & REQ_BLOCK_PC) {
1220                         scsi_setup_blk_pc_cmnd(cmd);
1221                 } else if (req->rq_disk) {
1222                         struct scsi_driver *drv;
1223
1224                         drv = *(struct scsi_driver **)req->rq_disk->private_data;
1225                         if (unlikely(!drv->init_command(cmd))) {
1226                                 scsi_release_buffers(cmd);
1227                                 scsi_put_command(cmd);
1228                                 goto kill;
1229                         }
1230                 }
1231         }
1232
1233         /*
1234          * The request is now prepped, no need to come back here
1235          */
1236         req->flags |= REQ_DONTPREP;
1237         return BLKPREP_OK;
1238
1239  defer:
1240         /* If we defer, the elv_next_request() returns NULL, but the
1241          * queue must be restarted, so we plug here if no returning
1242          * command will automatically do that. */
1243         if (sdev->device_busy == 0)
1244                 blk_plug_device(q);
1245         return BLKPREP_DEFER;
1246  kill:
1247         req->errors = DID_NO_CONNECT << 16;
1248         return BLKPREP_KILL;
1249 }
1250
1251 /*
1252  * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
1253  * return 0.
1254  *
1255  * Called with the queue_lock held.
1256  */
1257 static inline int scsi_dev_queue_ready(struct request_queue *q,
1258                                   struct scsi_device *sdev)
1259 {
1260         if (sdev->device_busy >= sdev->queue_depth)
1261                 return 0;
1262         if (sdev->device_busy == 0 && sdev->device_blocked) {
1263                 /*
1264                  * unblock after device_blocked iterates to zero
1265                  */
1266                 if (--sdev->device_blocked == 0) {
1267                         SCSI_LOG_MLQUEUE(3,
1268                                    sdev_printk(KERN_INFO, sdev,
1269                                    "unblocking device at zero depth\n"));
1270                 } else {
1271                         blk_plug_device(q);
1272                         return 0;
1273                 }
1274         }
1275         if (sdev->device_blocked)
1276                 return 0;
1277
1278         return 1;
1279 }
1280
1281 /*
1282  * scsi_host_queue_ready: if we can send requests to shost, return 1 else
1283  * return 0. We must end up running the queue again whenever 0 is
1284  * returned, else IO can hang.
1285  *
1286  * Called with host_lock held.
1287  */
1288 static inline int scsi_host_queue_ready(struct request_queue *q,
1289                                    struct Scsi_Host *shost,
1290                                    struct scsi_device *sdev)
1291 {
1292         if (scsi_host_in_recovery(shost))
1293                 return 0;
1294         if (shost->host_busy == 0 && shost->host_blocked) {
1295                 /*
1296                  * unblock after host_blocked iterates to zero
1297                  */
1298                 if (--shost->host_blocked == 0) {
1299                         SCSI_LOG_MLQUEUE(3,
1300                                 printk("scsi%d unblocking host at zero depth\n",
1301                                         shost->host_no));
1302                 } else {
1303                         blk_plug_device(q);
1304                         return 0;
1305                 }
1306         }
1307         if ((shost->can_queue > 0 && shost->host_busy >= shost->can_queue) ||
1308             shost->host_blocked || shost->host_self_blocked) {
1309                 if (list_empty(&sdev->starved_entry))
1310                         list_add_tail(&sdev->starved_entry, &shost->starved_list);
1311                 return 0;
1312         }
1313
1314         /* We're OK to process the command, so we can't be starved */
1315         if (!list_empty(&sdev->starved_entry))
1316                 list_del_init(&sdev->starved_entry);
1317
1318         return 1;
1319 }
1320
1321 /*
1322  * Kill a request for a dead device
1323  */
1324 static void scsi_kill_request(struct request *req, request_queue_t *q)
1325 {
1326         struct scsi_cmnd *cmd = req->special;
1327         struct scsi_device *sdev = cmd->device;
1328         struct Scsi_Host *shost = sdev->host;
1329
1330         blkdev_dequeue_request(req);
1331
1332         if (unlikely(cmd == NULL)) {
1333                 printk(KERN_CRIT "impossible request in %s.\n",
1334                                  __FUNCTION__);
1335                 BUG();
1336         }
1337
1338         scsi_init_cmd_errh(cmd);
1339         cmd->result = DID_NO_CONNECT << 16;
1340         atomic_inc(&cmd->device->iorequest_cnt);
1341
1342         /*
1343          * SCSI request completion path will do scsi_device_unbusy(),
1344          * bump busy counts.  To bump the counters, we need to dance
1345          * with the locks as normal issue path does.
1346          */
1347         sdev->device_busy++;
1348         spin_unlock(sdev->request_queue->queue_lock);
1349         spin_lock(shost->host_lock);
1350         shost->host_busy++;
1351         spin_unlock(shost->host_lock);
1352         spin_lock(sdev->request_queue->queue_lock);
1353
1354         __scsi_done(cmd);
1355 }
1356
1357 static void scsi_softirq_done(struct request *rq)
1358 {
1359         struct scsi_cmnd *cmd = rq->completion_data;
1360         unsigned long wait_for = (cmd->allowed + 1) * cmd->timeout_per_command;
1361         int disposition;
1362
1363         INIT_LIST_HEAD(&cmd->eh_entry);
1364
1365         disposition = scsi_decide_disposition(cmd);
1366         if (disposition != SUCCESS &&
1367             time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
1368                 sdev_printk(KERN_ERR, cmd->device,
1369                             "timing out command, waited %lus\n",
1370                             wait_for/HZ);
1371                 disposition = SUCCESS;
1372         }
1373                         
1374         scsi_log_completion(cmd, disposition);
1375
1376         switch (disposition) {
1377                 case SUCCESS:
1378                         scsi_finish_command(cmd);
1379                         break;
1380                 case NEEDS_RETRY:
1381                         scsi_retry_command(cmd);
1382                         break;
1383                 case ADD_TO_MLQUEUE:
1384                         scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
1385                         break;
1386                 default:
1387                         if (!scsi_eh_scmd_add(cmd, 0))
1388                                 scsi_finish_command(cmd);
1389         }
1390 }
1391
1392 /*
1393  * Function:    scsi_request_fn()
1394  *
1395  * Purpose:     Main strategy routine for SCSI.
1396  *
1397  * Arguments:   q       - Pointer to actual queue.
1398  *
1399  * Returns:     Nothing
1400  *
1401  * Lock status: IO request lock assumed to be held when called.
1402  */
1403 static void scsi_request_fn(struct request_queue *q)
1404 {
1405         struct scsi_device *sdev = q->queuedata;
1406         struct Scsi_Host *shost;
1407         struct scsi_cmnd *cmd;
1408         struct request *req;
1409
1410         if (!sdev) {
1411                 printk("scsi: killing requests for dead queue\n");
1412                 while ((req = elv_next_request(q)) != NULL)
1413                         scsi_kill_request(req, q);
1414                 return;
1415         }
1416
1417         if(!get_device(&sdev->sdev_gendev))
1418                 /* We must be tearing the block queue down already */
1419                 return;
1420
1421         /*
1422          * To start with, we keep looping until the queue is empty, or until
1423          * the host is no longer able to accept any more requests.
1424          */
1425         shost = sdev->host;
1426         while (!blk_queue_plugged(q)) {
1427                 int rtn;
1428                 /*
1429                  * get next queueable request.  We do this early to make sure
1430                  * that the request is fully prepared even if we cannot 
1431                  * accept it.
1432                  */
1433                 req = elv_next_request(q);
1434                 if (!req || !scsi_dev_queue_ready(q, sdev))
1435                         break;
1436
1437                 if (unlikely(!scsi_device_online(sdev))) {
1438                         sdev_printk(KERN_ERR, sdev,
1439                                     "rejecting I/O to offline device\n");
1440                         scsi_kill_request(req, q);
1441                         continue;
1442                 }
1443
1444
1445                 /*
1446                  * Remove the request from the request list.
1447                  */
1448                 if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
1449                         blkdev_dequeue_request(req);
1450                 sdev->device_busy++;
1451
1452                 spin_unlock(q->queue_lock);
1453                 cmd = req->special;
1454                 if (unlikely(cmd == NULL)) {
1455                         printk(KERN_CRIT "impossible request in %s.\n"
1456                                          "please mail a stack trace to "
1457                                          "linux-scsi@vger.kernel.org",
1458                                          __FUNCTION__);
1459                         BUG();
1460                 }
1461                 spin_lock(shost->host_lock);
1462
1463                 if (!scsi_host_queue_ready(q, shost, sdev))
1464                         goto not_ready;
1465                 if (sdev->single_lun) {
1466                         if (scsi_target(sdev)->starget_sdev_user &&
1467                             scsi_target(sdev)->starget_sdev_user != sdev)
1468                                 goto not_ready;
1469                         scsi_target(sdev)->starget_sdev_user = sdev;
1470                 }
1471                 shost->host_busy++;
1472
1473                 /*
1474                  * XXX(hch): This is rather suboptimal, scsi_dispatch_cmd will
1475                  *              take the lock again.
1476                  */
1477                 spin_unlock_irq(shost->host_lock);
1478
1479                 /*
1480                  * Finally, initialize any error handling parameters, and set up
1481                  * the timers for timeouts.
1482                  */
1483                 scsi_init_cmd_errh(cmd);
1484
1485                 /*
1486                  * Dispatch the command to the low-level driver.
1487                  */
1488                 rtn = scsi_dispatch_cmd(cmd);
1489                 spin_lock_irq(q->queue_lock);
1490                 if(rtn) {
1491                         /* we're refusing the command; because of
1492                          * the way locks get dropped, we need to 
1493                          * check here if plugging is required */
1494                         if(sdev->device_busy == 0)
1495                                 blk_plug_device(q);
1496
1497                         break;
1498                 }
1499         }
1500
1501         goto out;
1502
1503  not_ready:
1504         spin_unlock_irq(shost->host_lock);
1505
1506         /*
1507          * lock q, handle tag, requeue req, and decrement device_busy. We
1508          * must return with queue_lock held.
1509          *
1510          * Decrementing device_busy without checking it is OK, as all such
1511          * cases (host limits or settings) should run the queue at some
1512          * later time.
1513          */
1514         spin_lock_irq(q->queue_lock);
1515         blk_requeue_request(q, req);
1516         sdev->device_busy--;
1517         if(sdev->device_busy == 0)
1518                 blk_plug_device(q);
1519  out:
1520         /* must be careful here...if we trigger the ->remove() function
1521          * we cannot be holding the q lock */
1522         spin_unlock_irq(q->queue_lock);
1523         put_device(&sdev->sdev_gendev);
1524         spin_lock_irq(q->queue_lock);
1525 }
1526
1527 u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
1528 {
1529         struct device *host_dev;
1530         u64 bounce_limit = 0xffffffff;
1531
1532         if (shost->unchecked_isa_dma)
1533                 return BLK_BOUNCE_ISA;
1534         /*
1535          * Platforms with virtual-DMA translation
1536          * hardware have no practical limit.
1537          */
1538         if (!PCI_DMA_BUS_IS_PHYS)
1539                 return BLK_BOUNCE_ANY;
1540
1541         host_dev = scsi_get_device(shost);
1542         if (host_dev && host_dev->dma_mask)
1543                 bounce_limit = *host_dev->dma_mask;
1544
1545         return bounce_limit;
1546 }
1547 EXPORT_SYMBOL(scsi_calculate_bounce_limit);
1548
1549 struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
1550 {
1551         struct Scsi_Host *shost = sdev->host;
1552         struct request_queue *q;
1553
1554         q = blk_init_queue(scsi_request_fn, NULL);
1555         if (!q)
1556                 return NULL;
1557
1558         blk_queue_prep_rq(q, scsi_prep_fn);
1559
1560         blk_queue_max_hw_segments(q, shost->sg_tablesize);
1561         blk_queue_max_phys_segments(q, SCSI_MAX_PHYS_SEGMENTS);
1562         blk_queue_max_sectors(q, shost->max_sectors);
1563         blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
1564         blk_queue_segment_boundary(q, shost->dma_boundary);
1565         blk_queue_issue_flush_fn(q, scsi_issue_flush_fn);
1566         blk_queue_softirq_done(q, scsi_softirq_done);
1567
1568         if (!shost->use_clustering)
1569                 clear_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags);
1570         return q;
1571 }
1572
1573 void scsi_free_queue(struct request_queue *q)
1574 {
1575         blk_cleanup_queue(q);
1576 }
1577
1578 /*
1579  * Function:    scsi_block_requests()
1580  *
1581  * Purpose:     Utility function used by low-level drivers to prevent further
1582  *              commands from being queued to the device.
1583  *
1584  * Arguments:   shost       - Host in question
1585  *
1586  * Returns:     Nothing
1587  *
1588  * Lock status: No locks are assumed held.
1589  *
1590  * Notes:       There is no timer nor any other means by which the requests
1591  *              get unblocked other than the low-level driver calling
1592  *              scsi_unblock_requests().
1593  */
1594 void scsi_block_requests(struct Scsi_Host *shost)
1595 {
1596         shost->host_self_blocked = 1;
1597 }
1598 EXPORT_SYMBOL(scsi_block_requests);
1599
1600 /*
1601  * Function:    scsi_unblock_requests()
1602  *
1603  * Purpose:     Utility function used by low-level drivers to allow further
1604  *              commands from being queued to the device.
1605  *
1606  * Arguments:   shost       - Host in question
1607  *
1608  * Returns:     Nothing
1609  *
1610  * Lock status: No locks are assumed held.
1611  *
1612  * Notes:       There is no timer nor any other means by which the requests
1613  *              get unblocked other than the low-level driver calling
1614  *              scsi_unblock_requests().
1615  *
1616  *              This is done as an API function so that changes to the
1617  *              internals of the scsi mid-layer won't require wholesale
1618  *              changes to drivers that use this feature.
1619  */
1620 void scsi_unblock_requests(struct Scsi_Host *shost)
1621 {
1622         shost->host_self_blocked = 0;
1623         scsi_run_host_queues(shost);
1624 }
1625 EXPORT_SYMBOL(scsi_unblock_requests);
1626
1627 int __init scsi_init_queue(void)
1628 {
1629         int i;
1630
1631         scsi_io_context_cache = kmem_cache_create("scsi_io_context",
1632                                         sizeof(struct scsi_io_context),
1633                                         0, 0, NULL, NULL);
1634         if (!scsi_io_context_cache) {
1635                 printk(KERN_ERR "SCSI: can't init scsi io context cache\n");
1636                 return -ENOMEM;
1637         }
1638
1639         for (i = 0; i < SG_MEMPOOL_NR; i++) {
1640                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1641                 int size = sgp->size * sizeof(struct scatterlist);
1642
1643                 sgp->slab = kmem_cache_create(sgp->name, size, 0,
1644                                 SLAB_HWCACHE_ALIGN, NULL, NULL);
1645                 if (!sgp->slab) {
1646                         printk(KERN_ERR "SCSI: can't init sg slab %s\n",
1647                                         sgp->name);
1648                 }
1649
1650                 sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
1651                                                      sgp->slab);
1652                 if (!sgp->pool) {
1653                         printk(KERN_ERR "SCSI: can't init sg mempool %s\n",
1654                                         sgp->name);
1655                 }
1656         }
1657
1658         return 0;
1659 }
1660
1661 void scsi_exit_queue(void)
1662 {
1663         int i;
1664
1665         kmem_cache_destroy(scsi_io_context_cache);
1666
1667         for (i = 0; i < SG_MEMPOOL_NR; i++) {
1668                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1669                 mempool_destroy(sgp->pool);
1670                 kmem_cache_destroy(sgp->slab);
1671         }
1672 }
1673
1674 /**
1675  *      scsi_mode_select - issue a mode select
1676  *      @sdev:  SCSI device to be queried
1677  *      @pf:    Page format bit (1 == standard, 0 == vendor specific)
1678  *      @sp:    Save page bit (0 == don't save, 1 == save)
1679  *      @modepage: mode page being requested
1680  *      @buffer: request buffer (may not be smaller than eight bytes)
1681  *      @len:   length of request buffer.
1682  *      @timeout: command timeout
1683  *      @retries: number of retries before failing
1684  *      @data: returns a structure abstracting the mode header data
1685  *      @sense: place to put sense data (or NULL if no sense to be collected).
1686  *              must be SCSI_SENSE_BUFFERSIZE big.
1687  *
1688  *      Returns zero if successful; negative error number or scsi
1689  *      status on error
1690  *
1691  */
1692 int
1693 scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage,
1694                  unsigned char *buffer, int len, int timeout, int retries,
1695                  struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
1696 {
1697         unsigned char cmd[10];
1698         unsigned char *real_buffer;
1699         int ret;
1700
1701         memset(cmd, 0, sizeof(cmd));
1702         cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0);
1703
1704         if (sdev->use_10_for_ms) {
1705                 if (len > 65535)
1706                         return -EINVAL;
1707                 real_buffer = kmalloc(8 + len, GFP_KERNEL);
1708                 if (!real_buffer)
1709                         return -ENOMEM;
1710                 memcpy(real_buffer + 8, buffer, len);
1711                 len += 8;
1712                 real_buffer[0] = 0;
1713                 real_buffer[1] = 0;
1714                 real_buffer[2] = data->medium_type;
1715                 real_buffer[3] = data->device_specific;
1716                 real_buffer[4] = data->longlba ? 0x01 : 0;
1717                 real_buffer[5] = 0;
1718                 real_buffer[6] = data->block_descriptor_length >> 8;
1719                 real_buffer[7] = data->block_descriptor_length;
1720
1721                 cmd[0] = MODE_SELECT_10;
1722                 cmd[7] = len >> 8;
1723                 cmd[8] = len;
1724         } else {
1725                 if (len > 255 || data->block_descriptor_length > 255 ||
1726                     data->longlba)
1727                         return -EINVAL;
1728
1729                 real_buffer = kmalloc(4 + len, GFP_KERNEL);
1730                 if (!real_buffer)
1731                         return -ENOMEM;
1732                 memcpy(real_buffer + 4, buffer, len);
1733                 len += 4;
1734                 real_buffer[0] = 0;
1735                 real_buffer[1] = data->medium_type;
1736                 real_buffer[2] = data->device_specific;
1737                 real_buffer[3] = data->block_descriptor_length;
1738                 
1739
1740                 cmd[0] = MODE_SELECT;
1741                 cmd[4] = len;
1742         }
1743
1744         ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len,
1745                                sshdr, timeout, retries);
1746         kfree(real_buffer);
1747         return ret;
1748 }
1749 EXPORT_SYMBOL_GPL(scsi_mode_select);
1750
1751 /**
1752  *      scsi_mode_sense - issue a mode sense, falling back from 10 to 
1753  *              six bytes if necessary.
1754  *      @sdev:  SCSI device to be queried
1755  *      @dbd:   set if mode sense will allow block descriptors to be returned
1756  *      @modepage: mode page being requested
1757  *      @buffer: request buffer (may not be smaller than eight bytes)
1758  *      @len:   length of request buffer.
1759  *      @timeout: command timeout
1760  *      @retries: number of retries before failing
1761  *      @data: returns a structure abstracting the mode header data
1762  *      @sense: place to put sense data (or NULL if no sense to be collected).
1763  *              must be SCSI_SENSE_BUFFERSIZE big.
1764  *
1765  *      Returns zero if unsuccessful, or the header offset (either 4
1766  *      or 8 depending on whether a six or ten byte command was
1767  *      issued) if successful.
1768  **/
1769 int
1770 scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
1771                   unsigned char *buffer, int len, int timeout, int retries,
1772                   struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
1773 {
1774         unsigned char cmd[12];
1775         int use_10_for_ms;
1776         int header_length;
1777         int result;
1778         struct scsi_sense_hdr my_sshdr;
1779
1780         memset(data, 0, sizeof(*data));
1781         memset(&cmd[0], 0, 12);
1782         cmd[1] = dbd & 0x18;    /* allows DBD and LLBA bits */
1783         cmd[2] = modepage;
1784
1785         /* caller might not be interested in sense, but we need it */
1786         if (!sshdr)
1787                 sshdr = &my_sshdr;
1788
1789  retry:
1790         use_10_for_ms = sdev->use_10_for_ms;
1791
1792         if (use_10_for_ms) {
1793                 if (len < 8)
1794                         len = 8;
1795
1796                 cmd[0] = MODE_SENSE_10;
1797                 cmd[8] = len;
1798                 header_length = 8;
1799         } else {
1800                 if (len < 4)
1801                         len = 4;
1802
1803                 cmd[0] = MODE_SENSE;
1804                 cmd[4] = len;
1805                 header_length = 4;
1806         }
1807
1808         memset(buffer, 0, len);
1809
1810         result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len,
1811                                   sshdr, timeout, retries);
1812
1813         /* This code looks awful: what it's doing is making sure an
1814          * ILLEGAL REQUEST sense return identifies the actual command
1815          * byte as the problem.  MODE_SENSE commands can return
1816          * ILLEGAL REQUEST if the code page isn't supported */
1817
1818         if (use_10_for_ms && !scsi_status_is_good(result) &&
1819             (driver_byte(result) & DRIVER_SENSE)) {
1820                 if (scsi_sense_valid(sshdr)) {
1821                         if ((sshdr->sense_key == ILLEGAL_REQUEST) &&
1822                             (sshdr->asc == 0x20) && (sshdr->ascq == 0)) {
1823                                 /* 
1824                                  * Invalid command operation code
1825                                  */
1826                                 sdev->use_10_for_ms = 0;
1827                                 goto retry;
1828                         }
1829                 }
1830         }
1831
1832         if(scsi_status_is_good(result)) {
1833                 if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b &&
1834                              (modepage == 6 || modepage == 8))) {
1835                         /* Initio breakage? */
1836                         header_length = 0;
1837                         data->length = 13;
1838                         data->medium_type = 0;
1839                         data->device_specific = 0;
1840                         data->longlba = 0;
1841                         data->block_descriptor_length = 0;
1842                 } else if(use_10_for_ms) {
1843                         data->length = buffer[0]*256 + buffer[1] + 2;
1844                         data->medium_type = buffer[2];
1845                         data->device_specific = buffer[3];
1846                         data->longlba = buffer[4] & 0x01;
1847                         data->block_descriptor_length = buffer[6]*256
1848                                 + buffer[7];
1849                 } else {
1850                         data->length = buffer[0] + 1;
1851                         data->medium_type = buffer[1];
1852                         data->device_specific = buffer[2];
1853                         data->block_descriptor_length = buffer[3];
1854                 }
1855                 data->header_length = header_length;
1856         }
1857
1858         return result;
1859 }
1860 EXPORT_SYMBOL(scsi_mode_sense);
1861
1862 int
1863 scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries)
1864 {
1865         char cmd[] = {
1866                 TEST_UNIT_READY, 0, 0, 0, 0, 0,
1867         };
1868         struct scsi_sense_hdr sshdr;
1869         int result;
1870         
1871         result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, &sshdr,
1872                                   timeout, retries);
1873
1874         if ((driver_byte(result) & DRIVER_SENSE) && sdev->removable) {
1875
1876                 if ((scsi_sense_valid(&sshdr)) &&
1877                     ((sshdr.sense_key == UNIT_ATTENTION) ||
1878                      (sshdr.sense_key == NOT_READY))) {
1879                         sdev->changed = 1;
1880                         result = 0;
1881                 }
1882         }
1883         return result;
1884 }
1885 EXPORT_SYMBOL(scsi_test_unit_ready);
1886
1887 /**
1888  *      scsi_device_set_state - Take the given device through the device
1889  *              state model.
1890  *      @sdev:  scsi device to change the state of.
1891  *      @state: state to change to.
1892  *
1893  *      Returns zero if unsuccessful or an error if the requested 
1894  *      transition is illegal.
1895  **/
1896 int
1897 scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
1898 {
1899         enum scsi_device_state oldstate = sdev->sdev_state;
1900
1901         if (state == oldstate)
1902                 return 0;
1903
1904         switch (state) {
1905         case SDEV_CREATED:
1906                 /* There are no legal states that come back to
1907                  * created.  This is the manually initialised start
1908                  * state */
1909                 goto illegal;
1910                         
1911         case SDEV_RUNNING:
1912                 switch (oldstate) {
1913                 case SDEV_CREATED:
1914                 case SDEV_OFFLINE:
1915                 case SDEV_QUIESCE:
1916                 case SDEV_BLOCK:
1917                         break;
1918                 default:
1919                         goto illegal;
1920                 }
1921                 break;
1922
1923         case SDEV_QUIESCE:
1924                 switch (oldstate) {
1925                 case SDEV_RUNNING:
1926                 case SDEV_OFFLINE:
1927                         break;
1928                 default:
1929                         goto illegal;
1930                 }
1931                 break;
1932
1933         case SDEV_OFFLINE:
1934                 switch (oldstate) {
1935                 case SDEV_CREATED:
1936                 case SDEV_RUNNING:
1937                 case SDEV_QUIESCE:
1938                 case SDEV_BLOCK:
1939                         break;
1940                 default:
1941                         goto illegal;
1942                 }
1943                 break;
1944
1945         case SDEV_BLOCK:
1946                 switch (oldstate) {
1947                 case SDEV_CREATED:
1948                 case SDEV_RUNNING:
1949                         break;
1950                 default:
1951                         goto illegal;
1952                 }
1953                 break;
1954
1955         case SDEV_CANCEL:
1956                 switch (oldstate) {
1957                 case SDEV_CREATED:
1958                 case SDEV_RUNNING:
1959                 case SDEV_QUIESCE:
1960                 case SDEV_OFFLINE:
1961                 case SDEV_BLOCK:
1962                         break;
1963                 default:
1964                         goto illegal;
1965                 }
1966                 break;
1967
1968         case SDEV_DEL:
1969                 switch (oldstate) {
1970                 case SDEV_CREATED:
1971                 case SDEV_RUNNING:
1972                 case SDEV_OFFLINE:
1973                 case SDEV_CANCEL:
1974                         break;
1975                 default:
1976                         goto illegal;
1977                 }
1978                 break;
1979
1980         }
1981         sdev->sdev_state = state;
1982         return 0;
1983
1984  illegal:
1985         SCSI_LOG_ERROR_RECOVERY(1, 
1986                                 sdev_printk(KERN_ERR, sdev,
1987                                             "Illegal state transition %s->%s\n",
1988                                             scsi_device_state_name(oldstate),
1989                                             scsi_device_state_name(state))
1990                                 );
1991         return -EINVAL;
1992 }
1993 EXPORT_SYMBOL(scsi_device_set_state);
1994
1995 /**
1996  *      scsi_device_quiesce - Block user issued commands.
1997  *      @sdev:  scsi device to quiesce.
1998  *
1999  *      This works by trying to transition to the SDEV_QUIESCE state
2000  *      (which must be a legal transition).  When the device is in this
2001  *      state, only special requests will be accepted, all others will
2002  *      be deferred.  Since special requests may also be requeued requests,
2003  *      a successful return doesn't guarantee the device will be 
2004  *      totally quiescent.
2005  *
2006  *      Must be called with user context, may sleep.
2007  *
2008  *      Returns zero if unsuccessful or an error if not.
2009  **/
2010 int
2011 scsi_device_quiesce(struct scsi_device *sdev)
2012 {
2013         int err = scsi_device_set_state(sdev, SDEV_QUIESCE);
2014         if (err)
2015                 return err;
2016
2017         scsi_run_queue(sdev->request_queue);
2018         while (sdev->device_busy) {
2019                 msleep_interruptible(200);
2020                 scsi_run_queue(sdev->request_queue);
2021         }
2022         return 0;
2023 }
2024 EXPORT_SYMBOL(scsi_device_quiesce);
2025
2026 /**
2027  *      scsi_device_resume - Restart user issued commands to a quiesced device.
2028  *      @sdev:  scsi device to resume.
2029  *
2030  *      Moves the device from quiesced back to running and restarts the
2031  *      queues.
2032  *
2033  *      Must be called with user context, may sleep.
2034  **/
2035 void
2036 scsi_device_resume(struct scsi_device *sdev)
2037 {
2038         if(scsi_device_set_state(sdev, SDEV_RUNNING))
2039                 return;
2040         scsi_run_queue(sdev->request_queue);
2041 }
2042 EXPORT_SYMBOL(scsi_device_resume);
2043
2044 static void
2045 device_quiesce_fn(struct scsi_device *sdev, void *data)
2046 {
2047         scsi_device_quiesce(sdev);
2048 }
2049
2050 void
2051 scsi_target_quiesce(struct scsi_target *starget)
2052 {
2053         starget_for_each_device(starget, NULL, device_quiesce_fn);
2054 }
2055 EXPORT_SYMBOL(scsi_target_quiesce);
2056
2057 static void
2058 device_resume_fn(struct scsi_device *sdev, void *data)
2059 {
2060         scsi_device_resume(sdev);
2061 }
2062
2063 void
2064 scsi_target_resume(struct scsi_target *starget)
2065 {
2066         starget_for_each_device(starget, NULL, device_resume_fn);
2067 }
2068 EXPORT_SYMBOL(scsi_target_resume);
2069
2070 /**
2071  * scsi_internal_device_block - internal function to put a device
2072  *                              temporarily into the SDEV_BLOCK state
2073  * @sdev:       device to block
2074  *
2075  * Block request made by scsi lld's to temporarily stop all
2076  * scsi commands on the specified device.  Called from interrupt
2077  * or normal process context.
2078  *
2079  * Returns zero if successful or error if not
2080  *
2081  * Notes:       
2082  *      This routine transitions the device to the SDEV_BLOCK state
2083  *      (which must be a legal transition).  When the device is in this
2084  *      state, all commands are deferred until the scsi lld reenables
2085  *      the device with scsi_device_unblock or device_block_tmo fires.
2086  *      This routine assumes the host_lock is held on entry.
2087  **/
2088 int
2089 scsi_internal_device_block(struct scsi_device *sdev)
2090 {
2091         request_queue_t *q = sdev->request_queue;
2092         unsigned long flags;
2093         int err = 0;
2094
2095         err = scsi_device_set_state(sdev, SDEV_BLOCK);
2096         if (err)
2097                 return err;
2098
2099         /* 
2100          * The device has transitioned to SDEV_BLOCK.  Stop the
2101          * block layer from calling the midlayer with this device's
2102          * request queue. 
2103          */
2104         spin_lock_irqsave(q->queue_lock, flags);
2105         blk_stop_queue(q);
2106         spin_unlock_irqrestore(q->queue_lock, flags);
2107
2108         return 0;
2109 }
2110 EXPORT_SYMBOL_GPL(scsi_internal_device_block);
2111  
2112 /**
2113  * scsi_internal_device_unblock - resume a device after a block request
2114  * @sdev:       device to resume
2115  *
2116  * Called by scsi lld's or the midlayer to restart the device queue
2117  * for the previously suspended scsi device.  Called from interrupt or
2118  * normal process context.
2119  *
2120  * Returns zero if successful or error if not.
2121  *
2122  * Notes:       
2123  *      This routine transitions the device to the SDEV_RUNNING state
2124  *      (which must be a legal transition) allowing the midlayer to
2125  *      goose the queue for this device.  This routine assumes the 
2126  *      host_lock is held upon entry.
2127  **/
2128 int
2129 scsi_internal_device_unblock(struct scsi_device *sdev)
2130 {
2131         request_queue_t *q = sdev->request_queue; 
2132         int err;
2133         unsigned long flags;
2134         
2135         /* 
2136          * Try to transition the scsi device to SDEV_RUNNING
2137          * and goose the device queue if successful.  
2138          */
2139         err = scsi_device_set_state(sdev, SDEV_RUNNING);
2140         if (err)
2141                 return err;
2142
2143         spin_lock_irqsave(q->queue_lock, flags);
2144         blk_start_queue(q);
2145         spin_unlock_irqrestore(q->queue_lock, flags);
2146
2147         return 0;
2148 }
2149 EXPORT_SYMBOL_GPL(scsi_internal_device_unblock);
2150
2151 static void
2152 device_block(struct scsi_device *sdev, void *data)
2153 {
2154         scsi_internal_device_block(sdev);
2155 }
2156
2157 static int
2158 target_block(struct device *dev, void *data)
2159 {
2160         if (scsi_is_target_device(dev))
2161                 starget_for_each_device(to_scsi_target(dev), NULL,
2162                                         device_block);
2163         return 0;
2164 }
2165
2166 void
2167 scsi_target_block(struct device *dev)
2168 {
2169         if (scsi_is_target_device(dev))
2170                 starget_for_each_device(to_scsi_target(dev), NULL,
2171                                         device_block);
2172         else
2173                 device_for_each_child(dev, NULL, target_block);
2174 }
2175 EXPORT_SYMBOL_GPL(scsi_target_block);
2176
2177 static void
2178 device_unblock(struct scsi_device *sdev, void *data)
2179 {
2180         scsi_internal_device_unblock(sdev);
2181 }
2182
2183 static int
2184 target_unblock(struct device *dev, void *data)
2185 {
2186         if (scsi_is_target_device(dev))
2187                 starget_for_each_device(to_scsi_target(dev), NULL,
2188                                         device_unblock);
2189         return 0;
2190 }
2191
2192 void
2193 scsi_target_unblock(struct device *dev)
2194 {
2195         if (scsi_is_target_device(dev))
2196                 starget_for_each_device(to_scsi_target(dev), NULL,
2197                                         device_unblock);
2198         else
2199                 device_for_each_child(dev, NULL, target_unblock);
2200 }
2201 EXPORT_SYMBOL_GPL(scsi_target_unblock);
2202
2203 /**
2204  * scsi_kmap_atomic_sg - find and atomically map an sg-elemnt
2205  * @sg:         scatter-gather list
2206  * @sg_count:   number of segments in sg
2207  * @offset:     offset in bytes into sg, on return offset into the mapped area
2208  * @len:        bytes to map, on return number of bytes mapped
2209  *
2210  * Returns virtual address of the start of the mapped page
2211  */
2212 void *scsi_kmap_atomic_sg(struct scatterlist *sg, int sg_count,
2213                           size_t *offset, size_t *len)
2214 {
2215         int i;
2216         size_t sg_len = 0, len_complete = 0;
2217         struct page *page;
2218
2219         for (i = 0; i < sg_count; i++) {
2220                 len_complete = sg_len; /* Complete sg-entries */
2221                 sg_len += sg[i].length;
2222                 if (sg_len > *offset)
2223                         break;
2224         }
2225
2226         if (unlikely(i == sg_count)) {
2227                 printk(KERN_ERR "%s: Bytes in sg: %zu, requested offset %zu, "
2228                         "elements %d\n",
2229                        __FUNCTION__, sg_len, *offset, sg_count);
2230                 WARN_ON(1);
2231                 return NULL;
2232         }
2233
2234         /* Offset starting from the beginning of first page in this sg-entry */
2235         *offset = *offset - len_complete + sg[i].offset;
2236
2237         /* Assumption: contiguous pages can be accessed as "page + i" */
2238         page = nth_page(sg[i].page, (*offset >> PAGE_SHIFT));
2239         *offset &= ~PAGE_MASK;
2240
2241         /* Bytes in this sg-entry from *offset to the end of the page */
2242         sg_len = PAGE_SIZE - *offset;
2243         if (*len > sg_len)
2244                 *len = sg_len;
2245
2246         return kmap_atomic(page, KM_BIO_SRC_IRQ);
2247 }
2248 EXPORT_SYMBOL(scsi_kmap_atomic_sg);
2249
2250 /**
2251  * scsi_kunmap_atomic_sg - atomically unmap a virtual address, previously
2252  *                         mapped with scsi_kmap_atomic_sg
2253  * @virt:       virtual address to be unmapped
2254  */
2255 void scsi_kunmap_atomic_sg(void *virt)
2256 {
2257         kunmap_atomic(virt, KM_BIO_SRC_IRQ);
2258 }
2259 EXPORT_SYMBOL(scsi_kunmap_atomic_sg);