ide: add struct ide_taskfile (take 2)
[linux-2.6.git] / drivers / ide / ide-io.c
1 /*
2  *      IDE I/O functions
3  *
4  *      Basic PIO and command management functionality.
5  *
6  * This code was split off from ide.c. See ide.c for history and original
7  * copyrights.
8  *
9  * This program is free software; you can redistribute it and/or modify it
10  * under the terms of the GNU General Public License as published by the
11  * Free Software Foundation; either version 2, or (at your option) any
12  * later version.
13  *
14  * This program is distributed in the hope that it will be useful, but
15  * WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
17  * General Public License for more details.
18  *
19  * For the avoidance of doubt the "preferred form" of this code is one which
20  * is in an open non patent encumbered format. Where cryptographic key signing
21  * forms part of the process of creating an executable the information
22  * including keys needed to generate an equivalently functional executable
23  * are deemed to be part of the source code.
24  */
25  
26  
27 #include <linux/module.h>
28 #include <linux/types.h>
29 #include <linux/string.h>
30 #include <linux/kernel.h>
31 #include <linux/timer.h>
32 #include <linux/mm.h>
33 #include <linux/interrupt.h>
34 #include <linux/major.h>
35 #include <linux/errno.h>
36 #include <linux/genhd.h>
37 #include <linux/blkpg.h>
38 #include <linux/slab.h>
39 #include <linux/init.h>
40 #include <linux/pci.h>
41 #include <linux/delay.h>
42 #include <linux/ide.h>
43 #include <linux/completion.h>
44 #include <linux/reboot.h>
45 #include <linux/cdrom.h>
46 #include <linux/seq_file.h>
47 #include <linux/device.h>
48 #include <linux/kmod.h>
49 #include <linux/scatterlist.h>
50 #include <linux/bitops.h>
51
52 #include <asm/byteorder.h>
53 #include <asm/irq.h>
54 #include <asm/uaccess.h>
55 #include <asm/io.h>
56
57 static int __ide_end_request(ide_drive_t *drive, struct request *rq,
58                              int uptodate, unsigned int nr_bytes, int dequeue)
59 {
60         int ret = 1;
61
62         /*
63          * if failfast is set on a request, override number of sectors and
64          * complete the whole request right now
65          */
66         if (blk_noretry_request(rq) && end_io_error(uptodate))
67                 nr_bytes = rq->hard_nr_sectors << 9;
68
69         if (!blk_fs_request(rq) && end_io_error(uptodate) && !rq->errors)
70                 rq->errors = -EIO;
71
72         /*
73          * decide whether to reenable DMA -- 3 is a random magic for now,
74          * if we DMA timeout more than 3 times, just stay in PIO
75          */
76         if (drive->state == DMA_PIO_RETRY && drive->retry_pio <= 3) {
77                 drive->state = 0;
78                 HWGROUP(drive)->hwif->ide_dma_on(drive);
79         }
80
81         if (!end_that_request_chunk(rq, uptodate, nr_bytes)) {
82                 add_disk_randomness(rq->rq_disk);
83                 if (dequeue) {
84                         if (!list_empty(&rq->queuelist))
85                                 blkdev_dequeue_request(rq);
86                         HWGROUP(drive)->rq = NULL;
87                 }
88                 end_that_request_last(rq, uptodate);
89                 ret = 0;
90         }
91
92         return ret;
93 }
94
95 /**
96  *      ide_end_request         -       complete an IDE I/O
97  *      @drive: IDE device for the I/O
98  *      @uptodate:
99  *      @nr_sectors: number of sectors completed
100  *
101  *      This is our end_request wrapper function. We complete the I/O
102  *      update random number input and dequeue the request, which if
103  *      it was tagged may be out of order.
104  */
105
106 int ide_end_request (ide_drive_t *drive, int uptodate, int nr_sectors)
107 {
108         unsigned int nr_bytes = nr_sectors << 9;
109         struct request *rq;
110         unsigned long flags;
111         int ret = 1;
112
113         /*
114          * room for locking improvements here, the calls below don't
115          * need the queue lock held at all
116          */
117         spin_lock_irqsave(&ide_lock, flags);
118         rq = HWGROUP(drive)->rq;
119
120         if (!nr_bytes) {
121                 if (blk_pc_request(rq))
122                         nr_bytes = rq->data_len;
123                 else
124                         nr_bytes = rq->hard_cur_sectors << 9;
125         }
126
127         ret = __ide_end_request(drive, rq, uptodate, nr_bytes, 1);
128
129         spin_unlock_irqrestore(&ide_lock, flags);
130         return ret;
131 }
132 EXPORT_SYMBOL(ide_end_request);
133
134 /*
135  * Power Management state machine. This one is rather trivial for now,
136  * we should probably add more, like switching back to PIO on suspend
137  * to help some BIOSes, re-do the door locking on resume, etc...
138  */
139
140 enum {
141         ide_pm_flush_cache      = ide_pm_state_start_suspend,
142         idedisk_pm_standby,
143
144         idedisk_pm_restore_pio  = ide_pm_state_start_resume,
145         idedisk_pm_idle,
146         ide_pm_restore_dma,
147 };
148
149 static void ide_complete_power_step(ide_drive_t *drive, struct request *rq, u8 stat, u8 error)
150 {
151         struct request_pm_state *pm = rq->data;
152
153         if (drive->media != ide_disk)
154                 return;
155
156         switch (pm->pm_step) {
157         case ide_pm_flush_cache:        /* Suspend step 1 (flush cache) complete */
158                 if (pm->pm_state == PM_EVENT_FREEZE)
159                         pm->pm_step = ide_pm_state_completed;
160                 else
161                         pm->pm_step = idedisk_pm_standby;
162                 break;
163         case idedisk_pm_standby:        /* Suspend step 2 (standby) complete */
164                 pm->pm_step = ide_pm_state_completed;
165                 break;
166         case idedisk_pm_restore_pio:    /* Resume step 1 complete */
167                 pm->pm_step = idedisk_pm_idle;
168                 break;
169         case idedisk_pm_idle:           /* Resume step 2 (idle) complete */
170                 pm->pm_step = ide_pm_restore_dma;
171                 break;
172         }
173 }
174
175 static ide_startstop_t ide_start_power_step(ide_drive_t *drive, struct request *rq)
176 {
177         struct request_pm_state *pm = rq->data;
178         ide_task_t *args = rq->special;
179
180         memset(args, 0, sizeof(*args));
181
182         switch (pm->pm_step) {
183         case ide_pm_flush_cache:        /* Suspend step 1 (flush cache) */
184                 if (drive->media != ide_disk)
185                         break;
186                 /* Not supported? Switch to next step now. */
187                 if (!drive->wcache || !ide_id_has_flush_cache(drive->id)) {
188                         ide_complete_power_step(drive, rq, 0, 0);
189                         return ide_stopped;
190                 }
191                 if (ide_id_has_flush_cache_ext(drive->id))
192                         args->tf.command = WIN_FLUSH_CACHE_EXT;
193                 else
194                         args->tf.command = WIN_FLUSH_CACHE;
195                 args->command_type = IDE_DRIVE_TASK_NO_DATA;
196                 args->handler      = &task_no_data_intr;
197                 return do_rw_taskfile(drive, args);
198
199         case idedisk_pm_standby:        /* Suspend step 2 (standby) */
200                 args->tf.command = WIN_STANDBYNOW1;
201                 args->command_type = IDE_DRIVE_TASK_NO_DATA;
202                 args->handler      = &task_no_data_intr;
203                 return do_rw_taskfile(drive, args);
204
205         case idedisk_pm_restore_pio:    /* Resume step 1 (restore PIO) */
206                 ide_set_max_pio(drive);
207                 /*
208                  * skip idedisk_pm_idle for ATAPI devices
209                  */
210                 if (drive->media != ide_disk)
211                         pm->pm_step = ide_pm_restore_dma;
212                 else
213                         ide_complete_power_step(drive, rq, 0, 0);
214                 return ide_stopped;
215
216         case idedisk_pm_idle:           /* Resume step 2 (idle) */
217                 args->tf.command = WIN_IDLEIMMEDIATE;
218                 args->command_type = IDE_DRIVE_TASK_NO_DATA;
219                 args->handler = task_no_data_intr;
220                 return do_rw_taskfile(drive, args);
221
222         case ide_pm_restore_dma:        /* Resume step 3 (restore DMA) */
223                 /*
224                  * Right now, all we do is call ide_set_dma(drive),
225                  * we could be smarter and check for current xfer_speed
226                  * in struct drive etc...
227                  */
228                 if (drive->hwif->ide_dma_on == NULL)
229                         break;
230                 drive->hwif->dma_off_quietly(drive);
231                 /*
232                  * TODO: respect ->using_dma setting
233                  */
234                 ide_set_dma(drive);
235                 break;
236         }
237         pm->pm_step = ide_pm_state_completed;
238         return ide_stopped;
239 }
240
241 /**
242  *      ide_end_dequeued_request        -       complete an IDE I/O
243  *      @drive: IDE device for the I/O
244  *      @uptodate:
245  *      @nr_sectors: number of sectors completed
246  *
247  *      Complete an I/O that is no longer on the request queue. This
248  *      typically occurs when we pull the request and issue a REQUEST_SENSE.
249  *      We must still finish the old request but we must not tamper with the
250  *      queue in the meantime.
251  *
252  *      NOTE: This path does not handle barrier, but barrier is not supported
253  *      on ide-cd anyway.
254  */
255
256 int ide_end_dequeued_request(ide_drive_t *drive, struct request *rq,
257                              int uptodate, int nr_sectors)
258 {
259         unsigned long flags;
260         int ret;
261
262         spin_lock_irqsave(&ide_lock, flags);
263         BUG_ON(!blk_rq_started(rq));
264         ret = __ide_end_request(drive, rq, uptodate, nr_sectors << 9, 0);
265         spin_unlock_irqrestore(&ide_lock, flags);
266
267         return ret;
268 }
269 EXPORT_SYMBOL_GPL(ide_end_dequeued_request);
270
271
272 /**
273  *      ide_complete_pm_request - end the current Power Management request
274  *      @drive: target drive
275  *      @rq: request
276  *
277  *      This function cleans up the current PM request and stops the queue
278  *      if necessary.
279  */
280 static void ide_complete_pm_request (ide_drive_t *drive, struct request *rq)
281 {
282         unsigned long flags;
283
284 #ifdef DEBUG_PM
285         printk("%s: completing PM request, %s\n", drive->name,
286                blk_pm_suspend_request(rq) ? "suspend" : "resume");
287 #endif
288         spin_lock_irqsave(&ide_lock, flags);
289         if (blk_pm_suspend_request(rq)) {
290                 blk_stop_queue(drive->queue);
291         } else {
292                 drive->blocked = 0;
293                 blk_start_queue(drive->queue);
294         }
295         blkdev_dequeue_request(rq);
296         HWGROUP(drive)->rq = NULL;
297         end_that_request_last(rq, 1);
298         spin_unlock_irqrestore(&ide_lock, flags);
299 }
300
301 /**
302  *      ide_end_drive_cmd       -       end an explicit drive command
303  *      @drive: command 
304  *      @stat: status bits
305  *      @err: error bits
306  *
307  *      Clean up after success/failure of an explicit drive command.
308  *      These get thrown onto the queue so they are synchronized with
309  *      real I/O operations on the drive.
310  *
311  *      In LBA48 mode we have to read the register set twice to get
312  *      all the extra information out.
313  */
314  
315 void ide_end_drive_cmd (ide_drive_t *drive, u8 stat, u8 err)
316 {
317         ide_hwif_t *hwif = HWIF(drive);
318         unsigned long flags;
319         struct request *rq;
320
321         spin_lock_irqsave(&ide_lock, flags);
322         rq = HWGROUP(drive)->rq;
323         spin_unlock_irqrestore(&ide_lock, flags);
324
325         if (rq->cmd_type == REQ_TYPE_ATA_CMD) {
326                 u8 *args = (u8 *) rq->buffer;
327                 if (rq->errors == 0)
328                         rq->errors = !OK_STAT(stat,READY_STAT,BAD_STAT);
329
330                 if (args) {
331                         args[0] = stat;
332                         args[1] = err;
333                         args[2] = hwif->INB(IDE_NSECTOR_REG);
334                 }
335         } else if (rq->cmd_type == REQ_TYPE_ATA_TASK) {
336                 u8 *args = (u8 *) rq->buffer;
337                 if (rq->errors == 0)
338                         rq->errors = !OK_STAT(stat,READY_STAT,BAD_STAT);
339
340                 if (args) {
341                         args[0] = stat;
342                         args[1] = err;
343                         /* be sure we're looking at the low order bits */
344                         hwif->OUTB(drive->ctl & ~0x80, IDE_CONTROL_REG);
345                         args[2] = hwif->INB(IDE_NSECTOR_REG);
346                         args[3] = hwif->INB(IDE_SECTOR_REG);
347                         args[4] = hwif->INB(IDE_LCYL_REG);
348                         args[5] = hwif->INB(IDE_HCYL_REG);
349                         args[6] = hwif->INB(IDE_SELECT_REG);
350                 }
351         } else if (rq->cmd_type == REQ_TYPE_ATA_TASKFILE) {
352                 ide_task_t *args = (ide_task_t *) rq->special;
353                 if (rq->errors == 0)
354                         rq->errors = !OK_STAT(stat,READY_STAT,BAD_STAT);
355                         
356                 if (args) {
357                         struct ide_taskfile *tf = &args->tf;
358
359                         if (args->tf_in_flags.b.data) {
360                                 u16 data = hwif->INW(IDE_DATA_REG);
361
362                                 tf->data = data & 0xff;
363                                 tf->hob_data = (data >> 8) & 0xff;
364                         }
365                         tf->error = err;
366                         /* be sure we're looking at the low order bits */
367                         hwif->OUTB(drive->ctl & ~0x80, IDE_CONTROL_REG);
368                         tf->nsect  = hwif->INB(IDE_NSECTOR_REG);
369                         tf->lbal   = hwif->INB(IDE_SECTOR_REG);
370                         tf->lbam   = hwif->INB(IDE_LCYL_REG);
371                         tf->lbah   = hwif->INB(IDE_HCYL_REG);
372                         tf->device = hwif->INB(IDE_SELECT_REG);
373                         tf->status = stat;
374
375                         if (drive->addressing == 1) {
376                                 hwif->OUTB(drive->ctl|0x80, IDE_CONTROL_REG);
377                                 tf->hob_feature = hwif->INB(IDE_FEATURE_REG);
378                                 tf->hob_nsect   = hwif->INB(IDE_NSECTOR_REG);
379                                 tf->hob_lbal    = hwif->INB(IDE_SECTOR_REG);
380                                 tf->hob_lbam    = hwif->INB(IDE_LCYL_REG);
381                                 tf->hob_lbah    = hwif->INB(IDE_HCYL_REG);
382                         }
383                 }
384         } else if (blk_pm_request(rq)) {
385                 struct request_pm_state *pm = rq->data;
386 #ifdef DEBUG_PM
387                 printk("%s: complete_power_step(step: %d, stat: %x, err: %x)\n",
388                         drive->name, rq->pm->pm_step, stat, err);
389 #endif
390                 ide_complete_power_step(drive, rq, stat, err);
391                 if (pm->pm_step == ide_pm_state_completed)
392                         ide_complete_pm_request(drive, rq);
393                 return;
394         }
395
396         spin_lock_irqsave(&ide_lock, flags);
397         blkdev_dequeue_request(rq);
398         HWGROUP(drive)->rq = NULL;
399         rq->errors = err;
400         end_that_request_last(rq, !rq->errors);
401         spin_unlock_irqrestore(&ide_lock, flags);
402 }
403
404 EXPORT_SYMBOL(ide_end_drive_cmd);
405
406 /**
407  *      try_to_flush_leftover_data      -       flush junk
408  *      @drive: drive to flush
409  *
410  *      try_to_flush_leftover_data() is invoked in response to a drive
411  *      unexpectedly having its DRQ_STAT bit set.  As an alternative to
412  *      resetting the drive, this routine tries to clear the condition
413  *      by read a sector's worth of data from the drive.  Of course,
414  *      this may not help if the drive is *waiting* for data from *us*.
415  */
416 static void try_to_flush_leftover_data (ide_drive_t *drive)
417 {
418         int i = (drive->mult_count ? drive->mult_count : 1) * SECTOR_WORDS;
419
420         if (drive->media != ide_disk)
421                 return;
422         while (i > 0) {
423                 u32 buffer[16];
424                 u32 wcount = (i > 16) ? 16 : i;
425
426                 i -= wcount;
427                 HWIF(drive)->ata_input_data(drive, buffer, wcount);
428         }
429 }
430
431 static void ide_kill_rq(ide_drive_t *drive, struct request *rq)
432 {
433         if (rq->rq_disk) {
434                 ide_driver_t *drv;
435
436                 drv = *(ide_driver_t **)rq->rq_disk->private_data;
437                 drv->end_request(drive, 0, 0);
438         } else
439                 ide_end_request(drive, 0, 0);
440 }
441
442 static ide_startstop_t ide_ata_error(ide_drive_t *drive, struct request *rq, u8 stat, u8 err)
443 {
444         ide_hwif_t *hwif = drive->hwif;
445
446         if (stat & BUSY_STAT || ((stat & WRERR_STAT) && !drive->nowerr)) {
447                 /* other bits are useless when BUSY */
448                 rq->errors |= ERROR_RESET;
449         } else if (stat & ERR_STAT) {
450                 /* err has different meaning on cdrom and tape */
451                 if (err == ABRT_ERR) {
452                         if (drive->select.b.lba &&
453                             /* some newer drives don't support WIN_SPECIFY */
454                             hwif->INB(IDE_COMMAND_REG) == WIN_SPECIFY)
455                                 return ide_stopped;
456                 } else if ((err & BAD_CRC) == BAD_CRC) {
457                         /* UDMA crc error, just retry the operation */
458                         drive->crc_count++;
459                 } else if (err & (BBD_ERR | ECC_ERR)) {
460                         /* retries won't help these */
461                         rq->errors = ERROR_MAX;
462                 } else if (err & TRK0_ERR) {
463                         /* help it find track zero */
464                         rq->errors |= ERROR_RECAL;
465                 }
466         }
467
468         if ((stat & DRQ_STAT) && rq_data_dir(rq) == READ &&
469             (hwif->host_flags & IDE_HFLAG_ERROR_STOPS_FIFO) == 0)
470                 try_to_flush_leftover_data(drive);
471
472         if (rq->errors >= ERROR_MAX || blk_noretry_request(rq)) {
473                 ide_kill_rq(drive, rq);
474                 return ide_stopped;
475         }
476
477         if (hwif->INB(IDE_STATUS_REG) & (BUSY_STAT|DRQ_STAT))
478                 rq->errors |= ERROR_RESET;
479
480         if ((rq->errors & ERROR_RESET) == ERROR_RESET) {
481                 ++rq->errors;
482                 return ide_do_reset(drive);
483         }
484
485         if ((rq->errors & ERROR_RECAL) == ERROR_RECAL)
486                 drive->special.b.recalibrate = 1;
487
488         ++rq->errors;
489
490         return ide_stopped;
491 }
492
493 static ide_startstop_t ide_atapi_error(ide_drive_t *drive, struct request *rq, u8 stat, u8 err)
494 {
495         ide_hwif_t *hwif = drive->hwif;
496
497         if (stat & BUSY_STAT || ((stat & WRERR_STAT) && !drive->nowerr)) {
498                 /* other bits are useless when BUSY */
499                 rq->errors |= ERROR_RESET;
500         } else {
501                 /* add decoding error stuff */
502         }
503
504         if (hwif->INB(IDE_STATUS_REG) & (BUSY_STAT|DRQ_STAT))
505                 /* force an abort */
506                 hwif->OUTB(WIN_IDLEIMMEDIATE, IDE_COMMAND_REG);
507
508         if (rq->errors >= ERROR_MAX) {
509                 ide_kill_rq(drive, rq);
510         } else {
511                 if ((rq->errors & ERROR_RESET) == ERROR_RESET) {
512                         ++rq->errors;
513                         return ide_do_reset(drive);
514                 }
515                 ++rq->errors;
516         }
517
518         return ide_stopped;
519 }
520
521 ide_startstop_t
522 __ide_error(ide_drive_t *drive, struct request *rq, u8 stat, u8 err)
523 {
524         if (drive->media == ide_disk)
525                 return ide_ata_error(drive, rq, stat, err);
526         return ide_atapi_error(drive, rq, stat, err);
527 }
528
529 EXPORT_SYMBOL_GPL(__ide_error);
530
531 /**
532  *      ide_error       -       handle an error on the IDE
533  *      @drive: drive the error occurred on
534  *      @msg: message to report
535  *      @stat: status bits
536  *
537  *      ide_error() takes action based on the error returned by the drive.
538  *      For normal I/O that may well include retries. We deal with
539  *      both new-style (taskfile) and old style command handling here.
540  *      In the case of taskfile command handling there is work left to
541  *      do
542  */
543  
544 ide_startstop_t ide_error (ide_drive_t *drive, const char *msg, u8 stat)
545 {
546         struct request *rq;
547         u8 err;
548
549         err = ide_dump_status(drive, msg, stat);
550
551         if ((rq = HWGROUP(drive)->rq) == NULL)
552                 return ide_stopped;
553
554         /* retry only "normal" I/O: */
555         if (!blk_fs_request(rq)) {
556                 rq->errors = 1;
557                 ide_end_drive_cmd(drive, stat, err);
558                 return ide_stopped;
559         }
560
561         if (rq->rq_disk) {
562                 ide_driver_t *drv;
563
564                 drv = *(ide_driver_t **)rq->rq_disk->private_data;
565                 return drv->error(drive, rq, stat, err);
566         } else
567                 return __ide_error(drive, rq, stat, err);
568 }
569
570 EXPORT_SYMBOL_GPL(ide_error);
571
572 ide_startstop_t __ide_abort(ide_drive_t *drive, struct request *rq)
573 {
574         if (drive->media != ide_disk)
575                 rq->errors |= ERROR_RESET;
576
577         ide_kill_rq(drive, rq);
578
579         return ide_stopped;
580 }
581
582 EXPORT_SYMBOL_GPL(__ide_abort);
583
584 /**
585  *      ide_abort       -       abort pending IDE operations
586  *      @drive: drive the error occurred on
587  *      @msg: message to report
588  *
589  *      ide_abort kills and cleans up when we are about to do a 
590  *      host initiated reset on active commands. Longer term we
591  *      want handlers to have sensible abort handling themselves
592  *
593  *      This differs fundamentally from ide_error because in 
594  *      this case the command is doing just fine when we
595  *      blow it away.
596  */
597  
598 ide_startstop_t ide_abort(ide_drive_t *drive, const char *msg)
599 {
600         struct request *rq;
601
602         if (drive == NULL || (rq = HWGROUP(drive)->rq) == NULL)
603                 return ide_stopped;
604
605         /* retry only "normal" I/O: */
606         if (!blk_fs_request(rq)) {
607                 rq->errors = 1;
608                 ide_end_drive_cmd(drive, BUSY_STAT, 0);
609                 return ide_stopped;
610         }
611
612         if (rq->rq_disk) {
613                 ide_driver_t *drv;
614
615                 drv = *(ide_driver_t **)rq->rq_disk->private_data;
616                 return drv->abort(drive, rq);
617         } else
618                 return __ide_abort(drive, rq);
619 }
620
621 /**
622  *      ide_cmd         -       issue a simple drive command
623  *      @drive: drive the command is for
624  *      @cmd: command byte
625  *      @nsect: sector byte
626  *      @handler: handler for the command completion
627  *
628  *      Issue a simple drive command with interrupts.
629  *      The drive must be selected beforehand.
630  */
631
632 static void ide_cmd (ide_drive_t *drive, u8 cmd, u8 nsect,
633                 ide_handler_t *handler)
634 {
635         ide_hwif_t *hwif = HWIF(drive);
636         if (IDE_CONTROL_REG)
637                 hwif->OUTB(drive->ctl,IDE_CONTROL_REG); /* clear nIEN */
638         SELECT_MASK(drive,0);
639         hwif->OUTB(nsect,IDE_NSECTOR_REG);
640         ide_execute_command(drive, cmd, handler, WAIT_CMD, NULL);
641 }
642
643 /**
644  *      drive_cmd_intr          -       drive command completion interrupt
645  *      @drive: drive the completion interrupt occurred on
646  *
647  *      drive_cmd_intr() is invoked on completion of a special DRIVE_CMD.
648  *      We do any necessary data reading and then wait for the drive to
649  *      go non busy. At that point we may read the error data and complete
650  *      the request
651  */
652  
653 static ide_startstop_t drive_cmd_intr (ide_drive_t *drive)
654 {
655         struct request *rq = HWGROUP(drive)->rq;
656         ide_hwif_t *hwif = HWIF(drive);
657         u8 *args = (u8 *) rq->buffer;
658         u8 stat = hwif->INB(IDE_STATUS_REG);
659         int retries = 10;
660
661         local_irq_enable_in_hardirq();
662         if (rq->cmd_type == REQ_TYPE_ATA_CMD &&
663             (stat & DRQ_STAT) && args && args[3]) {
664                 u8 io_32bit = drive->io_32bit;
665                 drive->io_32bit = 0;
666                 hwif->ata_input_data(drive, &args[4], args[3] * SECTOR_WORDS);
667                 drive->io_32bit = io_32bit;
668                 while (((stat = hwif->INB(IDE_STATUS_REG)) & BUSY_STAT) && retries--)
669                         udelay(100);
670         }
671
672         if (!OK_STAT(stat, READY_STAT, BAD_STAT))
673                 return ide_error(drive, "drive_cmd", stat);
674                 /* calls ide_end_drive_cmd */
675         ide_end_drive_cmd(drive, stat, hwif->INB(IDE_ERROR_REG));
676         return ide_stopped;
677 }
678
679 static void ide_init_specify_cmd(ide_drive_t *drive, ide_task_t *task)
680 {
681         task->tf.nsect   = drive->sect;
682         task->tf.lbal    = drive->sect;
683         task->tf.lbam    = drive->cyl;
684         task->tf.lbah    = drive->cyl >> 8;
685         task->tf.device  = ((drive->head - 1) | drive->select.all) & ~ATA_LBA;
686         task->tf.command = WIN_SPECIFY;
687
688         task->handler = &set_geometry_intr;
689 }
690
691 static void ide_init_restore_cmd(ide_drive_t *drive, ide_task_t *task)
692 {
693         task->tf.nsect   = drive->sect;
694         task->tf.command = WIN_RESTORE;
695
696         task->handler = &recal_intr;
697 }
698
699 static void ide_init_setmult_cmd(ide_drive_t *drive, ide_task_t *task)
700 {
701         task->tf.nsect   = drive->mult_req;
702         task->tf.command = WIN_SETMULT;
703
704         task->handler = &set_multmode_intr;
705 }
706
707 static ide_startstop_t ide_disk_special(ide_drive_t *drive)
708 {
709         special_t *s = &drive->special;
710         ide_task_t args;
711
712         memset(&args, 0, sizeof(ide_task_t));
713         args.command_type = IDE_DRIVE_TASK_NO_DATA;
714
715         if (s->b.set_geometry) {
716                 s->b.set_geometry = 0;
717                 ide_init_specify_cmd(drive, &args);
718         } else if (s->b.recalibrate) {
719                 s->b.recalibrate = 0;
720                 ide_init_restore_cmd(drive, &args);
721         } else if (s->b.set_multmode) {
722                 s->b.set_multmode = 0;
723                 if (drive->mult_req > drive->id->max_multsect)
724                         drive->mult_req = drive->id->max_multsect;
725                 ide_init_setmult_cmd(drive, &args);
726         } else if (s->all) {
727                 int special = s->all;
728                 s->all = 0;
729                 printk(KERN_ERR "%s: bad special flag: 0x%02x\n", drive->name, special);
730                 return ide_stopped;
731         }
732
733         do_rw_taskfile(drive, &args);
734
735         return ide_started;
736 }
737
738 /*
739  * handle HDIO_SET_PIO_MODE ioctl abusers here, eventually it will go away
740  */
741 static int set_pio_mode_abuse(ide_hwif_t *hwif, u8 req_pio)
742 {
743         switch (req_pio) {
744         case 202:
745         case 201:
746         case 200:
747         case 102:
748         case 101:
749         case 100:
750                 return (hwif->host_flags & IDE_HFLAG_ABUSE_DMA_MODES) ? 1 : 0;
751         case 9:
752         case 8:
753                 return (hwif->host_flags & IDE_HFLAG_ABUSE_PREFETCH) ? 1 : 0;
754         case 7:
755         case 6:
756                 return (hwif->host_flags & IDE_HFLAG_ABUSE_FAST_DEVSEL) ? 1 : 0;
757         default:
758                 return 0;
759         }
760 }
761
762 /**
763  *      do_special              -       issue some special commands
764  *      @drive: drive the command is for
765  *
766  *      do_special() is used to issue WIN_SPECIFY, WIN_RESTORE, and WIN_SETMULT
767  *      commands to a drive.  It used to do much more, but has been scaled
768  *      back.
769  */
770
771 static ide_startstop_t do_special (ide_drive_t *drive)
772 {
773         special_t *s = &drive->special;
774
775 #ifdef DEBUG
776         printk("%s: do_special: 0x%02x\n", drive->name, s->all);
777 #endif
778         if (s->b.set_tune) {
779                 ide_hwif_t *hwif = drive->hwif;
780                 u8 req_pio = drive->tune_req;
781
782                 s->b.set_tune = 0;
783
784                 if (set_pio_mode_abuse(drive->hwif, req_pio)) {
785
786                         if (hwif->set_pio_mode == NULL)
787                                 return ide_stopped;
788
789                         /*
790                          * take ide_lock for drive->[no_]unmask/[no_]io_32bit
791                          */
792                         if (req_pio == 8 || req_pio == 9) {
793                                 unsigned long flags;
794
795                                 spin_lock_irqsave(&ide_lock, flags);
796                                 hwif->set_pio_mode(drive, req_pio);
797                                 spin_unlock_irqrestore(&ide_lock, flags);
798                         } else
799                                 hwif->set_pio_mode(drive, req_pio);
800                 } else {
801                         int keep_dma = drive->using_dma;
802
803                         ide_set_pio(drive, req_pio);
804
805                         if (hwif->host_flags & IDE_HFLAG_SET_PIO_MODE_KEEP_DMA) {
806                                 if (keep_dma)
807                                         hwif->ide_dma_on(drive);
808                         }
809                 }
810
811                 return ide_stopped;
812         } else {
813                 if (drive->media == ide_disk)
814                         return ide_disk_special(drive);
815
816                 s->all = 0;
817                 drive->mult_req = 0;
818                 return ide_stopped;
819         }
820 }
821
822 void ide_map_sg(ide_drive_t *drive, struct request *rq)
823 {
824         ide_hwif_t *hwif = drive->hwif;
825         struct scatterlist *sg = hwif->sg_table;
826
827         if (hwif->sg_mapped)    /* needed by ide-scsi */
828                 return;
829
830         if (rq->cmd_type != REQ_TYPE_ATA_TASKFILE) {
831                 hwif->sg_nents = blk_rq_map_sg(drive->queue, rq, sg);
832         } else {
833                 sg_init_one(sg, rq->buffer, rq->nr_sectors * SECTOR_SIZE);
834                 hwif->sg_nents = 1;
835         }
836 }
837
838 EXPORT_SYMBOL_GPL(ide_map_sg);
839
840 void ide_init_sg_cmd(ide_drive_t *drive, struct request *rq)
841 {
842         ide_hwif_t *hwif = drive->hwif;
843
844         hwif->nsect = hwif->nleft = rq->nr_sectors;
845         hwif->cursg_ofs = 0;
846         hwif->cursg = NULL;
847 }
848
849 EXPORT_SYMBOL_GPL(ide_init_sg_cmd);
850
851 /**
852  *      execute_drive_command   -       issue special drive command
853  *      @drive: the drive to issue the command on
854  *      @rq: the request structure holding the command
855  *
856  *      execute_drive_cmd() issues a special drive command,  usually 
857  *      initiated by ioctl() from the external hdparm program. The
858  *      command can be a drive command, drive task or taskfile 
859  *      operation. Weirdly you can call it with NULL to wait for
860  *      all commands to finish. Don't do this as that is due to change
861  */
862
863 static ide_startstop_t execute_drive_cmd (ide_drive_t *drive,
864                 struct request *rq)
865 {
866         ide_hwif_t *hwif = HWIF(drive);
867         if (rq->cmd_type == REQ_TYPE_ATA_TASKFILE) {
868                 ide_task_t *args = rq->special;
869  
870                 if (!args)
871                         goto done;
872
873                 hwif->data_phase = args->data_phase;
874
875                 switch (hwif->data_phase) {
876                 case TASKFILE_MULTI_OUT:
877                 case TASKFILE_OUT:
878                 case TASKFILE_MULTI_IN:
879                 case TASKFILE_IN:
880                         ide_init_sg_cmd(drive, rq);
881                         ide_map_sg(drive, rq);
882                 default:
883                         break;
884                 }
885
886                 if (args->tf_out_flags.all != 0) 
887                         return flagged_taskfile(drive, args);
888                 return do_rw_taskfile(drive, args);
889         } else if (rq->cmd_type == REQ_TYPE_ATA_TASK) {
890                 u8 *args = rq->buffer;
891  
892                 if (!args)
893                         goto done;
894 #ifdef DEBUG
895                 printk("%s: DRIVE_TASK_CMD ", drive->name);
896                 printk("cmd=0x%02x ", args[0]);
897                 printk("fr=0x%02x ", args[1]);
898                 printk("ns=0x%02x ", args[2]);
899                 printk("sc=0x%02x ", args[3]);
900                 printk("lcyl=0x%02x ", args[4]);
901                 printk("hcyl=0x%02x ", args[5]);
902                 printk("sel=0x%02x\n", args[6]);
903 #endif
904                 hwif->OUTB(args[1], IDE_FEATURE_REG);
905                 hwif->OUTB(args[3], IDE_SECTOR_REG);
906                 hwif->OUTB(args[4], IDE_LCYL_REG);
907                 hwif->OUTB(args[5], IDE_HCYL_REG);
908                 hwif->OUTB((args[6] & 0xEF)|drive->select.all, IDE_SELECT_REG);
909                 ide_cmd(drive, args[0], args[2], &drive_cmd_intr);
910                 return ide_started;
911         } else if (rq->cmd_type == REQ_TYPE_ATA_CMD) {
912                 u8 *args = rq->buffer;
913
914                 if (!args)
915                         goto done;
916 #ifdef DEBUG
917                 printk("%s: DRIVE_CMD ", drive->name);
918                 printk("cmd=0x%02x ", args[0]);
919                 printk("sc=0x%02x ", args[1]);
920                 printk("fr=0x%02x ", args[2]);
921                 printk("xx=0x%02x\n", args[3]);
922 #endif
923                 if (args[0] == WIN_SMART) {
924                         hwif->OUTB(0x4f, IDE_LCYL_REG);
925                         hwif->OUTB(0xc2, IDE_HCYL_REG);
926                         hwif->OUTB(args[2],IDE_FEATURE_REG);
927                         hwif->OUTB(args[1],IDE_SECTOR_REG);
928                         ide_cmd(drive, args[0], args[3], &drive_cmd_intr);
929                         return ide_started;
930                 }
931                 hwif->OUTB(args[2],IDE_FEATURE_REG);
932                 ide_cmd(drive, args[0], args[1], &drive_cmd_intr);
933                 return ide_started;
934         }
935
936 done:
937         /*
938          * NULL is actually a valid way of waiting for
939          * all current requests to be flushed from the queue.
940          */
941 #ifdef DEBUG
942         printk("%s: DRIVE_CMD (null)\n", drive->name);
943 #endif
944         ide_end_drive_cmd(drive,
945                         hwif->INB(IDE_STATUS_REG),
946                         hwif->INB(IDE_ERROR_REG));
947         return ide_stopped;
948 }
949
950 static void ide_check_pm_state(ide_drive_t *drive, struct request *rq)
951 {
952         struct request_pm_state *pm = rq->data;
953
954         if (blk_pm_suspend_request(rq) &&
955             pm->pm_step == ide_pm_state_start_suspend)
956                 /* Mark drive blocked when starting the suspend sequence. */
957                 drive->blocked = 1;
958         else if (blk_pm_resume_request(rq) &&
959                  pm->pm_step == ide_pm_state_start_resume) {
960                 /* 
961                  * The first thing we do on wakeup is to wait for BSY bit to
962                  * go away (with a looong timeout) as a drive on this hwif may
963                  * just be POSTing itself.
964                  * We do that before even selecting as the "other" device on
965                  * the bus may be broken enough to walk on our toes at this
966                  * point.
967                  */
968                 int rc;
969 #ifdef DEBUG_PM
970                 printk("%s: Wakeup request inited, waiting for !BSY...\n", drive->name);
971 #endif
972                 rc = ide_wait_not_busy(HWIF(drive), 35000);
973                 if (rc)
974                         printk(KERN_WARNING "%s: bus not ready on wakeup\n", drive->name);
975                 SELECT_DRIVE(drive);
976                 if (IDE_CONTROL_REG)
977                         HWIF(drive)->OUTB(drive->ctl, IDE_CONTROL_REG);
978                 rc = ide_wait_not_busy(HWIF(drive), 100000);
979                 if (rc)
980                         printk(KERN_WARNING "%s: drive not ready on wakeup\n", drive->name);
981         }
982 }
983
984 /**
985  *      start_request   -       start of I/O and command issuing for IDE
986  *
987  *      start_request() initiates handling of a new I/O request. It
988  *      accepts commands and I/O (read/write) requests. It also does
989  *      the final remapping for weird stuff like EZDrive. Once 
990  *      device mapper can work sector level the EZDrive stuff can go away
991  *
992  *      FIXME: this function needs a rename
993  */
994  
995 static ide_startstop_t start_request (ide_drive_t *drive, struct request *rq)
996 {
997         ide_startstop_t startstop;
998         sector_t block;
999
1000         BUG_ON(!blk_rq_started(rq));
1001
1002 #ifdef DEBUG
1003         printk("%s: start_request: current=0x%08lx\n",
1004                 HWIF(drive)->name, (unsigned long) rq);
1005 #endif
1006
1007         /* bail early if we've exceeded max_failures */
1008         if (drive->max_failures && (drive->failures > drive->max_failures)) {
1009                 rq->cmd_flags |= REQ_FAILED;
1010                 goto kill_rq;
1011         }
1012
1013         block    = rq->sector;
1014         if (blk_fs_request(rq) &&
1015             (drive->media == ide_disk || drive->media == ide_floppy)) {
1016                 block += drive->sect0;
1017         }
1018         /* Yecch - this will shift the entire interval,
1019            possibly killing some innocent following sector */
1020         if (block == 0 && drive->remap_0_to_1 == 1)
1021                 block = 1;  /* redirect MBR access to EZ-Drive partn table */
1022
1023         if (blk_pm_request(rq))
1024                 ide_check_pm_state(drive, rq);
1025
1026         SELECT_DRIVE(drive);
1027         if (ide_wait_stat(&startstop, drive, drive->ready_stat, BUSY_STAT|DRQ_STAT, WAIT_READY)) {
1028                 printk(KERN_ERR "%s: drive not ready for command\n", drive->name);
1029                 return startstop;
1030         }
1031         if (!drive->special.all) {
1032                 ide_driver_t *drv;
1033
1034                 /*
1035                  * We reset the drive so we need to issue a SETFEATURES.
1036                  * Do it _after_ do_special() restored device parameters.
1037                  */
1038                 if (drive->current_speed == 0xff)
1039                         ide_config_drive_speed(drive, drive->desired_speed);
1040
1041                 if (rq->cmd_type == REQ_TYPE_ATA_CMD ||
1042                     rq->cmd_type == REQ_TYPE_ATA_TASK ||
1043                     rq->cmd_type == REQ_TYPE_ATA_TASKFILE)
1044                         return execute_drive_cmd(drive, rq);
1045                 else if (blk_pm_request(rq)) {
1046                         struct request_pm_state *pm = rq->data;
1047 #ifdef DEBUG_PM
1048                         printk("%s: start_power_step(step: %d)\n",
1049                                 drive->name, rq->pm->pm_step);
1050 #endif
1051                         startstop = ide_start_power_step(drive, rq);
1052                         if (startstop == ide_stopped &&
1053                             pm->pm_step == ide_pm_state_completed)
1054                                 ide_complete_pm_request(drive, rq);
1055                         return startstop;
1056                 }
1057
1058                 drv = *(ide_driver_t **)rq->rq_disk->private_data;
1059                 return drv->do_request(drive, rq, block);
1060         }
1061         return do_special(drive);
1062 kill_rq:
1063         ide_kill_rq(drive, rq);
1064         return ide_stopped;
1065 }
1066
1067 /**
1068  *      ide_stall_queue         -       pause an IDE device
1069  *      @drive: drive to stall
1070  *      @timeout: time to stall for (jiffies)
1071  *
1072  *      ide_stall_queue() can be used by a drive to give excess bandwidth back
1073  *      to the hwgroup by sleeping for timeout jiffies.
1074  */
1075  
1076 void ide_stall_queue (ide_drive_t *drive, unsigned long timeout)
1077 {
1078         if (timeout > WAIT_WORSTCASE)
1079                 timeout = WAIT_WORSTCASE;
1080         drive->sleep = timeout + jiffies;
1081         drive->sleeping = 1;
1082 }
1083
1084 EXPORT_SYMBOL(ide_stall_queue);
1085
1086 #define WAKEUP(drive)   ((drive)->service_start + 2 * (drive)->service_time)
1087
1088 /**
1089  *      choose_drive            -       select a drive to service
1090  *      @hwgroup: hardware group to select on
1091  *
1092  *      choose_drive() selects the next drive which will be serviced.
1093  *      This is necessary because the IDE layer can't issue commands
1094  *      to both drives on the same cable, unlike SCSI.
1095  */
1096  
1097 static inline ide_drive_t *choose_drive (ide_hwgroup_t *hwgroup)
1098 {
1099         ide_drive_t *drive, *best;
1100
1101 repeat: 
1102         best = NULL;
1103         drive = hwgroup->drive;
1104
1105         /*
1106          * drive is doing pre-flush, ordered write, post-flush sequence. even
1107          * though that is 3 requests, it must be seen as a single transaction.
1108          * we must not preempt this drive until that is complete
1109          */
1110         if (blk_queue_flushing(drive->queue)) {
1111                 /*
1112                  * small race where queue could get replugged during
1113                  * the 3-request flush cycle, just yank the plug since
1114                  * we want it to finish asap
1115                  */
1116                 blk_remove_plug(drive->queue);
1117                 return drive;
1118         }
1119
1120         do {
1121                 if ((!drive->sleeping || time_after_eq(jiffies, drive->sleep))
1122                     && !elv_queue_empty(drive->queue)) {
1123                         if (!best
1124                          || (drive->sleeping && (!best->sleeping || time_before(drive->sleep, best->sleep)))
1125                          || (!best->sleeping && time_before(WAKEUP(drive), WAKEUP(best))))
1126                         {
1127                                 if (!blk_queue_plugged(drive->queue))
1128                                         best = drive;
1129                         }
1130                 }
1131         } while ((drive = drive->next) != hwgroup->drive);
1132         if (best && best->nice1 && !best->sleeping && best != hwgroup->drive && best->service_time > WAIT_MIN_SLEEP) {
1133                 long t = (signed long)(WAKEUP(best) - jiffies);
1134                 if (t >= WAIT_MIN_SLEEP) {
1135                 /*
1136                  * We *may* have some time to spare, but first let's see if
1137                  * someone can potentially benefit from our nice mood today..
1138                  */
1139                         drive = best->next;
1140                         do {
1141                                 if (!drive->sleeping
1142                                  && time_before(jiffies - best->service_time, WAKEUP(drive))
1143                                  && time_before(WAKEUP(drive), jiffies + t))
1144                                 {
1145                                         ide_stall_queue(best, min_t(long, t, 10 * WAIT_MIN_SLEEP));
1146                                         goto repeat;
1147                                 }
1148                         } while ((drive = drive->next) != best);
1149                 }
1150         }
1151         return best;
1152 }
1153
1154 /*
1155  * Issue a new request to a drive from hwgroup
1156  * Caller must have already done spin_lock_irqsave(&ide_lock, ..);
1157  *
1158  * A hwgroup is a serialized group of IDE interfaces.  Usually there is
1159  * exactly one hwif (interface) per hwgroup, but buggy controllers (eg. CMD640)
1160  * may have both interfaces in a single hwgroup to "serialize" access.
1161  * Or possibly multiple ISA interfaces can share a common IRQ by being grouped
1162  * together into one hwgroup for serialized access.
1163  *
1164  * Note also that several hwgroups can end up sharing a single IRQ,
1165  * possibly along with many other devices.  This is especially common in
1166  * PCI-based systems with off-board IDE controller cards.
1167  *
1168  * The IDE driver uses the single global ide_lock spinlock to protect
1169  * access to the request queues, and to protect the hwgroup->busy flag.
1170  *
1171  * The first thread into the driver for a particular hwgroup sets the
1172  * hwgroup->busy flag to indicate that this hwgroup is now active,
1173  * and then initiates processing of the top request from the request queue.
1174  *
1175  * Other threads attempting entry notice the busy setting, and will simply
1176  * queue their new requests and exit immediately.  Note that hwgroup->busy
1177  * remains set even when the driver is merely awaiting the next interrupt.
1178  * Thus, the meaning is "this hwgroup is busy processing a request".
1179  *
1180  * When processing of a request completes, the completing thread or IRQ-handler
1181  * will start the next request from the queue.  If no more work remains,
1182  * the driver will clear the hwgroup->busy flag and exit.
1183  *
1184  * The ide_lock (spinlock) is used to protect all access to the
1185  * hwgroup->busy flag, but is otherwise not needed for most processing in
1186  * the driver.  This makes the driver much more friendlier to shared IRQs
1187  * than previous designs, while remaining 100% (?) SMP safe and capable.
1188  */
1189 static void ide_do_request (ide_hwgroup_t *hwgroup, int masked_irq)
1190 {
1191         ide_drive_t     *drive;
1192         ide_hwif_t      *hwif;
1193         struct request  *rq;
1194         ide_startstop_t startstop;
1195         int             loops = 0;
1196
1197         /* for atari only: POSSIBLY BROKEN HERE(?) */
1198         ide_get_lock(ide_intr, hwgroup);
1199
1200         /* caller must own ide_lock */
1201         BUG_ON(!irqs_disabled());
1202
1203         while (!hwgroup->busy) {
1204                 hwgroup->busy = 1;
1205                 drive = choose_drive(hwgroup);
1206                 if (drive == NULL) {
1207                         int sleeping = 0;
1208                         unsigned long sleep = 0; /* shut up, gcc */
1209                         hwgroup->rq = NULL;
1210                         drive = hwgroup->drive;
1211                         do {
1212                                 if (drive->sleeping && (!sleeping || time_before(drive->sleep, sleep))) {
1213                                         sleeping = 1;
1214                                         sleep = drive->sleep;
1215                                 }
1216                         } while ((drive = drive->next) != hwgroup->drive);
1217                         if (sleeping) {
1218                 /*
1219                  * Take a short snooze, and then wake up this hwgroup again.
1220                  * This gives other hwgroups on the same a chance to
1221                  * play fairly with us, just in case there are big differences
1222                  * in relative throughputs.. don't want to hog the cpu too much.
1223                  */
1224                                 if (time_before(sleep, jiffies + WAIT_MIN_SLEEP))
1225                                         sleep = jiffies + WAIT_MIN_SLEEP;
1226 #if 1
1227                                 if (timer_pending(&hwgroup->timer))
1228                                         printk(KERN_CRIT "ide_set_handler: timer already active\n");
1229 #endif
1230                                 /* so that ide_timer_expiry knows what to do */
1231                                 hwgroup->sleeping = 1;
1232                                 hwgroup->req_gen_timer = hwgroup->req_gen;
1233                                 mod_timer(&hwgroup->timer, sleep);
1234                                 /* we purposely leave hwgroup->busy==1
1235                                  * while sleeping */
1236                         } else {
1237                                 /* Ugly, but how can we sleep for the lock
1238                                  * otherwise? perhaps from tq_disk?
1239                                  */
1240
1241                                 /* for atari only */
1242                                 ide_release_lock();
1243                                 hwgroup->busy = 0;
1244                         }
1245
1246                         /* no more work for this hwgroup (for now) */
1247                         return;
1248                 }
1249         again:
1250                 hwif = HWIF(drive);
1251                 if (hwgroup->hwif->sharing_irq &&
1252                     hwif != hwgroup->hwif &&
1253                     hwif->io_ports[IDE_CONTROL_OFFSET]) {
1254                         /* set nIEN for previous hwif */
1255                         SELECT_INTERRUPT(drive);
1256                 }
1257                 hwgroup->hwif = hwif;
1258                 hwgroup->drive = drive;
1259                 drive->sleeping = 0;
1260                 drive->service_start = jiffies;
1261
1262                 if (blk_queue_plugged(drive->queue)) {
1263                         printk(KERN_ERR "ide: huh? queue was plugged!\n");
1264                         break;
1265                 }
1266
1267                 /*
1268                  * we know that the queue isn't empty, but this can happen
1269                  * if the q->prep_rq_fn() decides to kill a request
1270                  */
1271                 rq = elv_next_request(drive->queue);
1272                 if (!rq) {
1273                         hwgroup->busy = 0;
1274                         break;
1275                 }
1276
1277                 /*
1278                  * Sanity: don't accept a request that isn't a PM request
1279                  * if we are currently power managed. This is very important as
1280                  * blk_stop_queue() doesn't prevent the elv_next_request()
1281                  * above to return us whatever is in the queue. Since we call
1282                  * ide_do_request() ourselves, we end up taking requests while
1283                  * the queue is blocked...
1284                  * 
1285                  * We let requests forced at head of queue with ide-preempt
1286                  * though. I hope that doesn't happen too much, hopefully not
1287                  * unless the subdriver triggers such a thing in its own PM
1288                  * state machine.
1289                  *
1290                  * We count how many times we loop here to make sure we service
1291                  * all drives in the hwgroup without looping for ever
1292                  */
1293                 if (drive->blocked && !blk_pm_request(rq) && !(rq->cmd_flags & REQ_PREEMPT)) {
1294                         drive = drive->next ? drive->next : hwgroup->drive;
1295                         if (loops++ < 4 && !blk_queue_plugged(drive->queue))
1296                                 goto again;
1297                         /* We clear busy, there should be no pending ATA command at this point. */
1298                         hwgroup->busy = 0;
1299                         break;
1300                 }
1301
1302                 hwgroup->rq = rq;
1303
1304                 /*
1305                  * Some systems have trouble with IDE IRQs arriving while
1306                  * the driver is still setting things up.  So, here we disable
1307                  * the IRQ used by this interface while the request is being started.
1308                  * This may look bad at first, but pretty much the same thing
1309                  * happens anyway when any interrupt comes in, IDE or otherwise
1310                  *  -- the kernel masks the IRQ while it is being handled.
1311                  */
1312                 if (masked_irq != IDE_NO_IRQ && hwif->irq != masked_irq)
1313                         disable_irq_nosync(hwif->irq);
1314                 spin_unlock(&ide_lock);
1315                 local_irq_enable_in_hardirq();
1316                         /* allow other IRQs while we start this request */
1317                 startstop = start_request(drive, rq);
1318                 spin_lock_irq(&ide_lock);
1319                 if (masked_irq != IDE_NO_IRQ && hwif->irq != masked_irq)
1320                         enable_irq(hwif->irq);
1321                 if (startstop == ide_stopped)
1322                         hwgroup->busy = 0;
1323         }
1324 }
1325
1326 /*
1327  * Passes the stuff to ide_do_request
1328  */
1329 void do_ide_request(struct request_queue *q)
1330 {
1331         ide_drive_t *drive = q->queuedata;
1332
1333         ide_do_request(HWGROUP(drive), IDE_NO_IRQ);
1334 }
1335
1336 /*
1337  * un-busy the hwgroup etc, and clear any pending DMA status. we want to
1338  * retry the current request in pio mode instead of risking tossing it
1339  * all away
1340  */
1341 static ide_startstop_t ide_dma_timeout_retry(ide_drive_t *drive, int error)
1342 {
1343         ide_hwif_t *hwif = HWIF(drive);
1344         struct request *rq;
1345         ide_startstop_t ret = ide_stopped;
1346
1347         /*
1348          * end current dma transaction
1349          */
1350
1351         if (error < 0) {
1352                 printk(KERN_WARNING "%s: DMA timeout error\n", drive->name);
1353                 (void)HWIF(drive)->ide_dma_end(drive);
1354                 ret = ide_error(drive, "dma timeout error",
1355                                                 hwif->INB(IDE_STATUS_REG));
1356         } else {
1357                 printk(KERN_WARNING "%s: DMA timeout retry\n", drive->name);
1358                 hwif->dma_timeout(drive);
1359         }
1360
1361         /*
1362          * disable dma for now, but remember that we did so because of
1363          * a timeout -- we'll reenable after we finish this next request
1364          * (or rather the first chunk of it) in pio.
1365          */
1366         drive->retry_pio++;
1367         drive->state = DMA_PIO_RETRY;
1368         hwif->dma_off_quietly(drive);
1369
1370         /*
1371          * un-busy drive etc (hwgroup->busy is cleared on return) and
1372          * make sure request is sane
1373          */
1374         rq = HWGROUP(drive)->rq;
1375
1376         if (!rq)
1377                 goto out;
1378
1379         HWGROUP(drive)->rq = NULL;
1380
1381         rq->errors = 0;
1382
1383         if (!rq->bio)
1384                 goto out;
1385
1386         rq->sector = rq->bio->bi_sector;
1387         rq->current_nr_sectors = bio_iovec(rq->bio)->bv_len >> 9;
1388         rq->hard_cur_sectors = rq->current_nr_sectors;
1389         rq->buffer = bio_data(rq->bio);
1390 out:
1391         return ret;
1392 }
1393
1394 /**
1395  *      ide_timer_expiry        -       handle lack of an IDE interrupt
1396  *      @data: timer callback magic (hwgroup)
1397  *
1398  *      An IDE command has timed out before the expected drive return
1399  *      occurred. At this point we attempt to clean up the current
1400  *      mess. If the current handler includes an expiry handler then
1401  *      we invoke the expiry handler, and providing it is happy the
1402  *      work is done. If that fails we apply generic recovery rules
1403  *      invoking the handler and checking the drive DMA status. We
1404  *      have an excessively incestuous relationship with the DMA
1405  *      logic that wants cleaning up.
1406  */
1407  
1408 void ide_timer_expiry (unsigned long data)
1409 {
1410         ide_hwgroup_t   *hwgroup = (ide_hwgroup_t *) data;
1411         ide_handler_t   *handler;
1412         ide_expiry_t    *expiry;
1413         unsigned long   flags;
1414         unsigned long   wait = -1;
1415
1416         spin_lock_irqsave(&ide_lock, flags);
1417
1418         if (((handler = hwgroup->handler) == NULL) ||
1419             (hwgroup->req_gen != hwgroup->req_gen_timer)) {
1420                 /*
1421                  * Either a marginal timeout occurred
1422                  * (got the interrupt just as timer expired),
1423                  * or we were "sleeping" to give other devices a chance.
1424                  * Either way, we don't really want to complain about anything.
1425                  */
1426                 if (hwgroup->sleeping) {
1427                         hwgroup->sleeping = 0;
1428                         hwgroup->busy = 0;
1429                 }
1430         } else {
1431                 ide_drive_t *drive = hwgroup->drive;
1432                 if (!drive) {
1433                         printk(KERN_ERR "ide_timer_expiry: hwgroup->drive was NULL\n");
1434                         hwgroup->handler = NULL;
1435                 } else {
1436                         ide_hwif_t *hwif;
1437                         ide_startstop_t startstop = ide_stopped;
1438                         if (!hwgroup->busy) {
1439                                 hwgroup->busy = 1;      /* paranoia */
1440                                 printk(KERN_ERR "%s: ide_timer_expiry: hwgroup->busy was 0 ??\n", drive->name);
1441                         }
1442                         if ((expiry = hwgroup->expiry) != NULL) {
1443                                 /* continue */
1444                                 if ((wait = expiry(drive)) > 0) {
1445                                         /* reset timer */
1446                                         hwgroup->timer.expires  = jiffies + wait;
1447                                         hwgroup->req_gen_timer = hwgroup->req_gen;
1448                                         add_timer(&hwgroup->timer);
1449                                         spin_unlock_irqrestore(&ide_lock, flags);
1450                                         return;
1451                                 }
1452                         }
1453                         hwgroup->handler = NULL;
1454                         /*
1455                          * We need to simulate a real interrupt when invoking
1456                          * the handler() function, which means we need to
1457                          * globally mask the specific IRQ:
1458                          */
1459                         spin_unlock(&ide_lock);
1460                         hwif  = HWIF(drive);
1461 #if DISABLE_IRQ_NOSYNC
1462                         disable_irq_nosync(hwif->irq);
1463 #else
1464                         /* disable_irq_nosync ?? */
1465                         disable_irq(hwif->irq);
1466 #endif /* DISABLE_IRQ_NOSYNC */
1467                         /* local CPU only,
1468                          * as if we were handling an interrupt */
1469                         local_irq_disable();
1470                         if (hwgroup->polling) {
1471                                 startstop = handler(drive);
1472                         } else if (drive_is_ready(drive)) {
1473                                 if (drive->waiting_for_dma)
1474                                         hwgroup->hwif->dma_lost_irq(drive);
1475                                 (void)ide_ack_intr(hwif);
1476                                 printk(KERN_WARNING "%s: lost interrupt\n", drive->name);
1477                                 startstop = handler(drive);
1478                         } else {
1479                                 if (drive->waiting_for_dma) {
1480                                         startstop = ide_dma_timeout_retry(drive, wait);
1481                                 } else
1482                                         startstop =
1483                                         ide_error(drive, "irq timeout", hwif->INB(IDE_STATUS_REG));
1484                         }
1485                         drive->service_time = jiffies - drive->service_start;
1486                         spin_lock_irq(&ide_lock);
1487                         enable_irq(hwif->irq);
1488                         if (startstop == ide_stopped)
1489                                 hwgroup->busy = 0;
1490                 }
1491         }
1492         ide_do_request(hwgroup, IDE_NO_IRQ);
1493         spin_unlock_irqrestore(&ide_lock, flags);
1494 }
1495
1496 /**
1497  *      unexpected_intr         -       handle an unexpected IDE interrupt
1498  *      @irq: interrupt line
1499  *      @hwgroup: hwgroup being processed
1500  *
1501  *      There's nothing really useful we can do with an unexpected interrupt,
1502  *      other than reading the status register (to clear it), and logging it.
1503  *      There should be no way that an irq can happen before we're ready for it,
1504  *      so we needn't worry much about losing an "important" interrupt here.
1505  *
1506  *      On laptops (and "green" PCs), an unexpected interrupt occurs whenever
1507  *      the drive enters "idle", "standby", or "sleep" mode, so if the status
1508  *      looks "good", we just ignore the interrupt completely.
1509  *
1510  *      This routine assumes __cli() is in effect when called.
1511  *
1512  *      If an unexpected interrupt happens on irq15 while we are handling irq14
1513  *      and if the two interfaces are "serialized" (CMD640), then it looks like
1514  *      we could screw up by interfering with a new request being set up for 
1515  *      irq15.
1516  *
1517  *      In reality, this is a non-issue.  The new command is not sent unless 
1518  *      the drive is ready to accept one, in which case we know the drive is
1519  *      not trying to interrupt us.  And ide_set_handler() is always invoked
1520  *      before completing the issuance of any new drive command, so we will not
1521  *      be accidentally invoked as a result of any valid command completion
1522  *      interrupt.
1523  *
1524  *      Note that we must walk the entire hwgroup here. We know which hwif
1525  *      is doing the current command, but we don't know which hwif burped
1526  *      mysteriously.
1527  */
1528  
1529 static void unexpected_intr (int irq, ide_hwgroup_t *hwgroup)
1530 {
1531         u8 stat;
1532         ide_hwif_t *hwif = hwgroup->hwif;
1533
1534         /*
1535          * handle the unexpected interrupt
1536          */
1537         do {
1538                 if (hwif->irq == irq) {
1539                         stat = hwif->INB(hwif->io_ports[IDE_STATUS_OFFSET]);
1540                         if (!OK_STAT(stat, READY_STAT, BAD_STAT)) {
1541                                 /* Try to not flood the console with msgs */
1542                                 static unsigned long last_msgtime, count;
1543                                 ++count;
1544                                 if (time_after(jiffies, last_msgtime + HZ)) {
1545                                         last_msgtime = jiffies;
1546                                         printk(KERN_ERR "%s%s: unexpected interrupt, "
1547                                                 "status=0x%02x, count=%ld\n",
1548                                                 hwif->name,
1549                                                 (hwif->next==hwgroup->hwif) ? "" : "(?)", stat, count);
1550                                 }
1551                         }
1552                 }
1553         } while ((hwif = hwif->next) != hwgroup->hwif);
1554 }
1555
1556 /**
1557  *      ide_intr        -       default IDE interrupt handler
1558  *      @irq: interrupt number
1559  *      @dev_id: hwif group
1560  *      @regs: unused weirdness from the kernel irq layer
1561  *
1562  *      This is the default IRQ handler for the IDE layer. You should
1563  *      not need to override it. If you do be aware it is subtle in
1564  *      places
1565  *
1566  *      hwgroup->hwif is the interface in the group currently performing
1567  *      a command. hwgroup->drive is the drive and hwgroup->handler is
1568  *      the IRQ handler to call. As we issue a command the handlers
1569  *      step through multiple states, reassigning the handler to the
1570  *      next step in the process. Unlike a smart SCSI controller IDE
1571  *      expects the main processor to sequence the various transfer
1572  *      stages. We also manage a poll timer to catch up with most
1573  *      timeout situations. There are still a few where the handlers
1574  *      don't ever decide to give up.
1575  *
1576  *      The handler eventually returns ide_stopped to indicate the
1577  *      request completed. At this point we issue the next request
1578  *      on the hwgroup and the process begins again.
1579  */
1580  
1581 irqreturn_t ide_intr (int irq, void *dev_id)
1582 {
1583         unsigned long flags;
1584         ide_hwgroup_t *hwgroup = (ide_hwgroup_t *)dev_id;
1585         ide_hwif_t *hwif;
1586         ide_drive_t *drive;
1587         ide_handler_t *handler;
1588         ide_startstop_t startstop;
1589
1590         spin_lock_irqsave(&ide_lock, flags);
1591         hwif = hwgroup->hwif;
1592
1593         if (!ide_ack_intr(hwif)) {
1594                 spin_unlock_irqrestore(&ide_lock, flags);
1595                 return IRQ_NONE;
1596         }
1597
1598         if ((handler = hwgroup->handler) == NULL || hwgroup->polling) {
1599                 /*
1600                  * Not expecting an interrupt from this drive.
1601                  * That means this could be:
1602                  *      (1) an interrupt from another PCI device
1603                  *      sharing the same PCI INT# as us.
1604                  * or   (2) a drive just entered sleep or standby mode,
1605                  *      and is interrupting to let us know.
1606                  * or   (3) a spurious interrupt of unknown origin.
1607                  *
1608                  * For PCI, we cannot tell the difference,
1609                  * so in that case we just ignore it and hope it goes away.
1610                  *
1611                  * FIXME: unexpected_intr should be hwif-> then we can
1612                  * remove all the ifdef PCI crap
1613                  */
1614 #ifdef CONFIG_BLK_DEV_IDEPCI
1615                 if (hwif->pci_dev && !hwif->pci_dev->vendor)
1616 #endif  /* CONFIG_BLK_DEV_IDEPCI */
1617                 {
1618                         /*
1619                          * Probably not a shared PCI interrupt,
1620                          * so we can safely try to do something about it:
1621                          */
1622                         unexpected_intr(irq, hwgroup);
1623 #ifdef CONFIG_BLK_DEV_IDEPCI
1624                 } else {
1625                         /*
1626                          * Whack the status register, just in case
1627                          * we have a leftover pending IRQ.
1628                          */
1629                         (void) hwif->INB(hwif->io_ports[IDE_STATUS_OFFSET]);
1630 #endif /* CONFIG_BLK_DEV_IDEPCI */
1631                 }
1632                 spin_unlock_irqrestore(&ide_lock, flags);
1633                 return IRQ_NONE;
1634         }
1635         drive = hwgroup->drive;
1636         if (!drive) {
1637                 /*
1638                  * This should NEVER happen, and there isn't much
1639                  * we could do about it here.
1640                  *
1641                  * [Note - this can occur if the drive is hot unplugged]
1642                  */
1643                 spin_unlock_irqrestore(&ide_lock, flags);
1644                 return IRQ_HANDLED;
1645         }
1646         if (!drive_is_ready(drive)) {
1647                 /*
1648                  * This happens regularly when we share a PCI IRQ with
1649                  * another device.  Unfortunately, it can also happen
1650                  * with some buggy drives that trigger the IRQ before
1651                  * their status register is up to date.  Hopefully we have
1652                  * enough advance overhead that the latter isn't a problem.
1653                  */
1654                 spin_unlock_irqrestore(&ide_lock, flags);
1655                 return IRQ_NONE;
1656         }
1657         if (!hwgroup->busy) {
1658                 hwgroup->busy = 1;      /* paranoia */
1659                 printk(KERN_ERR "%s: ide_intr: hwgroup->busy was 0 ??\n", drive->name);
1660         }
1661         hwgroup->handler = NULL;
1662         hwgroup->req_gen++;
1663         del_timer(&hwgroup->timer);
1664         spin_unlock(&ide_lock);
1665
1666         /* Some controllers might set DMA INTR no matter DMA or PIO;
1667          * bmdma status might need to be cleared even for
1668          * PIO interrupts to prevent spurious/lost irq.
1669          */
1670         if (hwif->ide_dma_clear_irq && !(drive->waiting_for_dma))
1671                 /* ide_dma_end() needs bmdma status for error checking.
1672                  * So, skip clearing bmdma status here and leave it
1673                  * to ide_dma_end() if this is dma interrupt.
1674                  */
1675                 hwif->ide_dma_clear_irq(drive);
1676
1677         if (drive->unmask)
1678                 local_irq_enable_in_hardirq();
1679         /* service this interrupt, may set handler for next interrupt */
1680         startstop = handler(drive);
1681         spin_lock_irq(&ide_lock);
1682
1683         /*
1684          * Note that handler() may have set things up for another
1685          * interrupt to occur soon, but it cannot happen until
1686          * we exit from this routine, because it will be the
1687          * same irq as is currently being serviced here, and Linux
1688          * won't allow another of the same (on any CPU) until we return.
1689          */
1690         drive->service_time = jiffies - drive->service_start;
1691         if (startstop == ide_stopped) {
1692                 if (hwgroup->handler == NULL) { /* paranoia */
1693                         hwgroup->busy = 0;
1694                         ide_do_request(hwgroup, hwif->irq);
1695                 } else {
1696                         printk(KERN_ERR "%s: ide_intr: huh? expected NULL handler "
1697                                 "on exit\n", drive->name);
1698                 }
1699         }
1700         spin_unlock_irqrestore(&ide_lock, flags);
1701         return IRQ_HANDLED;
1702 }
1703
1704 /**
1705  *      ide_init_drive_cmd      -       initialize a drive command request
1706  *      @rq: request object
1707  *
1708  *      Initialize a request before we fill it in and send it down to
1709  *      ide_do_drive_cmd. Commands must be set up by this function. Right
1710  *      now it doesn't do a lot, but if that changes abusers will have a
1711  *      nasty surprise.
1712  */
1713
1714 void ide_init_drive_cmd (struct request *rq)
1715 {
1716         memset(rq, 0, sizeof(*rq));
1717         rq->cmd_type = REQ_TYPE_ATA_CMD;
1718         rq->ref_count = 1;
1719 }
1720
1721 EXPORT_SYMBOL(ide_init_drive_cmd);
1722
1723 /**
1724  *      ide_do_drive_cmd        -       issue IDE special command
1725  *      @drive: device to issue command
1726  *      @rq: request to issue
1727  *      @action: action for processing
1728  *
1729  *      This function issues a special IDE device request
1730  *      onto the request queue.
1731  *
1732  *      If action is ide_wait, then the rq is queued at the end of the
1733  *      request queue, and the function sleeps until it has been processed.
1734  *      This is for use when invoked from an ioctl handler.
1735  *
1736  *      If action is ide_preempt, then the rq is queued at the head of
1737  *      the request queue, displacing the currently-being-processed
1738  *      request and this function returns immediately without waiting
1739  *      for the new rq to be completed.  This is VERY DANGEROUS, and is
1740  *      intended for careful use by the ATAPI tape/cdrom driver code.
1741  *
1742  *      If action is ide_end, then the rq is queued at the end of the
1743  *      request queue, and the function returns immediately without waiting
1744  *      for the new rq to be completed. This is again intended for careful
1745  *      use by the ATAPI tape/cdrom driver code.
1746  */
1747  
1748 int ide_do_drive_cmd (ide_drive_t *drive, struct request *rq, ide_action_t action)
1749 {
1750         unsigned long flags;
1751         ide_hwgroup_t *hwgroup = HWGROUP(drive);
1752         DECLARE_COMPLETION_ONSTACK(wait);
1753         int where = ELEVATOR_INSERT_BACK, err;
1754         int must_wait = (action == ide_wait || action == ide_head_wait);
1755
1756         rq->errors = 0;
1757
1758         /*
1759          * we need to hold an extra reference to request for safe inspection
1760          * after completion
1761          */
1762         if (must_wait) {
1763                 rq->ref_count++;
1764                 rq->end_io_data = &wait;
1765                 rq->end_io = blk_end_sync_rq;
1766         }
1767
1768         spin_lock_irqsave(&ide_lock, flags);
1769         if (action == ide_preempt)
1770                 hwgroup->rq = NULL;
1771         if (action == ide_preempt || action == ide_head_wait) {
1772                 where = ELEVATOR_INSERT_FRONT;
1773                 rq->cmd_flags |= REQ_PREEMPT;
1774         }
1775         __elv_add_request(drive->queue, rq, where, 0);
1776         ide_do_request(hwgroup, IDE_NO_IRQ);
1777         spin_unlock_irqrestore(&ide_lock, flags);
1778
1779         err = 0;
1780         if (must_wait) {
1781                 wait_for_completion(&wait);
1782                 if (rq->errors)
1783                         err = -EIO;
1784
1785                 blk_put_request(rq);
1786         }
1787
1788         return err;
1789 }
1790
1791 EXPORT_SYMBOL(ide_do_drive_cmd);