IRQ: Maintain regs pointer globally rather than passing to IRQ handlers
[linux-2.6.git] / drivers / ide / ide-io.c
1 /*
2  *      IDE I/O functions
3  *
4  *      Basic PIO and command management functionality.
5  *
6  * This code was split off from ide.c. See ide.c for history and original
7  * copyrights.
8  *
9  * This program is free software; you can redistribute it and/or modify it
10  * under the terms of the GNU General Public License as published by the
11  * Free Software Foundation; either version 2, or (at your option) any
12  * later version.
13  *
14  * This program is distributed in the hope that it will be useful, but
15  * WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
17  * General Public License for more details.
18  *
19  * For the avoidance of doubt the "preferred form" of this code is one which
20  * is in an open non patent encumbered format. Where cryptographic key signing
21  * forms part of the process of creating an executable the information
22  * including keys needed to generate an equivalently functional executable
23  * are deemed to be part of the source code.
24  */
25  
26  
27 #include <linux/module.h>
28 #include <linux/types.h>
29 #include <linux/string.h>
30 #include <linux/kernel.h>
31 #include <linux/timer.h>
32 #include <linux/mm.h>
33 #include <linux/interrupt.h>
34 #include <linux/major.h>
35 #include <linux/errno.h>
36 #include <linux/genhd.h>
37 #include <linux/blkpg.h>
38 #include <linux/slab.h>
39 #include <linux/init.h>
40 #include <linux/pci.h>
41 #include <linux/delay.h>
42 #include <linux/ide.h>
43 #include <linux/completion.h>
44 #include <linux/reboot.h>
45 #include <linux/cdrom.h>
46 #include <linux/seq_file.h>
47 #include <linux/device.h>
48 #include <linux/kmod.h>
49 #include <linux/scatterlist.h>
50
51 #include <asm/byteorder.h>
52 #include <asm/irq.h>
53 #include <asm/uaccess.h>
54 #include <asm/io.h>
55 #include <asm/bitops.h>
56
57 static int __ide_end_request(ide_drive_t *drive, struct request *rq,
58                              int uptodate, int nr_sectors)
59 {
60         int ret = 1;
61
62         /*
63          * if failfast is set on a request, override number of sectors and
64          * complete the whole request right now
65          */
66         if (blk_noretry_request(rq) && end_io_error(uptodate))
67                 nr_sectors = rq->hard_nr_sectors;
68
69         if (!blk_fs_request(rq) && end_io_error(uptodate) && !rq->errors)
70                 rq->errors = -EIO;
71
72         /*
73          * decide whether to reenable DMA -- 3 is a random magic for now,
74          * if we DMA timeout more than 3 times, just stay in PIO
75          */
76         if (drive->state == DMA_PIO_RETRY && drive->retry_pio <= 3) {
77                 drive->state = 0;
78                 HWGROUP(drive)->hwif->ide_dma_on(drive);
79         }
80
81         if (!end_that_request_first(rq, uptodate, nr_sectors)) {
82                 add_disk_randomness(rq->rq_disk);
83                 if (!list_empty(&rq->queuelist))
84                         blkdev_dequeue_request(rq);
85                 HWGROUP(drive)->rq = NULL;
86                 end_that_request_last(rq, uptodate);
87                 ret = 0;
88         }
89
90         return ret;
91 }
92
93 /**
94  *      ide_end_request         -       complete an IDE I/O
95  *      @drive: IDE device for the I/O
96  *      @uptodate:
97  *      @nr_sectors: number of sectors completed
98  *
99  *      This is our end_request wrapper function. We complete the I/O
100  *      update random number input and dequeue the request, which if
101  *      it was tagged may be out of order.
102  */
103
104 int ide_end_request (ide_drive_t *drive, int uptodate, int nr_sectors)
105 {
106         struct request *rq;
107         unsigned long flags;
108         int ret = 1;
109
110         /*
111          * room for locking improvements here, the calls below don't
112          * need the queue lock held at all
113          */
114         spin_lock_irqsave(&ide_lock, flags);
115         rq = HWGROUP(drive)->rq;
116
117         if (!nr_sectors)
118                 nr_sectors = rq->hard_cur_sectors;
119
120         ret = __ide_end_request(drive, rq, uptodate, nr_sectors);
121
122         spin_unlock_irqrestore(&ide_lock, flags);
123         return ret;
124 }
125 EXPORT_SYMBOL(ide_end_request);
126
127 /*
128  * Power Management state machine. This one is rather trivial for now,
129  * we should probably add more, like switching back to PIO on suspend
130  * to help some BIOSes, re-do the door locking on resume, etc...
131  */
132
133 enum {
134         ide_pm_flush_cache      = ide_pm_state_start_suspend,
135         idedisk_pm_standby,
136
137         idedisk_pm_restore_pio  = ide_pm_state_start_resume,
138         idedisk_pm_idle,
139         ide_pm_restore_dma,
140 };
141
142 static void ide_complete_power_step(ide_drive_t *drive, struct request *rq, u8 stat, u8 error)
143 {
144         struct request_pm_state *pm = rq->data;
145
146         if (drive->media != ide_disk)
147                 return;
148
149         switch (pm->pm_step) {
150         case ide_pm_flush_cache:        /* Suspend step 1 (flush cache) complete */
151                 if (pm->pm_state == PM_EVENT_FREEZE)
152                         pm->pm_step = ide_pm_state_completed;
153                 else
154                         pm->pm_step = idedisk_pm_standby;
155                 break;
156         case idedisk_pm_standby:        /* Suspend step 2 (standby) complete */
157                 pm->pm_step = ide_pm_state_completed;
158                 break;
159         case idedisk_pm_restore_pio:    /* Resume step 1 complete */
160                 pm->pm_step = idedisk_pm_idle;
161                 break;
162         case idedisk_pm_idle:           /* Resume step 2 (idle) complete */
163                 pm->pm_step = ide_pm_restore_dma;
164                 break;
165         }
166 }
167
168 static ide_startstop_t ide_start_power_step(ide_drive_t *drive, struct request *rq)
169 {
170         struct request_pm_state *pm = rq->data;
171         ide_task_t *args = rq->special;
172
173         memset(args, 0, sizeof(*args));
174
175         if (drive->media != ide_disk) {
176                 /*
177                  * skip idedisk_pm_restore_pio and idedisk_pm_idle for ATAPI
178                  * devices
179                  */
180                 if (pm->pm_step == idedisk_pm_restore_pio)
181                         pm->pm_step = ide_pm_restore_dma;
182         }
183
184         switch (pm->pm_step) {
185         case ide_pm_flush_cache:        /* Suspend step 1 (flush cache) */
186                 if (drive->media != ide_disk)
187                         break;
188                 /* Not supported? Switch to next step now. */
189                 if (!drive->wcache || !ide_id_has_flush_cache(drive->id)) {
190                         ide_complete_power_step(drive, rq, 0, 0);
191                         return ide_stopped;
192                 }
193                 if (ide_id_has_flush_cache_ext(drive->id))
194                         args->tfRegister[IDE_COMMAND_OFFSET] = WIN_FLUSH_CACHE_EXT;
195                 else
196                         args->tfRegister[IDE_COMMAND_OFFSET] = WIN_FLUSH_CACHE;
197                 args->command_type = IDE_DRIVE_TASK_NO_DATA;
198                 args->handler      = &task_no_data_intr;
199                 return do_rw_taskfile(drive, args);
200
201         case idedisk_pm_standby:        /* Suspend step 2 (standby) */
202                 args->tfRegister[IDE_COMMAND_OFFSET] = WIN_STANDBYNOW1;
203                 args->command_type = IDE_DRIVE_TASK_NO_DATA;
204                 args->handler      = &task_no_data_intr;
205                 return do_rw_taskfile(drive, args);
206
207         case idedisk_pm_restore_pio:    /* Resume step 1 (restore PIO) */
208                 if (drive->hwif->tuneproc != NULL)
209                         drive->hwif->tuneproc(drive, 255);
210                 ide_complete_power_step(drive, rq, 0, 0);
211                 return ide_stopped;
212
213         case idedisk_pm_idle:           /* Resume step 2 (idle) */
214                 args->tfRegister[IDE_COMMAND_OFFSET] = WIN_IDLEIMMEDIATE;
215                 args->command_type = IDE_DRIVE_TASK_NO_DATA;
216                 args->handler = task_no_data_intr;
217                 return do_rw_taskfile(drive, args);
218
219         case ide_pm_restore_dma:        /* Resume step 3 (restore DMA) */
220                 /*
221                  * Right now, all we do is call hwif->ide_dma_check(drive),
222                  * we could be smarter and check for current xfer_speed
223                  * in struct drive etc...
224                  */
225                 if ((drive->id->capability & 1) == 0)
226                         break;
227                 if (drive->hwif->ide_dma_check == NULL)
228                         break;
229                 drive->hwif->ide_dma_check(drive);
230                 break;
231         }
232         pm->pm_step = ide_pm_state_completed;
233         return ide_stopped;
234 }
235
236 /**
237  *      ide_end_dequeued_request        -       complete an IDE I/O
238  *      @drive: IDE device for the I/O
239  *      @uptodate:
240  *      @nr_sectors: number of sectors completed
241  *
242  *      Complete an I/O that is no longer on the request queue. This
243  *      typically occurs when we pull the request and issue a REQUEST_SENSE.
244  *      We must still finish the old request but we must not tamper with the
245  *      queue in the meantime.
246  *
247  *      NOTE: This path does not handle barrier, but barrier is not supported
248  *      on ide-cd anyway.
249  */
250
251 int ide_end_dequeued_request(ide_drive_t *drive, struct request *rq,
252                              int uptodate, int nr_sectors)
253 {
254         unsigned long flags;
255         int ret = 1;
256
257         spin_lock_irqsave(&ide_lock, flags);
258
259         BUG_ON(!blk_rq_started(rq));
260
261         /*
262          * if failfast is set on a request, override number of sectors and
263          * complete the whole request right now
264          */
265         if (blk_noretry_request(rq) && end_io_error(uptodate))
266                 nr_sectors = rq->hard_nr_sectors;
267
268         if (!blk_fs_request(rq) && end_io_error(uptodate) && !rq->errors)
269                 rq->errors = -EIO;
270
271         /*
272          * decide whether to reenable DMA -- 3 is a random magic for now,
273          * if we DMA timeout more than 3 times, just stay in PIO
274          */
275         if (drive->state == DMA_PIO_RETRY && drive->retry_pio <= 3) {
276                 drive->state = 0;
277                 HWGROUP(drive)->hwif->ide_dma_on(drive);
278         }
279
280         if (!end_that_request_first(rq, uptodate, nr_sectors)) {
281                 add_disk_randomness(rq->rq_disk);
282                 if (blk_rq_tagged(rq))
283                         blk_queue_end_tag(drive->queue, rq);
284                 end_that_request_last(rq, uptodate);
285                 ret = 0;
286         }
287         spin_unlock_irqrestore(&ide_lock, flags);
288         return ret;
289 }
290 EXPORT_SYMBOL_GPL(ide_end_dequeued_request);
291
292
293 /**
294  *      ide_complete_pm_request - end the current Power Management request
295  *      @drive: target drive
296  *      @rq: request
297  *
298  *      This function cleans up the current PM request and stops the queue
299  *      if necessary.
300  */
301 static void ide_complete_pm_request (ide_drive_t *drive, struct request *rq)
302 {
303         unsigned long flags;
304
305 #ifdef DEBUG_PM
306         printk("%s: completing PM request, %s\n", drive->name,
307                blk_pm_suspend_request(rq) ? "suspend" : "resume");
308 #endif
309         spin_lock_irqsave(&ide_lock, flags);
310         if (blk_pm_suspend_request(rq)) {
311                 blk_stop_queue(drive->queue);
312         } else {
313                 drive->blocked = 0;
314                 blk_start_queue(drive->queue);
315         }
316         blkdev_dequeue_request(rq);
317         HWGROUP(drive)->rq = NULL;
318         end_that_request_last(rq, 1);
319         spin_unlock_irqrestore(&ide_lock, flags);
320 }
321
322 /*
323  * FIXME: probably move this somewhere else, name is bad too :)
324  */
325 u64 ide_get_error_location(ide_drive_t *drive, char *args)
326 {
327         u32 high, low;
328         u8 hcyl, lcyl, sect;
329         u64 sector;
330
331         high = 0;
332         hcyl = args[5];
333         lcyl = args[4];
334         sect = args[3];
335
336         if (ide_id_has_flush_cache_ext(drive->id)) {
337                 low = (hcyl << 16) | (lcyl << 8) | sect;
338                 HWIF(drive)->OUTB(drive->ctl|0x80, IDE_CONTROL_REG);
339                 high = ide_read_24(drive);
340         } else {
341                 u8 cur = HWIF(drive)->INB(IDE_SELECT_REG);
342                 if (cur & 0x40) {
343                         high = cur & 0xf;
344                         low = (hcyl << 16) | (lcyl << 8) | sect;
345                 } else {
346                         low = hcyl * drive->head * drive->sect;
347                         low += lcyl * drive->sect;
348                         low += sect - 1;
349                 }
350         }
351
352         sector = ((u64) high << 24) | low;
353         return sector;
354 }
355 EXPORT_SYMBOL(ide_get_error_location);
356
357 /**
358  *      ide_end_drive_cmd       -       end an explicit drive command
359  *      @drive: command 
360  *      @stat: status bits
361  *      @err: error bits
362  *
363  *      Clean up after success/failure of an explicit drive command.
364  *      These get thrown onto the queue so they are synchronized with
365  *      real I/O operations on the drive.
366  *
367  *      In LBA48 mode we have to read the register set twice to get
368  *      all the extra information out.
369  */
370  
371 void ide_end_drive_cmd (ide_drive_t *drive, u8 stat, u8 err)
372 {
373         ide_hwif_t *hwif = HWIF(drive);
374         unsigned long flags;
375         struct request *rq;
376
377         spin_lock_irqsave(&ide_lock, flags);
378         rq = HWGROUP(drive)->rq;
379         spin_unlock_irqrestore(&ide_lock, flags);
380
381         if (rq->cmd_type == REQ_TYPE_ATA_CMD) {
382                 u8 *args = (u8 *) rq->buffer;
383                 if (rq->errors == 0)
384                         rq->errors = !OK_STAT(stat,READY_STAT,BAD_STAT);
385
386                 if (args) {
387                         args[0] = stat;
388                         args[1] = err;
389                         args[2] = hwif->INB(IDE_NSECTOR_REG);
390                 }
391         } else if (rq->cmd_type == REQ_TYPE_ATA_TASK) {
392                 u8 *args = (u8 *) rq->buffer;
393                 if (rq->errors == 0)
394                         rq->errors = !OK_STAT(stat,READY_STAT,BAD_STAT);
395
396                 if (args) {
397                         args[0] = stat;
398                         args[1] = err;
399                         args[2] = hwif->INB(IDE_NSECTOR_REG);
400                         args[3] = hwif->INB(IDE_SECTOR_REG);
401                         args[4] = hwif->INB(IDE_LCYL_REG);
402                         args[5] = hwif->INB(IDE_HCYL_REG);
403                         args[6] = hwif->INB(IDE_SELECT_REG);
404                 }
405         } else if (rq->cmd_type == REQ_TYPE_ATA_TASKFILE) {
406                 ide_task_t *args = (ide_task_t *) rq->special;
407                 if (rq->errors == 0)
408                         rq->errors = !OK_STAT(stat,READY_STAT,BAD_STAT);
409                         
410                 if (args) {
411                         if (args->tf_in_flags.b.data) {
412                                 u16 data                                = hwif->INW(IDE_DATA_REG);
413                                 args->tfRegister[IDE_DATA_OFFSET]       = (data) & 0xFF;
414                                 args->hobRegister[IDE_DATA_OFFSET]      = (data >> 8) & 0xFF;
415                         }
416                         args->tfRegister[IDE_ERROR_OFFSET]   = err;
417                         /* be sure we're looking at the low order bits */
418                         hwif->OUTB(drive->ctl & ~0x80, IDE_CONTROL_REG);
419                         args->tfRegister[IDE_NSECTOR_OFFSET] = hwif->INB(IDE_NSECTOR_REG);
420                         args->tfRegister[IDE_SECTOR_OFFSET]  = hwif->INB(IDE_SECTOR_REG);
421                         args->tfRegister[IDE_LCYL_OFFSET]    = hwif->INB(IDE_LCYL_REG);
422                         args->tfRegister[IDE_HCYL_OFFSET]    = hwif->INB(IDE_HCYL_REG);
423                         args->tfRegister[IDE_SELECT_OFFSET]  = hwif->INB(IDE_SELECT_REG);
424                         args->tfRegister[IDE_STATUS_OFFSET]  = stat;
425
426                         if (drive->addressing == 1) {
427                                 hwif->OUTB(drive->ctl|0x80, IDE_CONTROL_REG);
428                                 args->hobRegister[IDE_FEATURE_OFFSET]   = hwif->INB(IDE_FEATURE_REG);
429                                 args->hobRegister[IDE_NSECTOR_OFFSET]   = hwif->INB(IDE_NSECTOR_REG);
430                                 args->hobRegister[IDE_SECTOR_OFFSET]    = hwif->INB(IDE_SECTOR_REG);
431                                 args->hobRegister[IDE_LCYL_OFFSET]      = hwif->INB(IDE_LCYL_REG);
432                                 args->hobRegister[IDE_HCYL_OFFSET]      = hwif->INB(IDE_HCYL_REG);
433                         }
434                 }
435         } else if (blk_pm_request(rq)) {
436                 struct request_pm_state *pm = rq->data;
437 #ifdef DEBUG_PM
438                 printk("%s: complete_power_step(step: %d, stat: %x, err: %x)\n",
439                         drive->name, rq->pm->pm_step, stat, err);
440 #endif
441                 ide_complete_power_step(drive, rq, stat, err);
442                 if (pm->pm_step == ide_pm_state_completed)
443                         ide_complete_pm_request(drive, rq);
444                 return;
445         }
446
447         spin_lock_irqsave(&ide_lock, flags);
448         blkdev_dequeue_request(rq);
449         HWGROUP(drive)->rq = NULL;
450         rq->errors = err;
451         end_that_request_last(rq, !rq->errors);
452         spin_unlock_irqrestore(&ide_lock, flags);
453 }
454
455 EXPORT_SYMBOL(ide_end_drive_cmd);
456
457 /**
458  *      try_to_flush_leftover_data      -       flush junk
459  *      @drive: drive to flush
460  *
461  *      try_to_flush_leftover_data() is invoked in response to a drive
462  *      unexpectedly having its DRQ_STAT bit set.  As an alternative to
463  *      resetting the drive, this routine tries to clear the condition
464  *      by read a sector's worth of data from the drive.  Of course,
465  *      this may not help if the drive is *waiting* for data from *us*.
466  */
467 static void try_to_flush_leftover_data (ide_drive_t *drive)
468 {
469         int i = (drive->mult_count ? drive->mult_count : 1) * SECTOR_WORDS;
470
471         if (drive->media != ide_disk)
472                 return;
473         while (i > 0) {
474                 u32 buffer[16];
475                 u32 wcount = (i > 16) ? 16 : i;
476
477                 i -= wcount;
478                 HWIF(drive)->ata_input_data(drive, buffer, wcount);
479         }
480 }
481
482 static void ide_kill_rq(ide_drive_t *drive, struct request *rq)
483 {
484         if (rq->rq_disk) {
485                 ide_driver_t *drv;
486
487                 drv = *(ide_driver_t **)rq->rq_disk->private_data;
488                 drv->end_request(drive, 0, 0);
489         } else
490                 ide_end_request(drive, 0, 0);
491 }
492
493 static ide_startstop_t ide_ata_error(ide_drive_t *drive, struct request *rq, u8 stat, u8 err)
494 {
495         ide_hwif_t *hwif = drive->hwif;
496
497         if (stat & BUSY_STAT || ((stat & WRERR_STAT) && !drive->nowerr)) {
498                 /* other bits are useless when BUSY */
499                 rq->errors |= ERROR_RESET;
500         } else if (stat & ERR_STAT) {
501                 /* err has different meaning on cdrom and tape */
502                 if (err == ABRT_ERR) {
503                         if (drive->select.b.lba &&
504                             /* some newer drives don't support WIN_SPECIFY */
505                             hwif->INB(IDE_COMMAND_REG) == WIN_SPECIFY)
506                                 return ide_stopped;
507                 } else if ((err & BAD_CRC) == BAD_CRC) {
508                         /* UDMA crc error, just retry the operation */
509                         drive->crc_count++;
510                 } else if (err & (BBD_ERR | ECC_ERR)) {
511                         /* retries won't help these */
512                         rq->errors = ERROR_MAX;
513                 } else if (err & TRK0_ERR) {
514                         /* help it find track zero */
515                         rq->errors |= ERROR_RECAL;
516                 }
517         }
518
519         if ((stat & DRQ_STAT) && rq_data_dir(rq) == READ && hwif->err_stops_fifo == 0)
520                 try_to_flush_leftover_data(drive);
521
522         if (hwif->INB(IDE_STATUS_REG) & (BUSY_STAT|DRQ_STAT))
523                 /* force an abort */
524                 hwif->OUTB(WIN_IDLEIMMEDIATE, IDE_COMMAND_REG);
525
526         if (rq->errors >= ERROR_MAX || blk_noretry_request(rq))
527                 ide_kill_rq(drive, rq);
528         else {
529                 if ((rq->errors & ERROR_RESET) == ERROR_RESET) {
530                         ++rq->errors;
531                         return ide_do_reset(drive);
532                 }
533                 if ((rq->errors & ERROR_RECAL) == ERROR_RECAL)
534                         drive->special.b.recalibrate = 1;
535                 ++rq->errors;
536         }
537         return ide_stopped;
538 }
539
540 static ide_startstop_t ide_atapi_error(ide_drive_t *drive, struct request *rq, u8 stat, u8 err)
541 {
542         ide_hwif_t *hwif = drive->hwif;
543
544         if (stat & BUSY_STAT || ((stat & WRERR_STAT) && !drive->nowerr)) {
545                 /* other bits are useless when BUSY */
546                 rq->errors |= ERROR_RESET;
547         } else {
548                 /* add decoding error stuff */
549         }
550
551         if (hwif->INB(IDE_STATUS_REG) & (BUSY_STAT|DRQ_STAT))
552                 /* force an abort */
553                 hwif->OUTB(WIN_IDLEIMMEDIATE, IDE_COMMAND_REG);
554
555         if (rq->errors >= ERROR_MAX) {
556                 ide_kill_rq(drive, rq);
557         } else {
558                 if ((rq->errors & ERROR_RESET) == ERROR_RESET) {
559                         ++rq->errors;
560                         return ide_do_reset(drive);
561                 }
562                 ++rq->errors;
563         }
564
565         return ide_stopped;
566 }
567
568 ide_startstop_t
569 __ide_error(ide_drive_t *drive, struct request *rq, u8 stat, u8 err)
570 {
571         if (drive->media == ide_disk)
572                 return ide_ata_error(drive, rq, stat, err);
573         return ide_atapi_error(drive, rq, stat, err);
574 }
575
576 EXPORT_SYMBOL_GPL(__ide_error);
577
578 /**
579  *      ide_error       -       handle an error on the IDE
580  *      @drive: drive the error occurred on
581  *      @msg: message to report
582  *      @stat: status bits
583  *
584  *      ide_error() takes action based on the error returned by the drive.
585  *      For normal I/O that may well include retries. We deal with
586  *      both new-style (taskfile) and old style command handling here.
587  *      In the case of taskfile command handling there is work left to
588  *      do
589  */
590  
591 ide_startstop_t ide_error (ide_drive_t *drive, const char *msg, u8 stat)
592 {
593         struct request *rq;
594         u8 err;
595
596         err = ide_dump_status(drive, msg, stat);
597
598         if ((rq = HWGROUP(drive)->rq) == NULL)
599                 return ide_stopped;
600
601         /* retry only "normal" I/O: */
602         if (!blk_fs_request(rq)) {
603                 rq->errors = 1;
604                 ide_end_drive_cmd(drive, stat, err);
605                 return ide_stopped;
606         }
607
608         if (rq->rq_disk) {
609                 ide_driver_t *drv;
610
611                 drv = *(ide_driver_t **)rq->rq_disk->private_data;
612                 return drv->error(drive, rq, stat, err);
613         } else
614                 return __ide_error(drive, rq, stat, err);
615 }
616
617 EXPORT_SYMBOL_GPL(ide_error);
618
619 ide_startstop_t __ide_abort(ide_drive_t *drive, struct request *rq)
620 {
621         if (drive->media != ide_disk)
622                 rq->errors |= ERROR_RESET;
623
624         ide_kill_rq(drive, rq);
625
626         return ide_stopped;
627 }
628
629 EXPORT_SYMBOL_GPL(__ide_abort);
630
631 /**
632  *      ide_abort       -       abort pending IDE operations
633  *      @drive: drive the error occurred on
634  *      @msg: message to report
635  *
636  *      ide_abort kills and cleans up when we are about to do a 
637  *      host initiated reset on active commands. Longer term we
638  *      want handlers to have sensible abort handling themselves
639  *
640  *      This differs fundamentally from ide_error because in 
641  *      this case the command is doing just fine when we
642  *      blow it away.
643  */
644  
645 ide_startstop_t ide_abort(ide_drive_t *drive, const char *msg)
646 {
647         struct request *rq;
648
649         if (drive == NULL || (rq = HWGROUP(drive)->rq) == NULL)
650                 return ide_stopped;
651
652         /* retry only "normal" I/O: */
653         if (!blk_fs_request(rq)) {
654                 rq->errors = 1;
655                 ide_end_drive_cmd(drive, BUSY_STAT, 0);
656                 return ide_stopped;
657         }
658
659         if (rq->rq_disk) {
660                 ide_driver_t *drv;
661
662                 drv = *(ide_driver_t **)rq->rq_disk->private_data;
663                 return drv->abort(drive, rq);
664         } else
665                 return __ide_abort(drive, rq);
666 }
667
668 /**
669  *      ide_cmd         -       issue a simple drive command
670  *      @drive: drive the command is for
671  *      @cmd: command byte
672  *      @nsect: sector byte
673  *      @handler: handler for the command completion
674  *
675  *      Issue a simple drive command with interrupts.
676  *      The drive must be selected beforehand.
677  */
678
679 static void ide_cmd (ide_drive_t *drive, u8 cmd, u8 nsect,
680                 ide_handler_t *handler)
681 {
682         ide_hwif_t *hwif = HWIF(drive);
683         if (IDE_CONTROL_REG)
684                 hwif->OUTB(drive->ctl,IDE_CONTROL_REG); /* clear nIEN */
685         SELECT_MASK(drive,0);
686         hwif->OUTB(nsect,IDE_NSECTOR_REG);
687         ide_execute_command(drive, cmd, handler, WAIT_CMD, NULL);
688 }
689
690 /**
691  *      drive_cmd_intr          -       drive command completion interrupt
692  *      @drive: drive the completion interrupt occurred on
693  *
694  *      drive_cmd_intr() is invoked on completion of a special DRIVE_CMD.
695  *      We do any necessary data reading and then wait for the drive to
696  *      go non busy. At that point we may read the error data and complete
697  *      the request
698  */
699  
700 static ide_startstop_t drive_cmd_intr (ide_drive_t *drive)
701 {
702         struct request *rq = HWGROUP(drive)->rq;
703         ide_hwif_t *hwif = HWIF(drive);
704         u8 *args = (u8 *) rq->buffer;
705         u8 stat = hwif->INB(IDE_STATUS_REG);
706         int retries = 10;
707
708         local_irq_enable_in_hardirq();
709         if ((stat & DRQ_STAT) && args && args[3]) {
710                 u8 io_32bit = drive->io_32bit;
711                 drive->io_32bit = 0;
712                 hwif->ata_input_data(drive, &args[4], args[3] * SECTOR_WORDS);
713                 drive->io_32bit = io_32bit;
714                 while (((stat = hwif->INB(IDE_STATUS_REG)) & BUSY_STAT) && retries--)
715                         udelay(100);
716         }
717
718         if (!OK_STAT(stat, READY_STAT, BAD_STAT))
719                 return ide_error(drive, "drive_cmd", stat);
720                 /* calls ide_end_drive_cmd */
721         ide_end_drive_cmd(drive, stat, hwif->INB(IDE_ERROR_REG));
722         return ide_stopped;
723 }
724
725 static void ide_init_specify_cmd(ide_drive_t *drive, ide_task_t *task)
726 {
727         task->tfRegister[IDE_NSECTOR_OFFSET] = drive->sect;
728         task->tfRegister[IDE_SECTOR_OFFSET]  = drive->sect;
729         task->tfRegister[IDE_LCYL_OFFSET]    = drive->cyl;
730         task->tfRegister[IDE_HCYL_OFFSET]    = drive->cyl>>8;
731         task->tfRegister[IDE_SELECT_OFFSET]  = ((drive->head-1)|drive->select.all)&0xBF;
732         task->tfRegister[IDE_COMMAND_OFFSET] = WIN_SPECIFY;
733
734         task->handler = &set_geometry_intr;
735 }
736
737 static void ide_init_restore_cmd(ide_drive_t *drive, ide_task_t *task)
738 {
739         task->tfRegister[IDE_NSECTOR_OFFSET] = drive->sect;
740         task->tfRegister[IDE_COMMAND_OFFSET] = WIN_RESTORE;
741
742         task->handler = &recal_intr;
743 }
744
745 static void ide_init_setmult_cmd(ide_drive_t *drive, ide_task_t *task)
746 {
747         task->tfRegister[IDE_NSECTOR_OFFSET] = drive->mult_req;
748         task->tfRegister[IDE_COMMAND_OFFSET] = WIN_SETMULT;
749
750         task->handler = &set_multmode_intr;
751 }
752
753 static ide_startstop_t ide_disk_special(ide_drive_t *drive)
754 {
755         special_t *s = &drive->special;
756         ide_task_t args;
757
758         memset(&args, 0, sizeof(ide_task_t));
759         args.command_type = IDE_DRIVE_TASK_NO_DATA;
760
761         if (s->b.set_geometry) {
762                 s->b.set_geometry = 0;
763                 ide_init_specify_cmd(drive, &args);
764         } else if (s->b.recalibrate) {
765                 s->b.recalibrate = 0;
766                 ide_init_restore_cmd(drive, &args);
767         } else if (s->b.set_multmode) {
768                 s->b.set_multmode = 0;
769                 if (drive->mult_req > drive->id->max_multsect)
770                         drive->mult_req = drive->id->max_multsect;
771                 ide_init_setmult_cmd(drive, &args);
772         } else if (s->all) {
773                 int special = s->all;
774                 s->all = 0;
775                 printk(KERN_ERR "%s: bad special flag: 0x%02x\n", drive->name, special);
776                 return ide_stopped;
777         }
778
779         do_rw_taskfile(drive, &args);
780
781         return ide_started;
782 }
783
784 /**
785  *      do_special              -       issue some special commands
786  *      @drive: drive the command is for
787  *
788  *      do_special() is used to issue WIN_SPECIFY, WIN_RESTORE, and WIN_SETMULT
789  *      commands to a drive.  It used to do much more, but has been scaled
790  *      back.
791  */
792
793 static ide_startstop_t do_special (ide_drive_t *drive)
794 {
795         special_t *s = &drive->special;
796
797 #ifdef DEBUG
798         printk("%s: do_special: 0x%02x\n", drive->name, s->all);
799 #endif
800         if (s->b.set_tune) {
801                 s->b.set_tune = 0;
802                 if (HWIF(drive)->tuneproc != NULL)
803                         HWIF(drive)->tuneproc(drive, drive->tune_req);
804                 return ide_stopped;
805         } else {
806                 if (drive->media == ide_disk)
807                         return ide_disk_special(drive);
808
809                 s->all = 0;
810                 drive->mult_req = 0;
811                 return ide_stopped;
812         }
813 }
814
815 void ide_map_sg(ide_drive_t *drive, struct request *rq)
816 {
817         ide_hwif_t *hwif = drive->hwif;
818         struct scatterlist *sg = hwif->sg_table;
819
820         if (hwif->sg_mapped)    /* needed by ide-scsi */
821                 return;
822
823         if (rq->cmd_type != REQ_TYPE_ATA_TASKFILE) {
824                 hwif->sg_nents = blk_rq_map_sg(drive->queue, rq, sg);
825         } else {
826                 sg_init_one(sg, rq->buffer, rq->nr_sectors * SECTOR_SIZE);
827                 hwif->sg_nents = 1;
828         }
829 }
830
831 EXPORT_SYMBOL_GPL(ide_map_sg);
832
833 void ide_init_sg_cmd(ide_drive_t *drive, struct request *rq)
834 {
835         ide_hwif_t *hwif = drive->hwif;
836
837         hwif->nsect = hwif->nleft = rq->nr_sectors;
838         hwif->cursg = hwif->cursg_ofs = 0;
839 }
840
841 EXPORT_SYMBOL_GPL(ide_init_sg_cmd);
842
843 /**
844  *      execute_drive_command   -       issue special drive command
845  *      @drive: the drive to issue the command on
846  *      @rq: the request structure holding the command
847  *
848  *      execute_drive_cmd() issues a special drive command,  usually 
849  *      initiated by ioctl() from the external hdparm program. The
850  *      command can be a drive command, drive task or taskfile 
851  *      operation. Weirdly you can call it with NULL to wait for
852  *      all commands to finish. Don't do this as that is due to change
853  */
854
855 static ide_startstop_t execute_drive_cmd (ide_drive_t *drive,
856                 struct request *rq)
857 {
858         ide_hwif_t *hwif = HWIF(drive);
859         if (rq->cmd_type == REQ_TYPE_ATA_TASKFILE) {
860                 ide_task_t *args = rq->special;
861  
862                 if (!args)
863                         goto done;
864
865                 hwif->data_phase = args->data_phase;
866
867                 switch (hwif->data_phase) {
868                 case TASKFILE_MULTI_OUT:
869                 case TASKFILE_OUT:
870                 case TASKFILE_MULTI_IN:
871                 case TASKFILE_IN:
872                         ide_init_sg_cmd(drive, rq);
873                         ide_map_sg(drive, rq);
874                 default:
875                         break;
876                 }
877
878                 if (args->tf_out_flags.all != 0) 
879                         return flagged_taskfile(drive, args);
880                 return do_rw_taskfile(drive, args);
881         } else if (rq->cmd_type == REQ_TYPE_ATA_TASK) {
882                 u8 *args = rq->buffer;
883                 u8 sel;
884  
885                 if (!args)
886                         goto done;
887 #ifdef DEBUG
888                 printk("%s: DRIVE_TASK_CMD ", drive->name);
889                 printk("cmd=0x%02x ", args[0]);
890                 printk("fr=0x%02x ", args[1]);
891                 printk("ns=0x%02x ", args[2]);
892                 printk("sc=0x%02x ", args[3]);
893                 printk("lcyl=0x%02x ", args[4]);
894                 printk("hcyl=0x%02x ", args[5]);
895                 printk("sel=0x%02x\n", args[6]);
896 #endif
897                 hwif->OUTB(args[1], IDE_FEATURE_REG);
898                 hwif->OUTB(args[3], IDE_SECTOR_REG);
899                 hwif->OUTB(args[4], IDE_LCYL_REG);
900                 hwif->OUTB(args[5], IDE_HCYL_REG);
901                 sel = (args[6] & ~0x10);
902                 if (drive->select.b.unit)
903                         sel |= 0x10;
904                 hwif->OUTB(sel, IDE_SELECT_REG);
905                 ide_cmd(drive, args[0], args[2], &drive_cmd_intr);
906                 return ide_started;
907         } else if (rq->cmd_type == REQ_TYPE_ATA_CMD) {
908                 u8 *args = rq->buffer;
909
910                 if (!args)
911                         goto done;
912 #ifdef DEBUG
913                 printk("%s: DRIVE_CMD ", drive->name);
914                 printk("cmd=0x%02x ", args[0]);
915                 printk("sc=0x%02x ", args[1]);
916                 printk("fr=0x%02x ", args[2]);
917                 printk("xx=0x%02x\n", args[3]);
918 #endif
919                 if (args[0] == WIN_SMART) {
920                         hwif->OUTB(0x4f, IDE_LCYL_REG);
921                         hwif->OUTB(0xc2, IDE_HCYL_REG);
922                         hwif->OUTB(args[2],IDE_FEATURE_REG);
923                         hwif->OUTB(args[1],IDE_SECTOR_REG);
924                         ide_cmd(drive, args[0], args[3], &drive_cmd_intr);
925                         return ide_started;
926                 }
927                 hwif->OUTB(args[2],IDE_FEATURE_REG);
928                 ide_cmd(drive, args[0], args[1], &drive_cmd_intr);
929                 return ide_started;
930         }
931
932 done:
933         /*
934          * NULL is actually a valid way of waiting for
935          * all current requests to be flushed from the queue.
936          */
937 #ifdef DEBUG
938         printk("%s: DRIVE_CMD (null)\n", drive->name);
939 #endif
940         ide_end_drive_cmd(drive,
941                         hwif->INB(IDE_STATUS_REG),
942                         hwif->INB(IDE_ERROR_REG));
943         return ide_stopped;
944 }
945
946 static void ide_check_pm_state(ide_drive_t *drive, struct request *rq)
947 {
948         struct request_pm_state *pm = rq->data;
949
950         if (blk_pm_suspend_request(rq) &&
951             pm->pm_step == ide_pm_state_start_suspend)
952                 /* Mark drive blocked when starting the suspend sequence. */
953                 drive->blocked = 1;
954         else if (blk_pm_resume_request(rq) &&
955                  pm->pm_step == ide_pm_state_start_resume) {
956                 /* 
957                  * The first thing we do on wakeup is to wait for BSY bit to
958                  * go away (with a looong timeout) as a drive on this hwif may
959                  * just be POSTing itself.
960                  * We do that before even selecting as the "other" device on
961                  * the bus may be broken enough to walk on our toes at this
962                  * point.
963                  */
964                 int rc;
965 #ifdef DEBUG_PM
966                 printk("%s: Wakeup request inited, waiting for !BSY...\n", drive->name);
967 #endif
968                 rc = ide_wait_not_busy(HWIF(drive), 35000);
969                 if (rc)
970                         printk(KERN_WARNING "%s: bus not ready on wakeup\n", drive->name);
971                 SELECT_DRIVE(drive);
972                 HWIF(drive)->OUTB(8, HWIF(drive)->io_ports[IDE_CONTROL_OFFSET]);
973                 rc = ide_wait_not_busy(HWIF(drive), 100000);
974                 if (rc)
975                         printk(KERN_WARNING "%s: drive not ready on wakeup\n", drive->name);
976         }
977 }
978
979 /**
980  *      start_request   -       start of I/O and command issuing for IDE
981  *
982  *      start_request() initiates handling of a new I/O request. It
983  *      accepts commands and I/O (read/write) requests. It also does
984  *      the final remapping for weird stuff like EZDrive. Once 
985  *      device mapper can work sector level the EZDrive stuff can go away
986  *
987  *      FIXME: this function needs a rename
988  */
989  
990 static ide_startstop_t start_request (ide_drive_t *drive, struct request *rq)
991 {
992         ide_startstop_t startstop;
993         sector_t block;
994
995         BUG_ON(!blk_rq_started(rq));
996
997 #ifdef DEBUG
998         printk("%s: start_request: current=0x%08lx\n",
999                 HWIF(drive)->name, (unsigned long) rq);
1000 #endif
1001
1002         /* bail early if we've exceeded max_failures */
1003         if (drive->max_failures && (drive->failures > drive->max_failures)) {
1004                 goto kill_rq;
1005         }
1006
1007         block    = rq->sector;
1008         if (blk_fs_request(rq) &&
1009             (drive->media == ide_disk || drive->media == ide_floppy)) {
1010                 block += drive->sect0;
1011         }
1012         /* Yecch - this will shift the entire interval,
1013            possibly killing some innocent following sector */
1014         if (block == 0 && drive->remap_0_to_1 == 1)
1015                 block = 1;  /* redirect MBR access to EZ-Drive partn table */
1016
1017         if (blk_pm_request(rq))
1018                 ide_check_pm_state(drive, rq);
1019
1020         SELECT_DRIVE(drive);
1021         if (ide_wait_stat(&startstop, drive, drive->ready_stat, BUSY_STAT|DRQ_STAT, WAIT_READY)) {
1022                 printk(KERN_ERR "%s: drive not ready for command\n", drive->name);
1023                 return startstop;
1024         }
1025         if (!drive->special.all) {
1026                 ide_driver_t *drv;
1027
1028                 if (rq->cmd_type == REQ_TYPE_ATA_CMD ||
1029                     rq->cmd_type == REQ_TYPE_ATA_TASK ||
1030                     rq->cmd_type == REQ_TYPE_ATA_TASKFILE)
1031                         return execute_drive_cmd(drive, rq);
1032                 else if (blk_pm_request(rq)) {
1033                         struct request_pm_state *pm = rq->data;
1034 #ifdef DEBUG_PM
1035                         printk("%s: start_power_step(step: %d)\n",
1036                                 drive->name, rq->pm->pm_step);
1037 #endif
1038                         startstop = ide_start_power_step(drive, rq);
1039                         if (startstop == ide_stopped &&
1040                             pm->pm_step == ide_pm_state_completed)
1041                                 ide_complete_pm_request(drive, rq);
1042                         return startstop;
1043                 }
1044
1045                 drv = *(ide_driver_t **)rq->rq_disk->private_data;
1046                 return drv->do_request(drive, rq, block);
1047         }
1048         return do_special(drive);
1049 kill_rq:
1050         ide_kill_rq(drive, rq);
1051         return ide_stopped;
1052 }
1053
1054 /**
1055  *      ide_stall_queue         -       pause an IDE device
1056  *      @drive: drive to stall
1057  *      @timeout: time to stall for (jiffies)
1058  *
1059  *      ide_stall_queue() can be used by a drive to give excess bandwidth back
1060  *      to the hwgroup by sleeping for timeout jiffies.
1061  */
1062  
1063 void ide_stall_queue (ide_drive_t *drive, unsigned long timeout)
1064 {
1065         if (timeout > WAIT_WORSTCASE)
1066                 timeout = WAIT_WORSTCASE;
1067         drive->sleep = timeout + jiffies;
1068         drive->sleeping = 1;
1069 }
1070
1071 EXPORT_SYMBOL(ide_stall_queue);
1072
1073 #define WAKEUP(drive)   ((drive)->service_start + 2 * (drive)->service_time)
1074
1075 /**
1076  *      choose_drive            -       select a drive to service
1077  *      @hwgroup: hardware group to select on
1078  *
1079  *      choose_drive() selects the next drive which will be serviced.
1080  *      This is necessary because the IDE layer can't issue commands
1081  *      to both drives on the same cable, unlike SCSI.
1082  */
1083  
1084 static inline ide_drive_t *choose_drive (ide_hwgroup_t *hwgroup)
1085 {
1086         ide_drive_t *drive, *best;
1087
1088 repeat: 
1089         best = NULL;
1090         drive = hwgroup->drive;
1091
1092         /*
1093          * drive is doing pre-flush, ordered write, post-flush sequence. even
1094          * though that is 3 requests, it must be seen as a single transaction.
1095          * we must not preempt this drive until that is complete
1096          */
1097         if (blk_queue_flushing(drive->queue)) {
1098                 /*
1099                  * small race where queue could get replugged during
1100                  * the 3-request flush cycle, just yank the plug since
1101                  * we want it to finish asap
1102                  */
1103                 blk_remove_plug(drive->queue);
1104                 return drive;
1105         }
1106
1107         do {
1108                 if ((!drive->sleeping || time_after_eq(jiffies, drive->sleep))
1109                     && !elv_queue_empty(drive->queue)) {
1110                         if (!best
1111                          || (drive->sleeping && (!best->sleeping || time_before(drive->sleep, best->sleep)))
1112                          || (!best->sleeping && time_before(WAKEUP(drive), WAKEUP(best))))
1113                         {
1114                                 if (!blk_queue_plugged(drive->queue))
1115                                         best = drive;
1116                         }
1117                 }
1118         } while ((drive = drive->next) != hwgroup->drive);
1119         if (best && best->nice1 && !best->sleeping && best != hwgroup->drive && best->service_time > WAIT_MIN_SLEEP) {
1120                 long t = (signed long)(WAKEUP(best) - jiffies);
1121                 if (t >= WAIT_MIN_SLEEP) {
1122                 /*
1123                  * We *may* have some time to spare, but first let's see if
1124                  * someone can potentially benefit from our nice mood today..
1125                  */
1126                         drive = best->next;
1127                         do {
1128                                 if (!drive->sleeping
1129                                  && time_before(jiffies - best->service_time, WAKEUP(drive))
1130                                  && time_before(WAKEUP(drive), jiffies + t))
1131                                 {
1132                                         ide_stall_queue(best, min_t(long, t, 10 * WAIT_MIN_SLEEP));
1133                                         goto repeat;
1134                                 }
1135                         } while ((drive = drive->next) != best);
1136                 }
1137         }
1138         return best;
1139 }
1140
1141 /*
1142  * Issue a new request to a drive from hwgroup
1143  * Caller must have already done spin_lock_irqsave(&ide_lock, ..);
1144  *
1145  * A hwgroup is a serialized group of IDE interfaces.  Usually there is
1146  * exactly one hwif (interface) per hwgroup, but buggy controllers (eg. CMD640)
1147  * may have both interfaces in a single hwgroup to "serialize" access.
1148  * Or possibly multiple ISA interfaces can share a common IRQ by being grouped
1149  * together into one hwgroup for serialized access.
1150  *
1151  * Note also that several hwgroups can end up sharing a single IRQ,
1152  * possibly along with many other devices.  This is especially common in
1153  * PCI-based systems with off-board IDE controller cards.
1154  *
1155  * The IDE driver uses the single global ide_lock spinlock to protect
1156  * access to the request queues, and to protect the hwgroup->busy flag.
1157  *
1158  * The first thread into the driver for a particular hwgroup sets the
1159  * hwgroup->busy flag to indicate that this hwgroup is now active,
1160  * and then initiates processing of the top request from the request queue.
1161  *
1162  * Other threads attempting entry notice the busy setting, and will simply
1163  * queue their new requests and exit immediately.  Note that hwgroup->busy
1164  * remains set even when the driver is merely awaiting the next interrupt.
1165  * Thus, the meaning is "this hwgroup is busy processing a request".
1166  *
1167  * When processing of a request completes, the completing thread or IRQ-handler
1168  * will start the next request from the queue.  If no more work remains,
1169  * the driver will clear the hwgroup->busy flag and exit.
1170  *
1171  * The ide_lock (spinlock) is used to protect all access to the
1172  * hwgroup->busy flag, but is otherwise not needed for most processing in
1173  * the driver.  This makes the driver much more friendlier to shared IRQs
1174  * than previous designs, while remaining 100% (?) SMP safe and capable.
1175  */
1176 static void ide_do_request (ide_hwgroup_t *hwgroup, int masked_irq)
1177 {
1178         ide_drive_t     *drive;
1179         ide_hwif_t      *hwif;
1180         struct request  *rq;
1181         ide_startstop_t startstop;
1182         int             loops = 0;
1183
1184         /* for atari only: POSSIBLY BROKEN HERE(?) */
1185         ide_get_lock(ide_intr, hwgroup);
1186
1187         /* caller must own ide_lock */
1188         BUG_ON(!irqs_disabled());
1189
1190         while (!hwgroup->busy) {
1191                 hwgroup->busy = 1;
1192                 drive = choose_drive(hwgroup);
1193                 if (drive == NULL) {
1194                         int sleeping = 0;
1195                         unsigned long sleep = 0; /* shut up, gcc */
1196                         hwgroup->rq = NULL;
1197                         drive = hwgroup->drive;
1198                         do {
1199                                 if (drive->sleeping && (!sleeping || time_before(drive->sleep, sleep))) {
1200                                         sleeping = 1;
1201                                         sleep = drive->sleep;
1202                                 }
1203                         } while ((drive = drive->next) != hwgroup->drive);
1204                         if (sleeping) {
1205                 /*
1206                  * Take a short snooze, and then wake up this hwgroup again.
1207                  * This gives other hwgroups on the same a chance to
1208                  * play fairly with us, just in case there are big differences
1209                  * in relative throughputs.. don't want to hog the cpu too much.
1210                  */
1211                                 if (time_before(sleep, jiffies + WAIT_MIN_SLEEP))
1212                                         sleep = jiffies + WAIT_MIN_SLEEP;
1213 #if 1
1214                                 if (timer_pending(&hwgroup->timer))
1215                                         printk(KERN_CRIT "ide_set_handler: timer already active\n");
1216 #endif
1217                                 /* so that ide_timer_expiry knows what to do */
1218                                 hwgroup->sleeping = 1;
1219                                 mod_timer(&hwgroup->timer, sleep);
1220                                 /* we purposely leave hwgroup->busy==1
1221                                  * while sleeping */
1222                         } else {
1223                                 /* Ugly, but how can we sleep for the lock
1224                                  * otherwise? perhaps from tq_disk?
1225                                  */
1226
1227                                 /* for atari only */
1228                                 ide_release_lock();
1229                                 hwgroup->busy = 0;
1230                         }
1231
1232                         /* no more work for this hwgroup (for now) */
1233                         return;
1234                 }
1235         again:
1236                 hwif = HWIF(drive);
1237                 if (hwgroup->hwif->sharing_irq &&
1238                     hwif != hwgroup->hwif &&
1239                     hwif->io_ports[IDE_CONTROL_OFFSET]) {
1240                         /* set nIEN for previous hwif */
1241                         SELECT_INTERRUPT(drive);
1242                 }
1243                 hwgroup->hwif = hwif;
1244                 hwgroup->drive = drive;
1245                 drive->sleeping = 0;
1246                 drive->service_start = jiffies;
1247
1248                 if (blk_queue_plugged(drive->queue)) {
1249                         printk(KERN_ERR "ide: huh? queue was plugged!\n");
1250                         break;
1251                 }
1252
1253                 /*
1254                  * we know that the queue isn't empty, but this can happen
1255                  * if the q->prep_rq_fn() decides to kill a request
1256                  */
1257                 rq = elv_next_request(drive->queue);
1258                 if (!rq) {
1259                         hwgroup->busy = 0;
1260                         break;
1261                 }
1262
1263                 /*
1264                  * Sanity: don't accept a request that isn't a PM request
1265                  * if we are currently power managed. This is very important as
1266                  * blk_stop_queue() doesn't prevent the elv_next_request()
1267                  * above to return us whatever is in the queue. Since we call
1268                  * ide_do_request() ourselves, we end up taking requests while
1269                  * the queue is blocked...
1270                  * 
1271                  * We let requests forced at head of queue with ide-preempt
1272                  * though. I hope that doesn't happen too much, hopefully not
1273                  * unless the subdriver triggers such a thing in its own PM
1274                  * state machine.
1275                  *
1276                  * We count how many times we loop here to make sure we service
1277                  * all drives in the hwgroup without looping for ever
1278                  */
1279                 if (drive->blocked && !blk_pm_request(rq) && !(rq->cmd_flags & REQ_PREEMPT)) {
1280                         drive = drive->next ? drive->next : hwgroup->drive;
1281                         if (loops++ < 4 && !blk_queue_plugged(drive->queue))
1282                                 goto again;
1283                         /* We clear busy, there should be no pending ATA command at this point. */
1284                         hwgroup->busy = 0;
1285                         break;
1286                 }
1287
1288                 hwgroup->rq = rq;
1289
1290                 /*
1291                  * Some systems have trouble with IDE IRQs arriving while
1292                  * the driver is still setting things up.  So, here we disable
1293                  * the IRQ used by this interface while the request is being started.
1294                  * This may look bad at first, but pretty much the same thing
1295                  * happens anyway when any interrupt comes in, IDE or otherwise
1296                  *  -- the kernel masks the IRQ while it is being handled.
1297                  */
1298                 if (masked_irq != IDE_NO_IRQ && hwif->irq != masked_irq)
1299                         disable_irq_nosync(hwif->irq);
1300                 spin_unlock(&ide_lock);
1301                 local_irq_enable_in_hardirq();
1302                         /* allow other IRQs while we start this request */
1303                 startstop = start_request(drive, rq);
1304                 spin_lock_irq(&ide_lock);
1305                 if (masked_irq != IDE_NO_IRQ && hwif->irq != masked_irq)
1306                         enable_irq(hwif->irq);
1307                 if (startstop == ide_stopped)
1308                         hwgroup->busy = 0;
1309         }
1310 }
1311
1312 /*
1313  * Passes the stuff to ide_do_request
1314  */
1315 void do_ide_request(request_queue_t *q)
1316 {
1317         ide_drive_t *drive = q->queuedata;
1318
1319         ide_do_request(HWGROUP(drive), IDE_NO_IRQ);
1320 }
1321
1322 /*
1323  * un-busy the hwgroup etc, and clear any pending DMA status. we want to
1324  * retry the current request in pio mode instead of risking tossing it
1325  * all away
1326  */
1327 static ide_startstop_t ide_dma_timeout_retry(ide_drive_t *drive, int error)
1328 {
1329         ide_hwif_t *hwif = HWIF(drive);
1330         struct request *rq;
1331         ide_startstop_t ret = ide_stopped;
1332
1333         /*
1334          * end current dma transaction
1335          */
1336
1337         if (error < 0) {
1338                 printk(KERN_WARNING "%s: DMA timeout error\n", drive->name);
1339                 (void)HWIF(drive)->ide_dma_end(drive);
1340                 ret = ide_error(drive, "dma timeout error",
1341                                                 hwif->INB(IDE_STATUS_REG));
1342         } else {
1343                 printk(KERN_WARNING "%s: DMA timeout retry\n", drive->name);
1344                 (void) hwif->ide_dma_timeout(drive);
1345         }
1346
1347         /*
1348          * disable dma for now, but remember that we did so because of
1349          * a timeout -- we'll reenable after we finish this next request
1350          * (or rather the first chunk of it) in pio.
1351          */
1352         drive->retry_pio++;
1353         drive->state = DMA_PIO_RETRY;
1354         (void) hwif->ide_dma_off_quietly(drive);
1355
1356         /*
1357          * un-busy drive etc (hwgroup->busy is cleared on return) and
1358          * make sure request is sane
1359          */
1360         rq = HWGROUP(drive)->rq;
1361
1362         if (!rq)
1363                 goto out;
1364
1365         HWGROUP(drive)->rq = NULL;
1366
1367         rq->errors = 0;
1368
1369         if (!rq->bio)
1370                 goto out;
1371
1372         rq->sector = rq->bio->bi_sector;
1373         rq->current_nr_sectors = bio_iovec(rq->bio)->bv_len >> 9;
1374         rq->hard_cur_sectors = rq->current_nr_sectors;
1375         rq->buffer = bio_data(rq->bio);
1376 out:
1377         return ret;
1378 }
1379
1380 /**
1381  *      ide_timer_expiry        -       handle lack of an IDE interrupt
1382  *      @data: timer callback magic (hwgroup)
1383  *
1384  *      An IDE command has timed out before the expected drive return
1385  *      occurred. At this point we attempt to clean up the current
1386  *      mess. If the current handler includes an expiry handler then
1387  *      we invoke the expiry handler, and providing it is happy the
1388  *      work is done. If that fails we apply generic recovery rules
1389  *      invoking the handler and checking the drive DMA status. We
1390  *      have an excessively incestuous relationship with the DMA
1391  *      logic that wants cleaning up.
1392  */
1393  
1394 void ide_timer_expiry (unsigned long data)
1395 {
1396         ide_hwgroup_t   *hwgroup = (ide_hwgroup_t *) data;
1397         ide_handler_t   *handler;
1398         ide_expiry_t    *expiry;
1399         unsigned long   flags;
1400         unsigned long   wait = -1;
1401
1402         spin_lock_irqsave(&ide_lock, flags);
1403
1404         if ((handler = hwgroup->handler) == NULL) {
1405                 /*
1406                  * Either a marginal timeout occurred
1407                  * (got the interrupt just as timer expired),
1408                  * or we were "sleeping" to give other devices a chance.
1409                  * Either way, we don't really want to complain about anything.
1410                  */
1411                 if (hwgroup->sleeping) {
1412                         hwgroup->sleeping = 0;
1413                         hwgroup->busy = 0;
1414                 }
1415         } else {
1416                 ide_drive_t *drive = hwgroup->drive;
1417                 if (!drive) {
1418                         printk(KERN_ERR "ide_timer_expiry: hwgroup->drive was NULL\n");
1419                         hwgroup->handler = NULL;
1420                 } else {
1421                         ide_hwif_t *hwif;
1422                         ide_startstop_t startstop = ide_stopped;
1423                         if (!hwgroup->busy) {
1424                                 hwgroup->busy = 1;      /* paranoia */
1425                                 printk(KERN_ERR "%s: ide_timer_expiry: hwgroup->busy was 0 ??\n", drive->name);
1426                         }
1427                         if ((expiry = hwgroup->expiry) != NULL) {
1428                                 /* continue */
1429                                 if ((wait = expiry(drive)) > 0) {
1430                                         /* reset timer */
1431                                         hwgroup->timer.expires  = jiffies + wait;
1432                                         add_timer(&hwgroup->timer);
1433                                         spin_unlock_irqrestore(&ide_lock, flags);
1434                                         return;
1435                                 }
1436                         }
1437                         hwgroup->handler = NULL;
1438                         /*
1439                          * We need to simulate a real interrupt when invoking
1440                          * the handler() function, which means we need to
1441                          * globally mask the specific IRQ:
1442                          */
1443                         spin_unlock(&ide_lock);
1444                         hwif  = HWIF(drive);
1445 #if DISABLE_IRQ_NOSYNC
1446                         disable_irq_nosync(hwif->irq);
1447 #else
1448                         /* disable_irq_nosync ?? */
1449                         disable_irq(hwif->irq);
1450 #endif /* DISABLE_IRQ_NOSYNC */
1451                         /* local CPU only,
1452                          * as if we were handling an interrupt */
1453                         local_irq_disable();
1454                         if (hwgroup->polling) {
1455                                 startstop = handler(drive);
1456                         } else if (drive_is_ready(drive)) {
1457                                 if (drive->waiting_for_dma)
1458                                         (void) hwgroup->hwif->ide_dma_lostirq(drive);
1459                                 (void)ide_ack_intr(hwif);
1460                                 printk(KERN_WARNING "%s: lost interrupt\n", drive->name);
1461                                 startstop = handler(drive);
1462                         } else {
1463                                 if (drive->waiting_for_dma) {
1464                                         startstop = ide_dma_timeout_retry(drive, wait);
1465                                 } else
1466                                         startstop =
1467                                         ide_error(drive, "irq timeout", hwif->INB(IDE_STATUS_REG));
1468                         }
1469                         drive->service_time = jiffies - drive->service_start;
1470                         spin_lock_irq(&ide_lock);
1471                         enable_irq(hwif->irq);
1472                         if (startstop == ide_stopped)
1473                                 hwgroup->busy = 0;
1474                 }
1475         }
1476         ide_do_request(hwgroup, IDE_NO_IRQ);
1477         spin_unlock_irqrestore(&ide_lock, flags);
1478 }
1479
1480 /**
1481  *      unexpected_intr         -       handle an unexpected IDE interrupt
1482  *      @irq: interrupt line
1483  *      @hwgroup: hwgroup being processed
1484  *
1485  *      There's nothing really useful we can do with an unexpected interrupt,
1486  *      other than reading the status register (to clear it), and logging it.
1487  *      There should be no way that an irq can happen before we're ready for it,
1488  *      so we needn't worry much about losing an "important" interrupt here.
1489  *
1490  *      On laptops (and "green" PCs), an unexpected interrupt occurs whenever
1491  *      the drive enters "idle", "standby", or "sleep" mode, so if the status
1492  *      looks "good", we just ignore the interrupt completely.
1493  *
1494  *      This routine assumes __cli() is in effect when called.
1495  *
1496  *      If an unexpected interrupt happens on irq15 while we are handling irq14
1497  *      and if the two interfaces are "serialized" (CMD640), then it looks like
1498  *      we could screw up by interfering with a new request being set up for 
1499  *      irq15.
1500  *
1501  *      In reality, this is a non-issue.  The new command is not sent unless 
1502  *      the drive is ready to accept one, in which case we know the drive is
1503  *      not trying to interrupt us.  And ide_set_handler() is always invoked
1504  *      before completing the issuance of any new drive command, so we will not
1505  *      be accidentally invoked as a result of any valid command completion
1506  *      interrupt.
1507  *
1508  *      Note that we must walk the entire hwgroup here. We know which hwif
1509  *      is doing the current command, but we don't know which hwif burped
1510  *      mysteriously.
1511  */
1512  
1513 static void unexpected_intr (int irq, ide_hwgroup_t *hwgroup)
1514 {
1515         u8 stat;
1516         ide_hwif_t *hwif = hwgroup->hwif;
1517
1518         /*
1519          * handle the unexpected interrupt
1520          */
1521         do {
1522                 if (hwif->irq == irq) {
1523                         stat = hwif->INB(hwif->io_ports[IDE_STATUS_OFFSET]);
1524                         if (!OK_STAT(stat, READY_STAT, BAD_STAT)) {
1525                                 /* Try to not flood the console with msgs */
1526                                 static unsigned long last_msgtime, count;
1527                                 ++count;
1528                                 if (time_after(jiffies, last_msgtime + HZ)) {
1529                                         last_msgtime = jiffies;
1530                                         printk(KERN_ERR "%s%s: unexpected interrupt, "
1531                                                 "status=0x%02x, count=%ld\n",
1532                                                 hwif->name,
1533                                                 (hwif->next==hwgroup->hwif) ? "" : "(?)", stat, count);
1534                                 }
1535                         }
1536                 }
1537         } while ((hwif = hwif->next) != hwgroup->hwif);
1538 }
1539
1540 /**
1541  *      ide_intr        -       default IDE interrupt handler
1542  *      @irq: interrupt number
1543  *      @dev_id: hwif group
1544  *      @regs: unused weirdness from the kernel irq layer
1545  *
1546  *      This is the default IRQ handler for the IDE layer. You should
1547  *      not need to override it. If you do be aware it is subtle in
1548  *      places
1549  *
1550  *      hwgroup->hwif is the interface in the group currently performing
1551  *      a command. hwgroup->drive is the drive and hwgroup->handler is
1552  *      the IRQ handler to call. As we issue a command the handlers
1553  *      step through multiple states, reassigning the handler to the
1554  *      next step in the process. Unlike a smart SCSI controller IDE
1555  *      expects the main processor to sequence the various transfer
1556  *      stages. We also manage a poll timer to catch up with most
1557  *      timeout situations. There are still a few where the handlers
1558  *      don't ever decide to give up.
1559  *
1560  *      The handler eventually returns ide_stopped to indicate the
1561  *      request completed. At this point we issue the next request
1562  *      on the hwgroup and the process begins again.
1563  */
1564  
1565 irqreturn_t ide_intr (int irq, void *dev_id)
1566 {
1567         unsigned long flags;
1568         ide_hwgroup_t *hwgroup = (ide_hwgroup_t *)dev_id;
1569         ide_hwif_t *hwif;
1570         ide_drive_t *drive;
1571         ide_handler_t *handler;
1572         ide_startstop_t startstop;
1573
1574         spin_lock_irqsave(&ide_lock, flags);
1575         hwif = hwgroup->hwif;
1576
1577         if (!ide_ack_intr(hwif)) {
1578                 spin_unlock_irqrestore(&ide_lock, flags);
1579                 return IRQ_NONE;
1580         }
1581
1582         if ((handler = hwgroup->handler) == NULL || hwgroup->polling) {
1583                 /*
1584                  * Not expecting an interrupt from this drive.
1585                  * That means this could be:
1586                  *      (1) an interrupt from another PCI device
1587                  *      sharing the same PCI INT# as us.
1588                  * or   (2) a drive just entered sleep or standby mode,
1589                  *      and is interrupting to let us know.
1590                  * or   (3) a spurious interrupt of unknown origin.
1591                  *
1592                  * For PCI, we cannot tell the difference,
1593                  * so in that case we just ignore it and hope it goes away.
1594                  *
1595                  * FIXME: unexpected_intr should be hwif-> then we can
1596                  * remove all the ifdef PCI crap
1597                  */
1598 #ifdef CONFIG_BLK_DEV_IDEPCI
1599                 if (hwif->pci_dev && !hwif->pci_dev->vendor)
1600 #endif  /* CONFIG_BLK_DEV_IDEPCI */
1601                 {
1602                         /*
1603                          * Probably not a shared PCI interrupt,
1604                          * so we can safely try to do something about it:
1605                          */
1606                         unexpected_intr(irq, hwgroup);
1607 #ifdef CONFIG_BLK_DEV_IDEPCI
1608                 } else {
1609                         /*
1610                          * Whack the status register, just in case
1611                          * we have a leftover pending IRQ.
1612                          */
1613                         (void) hwif->INB(hwif->io_ports[IDE_STATUS_OFFSET]);
1614 #endif /* CONFIG_BLK_DEV_IDEPCI */
1615                 }
1616                 spin_unlock_irqrestore(&ide_lock, flags);
1617                 return IRQ_NONE;
1618         }
1619         drive = hwgroup->drive;
1620         if (!drive) {
1621                 /*
1622                  * This should NEVER happen, and there isn't much
1623                  * we could do about it here.
1624                  *
1625                  * [Note - this can occur if the drive is hot unplugged]
1626                  */
1627                 spin_unlock_irqrestore(&ide_lock, flags);
1628                 return IRQ_HANDLED;
1629         }
1630         if (!drive_is_ready(drive)) {
1631                 /*
1632                  * This happens regularly when we share a PCI IRQ with
1633                  * another device.  Unfortunately, it can also happen
1634                  * with some buggy drives that trigger the IRQ before
1635                  * their status register is up to date.  Hopefully we have
1636                  * enough advance overhead that the latter isn't a problem.
1637                  */
1638                 spin_unlock_irqrestore(&ide_lock, flags);
1639                 return IRQ_NONE;
1640         }
1641         if (!hwgroup->busy) {
1642                 hwgroup->busy = 1;      /* paranoia */
1643                 printk(KERN_ERR "%s: ide_intr: hwgroup->busy was 0 ??\n", drive->name);
1644         }
1645         hwgroup->handler = NULL;
1646         del_timer(&hwgroup->timer);
1647         spin_unlock(&ide_lock);
1648
1649         if (drive->unmask)
1650                 local_irq_enable_in_hardirq();
1651         /* service this interrupt, may set handler for next interrupt */
1652         startstop = handler(drive);
1653         spin_lock_irq(&ide_lock);
1654
1655         /*
1656          * Note that handler() may have set things up for another
1657          * interrupt to occur soon, but it cannot happen until
1658          * we exit from this routine, because it will be the
1659          * same irq as is currently being serviced here, and Linux
1660          * won't allow another of the same (on any CPU) until we return.
1661          */
1662         drive->service_time = jiffies - drive->service_start;
1663         if (startstop == ide_stopped) {
1664                 if (hwgroup->handler == NULL) { /* paranoia */
1665                         hwgroup->busy = 0;
1666                         ide_do_request(hwgroup, hwif->irq);
1667                 } else {
1668                         printk(KERN_ERR "%s: ide_intr: huh? expected NULL handler "
1669                                 "on exit\n", drive->name);
1670                 }
1671         }
1672         spin_unlock_irqrestore(&ide_lock, flags);
1673         return IRQ_HANDLED;
1674 }
1675
1676 /**
1677  *      ide_init_drive_cmd      -       initialize a drive command request
1678  *      @rq: request object
1679  *
1680  *      Initialize a request before we fill it in and send it down to
1681  *      ide_do_drive_cmd. Commands must be set up by this function. Right
1682  *      now it doesn't do a lot, but if that changes abusers will have a
1683  *      nasty surprise.
1684  */
1685
1686 void ide_init_drive_cmd (struct request *rq)
1687 {
1688         memset(rq, 0, sizeof(*rq));
1689         rq->cmd_type = REQ_TYPE_ATA_CMD;
1690         rq->ref_count = 1;
1691 }
1692
1693 EXPORT_SYMBOL(ide_init_drive_cmd);
1694
1695 /**
1696  *      ide_do_drive_cmd        -       issue IDE special command
1697  *      @drive: device to issue command
1698  *      @rq: request to issue
1699  *      @action: action for processing
1700  *
1701  *      This function issues a special IDE device request
1702  *      onto the request queue.
1703  *
1704  *      If action is ide_wait, then the rq is queued at the end of the
1705  *      request queue, and the function sleeps until it has been processed.
1706  *      This is for use when invoked from an ioctl handler.
1707  *
1708  *      If action is ide_preempt, then the rq is queued at the head of
1709  *      the request queue, displacing the currently-being-processed
1710  *      request and this function returns immediately without waiting
1711  *      for the new rq to be completed.  This is VERY DANGEROUS, and is
1712  *      intended for careful use by the ATAPI tape/cdrom driver code.
1713  *
1714  *      If action is ide_end, then the rq is queued at the end of the
1715  *      request queue, and the function returns immediately without waiting
1716  *      for the new rq to be completed. This is again intended for careful
1717  *      use by the ATAPI tape/cdrom driver code.
1718  */
1719  
1720 int ide_do_drive_cmd (ide_drive_t *drive, struct request *rq, ide_action_t action)
1721 {
1722         unsigned long flags;
1723         ide_hwgroup_t *hwgroup = HWGROUP(drive);
1724         DECLARE_COMPLETION_ONSTACK(wait);
1725         int where = ELEVATOR_INSERT_BACK, err;
1726         int must_wait = (action == ide_wait || action == ide_head_wait);
1727
1728         rq->errors = 0;
1729
1730         /*
1731          * we need to hold an extra reference to request for safe inspection
1732          * after completion
1733          */
1734         if (must_wait) {
1735                 rq->ref_count++;
1736                 rq->end_io_data = &wait;
1737                 rq->end_io = blk_end_sync_rq;
1738         }
1739
1740         spin_lock_irqsave(&ide_lock, flags);
1741         if (action == ide_preempt)
1742                 hwgroup->rq = NULL;
1743         if (action == ide_preempt || action == ide_head_wait) {
1744                 where = ELEVATOR_INSERT_FRONT;
1745                 rq->cmd_flags |= REQ_PREEMPT;
1746         }
1747         __elv_add_request(drive->queue, rq, where, 0);
1748         ide_do_request(hwgroup, IDE_NO_IRQ);
1749         spin_unlock_irqrestore(&ide_lock, flags);
1750
1751         err = 0;
1752         if (must_wait) {
1753                 wait_for_completion(&wait);
1754                 if (rq->errors)
1755                         err = -EIO;
1756
1757                 blk_put_request(rq);
1758         }
1759
1760         return err;
1761 }
1762
1763 EXPORT_SYMBOL(ide_do_drive_cmd);